diff --git a/drivers/crypto/rockchip/Kconfig b/drivers/crypto/rockchip/Kconfig index 79c04476ca89..21baebbce721 100644 --- a/drivers/crypto/rockchip/Kconfig +++ b/drivers/crypto/rockchip/Kconfig @@ -1,4 +1,16 @@ # SPDX-License-Identifier: GPL-2.0-only +if CRYPTO_DEV_ROCKCHIP + +config CRYPTO_DEV_ROCKCHIP_V1 + bool "crypto v1 for RV1108 RK3288 RK3368 RK3399" + default y if CPU_RV1108 || CPU_RK3288 || CPU_RK3368 || CPU_RK3399 + +config CRYPTO_DEV_ROCKCHIP_V2 + bool "crypto v2 for RV1109/RV1126 RK1808 RK3308 PX30/RK3326 RK356X RK3588" + default y if CPU_RV1126 || CPU_RK1808 || CPU_RK3308 || CPU_PX30 || CPU_RK3568 || CPU_RK3588 + +endif + config CRYPTO_DEV_ROCKCHIP_DEV tristate "Export rockchip crypto device for user space" depends on CRYPTO_DEV_ROCKCHIP diff --git a/drivers/crypto/rockchip/Makefile b/drivers/crypto/rockchip/Makefile index 886369eb5543..95da40fbe895 100644 --- a/drivers/crypto/rockchip/Makefile +++ b/drivers/crypto/rockchip/Makefile @@ -1,12 +1,21 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rk_crypto.o rk_crypto-objs := rk_crypto_core.o \ - rk_crypto_v1_ahash.o \ - rk_crypto_v1_skcipher.o \ - rk_crypto_v2_skcipher.o \ - rk_crypto_v2_ahash.o \ - rk_crypto_v2_akcipher.o \ - rk_crypto_v2_pka.o \ - rk_crypto_bignum.o + rk_crypto_utils.o \ + rk_crypto_ahash_utils.o \ + rk_crypto_skcipher_utils.o + +rk_crypto-$(CONFIG_CRYPTO_DEV_ROCKCHIP_V1) += \ + rk_crypto_v1.o \ + rk_crypto_v1_ahash.o \ + rk_crypto_v1_skcipher.o + +rk_crypto-$(CONFIG_CRYPTO_DEV_ROCKCHIP_V2) += \ + rk_crypto_v2.o \ + rk_crypto_v2_ahash.o \ + rk_crypto_v2_skcipher.o \ + rk_crypto_v2_akcipher.o \ + rk_crypto_v2_pka.o \ + rk_crypto_bignum.o obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP_DEV) += cryptodev_linux/ diff --git a/drivers/crypto/rockchip/rk_crypto_ahash_utils.c b/drivers/crypto/rockchip/rk_crypto_ahash_utils.c new file mode 100644 index 000000000000..a8e47a6d9711 --- /dev/null +++ b/drivers/crypto/rockchip/rk_crypto_ahash_utils.c @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Rockchip crypto hash uitls + * + * Copyright (c) 2022, Rockchip Electronics Co., Ltd + * + * Author: Lin Jinhan + * + */ + +#include "rk_crypto_core.h" +#include "rk_crypto_ahash_utils.h" + +static const char * const hash_algo2name[] = { + [HASH_ALGO_MD5] = "md5", + [HASH_ALGO_SHA1] = "sha1", + [HASH_ALGO_SHA224] = "sha224", + [HASH_ALGO_SHA256] = "sha256", + [HASH_ALGO_SHA384] = "sha384", + [HASH_ALGO_SHA512] = "sha512", + [HASH_ALGO_SM3] = "sm3", +}; + +static void rk_alg_ctx_clear(struct rk_alg_ctx *alg_ctx) +{ + alg_ctx->total = 0; + alg_ctx->left_bytes = 0; + alg_ctx->count = 0; + alg_ctx->sg_src = 0; + alg_ctx->req_src = 0; + alg_ctx->src_nents = 0; +} + +static void rk_ahash_ctx_clear(struct rk_ahash_ctx *ctx) +{ + rk_alg_ctx_clear(&ctx->algs_ctx); + + memset(ctx->hash_tmp, 0x00, RK_DMA_ALIGNMENT); + memset(ctx->lastc, 0x00, sizeof(ctx->lastc)); + + ctx->hash_tmp_len = 0; + ctx->calc_cnt = 0; + ctx->lastc_len = 0; +} + +struct rk_ahash_ctx *rk_ahash_ctx_cast(struct rk_crypto_dev *rk_dev) +{ + struct ahash_request *req = ahash_request_cast(rk_dev->async_req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + + return crypto_ahash_ctx(tfm); +} + +struct rk_alg_ctx *rk_ahash_alg_ctx(struct rk_crypto_dev *rk_dev) +{ + return &(rk_ahash_ctx_cast(rk_dev))->algs_ctx; +} + +struct rk_crypto_algt *rk_ahash_get_algt(struct crypto_ahash *tfm) +{ + struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); + + return container_of(alg, struct rk_crypto_algt, alg.hash); +} + +static int rk_ahash_set_data_start(struct rk_crypto_dev *rk_dev, uint32_t flag) +{ + int err; + struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(rk_dev); + + CRYPTO_TRACE(); + + err = rk_dev->load_data(rk_dev, alg_ctx->sg_src, alg_ctx->sg_dst); + if (!err) + err = alg_ctx->ops.hw_dma_start(rk_dev, flag); + + return err; +} + +static u32 rk_calc_lastc_new_len(u32 nbytes, u32 old_len) +{ + u32 total_len = nbytes + old_len; + + if (total_len <= RK_DMA_ALIGNMENT) + return nbytes; + + if (total_len % RK_DMA_ALIGNMENT) + return total_len % RK_DMA_ALIGNMENT; + + return RK_DMA_ALIGNMENT; +} + +static int rk_ahash_fallback_digest(const char *alg_name, bool is_hmac, + const u8 *key, u32 key_len, + const u8 *msg, u32 msg_len, + u8 *digest) +{ + struct crypto_ahash *ahash_tfm; + struct ahash_request *req; + struct crypto_wait wait; + struct scatterlist sg; + int ret; + + CRYPTO_TRACE("%s, is_hmac = %d, key_len = %u, msg_len = %u", + alg_name, is_hmac, key_len, msg_len); + + ahash_tfm = crypto_alloc_ahash(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ahash_tfm)) + return PTR_ERR(ahash_tfm); + + req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); + if (!req) { + crypto_free_ahash(ahash_tfm); + return -ENOMEM; + } + + init_completion(&wait.completion); + + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + + crypto_ahash_clear_flags(ahash_tfm, ~0); + + sg_init_one(&sg, msg, msg_len); + ahash_request_set_crypt(req, &sg, digest, msg_len); + + if (is_hmac) + crypto_ahash_setkey(ahash_tfm, key, key_len); + + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); + if (ret) { + CRYPTO_MSG("digest failed, ret = %d", ret); + goto exit; + } + +exit: + ahash_request_free(req); + crypto_free_ahash(ahash_tfm); + + return ret; +} + +static int rk_ahash_get_zero_result(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_crypto_algt *algt = rk_ahash_get_algt(tfm); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + return rk_ahash_fallback_digest(crypto_ahash_alg_name(tfm), + algt->type == ALG_TYPE_HMAC, + ctx->authkey, ctx->authkey_len, + NULL, 0, req->result); +} + +int rk_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) +{ + unsigned int blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + struct rk_crypto_algt *algt = rk_ahash_get_algt(tfm); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + const char *alg_name; + int ret = 0; + + CRYPTO_MSG(); + + if (algt->algo >= ARRAY_SIZE(hash_algo2name)) { + CRYPTO_MSG("hash algo %d invalid\n", algt->algo); + return -EINVAL; + } + + memset(ctx->authkey, 0, sizeof(ctx->authkey)); + + if (keylen <= blocksize) { + memcpy(ctx->authkey, key, keylen); + ctx->authkey_len = keylen; + goto exit; + } + + alg_name = hash_algo2name[algt->algo]; + + CRYPTO_TRACE("calc key digest %s", alg_name); + + ret = rk_ahash_fallback_digest(alg_name, false, NULL, 0, key, keylen, + ctx->authkey); + if (ret) { + CRYPTO_MSG("rk_ahash_fallback_digest error ret = %d\n", ret); + goto exit; + } + + ctx->authkey_len = crypto_ahash_digestsize(tfm); +exit: + return ret; +} + +int rk_ahash_init(struct ahash_request *req) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + CRYPTO_TRACE(); + + memset(rctx, 0x00, sizeof(*rctx)); + rk_ahash_ctx_clear(ctx); + + return 0; +} + +int rk_ahash_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct rk_crypto_dev *rk_dev = ctx->rk_dev; + + CRYPTO_TRACE("nbytes = %u", req->nbytes); + + memset(rctx, 0x00, sizeof(*rctx)); + + rctx->flag = RK_FLAG_UPDATE; + + return rk_dev->enqueue(rk_dev, &req->base); +} + +int rk_ahash_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct rk_crypto_dev *rk_dev = ctx->rk_dev; + + CRYPTO_TRACE(); + + memset(rctx, 0x00, sizeof(*rctx)); + + rctx->flag = RK_FLAG_FINAL; + + /* use fallback hash */ + if (ctx->calc_cnt == 0 && + ctx->hash_tmp_len == 0 && + ctx->lastc_len == 0) { + CRYPTO_TRACE("use fallback hash"); + return rk_ahash_get_zero_result(req); + } + + return rk_dev->enqueue(rk_dev, &req->base); +} + +int rk_ahash_finup(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct rk_crypto_dev *rk_dev = ctx->rk_dev; + + CRYPTO_TRACE("nbytes = %u", req->nbytes); + + memset(rctx, 0x00, sizeof(*rctx)); + + rctx->flag = RK_FLAG_UPDATE | RK_FLAG_FINAL; + + /* use fallback hash */ + if (req->nbytes == 0 && + ctx->calc_cnt == 0 && + ctx->hash_tmp_len == 0 && + ctx->lastc_len == 0) { + CRYPTO_TRACE("use fallback hash"); + return rk_ahash_get_zero_result(req); + } + + return rk_dev->enqueue(rk_dev, &req->base); +} + +int rk_ahash_digest(struct ahash_request *req) +{ + CRYPTO_TRACE("calc data %u bytes.", req->nbytes); + + return rk_ahash_init(req) ?: rk_ahash_finup(req); +} + +int rk_ahash_start(struct rk_crypto_dev *rk_dev) +{ + struct ahash_request *req = ahash_request_cast(rk_dev->async_req); + struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(rk_dev); + struct rk_ahash_ctx *ctx = rk_ahash_ctx_cast(rk_dev); + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_crypto_algt *algt = rk_ahash_get_algt(tfm); + struct scatterlist *src_sg; + unsigned long flags; + unsigned int nbytes; + int ret = 0; + + CRYPTO_TRACE("origin: old_len = %u, new_len = %u, nbytes = %u, flag = %d", + ctx->hash_tmp_len, ctx->lastc_len, req->nbytes, rctx->flag); + + /* update 0Byte do nothing */ + if (req->nbytes == 0 && !(rctx->flag & RK_FLAG_FINAL)) + goto no_calc; + + if (ctx->lastc_len) { + /* move lastc saved last time to the head of this calculation */ + memcpy(ctx->hash_tmp + ctx->hash_tmp_len, ctx->lastc, ctx->lastc_len); + ctx->hash_tmp_len = ctx->hash_tmp_len + ctx->lastc_len; + ctx->lastc_len = 0; + } + + CRYPTO_TRACE("hash_tmp_len = %u", ctx->hash_tmp_len); + + /* final request no need to save lastc_new */ + if ((rctx->flag & RK_FLAG_UPDATE) && (rctx->flag & RK_FLAG_FINAL)) { + nbytes = req->nbytes + ctx->hash_tmp_len; + + CRYPTO_TRACE("finup %u bytes", nbytes); + } else if (rctx->flag & RK_FLAG_UPDATE) { + ctx->lastc_len = rk_calc_lastc_new_len(req->nbytes, ctx->hash_tmp_len); + + CRYPTO_TRACE("nents = %u, ctx->lastc_len = %u, offset = %u", + sg_nents_for_len(req->src, req->nbytes), ctx->lastc_len, + req->nbytes - ctx->lastc_len); + + if (!sg_pcopy_to_buffer(req->src, sg_nents_for_len(req->src, req->nbytes), + ctx->lastc, ctx->lastc_len, req->nbytes - ctx->lastc_len)) { + ret = -EINVAL; + goto exit; + } + + nbytes = ctx->hash_tmp_len + req->nbytes - ctx->lastc_len; + + /* not enough data */ + if (nbytes < RK_DMA_ALIGNMENT) { + CRYPTO_TRACE("nbytes = %u, not enough data", nbytes); + memcpy(ctx->hash_tmp + ctx->hash_tmp_len, + ctx->lastc, ctx->lastc_len); + ctx->hash_tmp_len = ctx->hash_tmp_len + ctx->lastc_len; + ctx->lastc_len = 0; + goto no_calc; + } + + CRYPTO_TRACE("update nbytes = %u", nbytes); + } else { + /* final just calc lastc_old */ + nbytes = ctx->hash_tmp_len; + + CRYPTO_TRACE("final nbytes = %u", nbytes); + } + + if (ctx->hash_tmp_len) { + /* Concatenate old data to the header */ + sg_init_table(ctx->hash_sg, ARRAY_SIZE(ctx->hash_sg)); + sg_set_buf(ctx->hash_sg, ctx->hash_tmp, ctx->hash_tmp_len); + sg_chain(ctx->hash_sg, ARRAY_SIZE(ctx->hash_sg), req->src); + + src_sg = &ctx->hash_sg[0]; + ctx->hash_tmp_len = 0; + } else { + src_sg = req->src; + } + + alg_ctx->total = nbytes; + alg_ctx->left_bytes = nbytes; + alg_ctx->sg_src = src_sg; + alg_ctx->req_src = src_sg; + alg_ctx->src_nents = sg_nents_for_len(src_sg, nbytes); + + CRYPTO_TRACE("adjust: old_len = %u, new_len = %u, nbytes = %u", + ctx->hash_tmp_len, ctx->lastc_len, nbytes); + + if (nbytes) { + spin_lock_irqsave(&rk_dev->lock, flags); + if (ctx->calc_cnt == 0) + alg_ctx->ops.hw_init(rk_dev, algt->algo, algt->type); + + /* flush all 64byte key buffer for hmac */ + alg_ctx->ops.hw_write_key(ctx->rk_dev, ctx->authkey, sizeof(ctx->authkey)); + ret = rk_ahash_set_data_start(rk_dev, rctx->flag); + spin_unlock_irqrestore(&rk_dev->lock, flags); + } +exit: + return ret; +no_calc: + CRYPTO_TRACE("no calc"); + rk_alg_ctx_clear(alg_ctx); + + return 0; +} + +int rk_ahash_crypto_rx(struct rk_crypto_dev *rk_dev) +{ + int err = 0; + struct ahash_request *req = ahash_request_cast(rk_dev->async_req); + struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(rk_dev); + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct rk_ahash_ctx *ctx = rk_ahash_ctx_cast(rk_dev); + + CRYPTO_TRACE("left bytes = %u, flag = %d", alg_ctx->left_bytes, rctx->flag); + + err = rk_dev->unload_data(rk_dev); + if (err) + goto out_rx; + + ctx->calc_cnt += alg_ctx->count; + + if (alg_ctx->left_bytes) { + if (alg_ctx->aligned) { + if (sg_is_last(alg_ctx->sg_src)) { + dev_warn(rk_dev->dev, "[%s:%d], Lack of data\n", + __func__, __LINE__); + err = -ENOMEM; + goto out_rx; + } + alg_ctx->sg_src = sg_next(alg_ctx->sg_src); + } + err = rk_ahash_set_data_start(rk_dev, rctx->flag); + } else { + /* + * it will take some time to process date after last dma + * transmission. + */ + struct crypto_ahash *tfm; + + /* only final will get result */ + if (!(rctx->flag & RK_FLAG_FINAL)) + goto out_rx; + + if (!req->result) { + err = -EINVAL; + goto out_rx; + } + + tfm = crypto_ahash_reqtfm(req); + + err = alg_ctx->ops.hw_get_result(rk_dev, req->result, + crypto_ahash_digestsize(tfm)); + } + +out_rx: + return err; +} diff --git a/drivers/crypto/rockchip/rk_crypto_ahash_utils.h b/drivers/crypto/rockchip/rk_crypto_ahash_utils.h new file mode 100644 index 000000000000..46afd98a0252 --- /dev/null +++ b/drivers/crypto/rockchip/rk_crypto_ahash_utils.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2022 Rockchip Electronics Co. Ltd. */ + +#ifndef __RK_CRYPTO_AHASH_UTILS_H__ +#define __RK_CRYPTO_AHASH_UTILS_H__ + +#include + +#include "rk_crypto_core.h" +#include "rk_crypto_utils.h" + +struct rk_alg_ctx *rk_ahash_alg_ctx(struct rk_crypto_dev *rk_dev); + +struct rk_crypto_algt *rk_ahash_get_algt(struct crypto_ahash *tfm); + +struct rk_ahash_ctx *rk_ahash_ctx_cast(struct rk_crypto_dev *rk_dev); + +int rk_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen); + +int rk_ahash_init(struct ahash_request *req); + +int rk_ahash_update(struct ahash_request *req); + +int rk_ahash_final(struct ahash_request *req); + +int rk_ahash_finup(struct ahash_request *req); + +int rk_ahash_digest(struct ahash_request *req); + +int rk_ahash_crypto_rx(struct rk_crypto_dev *rk_dev); + +int rk_ahash_start(struct rk_crypto_dev *rk_dev); + +#endif diff --git a/drivers/crypto/rockchip/rk_crypto_core.c b/drivers/crypto/rockchip/rk_crypto_core.c index 9a8addeb50b0..dc95df53feb3 100644 --- a/drivers/crypto/rockchip/rk_crypto_core.c +++ b/drivers/crypto/rockchip/rk_crypto_core.c @@ -24,36 +24,6 @@ #include "rk_crypto_v2.h" #include "cryptodev_linux/rk_cryptodev.h" -#define RK_CRYPTO_V1_SOC_DATA_INIT(names) {\ - .crypto_ver = "CRYPTO V1.0.0.0",\ - .use_soft_aes192 = false,\ - .valid_algs_name = (names),\ - .valid_algs_num = ARRAY_SIZE(names),\ - .total_algs = crypto_v1_algs,\ - .total_algs_num = ARRAY_SIZE(crypto_v1_algs),\ - .rsts = crypto_v1_rsts,\ - .rsts_num = ARRAY_SIZE(crypto_v1_rsts),\ - .hw_init = rk_hw_crypto_v1_init,\ - .hw_deinit = rk_hw_crypto_v1_deinit,\ - .hw_info_size = sizeof(struct rk_hw_crypto_v1_info),\ - .default_pka_offset = 0,\ -} - -#define RK_CRYPTO_V2_SOC_DATA_INIT(names, soft_aes_192) {\ - .crypto_ver = "CRYPTO V2.0.0.0",\ - .use_soft_aes192 = soft_aes_192,\ - .valid_algs_name = (names),\ - .valid_algs_num = ARRAY_SIZE(names),\ - .total_algs = crypto_v2_algs,\ - .total_algs_num = ARRAY_SIZE(crypto_v2_algs),\ - .rsts = crypto_v2_rsts,\ - .rsts_num = ARRAY_SIZE(crypto_v2_rsts),\ - .hw_init = rk_hw_crypto_v2_init,\ - .hw_deinit = rk_hw_crypto_v2_deinit,\ - .hw_info_size = sizeof(struct rk_hw_crypto_v2_info),\ - .default_pka_offset = 0x0480,\ -} - static struct rk_alg_ctx *rk_alg_ctx_cast(struct crypto_async_request *async_req) { struct rk_cipher_ctx *ctx = crypto_tfm_ctx(async_req->tfm); @@ -161,6 +131,8 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev, struct device *dev = rk_dev->dev; struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev->async_req); + alg_ctx->count = 0; + /* 0 data input just do nothing */ if (alg_ctx->total == 0) return 0; @@ -441,10 +413,13 @@ static struct rk_crypto_algt *rk_crypto_find_algs(struct rk_crypto_dev *rk_dev, u32 i; struct rk_crypto_algt **algs; struct rk_crypto_algt *tmp_algs; + uint32_t total_algs_num = 0; - algs = rk_dev->soc_data->total_algs; + algs = rk_dev->soc_data->hw_get_algts(&total_algs_num); + if (!algs || total_algs_num == 0) + return NULL; - for (i = 0; i < rk_dev->soc_data->total_algs_num; i++, algs++) { + for (i = 0; i < total_algs_num; i++, algs++) { tmp_algs = *algs; tmp_algs->rk_dev = rk_dev; @@ -513,6 +488,9 @@ err_cipher_algs: for (k = 0; k < i; k++, algs_name++) { tmp_algs = rk_crypto_find_algs(rk_dev, *algs_name); + if (!tmp_algs) + continue; + if (tmp_algs->type == ALG_TYPE_CIPHER) crypto_unregister_skcipher(&tmp_algs->alg.crypto); else if (tmp_algs->type == ALG_TYPE_HASH || tmp_algs->type == ALG_TYPE_HMAC) @@ -533,6 +511,9 @@ static void rk_crypto_unregister(struct rk_crypto_dev *rk_dev) for (i = 0; i < rk_dev->soc_data->valid_algs_num; i++, algs_name++) { tmp_algs = rk_crypto_find_algs(rk_dev, *algs_name); + if (!tmp_algs) + continue; + if (tmp_algs->type == ALG_TYPE_CIPHER) crypto_unregister_skcipher(&tmp_algs->alg.crypto); else if (tmp_algs->type == ALG_TYPE_HASH || tmp_algs->type == ALG_TYPE_HMAC) @@ -564,52 +545,6 @@ static void rk_crypto_action(void *data) reset_control_assert(rk_dev->rst); } -static const char * const crypto_v2_rsts[] = { - "crypto-rst", -}; - -static struct rk_crypto_algt *crypto_v2_algs[] = { - &rk_v2_ecb_sm4_alg, /* ecb(sm4) */ - &rk_v2_cbc_sm4_alg, /* cbc(sm4) */ - &rk_v2_xts_sm4_alg, /* xts(sm4) */ - &rk_v2_cfb_sm4_alg, /* cfb(sm4) */ - &rk_v2_ofb_sm4_alg, /* ofb(sm4) */ - &rk_v2_ctr_sm4_alg, /* ctr(sm4) */ - - &rk_v2_ecb_aes_alg, /* ecb(aes) */ - &rk_v2_cbc_aes_alg, /* cbc(aes) */ - &rk_v2_xts_aes_alg, /* xts(aes) */ - &rk_v2_cfb_aes_alg, /* cfb(aes) */ - &rk_v2_ofb_aes_alg, /* ofb(aes) */ - &rk_v2_ctr_aes_alg, /* ctr(aes) */ - - &rk_v2_ecb_des_alg, /* ecb(des) */ - &rk_v2_cbc_des_alg, /* cbc(des) */ - &rk_v2_cfb_des_alg, /* cfb(des) */ - &rk_v2_ofb_des_alg, /* ofb(des) */ - - &rk_v2_ecb_des3_ede_alg, /* ecb(des3_ede) */ - &rk_v2_cbc_des3_ede_alg, /* cbc(des3_ede) */ - &rk_v2_cfb_des3_ede_alg, /* cfb(des3_ede) */ - &rk_v2_ofb_des3_ede_alg, /* ofb(des3_ede) */ - - &rk_v2_ahash_sha1, /* sha1 */ - &rk_v2_ahash_sha224, /* sha224 */ - &rk_v2_ahash_sha256, /* sha256 */ - &rk_v2_ahash_sha384, /* sha384 */ - &rk_v2_ahash_sha512, /* sha512 */ - &rk_v2_ahash_md5, /* md5 */ - &rk_v2_ahash_sm3, /* sm3 */ - - &rk_v2_hmac_sha1, /* hmac(sha1) */ - &rk_v2_hmac_sha256, /* hmac(sha256) */ - &rk_v2_hmac_sha512, /* hmac(sha512) */ - &rk_v2_hmac_md5, /* hmac(md5) */ - &rk_v2_hmac_sm3, /* hmac(sm3) */ - - &rk_v2_asym_rsa, /* rsa */ -}; - static char *crypto_no_sm_algs_name[] = { "ecb(aes)", "cbc(aes)", "cfb(aes)", "ofb(aes)", "ctr(aes)", "ecb(des)", "cbc(des)", "cfb(des)", "ofb(des)", @@ -648,25 +583,6 @@ static const struct rk_crypto_soc_data rv1126_soc_data = static const struct rk_crypto_soc_data full_soc_data = RK_CRYPTO_V2_SOC_DATA_INIT(crypto_full_algs_name, false); -static const char * const crypto_v1_rsts[] = { - "crypto-rst", -}; - -static struct rk_crypto_algt *crypto_v1_algs[] = { - &rk_v1_ecb_aes_alg, /* ecb(aes) */ - &rk_v1_cbc_aes_alg, /* cbc(aes) */ - - &rk_v1_ecb_des_alg, /* ecb(des) */ - &rk_v1_cbc_des_alg, /* cbc(des) */ - - &rk_v1_ecb_des3_ede_alg, /* ecb(des3_ede) */ - &rk_v1_cbc_des3_ede_alg, /* cbc(des3_ede) */ - - &rk_v1_ahash_sha1, /* sha1 */ - &rk_v1_ahash_sha256, /* sha256 */ - &rk_v1_ahash_md5, /* md5 */ -}; - static char *rk3288_cipher_algs[] = { "ecb(aes)", "cbc(aes)", "ecb(des)", "cbc(des)", @@ -678,6 +594,8 @@ static const struct rk_crypto_soc_data rk3288_soc_data = RK_CRYPTO_V1_SOC_DATA_INIT(rk3288_cipher_algs); static const struct of_device_id crypto_of_id_table[] = { + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V2) /* crypto v2 in belows */ { .compatible = "rockchip,px30-crypto", @@ -695,11 +613,16 @@ static const struct of_device_id crypto_of_id_table[] = { .compatible = "rockchip,rk3588-crypto", .data = (void *)&full_soc_data, }, +#endif + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V1) /* crypto v1 in belows */ { .compatible = "rockchip,rk3288-crypto", .data = (void *)&rk3288_soc_data, }, +#endif + { /* sentinel */ } }; @@ -713,6 +636,8 @@ static int rk_crypto_probe(struct platform_device *pdev) struct rk_crypto_soc_data *soc_data; const struct of_device_id *match; struct rk_crypto_dev *rk_dev; + const char * const *rsts; + uint32_t rst_num = 0; int err = 0; rk_dev = devm_kzalloc(&pdev->dev, @@ -726,9 +651,10 @@ static int rk_crypto_probe(struct platform_device *pdev) soc_data = (struct rk_crypto_soc_data *)match->data; rk_dev->soc_data = soc_data; - if (soc_data->rsts[0]) { + rsts = soc_data->hw_get_rsts(&rst_num); + if (rsts && rsts[0]) { rk_dev->rst = - devm_reset_control_get(dev, soc_data->rsts[0]); + devm_reset_control_get(dev, rsts[0]); if (IS_ERR(rk_dev->rst)) { err = PTR_ERR(rk_dev->rst); goto err_crypto; diff --git a/drivers/crypto/rockchip/rk_crypto_core.h b/drivers/crypto/rockchip/rk_crypto_core.h index ca2788df8f35..d855625b75d0 100644 --- a/drivers/crypto/rockchip/rk_crypto_core.h +++ b/drivers/crypto/rockchip/rk_crypto_core.h @@ -43,19 +43,20 @@ #define sha384_state sha512_state #define sha224_state sha256_state +#define RK_FLAG_FINAL BIT(0) +#define RK_FLAG_UPDATE BIT(1) + struct rk_crypto_soc_data { const char *crypto_ver; char **valid_algs_name; int valid_algs_num; - struct rk_crypto_algt **total_algs; - int total_algs_num; - const char * const *rsts; - int rsts_num; unsigned int hw_info_size; bool use_soft_aes192; int default_pka_offset; int (*hw_init)(struct device *dev, void *hw_info); void (*hw_deinit)(struct device *dev, void *hw_info); + const char * const *(*hw_get_rsts)(uint32_t *num); + struct rk_crypto_algt **(*hw_get_algts)(uint32_t *num); }; struct rk_crypto_dev { @@ -98,6 +99,12 @@ struct rk_alg_ops { int (*update)(struct rk_crypto_dev *rk_dev); void (*complete)(struct crypto_async_request *base, int err); int (*irq_handle)(int irq, void *dev_id); + + int (*hw_write_key)(struct rk_crypto_dev *rk_dev, const u8 *key, u32 key_len); + void (*hw_write_iv)(struct rk_crypto_dev *rk_dev, const u8 *iv, u32 iv_len); + int (*hw_init)(struct rk_crypto_dev *rk_dev, u32 algo, u32 type); + int (*hw_dma_start)(struct rk_crypto_dev *rk_dev, uint32_t flag); + int (*hw_get_result)(struct rk_crypto_dev *rk_dev, uint8_t *data, uint32_t data_len); }; struct rk_alg_ctx { @@ -132,10 +139,13 @@ struct rk_ahash_ctx { struct scatterlist hash_sg[2]; u8 *hash_tmp; u32 hash_tmp_len; + u32 calc_cnt; u8 lastc[RK_DMA_ALIGNMENT]; u32 lastc_len; + void *priv; + /* for fallback */ struct crypto_ahash *fallback_tfm; }; @@ -144,7 +154,7 @@ struct rk_ahash_ctx { struct rk_ahash_rctx { struct ahash_request fallback_req; u32 mode; - bool is_final; + u32 flag; }; /* the private variable of cipher */ @@ -156,6 +166,8 @@ struct rk_cipher_ctx { u32 mode; u8 iv[AES_BLOCK_SIZE]; u8 lastc[AES_BLOCK_SIZE]; + bool is_enc; + void *priv; /* for fallback */ bool fallback_key_inited; diff --git a/drivers/crypto/rockchip/rk_crypto_skcipher_utils.c b/drivers/crypto/rockchip/rk_crypto_skcipher_utils.c new file mode 100644 index 000000000000..e73895e38be7 --- /dev/null +++ b/drivers/crypto/rockchip/rk_crypto_skcipher_utils.c @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Rockchip crypto skcipher uitls + * + * Copyright (c) 2022, Rockchip Electronics Co., Ltd + * + * Author: Lin Jinhan + * + */ + +#include "rk_crypto_skcipher_utils.h" + +struct rk_crypto_algt *rk_cipher_get_algt(struct crypto_skcipher *tfm) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + + return container_of(alg, struct rk_crypto_algt, alg.crypto); +} + +static struct rk_cipher_ctx *rk_cipher_ctx_cast(struct rk_crypto_dev *rk_dev) +{ + struct skcipher_request *req = skcipher_request_cast(rk_dev->async_req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + + return ctx; +} + +struct rk_alg_ctx *rk_cipher_alg_ctx(struct rk_crypto_dev *rk_dev) +{ + return &(rk_cipher_ctx_cast(rk_dev)->algs_ctx); +} + +static bool is_no_multi_blocksize(struct skcipher_request *req) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher); + + return (algt->mode == CIPHER_MODE_CFB || + algt->mode == CIPHER_MODE_OFB || + algt->mode == CIPHER_MODE_CTR || + algt->mode == CIPHER_MODE_XTS) ? true : false; +} + +int rk_cipher_fallback(struct skcipher_request *req, struct rk_cipher_ctx *ctx, bool encrypt) +{ + int ret; + + CRYPTO_MSG("use fallback tfm"); + + if (!ctx->fallback_tfm) { + ret = -ENODEV; + CRYPTO_MSG("fallback_tfm is empty!\n"); + goto exit; + } + + if (!ctx->fallback_key_inited) { + ret = crypto_skcipher_setkey(ctx->fallback_tfm, + ctx->key, ctx->keylen); + if (ret) { + CRYPTO_MSG("fallback crypto_skcipher_setkey err = %d\n", + ret); + goto exit; + } + + ctx->fallback_key_inited = true; + } + + skcipher_request_set_tfm(&ctx->fallback_req, ctx->fallback_tfm); + skcipher_request_set_callback(&ctx->fallback_req, + req->base.flags, + req->base.complete, + req->base.data); + + skcipher_request_set_crypt(&ctx->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + + ret = encrypt ? crypto_skcipher_encrypt(&ctx->fallback_req) : + crypto_skcipher_decrypt(&ctx->fallback_req); + +exit: + return ret; +} + +/* increment counter (128-bit int) by 1 */ +static void rk_ctr128_inc(uint8_t *counter) +{ + u32 n = 16; + u8 c; + + do { + --n; + c = counter[n]; + ++c; + counter[n] = c; + if (c) + return; + } while (n); +} + +static void rk_ctr128_calc(uint8_t *counter, uint32_t data_len) +{ + u32 i; + u32 chunksize = AES_BLOCK_SIZE; + + for (i = 0; i < DIV_ROUND_UP(data_len, chunksize); i++) + rk_ctr128_inc(counter); +} + +static uint32_t rk_get_new_iv(struct rk_cipher_ctx *ctx, u32 mode, bool is_enc, uint8_t *iv) +{ + struct scatterlist *sg_dst; + struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx; + uint32_t ivsize = alg_ctx->chunk_size; + + if (!iv) + return 0; + + sg_dst = alg_ctx->aligned ? alg_ctx->sg_dst : &alg_ctx->sg_tmp; + + CRYPTO_TRACE("aligned = %u, count = %u, ivsize = %u, is_enc = %d\n", + alg_ctx->aligned, alg_ctx->count, ivsize, is_enc); + + switch (mode) { + case CIPHER_MODE_CTR: + rk_ctr128_calc(iv, alg_ctx->count); + break; + case CIPHER_MODE_CBC: + case CIPHER_MODE_CFB: + if (is_enc) + sg_pcopy_to_buffer(sg_dst, 1, iv, ivsize, alg_ctx->count - ivsize); + else + memcpy(iv, ctx->lastc, ivsize); + break; + case CIPHER_MODE_OFB: + sg_pcopy_to_buffer(sg_dst, 1, iv, ivsize, alg_ctx->count - ivsize); + crypto_xor(iv, ctx->lastc, ivsize); + break; + default: + return 0; + } + + return ivsize; +} + +static void rk_iv_copyback(struct rk_crypto_dev *rk_dev) +{ + uint32_t iv_size; + struct skcipher_request *req = skcipher_request_cast(rk_dev->async_req); + struct rk_cipher_ctx *ctx = rk_cipher_ctx_cast(rk_dev); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher); + + iv_size = rk_get_new_iv(ctx, algt->mode, ctx->is_enc, ctx->iv); + + if (iv_size && req->iv) + memcpy(req->iv, ctx->iv, iv_size); +} + +static void rk_update_iv(struct rk_crypto_dev *rk_dev) +{ + uint32_t iv_size; + struct rk_cipher_ctx *ctx = rk_cipher_ctx_cast(rk_dev); + struct rk_alg_ctx *algs_ctx = &ctx->algs_ctx; + struct skcipher_request *req = skcipher_request_cast(rk_dev->async_req); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher); + + iv_size = rk_get_new_iv(ctx, algt->mode, ctx->is_enc, ctx->iv); + + if (iv_size) + algs_ctx->ops.hw_write_iv(rk_dev, ctx->iv, iv_size); +} + +static int rk_set_data_start(struct rk_crypto_dev *rk_dev) +{ + int err; + struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev); + + err = rk_dev->load_data(rk_dev, alg_ctx->sg_src, alg_ctx->sg_dst); + if (!err) { + u32 ivsize = alg_ctx->chunk_size; + struct scatterlist *src_sg; + struct rk_cipher_ctx *ctx = rk_cipher_ctx_cast(rk_dev); + + memset(ctx->lastc, 0x00, sizeof(ctx->lastc)); + + src_sg = alg_ctx->aligned ? alg_ctx->sg_src : &alg_ctx->sg_tmp; + + ivsize = alg_ctx->count > ivsize ? ivsize : alg_ctx->count; + + sg_pcopy_to_buffer(src_sg, 1, ctx->lastc, ivsize, alg_ctx->count - ivsize); + + alg_ctx->ops.hw_dma_start(rk_dev, true); + } + + return err; +} + +int rk_cipher_setkey(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen) +{ + struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher); + struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + uint32_t key_factor; + int ret = -EINVAL; + + CRYPTO_MSG("algo = %x, mode = %x, key_len = %d\n", + algt->algo, algt->mode, keylen); + + /* The key length of XTS is twice the normal length */ + key_factor = algt->mode == CIPHER_MODE_XTS ? 2 : 1; + + switch (algt->algo) { + case CIPHER_ALGO_DES: + ret = verify_skcipher_des_key(cipher, key); + if (ret) + goto exit; + break; + case CIPHER_ALGO_DES3_EDE: + ret = verify_skcipher_des3_key(cipher, key); + if (ret) + goto exit; + break; + case CIPHER_ALGO_AES: + if (keylen != (AES_KEYSIZE_128 * key_factor) && + keylen != (AES_KEYSIZE_192 * key_factor) && + keylen != (AES_KEYSIZE_256 * key_factor)) + goto exit; + break; + case CIPHER_ALGO_SM4: + if (keylen != (SM4_KEY_SIZE * key_factor)) + goto exit; + break; + default: + ret = -EINVAL; + goto exit; + } + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + ctx->fallback_key_inited = false; + + ret = 0; +exit: + return ret; +} + +int rk_ablk_rx(struct rk_crypto_dev *rk_dev) +{ + int err = 0; + struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev); + + CRYPTO_TRACE("left_bytes = %u\n", alg_ctx->left_bytes); + + err = rk_dev->unload_data(rk_dev); + if (err) + goto out_rx; + + if (alg_ctx->left_bytes) { + rk_update_iv(rk_dev); + if (alg_ctx->aligned) { + if (sg_is_last(alg_ctx->sg_src)) { + dev_err(rk_dev->dev, "[%s:%d] Lack of data\n", + __func__, __LINE__); + err = -ENOMEM; + goto out_rx; + } + alg_ctx->sg_src = sg_next(alg_ctx->sg_src); + alg_ctx->sg_dst = sg_next(alg_ctx->sg_dst); + } + err = rk_set_data_start(rk_dev); + } else { + rk_iv_copyback(rk_dev); + } +out_rx: + return err; +} + +int rk_ablk_start(struct rk_crypto_dev *rk_dev) +{ + struct skcipher_request *req = + skcipher_request_cast(rk_dev->async_req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct rk_crypto_algt *algt = rk_cipher_get_algt(tfm); + struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev); + unsigned long flags; + int err = 0; + + alg_ctx->left_bytes = req->cryptlen; + alg_ctx->total = req->cryptlen; + alg_ctx->sg_src = req->src; + alg_ctx->req_src = req->src; + alg_ctx->src_nents = sg_nents_for_len(req->src, req->cryptlen); + alg_ctx->sg_dst = req->dst; + alg_ctx->req_dst = req->dst; + alg_ctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen); + + CRYPTO_TRACE("total = %u", alg_ctx->total); + + spin_lock_irqsave(&rk_dev->lock, flags); + alg_ctx->ops.hw_init(rk_dev, algt->algo, algt->mode); + err = rk_set_data_start(rk_dev); + spin_unlock_irqrestore(&rk_dev->lock, flags); + return err; +} + +int rk_skcipher_handle_req(struct rk_crypto_dev *rk_dev, struct skcipher_request *req) +{ + struct rk_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + + if (!IS_ALIGNED(req->cryptlen, ctx->algs_ctx.chunk_size) && + !is_no_multi_blocksize(req)) + return -EINVAL; + else + return rk_dev->enqueue(rk_dev, &req->base); +} diff --git a/drivers/crypto/rockchip/rk_crypto_skcipher_utils.h b/drivers/crypto/rockchip/rk_crypto_skcipher_utils.h new file mode 100644 index 000000000000..74ac04eef7af --- /dev/null +++ b/drivers/crypto/rockchip/rk_crypto_skcipher_utils.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2022 Rockchip Electronics Co. Ltd. */ + +#ifndef __RK_CRYPTO_SKCIPHER_UTILS_H__ +#define __RK_CRYPTO_SKCIPHER_UTILS_H__ + +#include +#include + +#include "rk_crypto_core.h" +#include "rk_crypto_utils.h" + +struct rk_crypto_algt *rk_cipher_get_algt(struct crypto_skcipher *tfm); + +struct rk_alg_ctx *rk_cipher_alg_ctx(struct rk_crypto_dev *rk_dev); + +int rk_cipher_fallback(struct skcipher_request *req, struct rk_cipher_ctx *ctx, bool encrypt); + +int rk_cipher_setkey(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen); + +int rk_ablk_rx(struct rk_crypto_dev *rk_dev); + +int rk_ablk_start(struct rk_crypto_dev *rk_dev); + +int rk_skcipher_handle_req(struct rk_crypto_dev *rk_dev, struct skcipher_request *req); + +#endif + diff --git a/drivers/crypto/rockchip/rk_crypto_utils.c b/drivers/crypto/rockchip/rk_crypto_utils.c new file mode 100644 index 000000000000..073d617ec819 --- /dev/null +++ b/drivers/crypto/rockchip/rk_crypto_utils.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Rockchip crypto uitls + * + * Copyright (c) 2022, Rockchip Electronics Co., Ltd + * + * Author: Lin Jinhan + * + */ + +#include "rk_crypto_core.h" +#include "rk_crypto_utils.h" + +static inline void word2byte_be(u32 word, u8 *ch) +{ + ch[0] = (word >> 24) & 0xff; + ch[1] = (word >> 16) & 0xff; + ch[2] = (word >> 8) & 0xff; + ch[3] = (word >> 0) & 0xff; +} + +static inline u32 byte2word_be(const u8 *ch) +{ + return (*ch << 24) + (*(ch + 1) << 16) + + (*(ch + 2) << 8) + *(ch + 3); +} + +void rk_crypto_write_regs(struct rk_crypto_dev *rk_dev, u32 base_addr, const u8 *data, u32 bytes) +{ + u32 i; + u8 tmp_buf[4]; + + for (i = 0; i < bytes / 4; i++, base_addr += 4) + CRYPTO_WRITE(rk_dev, base_addr, byte2word_be(data + i * 4)); + + if (bytes % 4) { + memset(tmp_buf, 0x00, sizeof(tmp_buf)); + memcpy((u8 *)tmp_buf, data + (bytes / 4) * 4, bytes % 4); + CRYPTO_WRITE(rk_dev, base_addr, byte2word_be(tmp_buf)); + } +} + +void rk_crypto_clear_regs(struct rk_crypto_dev *rk_dev, u32 base_addr, u32 words) +{ + u32 i; + + for (i = 0; i < words; i++, base_addr += 4) + CRYPTO_WRITE(rk_dev, base_addr, 0); +} + +void rk_crypto_read_regs(struct rk_crypto_dev *rk_dev, u32 base_addr, u8 *data, u32 bytes) +{ + u32 i; + + for (i = 0; i < bytes / 4; i++, base_addr += 4) + word2byte_be(CRYPTO_READ(rk_dev, base_addr), data + i * 4); + + if (bytes % 4) { + uint8_t tmp_buf[4]; + + word2byte_be(CRYPTO_READ(rk_dev, base_addr), tmp_buf); + memcpy(data + i * 4, tmp_buf, bytes % 4); + } +} diff --git a/drivers/crypto/rockchip/rk_crypto_utils.h b/drivers/crypto/rockchip/rk_crypto_utils.h new file mode 100644 index 000000000000..c1ea593dbaaf --- /dev/null +++ b/drivers/crypto/rockchip/rk_crypto_utils.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2022 Rockchip Electronics Co. Ltd. */ + +#ifndef __RK_CRYPTO_UTILS_H__ +#define __RK_CRYPTO_UTILS_H__ + +void rk_crypto_write_regs(struct rk_crypto_dev *rk_dev, u32 base_addr, const u8 *data, u32 bytes); + +void rk_crypto_clear_regs(struct rk_crypto_dev *rk_dev, u32 base_addr, u32 words); + +void rk_crypto_read_regs(struct rk_crypto_dev *rk_dev, u32 base_addr, u8 *data, u32 bytes); + +#endif + diff --git a/drivers/crypto/rockchip/rk_crypto_v1.c b/drivers/crypto/rockchip/rk_crypto_v1.c new file mode 100644 index 000000000000..fcc6eeb7383e --- /dev/null +++ b/drivers/crypto/rockchip/rk_crypto_v1.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Crypto acceleration support for Rockchip Crypto V1 + * + * Copyright (c) 2022, Rockchip Electronics Co., Ltd + * + * Author: Lin Jinhan + * + */ +#include "rk_crypto_core.h" +#include "rk_crypto_v1.h" + +static const char * const crypto_v1_rsts[] = { + "crypto-rst", +}; + +static struct rk_crypto_algt *crypto_v1_algs[] = { + &rk_v1_ecb_aes_alg, /* ecb(aes) */ + &rk_v1_cbc_aes_alg, /* cbc(aes) */ + + &rk_v1_ecb_des_alg, /* ecb(des) */ + &rk_v1_cbc_des_alg, /* cbc(des) */ + + &rk_v1_ecb_des3_ede_alg, /* ecb(des3_ede) */ + &rk_v1_cbc_des3_ede_alg, /* cbc(des3_ede) */ + + &rk_v1_ahash_sha1, /* sha1 */ + &rk_v1_ahash_sha256, /* sha256 */ + &rk_v1_ahash_md5, /* md5 */ +}; + +int rk_hw_crypto_v1_init(struct device *dev, void *hw_info) +{ + return 0; +} + +void rk_hw_crypto_v1_deinit(struct device *dev, void *hw_info) +{ + +} + +const char * const *rk_hw_crypto_v1_get_rsts(uint32_t *num) +{ + *num = ARRAY_SIZE(crypto_v1_rsts); + + return crypto_v1_rsts; +} + +struct rk_crypto_algt **rk_hw_crypto_v1_get_algts(uint32_t *num) +{ + *num = ARRAY_SIZE(crypto_v1_algs); + + return crypto_v1_algs; +} + diff --git a/drivers/crypto/rockchip/rk_crypto_v1.h b/drivers/crypto/rockchip/rk_crypto_v1.h index d02ad537871a..b7180fbc6574 100644 --- a/drivers/crypto/rockchip/rk_crypto_v1.h +++ b/drivers/crypto/rockchip/rk_crypto_v1.h @@ -11,6 +11,21 @@ struct rk_hw_crypto_v1_info { int reserved; }; +#define RK_CRYPTO_V1_SOC_DATA_INIT(names) {\ + .crypto_ver = "CRYPTO V1.0.0.0",\ + .use_soft_aes192 = false,\ + .valid_algs_name = (names),\ + .valid_algs_num = ARRAY_SIZE(names),\ + .hw_init = rk_hw_crypto_v1_init,\ + .hw_deinit = rk_hw_crypto_v1_deinit,\ + .hw_get_rsts = rk_hw_crypto_v1_get_rsts,\ + .hw_get_algts = rk_hw_crypto_v1_get_algts,\ + .hw_info_size = sizeof(struct rk_hw_crypto_v1_info),\ + .default_pka_offset = 0,\ +} + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V1) + extern struct rk_crypto_algt rk_v1_ecb_aes_alg; extern struct rk_crypto_algt rk_v1_cbc_aes_alg; @@ -26,5 +41,17 @@ extern struct rk_crypto_algt rk_v1_ahash_md5; int rk_hw_crypto_v1_init(struct device *dev, void *hw_info); void rk_hw_crypto_v1_deinit(struct device *dev, void *hw_info); -#endif +const char * const *rk_hw_crypto_v1_get_rsts(uint32_t *num); +struct rk_crypto_algt **rk_hw_crypto_v1_get_algts(uint32_t *num); + +#else + +static inline int rk_hw_crypto_v1_init(struct device *dev, void *hw_info) { return -EINVAL; } +static inline void rk_hw_crypto_v1_deinit(struct device *dev, void *hw_info) {} +static inline const char * const *rk_hw_crypto_v1_get_rsts(uint32_t *num) { return NULL; } +static inline struct rk_crypto_algt **rk_hw_crypto_v1_get_algts(uint32_t *num) { return NULL; } + +#endif /* end of IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V1) */ + +#endif /* end of __RK_CRYPTO_V1_H__ */ diff --git a/drivers/crypto/rockchip/rk_crypto_v1_skcipher.c b/drivers/crypto/rockchip/rk_crypto_v1_skcipher.c index 8e0751bc3d83..a4c4a9a5d461 100644 --- a/drivers/crypto/rockchip/rk_crypto_v1_skcipher.c +++ b/drivers/crypto/rockchip/rk_crypto_v1_skcipher.c @@ -407,16 +407,6 @@ static void rk_ablk_exit_tfm(struct crypto_skcipher *tfm) ctx->rk_dev->release_crypto(ctx->rk_dev, alg_name); } -int rk_hw_crypto_v1_init(struct device *dev, void *hw_info) -{ - return 0; -} - -void rk_hw_crypto_v1_deinit(struct device *dev, void *hw_info) -{ - -} - struct rk_crypto_algt rk_v1_ecb_aes_alg = RK_CIPHER_ALGO_INIT(AES, ECB, ecb(aes), ecb-aes-rk); diff --git a/drivers/crypto/rockchip/rk_crypto_v2.c b/drivers/crypto/rockchip/rk_crypto_v2.c new file mode 100644 index 000000000000..c7caa867632a --- /dev/null +++ b/drivers/crypto/rockchip/rk_crypto_v2.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Crypto acceleration support for Rockchip Crypto V2 + * + * Copyright (c) 2022, Rockchip Electronics Co., Ltd + * + * Author: Lin Jinhan + * + */ + +#include "rk_crypto_core.h" +#include "rk_crypto_v2.h" + +static const char * const crypto_v2_rsts[] = { + "crypto-rst", +}; + +static struct rk_crypto_algt *crypto_v2_algs[] = { + &rk_v2_ecb_sm4_alg, /* ecb(sm4) */ + &rk_v2_cbc_sm4_alg, /* cbc(sm4) */ + &rk_v2_xts_sm4_alg, /* xts(sm4) */ + &rk_v2_cfb_sm4_alg, /* cfb(sm4) */ + &rk_v2_ofb_sm4_alg, /* ofb(sm4) */ + &rk_v2_ctr_sm4_alg, /* ctr(sm4) */ + + &rk_v2_ecb_aes_alg, /* ecb(aes) */ + &rk_v2_cbc_aes_alg, /* cbc(aes) */ + &rk_v2_xts_aes_alg, /* xts(aes) */ + &rk_v2_cfb_aes_alg, /* cfb(aes) */ + &rk_v2_ofb_aes_alg, /* ofb(aes) */ + &rk_v2_ctr_aes_alg, /* ctr(aes) */ + + &rk_v2_ecb_des_alg, /* ecb(des) */ + &rk_v2_cbc_des_alg, /* cbc(des) */ + &rk_v2_cfb_des_alg, /* cfb(des) */ + &rk_v2_ofb_des_alg, /* ofb(des) */ + + &rk_v2_ecb_des3_ede_alg, /* ecb(des3_ede) */ + &rk_v2_cbc_des3_ede_alg, /* cbc(des3_ede) */ + &rk_v2_cfb_des3_ede_alg, /* cfb(des3_ede) */ + &rk_v2_ofb_des3_ede_alg, /* ofb(des3_ede) */ + + &rk_v2_ahash_sha1, /* sha1 */ + &rk_v2_ahash_sha224, /* sha224 */ + &rk_v2_ahash_sha256, /* sha256 */ + &rk_v2_ahash_sha384, /* sha384 */ + &rk_v2_ahash_sha512, /* sha512 */ + &rk_v2_ahash_md5, /* md5 */ + &rk_v2_ahash_sm3, /* sm3 */ + + &rk_v2_hmac_sha1, /* hmac(sha1) */ + &rk_v2_hmac_sha256, /* hmac(sha256) */ + &rk_v2_hmac_sha512, /* hmac(sha512) */ + &rk_v2_hmac_md5, /* hmac(md5) */ + &rk_v2_hmac_sm3, /* hmac(sm3) */ + + &rk_v2_asym_rsa, /* rsa */ +}; + +int rk_hw_crypto_v2_init(struct device *dev, void *hw_info) +{ + int err = 0; + struct rk_hw_crypto_v2_info *info = + (struct rk_hw_crypto_v2_info *)hw_info; + + info->desc = dma_alloc_coherent(dev, + sizeof(struct crypto_lli_desc), + &info->desc_dma, + GFP_KERNEL); + if (!info->desc) { + err = -ENOMEM; + goto end; + } + +end: + return err; +} + +void rk_hw_crypto_v2_deinit(struct device *dev, void *hw_info) +{ + struct rk_hw_crypto_v2_info *info = + (struct rk_hw_crypto_v2_info *)hw_info; + + if (info && info->desc) + dma_free_coherent(dev, sizeof(struct crypto_lli_desc), + info->desc, info->desc_dma); +} + +const char * const *rk_hw_crypto_v2_get_rsts(uint32_t *num) +{ + *num = ARRAY_SIZE(crypto_v2_rsts); + + return crypto_v2_rsts; +} + +struct rk_crypto_algt **rk_hw_crypto_v2_get_algts(uint32_t *num) +{ + *num = ARRAY_SIZE(crypto_v2_algs); + + return crypto_v2_algs; +} + diff --git a/drivers/crypto/rockchip/rk_crypto_v2.h b/drivers/crypto/rockchip/rk_crypto_v2.h index 8d5614b97fa3..fc1ac8a40886 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2.h +++ b/drivers/crypto/rockchip/rk_crypto_v2.h @@ -21,11 +21,23 @@ struct crypto_lli_desc { struct rk_hw_crypto_v2_info { struct crypto_lli_desc *desc; dma_addr_t desc_dma; - bool is_started; - u32 hash_calc_cnt; - u32 ciher_calc_cnt; }; +#define RK_CRYPTO_V2_SOC_DATA_INIT(names, soft_aes_192) {\ + .crypto_ver = "CRYPTO V2.0.0.0",\ + .use_soft_aes192 = soft_aes_192,\ + .valid_algs_name = (names),\ + .valid_algs_num = ARRAY_SIZE(names),\ + .hw_init = rk_hw_crypto_v2_init,\ + .hw_deinit = rk_hw_crypto_v2_deinit,\ + .hw_get_rsts = rk_hw_crypto_v2_get_rsts,\ + .hw_get_algts = rk_hw_crypto_v2_get_algts,\ + .hw_info_size = sizeof(struct rk_hw_crypto_v2_info),\ + .default_pka_offset = 0x0480,\ +} + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V2) + extern struct rk_crypto_algt rk_v2_ecb_sm4_alg; extern struct rk_crypto_algt rk_v2_cbc_sm4_alg; extern struct rk_crypto_algt rk_v2_xts_sm4_alg; @@ -68,12 +80,16 @@ extern struct rk_crypto_algt rk_v2_asym_rsa; int rk_hw_crypto_v2_init(struct device *dev, void *hw_info); void rk_hw_crypto_v2_deinit(struct device *dev, void *hw_info); +const char * const *rk_hw_crypto_v2_get_rsts(uint32_t *num); +struct rk_crypto_algt **rk_hw_crypto_v2_get_algts(uint32_t *num); -void rk_pka_set_crypto_base(void __iomem *base); +#else -int rk_pka_expt_mod(struct rk_bignum *in, - struct rk_bignum *e, - struct rk_bignum *n, - struct rk_bignum *out); +static inline int rk_hw_crypto_v2_init(struct device *dev, void *hw_info) { return -EINVAL; } +static inline void rk_hw_crypto_v2_deinit(struct device *dev, void *hw_info) {} +static inline const char * const *rk_hw_crypto_v2_get_rsts(uint32_t *num) { return NULL; } +static inline struct rk_crypto_algt **rk_hw_crypto_v2_get_algts(uint32_t *num) { return NULL; } -#endif +#endif /* end of IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V2) */ + +#endif /* end of __RK_CRYPTO_V2_H__ */ diff --git a/drivers/crypto/rockchip/rk_crypto_v2_ahash.c b/drivers/crypto/rockchip/rk_crypto_v2_ahash.c index ae003bb88421..1395cd350f42 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2_ahash.c +++ b/drivers/crypto/rockchip/rk_crypto_v2_ahash.c @@ -15,6 +15,7 @@ #include "rk_crypto_core.h" #include "rk_crypto_v2.h" #include "rk_crypto_v2_reg.h" +#include "rk_crypto_ahash_utils.h" #define RK_HASH_CTX_MAGIC 0x1A1A1A1A #define RK_POLL_PERIOD_US 100 @@ -35,64 +36,6 @@ static const u32 hash_algo2bc[] = { [HASH_ALGO_SM3] = CRYPTO_SM3, }; -const char *hash_algo2name[] = { - [HASH_ALGO_MD5] = "md5", - [HASH_ALGO_SHA1] = "sha1", - [HASH_ALGO_SHA224] = "sha224", - [HASH_ALGO_SHA256] = "sha256", - [HASH_ALGO_SHA384] = "sha384", - [HASH_ALGO_SHA512] = "sha512", - [HASH_ALGO_SM3] = "sm3", -}; - -static struct rk_ahash_ctx *rk_ahash_ctx_cast( - struct rk_crypto_dev *rk_dev) -{ - struct ahash_request *req = - ahash_request_cast(rk_dev->async_req); - - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - - return crypto_ahash_ctx(tfm); -} - -static struct rk_alg_ctx *rk_alg_ctx_cast( - struct rk_crypto_dev *rk_dev) -{ - return &(rk_ahash_ctx_cast(rk_dev))->algs_ctx; -} - -static void rk_alg_ctx_clear(struct rk_alg_ctx *alg_ctx) -{ - alg_ctx->total = 0; - alg_ctx->left_bytes = 0; - alg_ctx->count = 0; - alg_ctx->sg_src = 0; - alg_ctx->req_src = 0; - alg_ctx->src_nents = 0; -} - -static inline void word2byte_be(u32 word, u8 *ch) -{ - ch[0] = (word >> 24) & 0xff; - ch[1] = (word >> 16) & 0xff; - ch[2] = (word >> 8) & 0xff; - ch[3] = (word >> 0) & 0xff; -} - -static inline u32 byte2word_be(const u8 *ch) -{ - return (*ch << 24) + (*(ch + 1) << 16) + - (*(ch + 2) << 8) + *(ch + 3); -} - -static struct rk_crypto_algt *rk_ahash_get_algt(struct crypto_ahash *tfm) -{ - struct ahash_alg *alg = __crypto_ahash_alg(tfm->base.__crt_alg); - - return container_of(alg, struct rk_crypto_algt, alg.hash); -} - static void rk_hash_reset(struct rk_crypto_dev *rk_dev) { int ret; @@ -122,7 +65,7 @@ static int rk_crypto_irq_handle(int irq, void *dev_id) u32 interrupt_status; struct rk_hw_crypto_v2_info *hw_info = (struct rk_hw_crypto_v2_info *)rk_dev->hw_info; - struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); + struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(rk_dev); /* disable crypto irq */ CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0); @@ -166,7 +109,7 @@ static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err) struct ahash_request *req = ahash_request_cast(base); struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); - struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(ctx->rk_dev); + struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(ctx->rk_dev); struct rk_hw_crypto_v2_info *hw_info = ctx->rk_dev->hw_info; struct crypto_lli_desc *lli_desc = hw_info->desc; @@ -192,24 +135,15 @@ static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err) static inline void clear_hash_out_reg(struct rk_crypto_dev *rk_dev) { - int i; - - /*clear out register*/ - for (i = 0; i < 16; i++) - CRYPTO_WRITE(rk_dev, CRYPTO_HASH_DOUT_0 + 4 * i, 0); + rk_crypto_clear_regs(rk_dev, CRYPTO_HASH_DOUT_0, 16); } -static void write_key_reg(struct rk_crypto_dev *rk_dev, const u8 *key, +static int write_key_reg(struct rk_crypto_dev *rk_dev, const u8 *key, u32 key_len) { - u32 i; - u32 chn_base_addr; + rk_crypto_write_regs(rk_dev, CRYPTO_CH0_KEY_0, key, key_len); - chn_base_addr = CRYPTO_CH0_KEY_0; - - for (i = 0; i < key_len / 4; i++, chn_base_addr += 4) - CRYPTO_WRITE(rk_dev, chn_base_addr, - byte2word_be(key + i * 4)); + return 0; } static int rk_hw_hash_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 type) @@ -246,81 +180,6 @@ static void clean_hash_setting(struct rk_crypto_dev *rk_dev) CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, 0 | CRYPTO_WRITE_MASK_ALL); } -static int rk_ahash_init(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct rk_crypto_algt *algt = rk_ahash_get_algt(tfm); - struct rk_ahash_rctx *rctx = ahash_request_ctx(req); - struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); - - CRYPTO_TRACE(); - - memset(rctx, 0x00, sizeof(*rctx)); - - return rk_hw_hash_init(ctx->rk_dev, algt->algo, algt->type); -} - -static int rk_ahash_update(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); - struct rk_ahash_rctx *rctx = ahash_request_ctx(req); - struct rk_crypto_dev *rk_dev = ctx->rk_dev; - - CRYPTO_TRACE("nbytes = %u", req->nbytes); - - memset(rctx, 0x00, sizeof(*rctx)); - - return rk_dev->enqueue(rk_dev, &req->base); -} - -static int rk_ahash_final(struct ahash_request *req) -{ - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); - struct rk_ahash_rctx *rctx = ahash_request_ctx(req); - struct rk_crypto_dev *rk_dev = ctx->rk_dev; - - CRYPTO_TRACE(); - - memset(rctx, 0x00, sizeof(*rctx)); - - rctx->is_final = true; - - return rk_dev->enqueue(rk_dev, &req->base); -} - -static int rk_ahash_finup(struct ahash_request *req) -{ - int ret = 0; - crypto_completion_t complete_bak; - void *data_bak; - - DECLARE_CRYPTO_WAIT(wait); - - /* update not trigger user complete */ - complete_bak = req->base.complete; - data_bak = req->base.data; - - req->base.complete = crypto_req_done; - req->base.data = &wait; - - ret = crypto_wait_req(rk_ahash_update(req), &wait); - if (ret) { - CRYPTO_MSG("rk_ahash_update failed, ret = %d", ret); - goto exit; - } - - /* final will trigger user complete */ - req->base.complete = complete_bak; - req->base.data = data_bak; - - ret = rk_ahash_final(req); - -exit: - return ret; -} - static int rk_ahash_import(struct ahash_request *req, const void *in) { struct rk_ahash_expt_ctx state; @@ -347,116 +206,17 @@ static int rk_ahash_export(struct ahash_request *req, void *out) return 0; } -static int rk_ahash_digest(struct ahash_request *req) -{ - CRYPTO_TRACE("calc data %u bytes.", req->nbytes); - - return rk_ahash_init(req) ?: rk_ahash_finup(req); -} - -static int rk_ahash_fallback_digest(const char *alg_name, bool is_hmac, - const u8 *key, u32 key_len, - const u8 *msg, u32 msg_len, - u8 *digest, u32 digest_len) -{ - struct crypto_ahash *ahash_tfm; - struct ahash_request *req; - struct crypto_wait wait; - struct scatterlist sg; - int ret; - - CRYPTO_TRACE("%s, is_hmac = %d, key_len = %u, msg_len = %u, digest_len = %u", - alg_name, is_hmac, key_len, msg_len, digest_len); - - ahash_tfm = crypto_alloc_ahash(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ahash_tfm)) - return PTR_ERR(ahash_tfm); - - req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); - if (!req) { - crypto_free_ahash(ahash_tfm); - return -ENOMEM; - } - - init_completion(&wait.completion); - - ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, &wait); - - crypto_ahash_clear_flags(ahash_tfm, ~0); - - sg_init_one(&sg, msg, msg_len); - ahash_request_set_crypt(req, &sg, digest, msg_len); - - if (is_hmac) - crypto_ahash_setkey(ahash_tfm, key, key_len); - - ret = crypto_wait_req(crypto_ahash_digest(req), &wait); - if (ret) { - CRYPTO_MSG("digest failed, ret = %d", ret); - goto exit; - } - -exit: - ahash_request_free(req); - crypto_free_ahash(ahash_tfm); - - return ret; -} - -static int rk_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, - unsigned int keylen) -{ - unsigned int blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); - unsigned int digestsize = crypto_ahash_digestsize(tfm); - struct rk_crypto_algt *algt = rk_ahash_get_algt(tfm); - struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); - const char *alg_name; - int ret = 0; - - CRYPTO_MSG(); - - if (algt->algo >= ARRAY_SIZE(hash_algo2name)) { - CRYPTO_MSG("hash algo %d invalid\n", algt->algo); - return -EINVAL; - } - - memset(ctx->authkey, 0, sizeof(ctx->authkey)); - - if (keylen <= blocksize) { - memcpy(ctx->authkey, key, keylen); - ctx->authkey_len = keylen; - goto exit; - } - - alg_name = hash_algo2name[algt->algo]; - - CRYPTO_TRACE("calc key digest %s", alg_name); - - ret = rk_ahash_fallback_digest(alg_name, false, NULL, 0, key, keylen, - ctx->authkey, digestsize); - if (ret) { - CRYPTO_MSG("rk_ahash_fallback_digest error ret = %d\n", ret); - goto exit; - } - - ctx->authkey_len = digestsize; -exit: - if (ret == 0) - write_key_reg(ctx->rk_dev, ctx->authkey, sizeof(ctx->authkey)); - - return ret; -} - -static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, bool is_final) +static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) { struct rk_hw_crypto_v2_info *hw_info = (struct rk_hw_crypto_v2_info *)rk_dev->hw_info; - struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); - u32 dma_ctl = hw_info->is_started ? CRYPTO_DMA_RESTART : CRYPTO_DMA_START; + struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(rk_dev); + struct rk_ahash_ctx *ctx = rk_ahash_ctx_cast(rk_dev); + u32 dma_ctl = CRYPTO_DMA_RESTART; + bool is_final = flag & RK_FLAG_FINAL; - CRYPTO_TRACE("count %u Byte, is_started = %d, is_final = %d", - alg_ctx->count, hw_info->is_started, is_final); + CRYPTO_TRACE("ctx->calc_cnt = %u, count %u Byte, is_final = %d", + ctx->calc_cnt, alg_ctx->count, is_final); if (alg_ctx->count % RK_DMA_ALIGNMENT && !is_final) { dev_err(rk_dev->dev, "count = %u is not aligned with [%u]\n", @@ -479,8 +239,9 @@ static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, bool is_final) hw_info->desc->dma_ctrl = is_final ? LLI_DMA_CTRL_LAST : LLI_DMA_CTRL_PAUSE; hw_info->desc->dma_ctrl |= LLI_DMA_CTRL_SRC_DONE; - if (!hw_info->is_started) { - hw_info->is_started = true; + if (ctx->calc_cnt == 0) { + dma_ctl = CRYPTO_DMA_START; + hw_info->desc->user_define |= LLI_USER_CIPHER_START; hw_info->desc->user_define |= LLI_USER_STRING_START; @@ -488,20 +249,14 @@ static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, bool is_final) CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, (CRYPTO_HASH_ENABLE << CRYPTO_WRITE_MASK_SHIFT) | CRYPTO_HASH_ENABLE); - - hw_info->hash_calc_cnt = 0; } - if (is_final && alg_ctx->left_bytes == 0) { + if (is_final && alg_ctx->left_bytes == 0) hw_info->desc->user_define |= LLI_USER_STRING_LAST; - hw_info->is_started = false; - } CRYPTO_TRACE("dma_ctrl = %08x, user_define = %08x, len = %u", hw_info->desc->dma_ctrl, hw_info->desc->user_define, alg_ctx->count); - hw_info->hash_calc_cnt += alg_ctx->count; - dma_wmb(); /* enable crypto irq */ @@ -512,156 +267,11 @@ static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, bool is_final) return 0; } -static int rk_ahash_set_data_start(struct rk_crypto_dev *rk_dev, bool is_final) -{ - int err; - struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); - - CRYPTO_TRACE(); - - err = rk_dev->load_data(rk_dev, alg_ctx->sg_src, alg_ctx->sg_dst); - if (!err) - err = rk_ahash_dma_start(rk_dev, is_final); - - return err; -} - -static u32 rk_calc_lastc_new_len(u32 nbytes, u32 old_len) -{ - u32 total_len = nbytes + old_len; - - if (total_len <= RK_DMA_ALIGNMENT) - return nbytes; - - if (total_len % RK_DMA_ALIGNMENT) - return total_len % RK_DMA_ALIGNMENT; - - return RK_DMA_ALIGNMENT; -} - -static int rk_ahash_start(struct rk_crypto_dev *rk_dev) -{ - struct ahash_request *req = ahash_request_cast(rk_dev->async_req); - struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); - struct rk_ahash_ctx *ctx = rk_ahash_ctx_cast(rk_dev); - struct rk_ahash_rctx *rctx = ahash_request_ctx(req); - struct scatterlist *src_sg; - unsigned int nbytes; - int ret = 0; - - CRYPTO_TRACE("origin: old_len = %u, new_len = %u, nbytes = %u, is_final = %d", - ctx->hash_tmp_len, ctx->lastc_len, req->nbytes, rctx->is_final); - - /* update 0Byte do nothing */ - if (req->nbytes == 0 && !rctx->is_final) - goto no_calc; - - if (ctx->lastc_len) { - /* move lastc saved last time to the head of this calculation */ - memcpy(ctx->hash_tmp + ctx->hash_tmp_len, ctx->lastc, ctx->lastc_len); - ctx->hash_tmp_len = ctx->hash_tmp_len + ctx->lastc_len; - ctx->lastc_len = 0; - } - - CRYPTO_TRACE("hash_tmp_len = %u", ctx->hash_tmp_len); - - /* final request no need to save lastc_new */ - if (!rctx->is_final) { - ctx->lastc_len = rk_calc_lastc_new_len(req->nbytes, ctx->hash_tmp_len); - - CRYPTO_TRACE("nents = %u, ctx->lastc_len = %u, offset = %u", - sg_nents_for_len(req->src, req->nbytes), ctx->lastc_len, - req->nbytes - ctx->lastc_len); - - if (!sg_pcopy_to_buffer(req->src, sg_nents_for_len(req->src, req->nbytes), - ctx->lastc, ctx->lastc_len, req->nbytes - ctx->lastc_len)) { - ret = -EINVAL; - goto exit; - } - - nbytes = ctx->hash_tmp_len + req->nbytes - ctx->lastc_len; - - /* not enough data */ - if (nbytes < RK_DMA_ALIGNMENT) { - CRYPTO_TRACE("nbytes = %u, not enough data", nbytes); - memcpy(ctx->hash_tmp + ctx->hash_tmp_len, - ctx->lastc, ctx->lastc_len); - ctx->hash_tmp_len = ctx->hash_tmp_len + ctx->lastc_len; - ctx->lastc_len = 0; - goto no_calc; - } - } else { - /* final just calc lastc_old */ - nbytes = ctx->hash_tmp_len; - CRYPTO_TRACE("nbytes = %u", nbytes); - } - - if (ctx->hash_tmp_len) { - /* Concatenate old data to the header */ - sg_init_table(ctx->hash_sg, ARRAY_SIZE(ctx->hash_sg)); - sg_set_buf(ctx->hash_sg, ctx->hash_tmp, ctx->hash_tmp_len); - sg_chain(ctx->hash_sg, ARRAY_SIZE(ctx->hash_sg), req->src); - - src_sg = &ctx->hash_sg[0]; - ctx->hash_tmp_len = 0; - } else { - src_sg = req->src; - } - - alg_ctx->total = nbytes; - alg_ctx->left_bytes = nbytes; - alg_ctx->sg_src = src_sg; - alg_ctx->req_src = src_sg; - alg_ctx->src_nents = sg_nents_for_len(src_sg, nbytes); - - CRYPTO_TRACE("adjust: old_len = %u, new_len = %u, nbytes = %u", - ctx->hash_tmp_len, ctx->lastc_len, nbytes); - - if (nbytes) - ret = rk_ahash_set_data_start(rk_dev, rctx->is_final); - -exit: - return ret; -no_calc: - CRYPTO_TRACE("no calc"); - rk_alg_ctx_clear(alg_ctx); - - return 0; -} - -static int rk_ahash_get_zero_result(struct rk_crypto_dev *rk_dev, - uint8_t *data, uint32_t data_len) -{ - - struct ahash_request *req = - ahash_request_cast(rk_dev->async_req); - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct rk_crypto_algt *algt = rk_ahash_get_algt(tfm); - struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); - - return rk_ahash_fallback_digest(algt->name, algt->type == ALG_TYPE_HMAC, - ctx->authkey, ctx->authkey_len, - NULL, 0, - data, data_len); - -} static int rk_ahash_get_result(struct rk_crypto_dev *rk_dev, uint8_t *data, uint32_t data_len) { int ret = 0; - u32 i, offset; u32 reg_ctrl = 0; - struct rk_hw_crypto_v2_info *hw_info = - (struct rk_hw_crypto_v2_info *)rk_dev->hw_info; - - hw_info->is_started = false; - - /* use fallback hash */ - if (hw_info->hash_calc_cnt == 0) { - CRYPTO_TRACE("use fallback hash"); - ret = rk_ahash_get_zero_result(rk_dev, data, data_len); - goto exit; - } ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_HASH_VALID, reg_ctrl, @@ -671,75 +281,16 @@ static int rk_ahash_get_result(struct rk_crypto_dev *rk_dev, if (ret) goto exit; - offset = CRYPTO_HASH_DOUT_0; - for (i = 0; i < data_len / 4; i++, offset += 4) - word2byte_be(CRYPTO_READ(rk_dev, offset), - data + i * 4); - - if (data_len % 4) { - uint8_t tmp_buf[4]; - - word2byte_be(CRYPTO_READ(rk_dev, offset), tmp_buf); - memcpy(data + i * 4, tmp_buf, data_len % 4); - } + rk_crypto_read_regs(rk_dev, CRYPTO_HASH_DOUT_0, data, data_len); CRYPTO_WRITE(rk_dev, CRYPTO_HASH_VALID, CRYPTO_HASH_IS_VALID); exit: - hw_info->hash_calc_cnt = 0; + clean_hash_setting(rk_dev); + return ret; } -static int rk_ahash_crypto_rx(struct rk_crypto_dev *rk_dev) -{ - int err = 0; - struct ahash_request *req = ahash_request_cast(rk_dev->async_req); - struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); - struct rk_ahash_rctx *rctx = ahash_request_ctx(req); - - CRYPTO_TRACE("left bytes = %u, is_final = %d", alg_ctx->left_bytes, rctx->is_final); - - err = rk_dev->unload_data(rk_dev); - if (err) - goto out_rx; - - if (alg_ctx->left_bytes) { - if (alg_ctx->aligned) { - if (sg_is_last(alg_ctx->sg_src)) { - dev_warn(rk_dev->dev, "[%s:%d], Lack of data\n", - __func__, __LINE__); - err = -ENOMEM; - goto out_rx; - } - alg_ctx->sg_src = sg_next(alg_ctx->sg_src); - } - err = rk_ahash_set_data_start(rk_dev, rctx->is_final); - } else { - /* - * it will take some time to process date after last dma - * transmission. - */ - struct crypto_ahash *tfm; - - if (!rctx->is_final) - goto out_rx; - - if (!req->result) { - err = -EINVAL; - goto out_rx; - } - - tfm = crypto_ahash_reqtfm(req); - - err = rk_ahash_get_result(rk_dev, req->result, - crypto_ahash_digestsize(tfm)); - } - -out_rx: - return err; - -} - static int rk_cra_hash_init(struct crypto_tfm *tfm) { struct rk_crypto_algt *algt = @@ -763,6 +314,11 @@ static int rk_cra_hash_init(struct crypto_tfm *tfm) alg_ctx->ops.complete = rk_ahash_crypto_complete; alg_ctx->ops.irq_handle = rk_crypto_irq_handle; + alg_ctx->ops.hw_write_key = write_key_reg; + alg_ctx->ops.hw_init = rk_hw_hash_init; + alg_ctx->ops.hw_dma_start = rk_ahash_dma_start; + alg_ctx->ops.hw_get_result = rk_ahash_get_result; + ctx->rk_dev = rk_dev; ctx->hash_tmp = (u8 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); if (!ctx->hash_tmp) { @@ -785,8 +341,6 @@ static void rk_cra_hash_exit(struct crypto_tfm *tfm) CRYPTO_TRACE(); - clean_hash_setting(ctx->rk_dev); - if (ctx->hash_tmp) free_page((unsigned long)ctx->hash_tmp); diff --git a/drivers/crypto/rockchip/rk_crypto_v2_akcipher.c b/drivers/crypto/rockchip/rk_crypto_v2_akcipher.c index f9944a07538c..6ee2a5d40a00 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2_akcipher.c +++ b/drivers/crypto/rockchip/rk_crypto_v2_akcipher.c @@ -14,6 +14,7 @@ #include "rk_crypto_core.h" #include "rk_crypto_v2.h" #include "rk_crypto_v2_reg.h" +#include "rk_crypto_v2_pka.h" #define BG_WORDS2BYTES(words) ((words) * sizeof(u32)) #define BG_BYTES2WORDS(bytes) (((bytes) + sizeof(u32) - 1) / sizeof(u32)) diff --git a/drivers/crypto/rockchip/rk_crypto_v2_pka.h b/drivers/crypto/rockchip/rk_crypto_v2_pka.h new file mode 100644 index 000000000000..3c0b236f9d88 --- /dev/null +++ b/drivers/crypto/rockchip/rk_crypto_v2_pka.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2022 Rockchip Electronics Co. Ltd. */ + +#ifndef __RK_CRYPTO_V2_PKA_H__ +#define __RK_CRYPTO_V2_PKA_H__ + +#include "rk_crypto_bignum.h" + +void rk_pka_set_crypto_base(void __iomem *base); + +int rk_pka_expt_mod(struct rk_bignum *in, + struct rk_bignum *e, + struct rk_bignum *n, + struct rk_bignum *out); + +#endif diff --git a/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c b/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c index c3437077ab3a..2c8eb32d6099 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c +++ b/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c @@ -14,12 +14,10 @@ #include #include "rk_crypto_core.h" +#include "rk_crypto_skcipher_utils.h" #include "rk_crypto_v2.h" #include "rk_crypto_v2_reg.h" -#define MASK_BC_MODE(mode) ((mode) & 0x00f0) -#define IS_BC_DECRYPT(mode) (!!((mode) & CRYPTO_BC_DECRYPT)) - static const u32 cipher_algo2bc[] = { [CIPHER_ALGO_DES] = CRYPTO_BC_DES, [CIPHER_ALGO_DES3_EDE] = CRYPTO_BC_TDES, @@ -36,30 +34,13 @@ static const u32 cipher_mode2bc[] = { [CIPHER_MODE_XTS] = CRYPTO_BC_XTS, }; -static struct rk_cipher_ctx *rk_cipher_ctx_cast( - struct rk_crypto_dev *rk_dev) -{ - struct skcipher_request *req = - skcipher_request_cast(rk_dev->async_req); - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); - - return ctx; -} - -static struct rk_alg_ctx *rk_alg_ctx_cast( - struct rk_crypto_dev *rk_dev) -{ - return &(rk_cipher_ctx_cast(rk_dev)->algs_ctx); -} - static int rk_crypto_irq_handle(int irq, void *dev_id) { struct rk_crypto_dev *rk_dev = platform_get_drvdata(dev_id); u32 interrupt_status; struct rk_hw_crypto_v2_info *hw_info = (struct rk_hw_crypto_v2_info *)rk_dev->hw_info; - struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); + struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev); interrupt_status = CRYPTO_READ(rk_dev, CRYPTO_DMA_INT_ST); CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_ST, interrupt_status); @@ -95,41 +76,14 @@ static int rk_crypto_irq_handle(int irq, void *dev_id) return 0; } -static inline void word2byte_be(u32 word, u8 *ch) -{ - ch[0] = (word >> 24) & 0xff; - ch[1] = (word >> 16) & 0xff; - ch[2] = (word >> 8) & 0xff; - ch[3] = (word >> 0) & 0xff; -} - -static inline u32 byte2word_be(const u8 *ch) -{ - return (*ch << 24) + (*(ch + 1) << 16) + - (*(ch + 2) << 8) + *(ch + 3); -} - static void set_iv_reg(struct rk_crypto_dev *rk_dev, const u8 *iv, u32 iv_len) { - u32 i; - u8 tmp_buf[4]; - u32 base_iv; - if (!iv || iv_len == 0) return; CRYPTO_DUMPHEX("set iv", iv, iv_len); - base_iv = CRYPTO_CH0_IV_0; - /* write iv data to reg */ - for (i = 0; i < iv_len / 4; i++, base_iv += 4) - CRYPTO_WRITE(rk_dev, base_iv, byte2word_be(iv + i * 4)); - - if (iv_len % 4) { - memset(tmp_buf, 0x00, sizeof(tmp_buf)); - memcpy((u8 *)tmp_buf, iv + (iv_len / 4) * 4, iv_len % 4); - CRYPTO_WRITE(rk_dev, base_iv, byte2word_be(tmp_buf)); - } + rk_crypto_write_regs(rk_dev, CRYPTO_CH0_IV_0, iv, iv_len); CRYPTO_WRITE(rk_dev, CRYPTO_CH0_IV_LEN_0, iv_len); } @@ -137,50 +91,13 @@ static void set_iv_reg(struct rk_crypto_dev *rk_dev, const u8 *iv, u32 iv_len) static void write_key_reg(struct rk_crypto_dev *rk_dev, const u8 *key, u32 key_len) { - u32 i; - u8 tmp_buf[4]; - u32 base_addr; - - base_addr = CRYPTO_CH0_KEY_0; - - for (i = 0; i < key_len / 4; i++, base_addr += 4) - CRYPTO_WRITE(rk_dev, base_addr, - byte2word_be(key + i * 4)); - - if (key_len % 4) { - memset(tmp_buf, 0x00, sizeof(tmp_buf)); - memcpy((u8 *)tmp_buf, key + i * 4, key_len % 4); - CRYPTO_WRITE(rk_dev, base_addr, - byte2word_be(tmp_buf)); - } + rk_crypto_write_regs(rk_dev, CRYPTO_CH0_KEY_0, key, key_len); } static void write_tkey_reg(struct rk_crypto_dev *rk_dev, const u8 *key, u32 key_len) { - u32 i; - u8 tmp_buf[4]; - u32 base_addr; - - base_addr = CRYPTO_CH4_KEY_0; - - for (i = 0; i < key_len / 4; i++, base_addr += 4) - CRYPTO_WRITE(rk_dev, base_addr, - byte2word_be(key + i * 4)); - - if (key_len % 4) { - memset(tmp_buf, 0x00, sizeof(tmp_buf)); - memcpy((u8 *)tmp_buf, key + i * 4, key_len % 4); - CRYPTO_WRITE(rk_dev, base_addr, - byte2word_be(tmp_buf)); - } -} - -static struct rk_crypto_algt *rk_cipher_get_algt(struct crypto_skcipher *tfm) -{ - struct skcipher_alg *alg = crypto_skcipher_alg(tfm); - - return container_of(alg, struct rk_crypto_algt, alg.crypto); + rk_crypto_write_regs(rk_dev, CRYPTO_CH4_KEY_0, key, key_len); } static bool is_force_fallback(struct rk_crypto_algt *algt, uint32_t key_len) @@ -198,17 +115,6 @@ static bool is_force_fallback(struct rk_crypto_algt *algt, uint32_t key_len) return false; } -static bool is_no_multi_blocksize(struct skcipher_request *req) -{ - struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); - struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher); - - return (algt->mode == CIPHER_MODE_CFB || - algt->mode == CIPHER_MODE_OFB || - algt->mode == CIPHER_MODE_CTR || - algt->mode == CIPHER_MODE_XTS) ? true : false; -} - static bool is_calc_need_round_up(struct skcipher_request *req) { struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); @@ -269,109 +175,6 @@ static void rk_crypto_complete(struct crypto_async_request *base, int err) base->complete(base, err); } -static int rk_handle_req(struct rk_crypto_dev *rk_dev, - struct skcipher_request *req) -{ - struct rk_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - - if (!IS_ALIGNED(req->cryptlen, ctx->algs_ctx.chunk_size) && - !is_no_multi_blocksize(req)) - return -EINVAL; - else - return rk_dev->enqueue(rk_dev, &req->base); -} - -static int rk_cipher_setkey(struct crypto_skcipher *cipher, - const u8 *key, unsigned int keylen) -{ - struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher); - struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); - uint32_t key_factor; - int ret = -EINVAL; - - CRYPTO_MSG("algo = %x, mode = %x, key_len = %d\n", - algt->algo, algt->mode, keylen); - - /* The key length of XTS is twice the normal length */ - key_factor = algt->mode == CIPHER_MODE_XTS ? 2 : 1; - - switch (algt->algo) { - case CIPHER_ALGO_DES: - ret = verify_skcipher_des_key(cipher, key); - if (ret) - goto exit; - break; - case CIPHER_ALGO_DES3_EDE: - ret = verify_skcipher_des3_key(cipher, key); - if (ret) - goto exit; - break; - case CIPHER_ALGO_AES: - if (keylen != (AES_KEYSIZE_128 * key_factor) && - keylen != (AES_KEYSIZE_192 * key_factor) && - keylen != (AES_KEYSIZE_256 * key_factor)) - goto exit; - break; - case CIPHER_ALGO_SM4: - if (keylen != (SM4_KEY_SIZE * key_factor)) - goto exit; - break; - default: - ret = -EINVAL; - goto exit; - } - - memcpy(ctx->key, key, keylen); - ctx->keylen = keylen; - ctx->fallback_key_inited = false; - - ret = 0; -exit: - return ret; -} - -static int rk_cipher_fallback(struct skcipher_request *req, - struct rk_cipher_ctx *ctx, - bool encrypt) -{ - int ret; - - CRYPTO_MSG("use fallback tfm"); - - if (!ctx->fallback_tfm) { - ret = -ENODEV; - CRYPTO_MSG("fallback_tfm is empty!\n"); - goto exit; - } - - if (!ctx->fallback_key_inited) { - ret = crypto_skcipher_setkey(ctx->fallback_tfm, - ctx->key, ctx->keylen); - if (ret) { - CRYPTO_MSG("fallback crypto_skcipher_setkey err = %d\n", - ret); - goto exit; - } - - ctx->fallback_key_inited = true; - } - - skcipher_request_set_tfm(&ctx->fallback_req, ctx->fallback_tfm); - skcipher_request_set_callback(&ctx->fallback_req, - req->base.flags, - req->base.complete, - req->base.data); - - skcipher_request_set_crypt(&ctx->fallback_req, req->src, - req->dst, req->cryptlen, req->iv); - - ret = encrypt ? crypto_skcipher_encrypt(&ctx->fallback_req) : - crypto_skcipher_decrypt(&ctx->fallback_req); - -exit: - return ret; -} - static int rk_cipher_crypt(struct skcipher_request *req, bool encrypt) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -430,8 +233,10 @@ static int rk_cipher_crypt(struct skcipher_request *req, bool encrypt) if (req->iv) memcpy(ctx->iv, req->iv, crypto_skcipher_ivsize(tfm)); + ctx->is_enc = encrypt; + CRYPTO_MSG("ctx->mode = %x\n", ctx->mode); - return rk_handle_req(ctx->rk_dev, req); + return rk_skcipher_handle_req(ctx->rk_dev, req); } static int rk_cipher_encrypt(struct skcipher_request *req) @@ -444,7 +249,7 @@ static int rk_cipher_decrypt(struct skcipher_request *req) return rk_cipher_crypt(req, false); } -static void rk_ablk_hw_init(struct rk_crypto_dev *rk_dev) +static int rk_ablk_hw_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 mode) { struct skcipher_request *req = skcipher_request_cast(rk_dev->async_req); @@ -458,7 +263,7 @@ static void rk_ablk_hw_init(struct rk_crypto_dev *rk_dev) ivsize = crypto_skcipher_ivsize(cipher); - if (MASK_BC_MODE(ctx->mode) == CRYPTO_BC_XTS) { + if (mode == CIPHER_MODE_XTS) { uint32_t tmp_len = ctx->keylen / 2; write_key_reg(ctx->rk_dev, ctx->key, tmp_len); @@ -467,7 +272,7 @@ static void rk_ablk_hw_init(struct rk_crypto_dev *rk_dev) write_key_reg(ctx->rk_dev, ctx->key, ctx->keylen); } - if (MASK_BC_MODE(ctx->mode) != CRYPTO_BC_ECB) + if (mode != CIPHER_MODE_ECB) set_iv_reg(rk_dev, req->iv, ivsize); ctx->mode |= CRYPTO_BC_ENABLE; @@ -477,15 +282,17 @@ static void rk_ablk_hw_init(struct rk_crypto_dev *rk_dev) CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x7f); CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, ctx->mode | CRYPTO_WRITE_MASK_ALL); + + return 0; } -static void crypto_dma_start(struct rk_crypto_dev *rk_dev) +static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) { struct rk_hw_crypto_v2_info *hw_info = (struct rk_hw_crypto_v2_info *)rk_dev->hw_info; struct skcipher_request *req = skcipher_request_cast(rk_dev->async_req); - struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); + struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev); u32 calc_len = alg_ctx->count; u32 start_flag = CRYPTO_DMA_START; @@ -512,179 +319,8 @@ static void crypto_dma_start(struct rk_crypto_dev *rk_dev) CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma); CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, start_flag | (start_flag << WRITE_MASK)); -} -static int rk_set_data_start(struct rk_crypto_dev *rk_dev) -{ - int err; - struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); - - err = rk_dev->load_data(rk_dev, alg_ctx->sg_src, alg_ctx->sg_dst); - if (!err) { - u32 ivsize = alg_ctx->chunk_size; - struct scatterlist *src_sg; - struct rk_cipher_ctx *ctx = rk_cipher_ctx_cast(rk_dev); - - memset(ctx->lastc, 0x00, sizeof(ctx->lastc)); - - src_sg = alg_ctx->aligned ? alg_ctx->sg_src : &alg_ctx->sg_tmp; - - ivsize = alg_ctx->count > ivsize ? ivsize : alg_ctx->count; - - sg_pcopy_to_buffer(src_sg, 1, ctx->lastc, ivsize, alg_ctx->count - ivsize); - - crypto_dma_start(rk_dev); - } - return err; -} - -static int rk_ablk_start(struct rk_crypto_dev *rk_dev) -{ - struct skcipher_request *req = - skcipher_request_cast(rk_dev->async_req); - struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); - unsigned long flags; - int err = 0; - - alg_ctx->left_bytes = req->cryptlen; - alg_ctx->total = req->cryptlen; - alg_ctx->sg_src = req->src; - alg_ctx->req_src = req->src; - alg_ctx->src_nents = sg_nents_for_len(req->src, req->cryptlen); - alg_ctx->sg_dst = req->dst; - alg_ctx->req_dst = req->dst; - alg_ctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen); - - CRYPTO_TRACE("total = %u", alg_ctx->total); - - spin_lock_irqsave(&rk_dev->lock, flags); - rk_ablk_hw_init(rk_dev); - err = rk_set_data_start(rk_dev); - spin_unlock_irqrestore(&rk_dev->lock, flags); - return err; -} - -/* increment counter (128-bit int) by 1 */ -static void rk_ctr128_inc(uint8_t *counter) -{ - u32 n = 16; - u8 c; - - do { - --n; - c = counter[n]; - ++c; - counter[n] = c; - if (c) - return; - } while (n); -} - -static void rk_ctr128_calc(uint8_t *counter, uint32_t data_len) -{ - u32 i; - u32 chunksize = AES_BLOCK_SIZE; - - for (i = 0; i < DIV_ROUND_UP(data_len, chunksize); i++) - rk_ctr128_inc(counter); -} - -static uint32_t rk_get_new_iv(struct rk_cipher_ctx *ctx, - uint8_t *iv) -{ - u32 bc_mode = MASK_BC_MODE(ctx->mode); - int is_enc = !IS_BC_DECRYPT(ctx->mode); - struct scatterlist *sg_dst; - struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx; - uint32_t ivsize = alg_ctx->chunk_size; - - if (!iv) - return 0; - - sg_dst = alg_ctx->aligned ? alg_ctx->sg_dst : &alg_ctx->sg_tmp; - - CRYPTO_TRACE("aligned = %u, count = %u, ivsize = %u, is_enc = %d\n", - alg_ctx->aligned, alg_ctx->count, ivsize, is_enc); - - switch (bc_mode) { - case CRYPTO_BC_CTR: - rk_ctr128_calc(iv, alg_ctx->count); - break; - case CRYPTO_BC_CBC: - case CRYPTO_BC_CFB: - if (is_enc) - sg_pcopy_to_buffer(sg_dst, 1, iv, ivsize, alg_ctx->count - ivsize); - else - memcpy(iv, ctx->lastc, ivsize); - break; - case CRYPTO_BC_OFB: - sg_pcopy_to_buffer(sg_dst, 1, iv, ivsize, alg_ctx->count - ivsize); - crypto_xor(iv, ctx->lastc, ivsize); - break; - default: - return 0; - } - - return ivsize; -} - -static void rk_update_iv(struct rk_crypto_dev *rk_dev) -{ - uint32_t iv_size; - struct rk_cipher_ctx *ctx = rk_cipher_ctx_cast(rk_dev); - - iv_size = rk_get_new_iv(ctx, ctx->iv); - - if (iv_size) - set_iv_reg(rk_dev, ctx->iv, iv_size); -} - -static void rk_iv_copyback(struct rk_crypto_dev *rk_dev) -{ - uint32_t iv_size; - struct skcipher_request *req = - skcipher_request_cast(rk_dev->async_req); - struct rk_cipher_ctx *ctx = rk_cipher_ctx_cast(rk_dev); - - iv_size = rk_get_new_iv(ctx, ctx->iv); - - if (iv_size && req->iv) - memcpy(req->iv, ctx->iv, iv_size); -} - -/* return: - * true some err was occurred - * fault no err, continue - */ -static int rk_ablk_rx(struct rk_crypto_dev *rk_dev) -{ - int err = 0; - struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev); - - CRYPTO_TRACE("left_bytes = %u\n", alg_ctx->left_bytes); - - err = rk_dev->unload_data(rk_dev); - if (err) - goto out_rx; - - if (alg_ctx->left_bytes) { - rk_update_iv(rk_dev); - if (alg_ctx->aligned) { - if (sg_is_last(alg_ctx->sg_src)) { - dev_err(rk_dev->dev, "[%s:%d] Lack of data\n", - __func__, __LINE__); - err = -ENOMEM; - goto out_rx; - } - alg_ctx->sg_src = sg_next(alg_ctx->sg_src); - alg_ctx->sg_dst = sg_next(alg_ctx->sg_dst); - } - err = rk_set_data_start(rk_dev); - } else { - rk_iv_copyback(rk_dev); - } -out_rx: - return err; + return 0; } static int rk_ablk_init_tfm(struct crypto_skcipher *tfm) @@ -713,6 +349,10 @@ static int rk_ablk_init_tfm(struct crypto_skcipher *tfm) alg_ctx->ops.complete = rk_crypto_complete; alg_ctx->ops.irq_handle = rk_crypto_irq_handle; + alg_ctx->ops.hw_init = rk_ablk_hw_init; + alg_ctx->ops.hw_dma_start = crypto_dma_start; + alg_ctx->ops.hw_write_iv = set_iv_reg; + ctx->rk_dev = rk_dev; if (algt->alg.crypto.base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) { @@ -745,35 +385,6 @@ static void rk_ablk_exit_tfm(struct crypto_skcipher *tfm) ctx->rk_dev->release_crypto(ctx->rk_dev, alg_name); } -int rk_hw_crypto_v2_init(struct device *dev, void *hw_info) -{ - int err = 0; - struct rk_hw_crypto_v2_info *info = - (struct rk_hw_crypto_v2_info *)hw_info; - - info->desc = dma_alloc_coherent(dev, - sizeof(struct crypto_lli_desc), - &info->desc_dma, - GFP_KERNEL); - if (!info->desc) { - err = -ENOMEM; - goto end; - } - -end: - return err; -} - -void rk_hw_crypto_v2_deinit(struct device *dev, void *hw_info) -{ - struct rk_hw_crypto_v2_info *info = - (struct rk_hw_crypto_v2_info *)hw_info; - - if (info && info->desc) - dma_free_coherent(dev, sizeof(struct crypto_lli_desc), - info->desc, info->desc_dma); -} - struct rk_crypto_algt rk_v2_ecb_sm4_alg = RK_CIPHER_ALGO_INIT(SM4, ECB, ecb(sm4), ecb-sm4-rk);