From f68bea16c9a8fa2a2102a3fc9974fea5d748aa57 Mon Sep 17 00:00:00 2001 From: Lin Jinhan Date: Thu, 3 Mar 2022 16:02:45 +0800 Subject: [PATCH] crypto: rockchip: v3: add crypto V3 support Add a new generation of Crypto IP drivers and add multithreaded cipher/hash support. Signed-off-by: Lin Jinhan Change-Id: Iaefda28a1efc9c5af29b71cd3453d108b6e9260a --- drivers/crypto/rockchip/Kconfig | 4 + drivers/crypto/rockchip/Makefile | 8 + drivers/crypto/rockchip/rk_crypto_core.c | 21 + drivers/crypto/rockchip/rk_crypto_v3.c | 104 ++++ drivers/crypto/rockchip/rk_crypto_v3.h | 87 ++++ drivers/crypto/rockchip/rk_crypto_v3_ahash.c | 451 ++++++++++++++++++ drivers/crypto/rockchip/rk_crypto_v3_reg.h | 80 ++++ .../crypto/rockchip/rk_crypto_v3_skcipher.c | 446 +++++++++++++++++ 8 files changed, 1201 insertions(+) create mode 100644 drivers/crypto/rockchip/rk_crypto_v3.c create mode 100644 drivers/crypto/rockchip/rk_crypto_v3.h create mode 100644 drivers/crypto/rockchip/rk_crypto_v3_ahash.c create mode 100644 drivers/crypto/rockchip/rk_crypto_v3_reg.h create mode 100644 drivers/crypto/rockchip/rk_crypto_v3_skcipher.c diff --git a/drivers/crypto/rockchip/Kconfig b/drivers/crypto/rockchip/Kconfig index 21baebbce721..a25fbb077b62 100644 --- a/drivers/crypto/rockchip/Kconfig +++ b/drivers/crypto/rockchip/Kconfig @@ -9,6 +9,10 @@ config CRYPTO_DEV_ROCKCHIP_V2 bool "crypto v2 for RV1109/RV1126 RK1808 RK3308 PX30/RK3326 RK356X RK3588" default y if CPU_RV1126 || CPU_RK1808 || CPU_RK3308 || CPU_PX30 || CPU_RK3568 || CPU_RK3588 +config CRYPTO_DEV_ROCKCHIP_V3 + bool "crypto v3 for RV1106" + default y if CPU_RV1106 + endif config CRYPTO_DEV_ROCKCHIP_DEV diff --git a/drivers/crypto/rockchip/Makefile b/drivers/crypto/rockchip/Makefile index 95da40fbe895..96f800fc4775 100644 --- a/drivers/crypto/rockchip/Makefile +++ b/drivers/crypto/rockchip/Makefile @@ -18,4 +18,12 @@ rk_crypto-$(CONFIG_CRYPTO_DEV_ROCKCHIP_V2) += \ rk_crypto_v2_pka.o \ rk_crypto_bignum.o +rk_crypto-$(CONFIG_CRYPTO_DEV_ROCKCHIP_V3) += \ + rk_crypto_v3.o \ + rk_crypto_v3_ahash.o \ + rk_crypto_v3_skcipher.o \ + rk_crypto_v2_akcipher.o \ + rk_crypto_v2_pka.o \ + rk_crypto_bignum.o + obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP_DEV) += cryptodev_linux/ diff --git a/drivers/crypto/rockchip/rk_crypto_core.c b/drivers/crypto/rockchip/rk_crypto_core.c index dc95df53feb3..d045c49ba32e 100644 --- a/drivers/crypto/rockchip/rk_crypto_core.c +++ b/drivers/crypto/rockchip/rk_crypto_core.c @@ -22,6 +22,7 @@ #include "rk_crypto_core.h" #include "rk_crypto_v1.h" #include "rk_crypto_v2.h" +#include "rk_crypto_v3.h" #include "cryptodev_linux/rk_cryptodev.h" static struct rk_alg_ctx *rk_alg_ctx_cast(struct crypto_async_request *async_req) @@ -574,6 +575,15 @@ static char *crypto_full_algs_name[] = { "rsa" }; +static char *crypto_rv1106_algs_name[] = { + "ecb(aes)", "cbc(aes)", "cfb(aes)", "ofb(aes)", "ctr(aes)", + "ecb(des)", "cbc(des)", "cfb(des)", "ofb(des)", + "ecb(des3_ede)", "cbc(des3_ede)", "cfb(des3_ede)", "ofb(des3_ede)", + "sha1", "sha224", "sha256", "md5", + "hmac(sha1)", "hmac(sha256)", "hmac(md5)", + "rsa" +}; + static const struct rk_crypto_soc_data px30_soc_data = RK_CRYPTO_V2_SOC_DATA_INIT(crypto_no_sm_algs_name, false); @@ -583,6 +593,9 @@ static const struct rk_crypto_soc_data rv1126_soc_data = static const struct rk_crypto_soc_data full_soc_data = RK_CRYPTO_V2_SOC_DATA_INIT(crypto_full_algs_name, false); +static const struct rk_crypto_soc_data cryto_v3_soc_data = + RK_CRYPTO_V3_SOC_DATA_INIT(crypto_rv1106_algs_name); + static char *rk3288_cipher_algs[] = { "ecb(aes)", "cbc(aes)", "ecb(des)", "cbc(des)", @@ -595,6 +608,14 @@ static const struct rk_crypto_soc_data rk3288_soc_data = static const struct of_device_id crypto_of_id_table[] = { +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V3) + /* crypto v3 in belows */ + { + .compatible = "rockchip,crypto-v3", + .data = (void *)&cryto_v3_soc_data, + }, +#endif + #if IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V2) /* crypto v2 in belows */ { diff --git a/drivers/crypto/rockchip/rk_crypto_v3.c b/drivers/crypto/rockchip/rk_crypto_v3.c new file mode 100644 index 000000000000..aa0544f8d6b6 --- /dev/null +++ b/drivers/crypto/rockchip/rk_crypto_v3.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Crypto acceleration support for Rockchip Crypto V3 + * + * Copyright (c) 2022, Rockchip Electronics Co., Ltd + * + * Author: Lin Jinhan + * + */ + +#include "rk_crypto_core.h" +#include "rk_crypto_v3.h" +#include "rk_crypto_v3_reg.h" + +static const char * const crypto_v3_rsts[] = { + "crypto-rst", +}; + +static struct rk_crypto_algt *crypto_v3_algs[] = { + &rk_v3_ecb_sm4_alg, /* ecb(sm4) */ + &rk_v3_cbc_sm4_alg, /* cbc(sm4) */ + &rk_v3_xts_sm4_alg, /* xts(sm4) */ + &rk_v3_cfb_sm4_alg, /* cfb(sm4) */ + &rk_v3_ofb_sm4_alg, /* ofb(sm4) */ + &rk_v3_ctr_sm4_alg, /* ctr(sm4) */ + + &rk_v3_ecb_aes_alg, /* ecb(aes) */ + &rk_v3_cbc_aes_alg, /* cbc(aes) */ + &rk_v3_xts_aes_alg, /* xts(aes) */ + &rk_v3_cfb_aes_alg, /* cfb(aes) */ + &rk_v3_ofb_aes_alg, /* ofb(aes) */ + &rk_v3_ctr_aes_alg, /* ctr(aes) */ + + &rk_v3_ecb_des_alg, /* ecb(des) */ + &rk_v3_cbc_des_alg, /* cbc(des) */ + &rk_v3_cfb_des_alg, /* cfb(des) */ + &rk_v3_ofb_des_alg, /* ofb(des) */ + + &rk_v3_ecb_des3_ede_alg, /* ecb(des3_ede) */ + &rk_v3_cbc_des3_ede_alg, /* cbc(des3_ede) */ + &rk_v3_cfb_des3_ede_alg, /* cfb(des3_ede) */ + &rk_v3_ofb_des3_ede_alg, /* ofb(des3_ede) */ + + &rk_v3_ahash_sha1, /* sha1 */ + &rk_v3_ahash_sha224, /* sha224 */ + &rk_v3_ahash_sha256, /* sha256 */ + &rk_v3_ahash_sha384, /* sha384 */ + &rk_v3_ahash_sha512, /* sha512 */ + &rk_v3_ahash_md5, /* md5 */ + &rk_v3_ahash_sm3, /* sm3 */ + + &rk_v3_hmac_sha1, /* hmac(sha1) */ + &rk_v3_hmac_sha256, /* hmac(sha256) */ + &rk_v3_hmac_sha512, /* hmac(sha512) */ + &rk_v3_hmac_md5, /* hmac(md5) */ + &rk_v3_hmac_sm3, /* hmac(sm3) */ + + /* Shared v2 version implementation */ + &rk_v2_asym_rsa, /* rsa */ +}; + +int rk_hw_crypto_v3_init(struct device *dev, void *hw_info) +{ + int err = 0; + struct rk_hw_crypto_v3_info *info = + (struct rk_hw_crypto_v3_info *)hw_info; + + info->desc = dma_alloc_coherent(dev, + sizeof(struct crypto_lli_desc), + &info->desc_dma, + GFP_KERNEL); + if (!info->desc) { + err = -ENOMEM; + goto end; + } + +end: + return err; +} + +void rk_hw_crypto_v3_deinit(struct device *dev, void *hw_info) +{ + struct rk_hw_crypto_v3_info *info = + (struct rk_hw_crypto_v3_info *)hw_info; + + if (info && info->desc) + dma_free_coherent(dev, sizeof(struct crypto_lli_desc), + info->desc, info->desc_dma); +} + +const char * const *rk_hw_crypto_v3_get_rsts(uint32_t *num) +{ + *num = ARRAY_SIZE(crypto_v3_rsts); + + return crypto_v3_rsts; +} + +struct rk_crypto_algt **rk_hw_crypto_v3_get_algts(uint32_t *num) +{ + *num = ARRAY_SIZE(crypto_v3_algs); + + return crypto_v3_algs; +} + diff --git a/drivers/crypto/rockchip/rk_crypto_v3.h b/drivers/crypto/rockchip/rk_crypto_v3.h new file mode 100644 index 000000000000..91939e6807e3 --- /dev/null +++ b/drivers/crypto/rockchip/rk_crypto_v3.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2022 Rockchip Electronics Co. Ltd. */ + +#ifndef __RK_CRYPTO_V3_H__ +#define __RK_CRYPTO_V3_H__ + +#include + +#include "rk_crypto_v2.h" + +struct rk_hw_crypto_v3_info { + struct crypto_lli_desc *desc; + dma_addr_t desc_dma; +}; + +#define RK_CRYPTO_V3_SOC_DATA_INIT(names) {\ + .crypto_ver = "CRYPTO V3.0.0.0 multi",\ + .use_soft_aes192 = false,\ + .valid_algs_name = (names),\ + .valid_algs_num = ARRAY_SIZE(names),\ + .hw_init = rk_hw_crypto_v3_init,\ + .hw_deinit = rk_hw_crypto_v3_deinit,\ + .hw_get_rsts = rk_hw_crypto_v3_get_rsts,\ + .hw_get_algts = rk_hw_crypto_v3_get_algts,\ + .hw_info_size = sizeof(struct rk_hw_crypto_v3_info),\ + .default_pka_offset = 0x0480,\ +} + +#if IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V3) + +extern struct rk_crypto_algt rk_v3_ecb_sm4_alg; +extern struct rk_crypto_algt rk_v3_cbc_sm4_alg; +extern struct rk_crypto_algt rk_v3_xts_sm4_alg; +extern struct rk_crypto_algt rk_v3_cfb_sm4_alg; +extern struct rk_crypto_algt rk_v3_ofb_sm4_alg; +extern struct rk_crypto_algt rk_v3_ctr_sm4_alg; + +extern struct rk_crypto_algt rk_v3_ecb_aes_alg; +extern struct rk_crypto_algt rk_v3_cbc_aes_alg; +extern struct rk_crypto_algt rk_v3_xts_aes_alg; +extern struct rk_crypto_algt rk_v3_cfb_aes_alg; +extern struct rk_crypto_algt rk_v3_ofb_aes_alg; +extern struct rk_crypto_algt rk_v3_ctr_aes_alg; + +extern struct rk_crypto_algt rk_v3_ecb_des_alg; +extern struct rk_crypto_algt rk_v3_cbc_des_alg; +extern struct rk_crypto_algt rk_v3_cfb_des_alg; +extern struct rk_crypto_algt rk_v3_ofb_des_alg; + +extern struct rk_crypto_algt rk_v3_ecb_des3_ede_alg; +extern struct rk_crypto_algt rk_v3_cbc_des3_ede_alg; +extern struct rk_crypto_algt rk_v3_cfb_des3_ede_alg; +extern struct rk_crypto_algt rk_v3_ofb_des3_ede_alg; + +extern struct rk_crypto_algt rk_v3_ahash_sha1; +extern struct rk_crypto_algt rk_v3_ahash_sha224; +extern struct rk_crypto_algt rk_v3_ahash_sha256; +extern struct rk_crypto_algt rk_v3_ahash_sha384; +extern struct rk_crypto_algt rk_v3_ahash_sha512; +extern struct rk_crypto_algt rk_v3_ahash_md5; +extern struct rk_crypto_algt rk_v3_ahash_sm3; + +extern struct rk_crypto_algt rk_v3_hmac_md5; +extern struct rk_crypto_algt rk_v3_hmac_sha1; +extern struct rk_crypto_algt rk_v3_hmac_sha256; +extern struct rk_crypto_algt rk_v3_hmac_sha512; +extern struct rk_crypto_algt rk_v3_hmac_sm3; + +/* Shared v2 version implementation */ +extern struct rk_crypto_algt rk_v2_asym_rsa; + +int rk_hw_crypto_v3_init(struct device *dev, void *hw_info); +void rk_hw_crypto_v3_deinit(struct device *dev, void *hw_info); +const char * const *rk_hw_crypto_v3_get_rsts(uint32_t *num); +struct rk_crypto_algt **rk_hw_crypto_v3_get_algts(uint32_t *num); + +#else + +static inline int rk_hw_crypto_v3_init(struct device *dev, void *hw_info) { return -EINVAL; } +static inline void rk_hw_crypto_v3_deinit(struct device *dev, void *hw_info) {} +static inline const char * const *rk_hw_crypto_v3_get_rsts(uint32_t *num) { return NULL; } +static inline struct rk_crypto_algt **rk_hw_crypto_v3_get_algts(uint32_t *num) { return NULL; } + +#endif /* end of IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V3) */ + +#endif /* end of __RK_CRYPTO_V3_H__ */ diff --git a/drivers/crypto/rockchip/rk_crypto_v3_ahash.c b/drivers/crypto/rockchip/rk_crypto_v3_ahash.c new file mode 100644 index 000000000000..7fe53ac71a65 --- /dev/null +++ b/drivers/crypto/rockchip/rk_crypto_v3_ahash.c @@ -0,0 +1,451 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Hash acceleration support for Rockchip Crypto v3 + * + * Copyright (c) 2022, Rockchip Electronics Co., Ltd + * + * Author: Lin Jinhan + * + */ + +#include +#include + +#include "rk_crypto_core.h" +#include "rk_crypto_v3.h" +#include "rk_crypto_v3_reg.h" +#include "rk_crypto_ahash_utils.h" + +#define RK_HASH_CTX_MAGIC 0x1A1A1A1A +#define RK_POLL_PERIOD_US 100 +#define RK_POLL_TIMEOUT_US 50000 + +struct rk_ahash_expt_ctx { + struct rk_ahash_ctx ctx; + u8 lastc[RK_DMA_ALIGNMENT]; +}; + +struct rk_hash_mid_data { + u32 valid_flag; + u32 hash_ctl; + u32 data[CRYPTO_HASH_MID_WORD_SIZE]; +}; + +static const u32 hash_algo2bc[] = { + [HASH_ALGO_MD5] = CRYPTO_MD5, + [HASH_ALGO_SHA1] = CRYPTO_SHA1, + [HASH_ALGO_SHA224] = CRYPTO_SHA224, + [HASH_ALGO_SHA256] = CRYPTO_SHA256, + [HASH_ALGO_SHA384] = CRYPTO_SHA384, + [HASH_ALGO_SHA512] = CRYPTO_SHA512, + [HASH_ALGO_SM3] = CRYPTO_SM3, +}; + +static void rk_hash_reset(struct rk_crypto_dev *rk_dev) +{ + int ret; + u32 tmp = 0, tmp_mask = 0; + unsigned int pool_timeout_us = 1000; + + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x00); + + tmp = CRYPTO_SW_CC_RESET; + tmp_mask = tmp << CRYPTO_WRITE_MASK_SHIFT; + + CRYPTO_WRITE(rk_dev, CRYPTO_RST_CTL, tmp | tmp_mask); + + /* This is usually done in 20 clock cycles */ + ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_RST_CTL, + tmp, !tmp, 0, pool_timeout_us); + if (ret) + dev_err(rk_dev->dev, "cipher reset pool timeout %ums.", + pool_timeout_us); + + CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, 0xffff0000); +} + +static int rk_hash_mid_data_store(struct rk_crypto_dev *rk_dev, struct rk_hash_mid_data *mid_data) +{ + int ret; + uint32_t reg_ctrl; + + CRYPTO_TRACE(); + + ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_MID_VALID, + reg_ctrl, + reg_ctrl & CRYPTO_HASH_MID_IS_VALID, + RK_POLL_PERIOD_US, + RK_POLL_TIMEOUT_US); + CRYPTO_WRITE(rk_dev, CRYPTO_MID_VALID_SWITCH, + CRYPTO_MID_VALID_ENABLE << CRYPTO_WRITE_MASK_SHIFT); + if (ret) { + CRYPTO_TRACE("CRYPTO_MID_VALID timeout."); + goto exit; + } + + CRYPTO_WRITE(rk_dev, CRYPTO_MID_VALID, + CRYPTO_HASH_MID_IS_VALID | + CRYPTO_HASH_MID_IS_VALID << CRYPTO_WRITE_MASK_SHIFT); + + rk_crypto_read_regs(rk_dev, CRYPTO_HASH_MID_DATA_0, + (u8 *)mid_data->data, sizeof(mid_data->data)); + + mid_data->hash_ctl = CRYPTO_READ(rk_dev, CRYPTO_HASH_CTL); + mid_data->valid_flag = 1; + + CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, 0 | CRYPTO_WRITE_MASK_ALL); + +exit: + return ret; +} + +static int rk_hash_mid_data_restore(struct rk_crypto_dev *rk_dev, struct rk_hash_mid_data *mid_data) +{ + CRYPTO_TRACE(); + + CRYPTO_WRITE(rk_dev, CRYPTO_MID_VALID_SWITCH, + CRYPTO_MID_VALID_ENABLE | CRYPTO_MID_VALID_ENABLE << CRYPTO_WRITE_MASK_SHIFT); + + CRYPTO_WRITE(rk_dev, CRYPTO_MID_VALID, + CRYPTO_HASH_MID_IS_VALID | + CRYPTO_HASH_MID_IS_VALID << CRYPTO_WRITE_MASK_SHIFT); + + if (!mid_data->valid_flag) { + CRYPTO_TRACE("clear mid data"); + rk_crypto_clear_regs(rk_dev, CRYPTO_HASH_MID_DATA_0, ARRAY_SIZE(mid_data->data)); + return 0; + } + + rk_crypto_write_regs(rk_dev, CRYPTO_HASH_MID_DATA_0, + (u8 *)mid_data->data, sizeof(mid_data->data)); + + CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, mid_data->hash_ctl | CRYPTO_WRITE_MASK_ALL); + + return 0; +} + +static int rk_crypto_irq_handle(int irq, void *dev_id) +{ + struct rk_crypto_dev *rk_dev = platform_get_drvdata(dev_id); + u32 interrupt_status; + struct rk_hw_crypto_v3_info *hw_info = + (struct rk_hw_crypto_v3_info *)rk_dev->hw_info; + struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(rk_dev); + + /* disable crypto irq */ + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0); + + interrupt_status = CRYPTO_READ(rk_dev, CRYPTO_DMA_INT_ST); + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_ST, interrupt_status); + + interrupt_status &= CRYPTO_LOCKSTEP_MASK; + + if (interrupt_status != CRYPTO_SRC_ITEM_DONE_INT_ST) { + dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->desc); + dev_err(rk_dev->dev, "DMA addr_in = %08x\n", + (u32)alg_ctx->addr_in); + dev_err(rk_dev->dev, "DMA addr_out = %08x\n", + (u32)alg_ctx->addr_out); + dev_err(rk_dev->dev, "DMA count = %08x\n", alg_ctx->count); + dev_err(rk_dev->dev, "DMA desc_dma = %08x\n", + (u32)hw_info->desc_dma); + dev_err(rk_dev->dev, "DMA Error status = %08x\n", + interrupt_status); + dev_err(rk_dev->dev, "DMA CRYPTO_DMA_LLI_ADDR status = %08x\n", + CRYPTO_READ(rk_dev, CRYPTO_DMA_LLI_ADDR)); + dev_err(rk_dev->dev, "DMA CRYPTO_DMA_ST status = %08x\n", + CRYPTO_READ(rk_dev, CRYPTO_DMA_ST)); + dev_err(rk_dev->dev, "DMA CRYPTO_DMA_STATE status = %08x\n", + CRYPTO_READ(rk_dev, CRYPTO_DMA_STATE)); + dev_err(rk_dev->dev, "DMA CRYPTO_DMA_LLI_RADDR status = %08x\n", + CRYPTO_READ(rk_dev, CRYPTO_DMA_LLI_RADDR)); + dev_err(rk_dev->dev, "DMA CRYPTO_DMA_SRC_RADDR status = %08x\n", + CRYPTO_READ(rk_dev, CRYPTO_DMA_SRC_RADDR)); + dev_err(rk_dev->dev, "DMA CRYPTO_DMA_DST_RADDR status = %08x\n", + CRYPTO_READ(rk_dev, CRYPTO_DMA_DST_RADDR)); + rk_dev->err = -EFAULT; + } + + return 0; +} + +static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err) +{ + struct ahash_request *req = ahash_request_cast(base); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(ctx->rk_dev); + + struct rk_hw_crypto_v3_info *hw_info = ctx->rk_dev->hw_info; + struct crypto_lli_desc *lli_desc = hw_info->desc; + + if (err) { + rk_hash_reset(ctx->rk_dev); + pr_err("aligned = %u, align_size = %u\n", + alg_ctx->aligned, alg_ctx->align_size); + pr_err("total = %u, left = %u, count = %u\n", + alg_ctx->total, alg_ctx->left_bytes, alg_ctx->count); + pr_err("lli->src = %08x\n", lli_desc->src_addr); + pr_err("lli->src_len = %08x\n", lli_desc->src_len); + pr_err("lli->dst = %08x\n", lli_desc->dst_addr); + pr_err("lli->dst_len = %08x\n", lli_desc->dst_len); + pr_err("lli->dma_ctl = %08x\n", lli_desc->dma_ctrl); + pr_err("lli->usr_def = %08x\n", lli_desc->user_define); + pr_err("lli->next = %08x\n\n\n", lli_desc->next_addr); + } + + if (alg_ctx->total) + rk_hash_mid_data_store(ctx->rk_dev, (struct rk_hash_mid_data *)ctx->priv); + + if (base->complete) + base->complete(base, err); +} + +static inline void clear_hash_out_reg(struct rk_crypto_dev *rk_dev) +{ + rk_crypto_clear_regs(rk_dev, CRYPTO_HASH_DOUT_0, 16); +} + +static int write_key_reg(struct rk_crypto_dev *rk_dev, const u8 *key, + u32 key_len) +{ + rk_crypto_write_regs(rk_dev, CRYPTO_CH0_KEY_0, key, key_len); + + return 0; +} + +static int rk_hw_hash_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 type) +{ + u32 reg_ctrl = 0; + struct rk_ahash_ctx *ctx = rk_ahash_ctx_cast(rk_dev); + struct rk_hash_mid_data *mid_data = (struct rk_hash_mid_data *)ctx->priv; + + if (algo >= ARRAY_SIZE(hash_algo2bc)) + goto exit; + + rk_hash_reset(rk_dev); + + clear_hash_out_reg(rk_dev); + + reg_ctrl = hash_algo2bc[algo] | CRYPTO_HW_PAD_ENABLE; + + if (IS_TYPE_HMAC(type)) { + CRYPTO_TRACE("this is hmac"); + reg_ctrl |= CRYPTO_HMAC_ENABLE; + } + + CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, reg_ctrl | CRYPTO_WRITE_MASK_ALL); + CRYPTO_WRITE(rk_dev, CRYPTO_FIFO_CTL, 0x00030003); + + memset(mid_data, 0x00, sizeof(*mid_data)); + + return 0; +exit: + CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, 0 | CRYPTO_WRITE_MASK_ALL); + + return -EINVAL; +} + +static void clean_hash_setting(struct rk_crypto_dev *rk_dev) +{ + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0); + CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, 0 | CRYPTO_WRITE_MASK_ALL); +} + +static int rk_ahash_import(struct ahash_request *req, const void *in) +{ + struct rk_ahash_expt_ctx state; + + /* 'in' may not be aligned so memcpy to local variable */ + memcpy(&state, in, sizeof(state)); + + ///TODO: deal with import + + return 0; +} + +static int rk_ahash_export(struct ahash_request *req, void *out) +{ + struct rk_ahash_expt_ctx state; + + /* Don't let anything leak to 'out' */ + memset(&state, 0, sizeof(state)); + + ///TODO: deal with import + + memcpy(out, &state, sizeof(state)); + + return 0; +} + +static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) +{ + struct rk_hw_crypto_v3_info *hw_info = + (struct rk_hw_crypto_v3_info *)rk_dev->hw_info; + struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(rk_dev); + struct rk_ahash_ctx *ctx = rk_ahash_ctx_cast(rk_dev); + u32 dma_ctl = CRYPTO_DMA_RESTART; + bool is_final = flag & RK_FLAG_FINAL; + + CRYPTO_TRACE("ctx->calc_cnt = %u, count %u Byte, is_final = %d", + ctx->calc_cnt, alg_ctx->count, is_final); + + if (alg_ctx->count % RK_DMA_ALIGNMENT && !is_final) { + dev_err(rk_dev->dev, "count = %u is not aligned with [%u]\n", + alg_ctx->count, RK_DMA_ALIGNMENT); + return -EINVAL; + } + + if (alg_ctx->count == 0) { + /* do nothing */ + CRYPTO_TRACE("empty calc"); + return 0; + } + + if (alg_ctx->total == alg_ctx->left_bytes + alg_ctx->count) + rk_hash_mid_data_restore(rk_dev, (struct rk_hash_mid_data *)ctx->priv); + + memset(hw_info->desc, 0x00, sizeof(*hw_info->desc)); + + hw_info->desc->src_addr = alg_ctx->addr_in; + hw_info->desc->src_len = alg_ctx->count; + hw_info->desc->next_addr = hw_info->desc_dma; + + hw_info->desc->dma_ctrl = is_final ? LLI_DMA_CTRL_LAST : LLI_DMA_CTRL_PAUSE; + hw_info->desc->dma_ctrl |= LLI_DMA_CTRL_SRC_DONE; + + if (ctx->calc_cnt == 0) { + dma_ctl = CRYPTO_DMA_START; + + hw_info->desc->user_define |= LLI_USER_CIPHER_START; + hw_info->desc->user_define |= LLI_USER_STRING_START; + + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma); + CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, + (CRYPTO_HASH_ENABLE << CRYPTO_WRITE_MASK_SHIFT) | + CRYPTO_HASH_ENABLE); + } + + if (is_final && alg_ctx->left_bytes == 0) + hw_info->desc->user_define |= LLI_USER_STRING_LAST; + + CRYPTO_TRACE("dma_ctrl = %08x, user_define = %08x, len = %u", + hw_info->desc->dma_ctrl, hw_info->desc->user_define, alg_ctx->count); + + dma_wmb(); + + /* enable crypto irq */ + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x7f); + + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, dma_ctl | dma_ctl << CRYPTO_WRITE_MASK_SHIFT); + + return 0; +} + +static int rk_ahash_get_result(struct rk_crypto_dev *rk_dev, + uint8_t *data, uint32_t data_len) +{ + int ret = 0; + u32 reg_ctrl = 0; + struct rk_ahash_ctx *ctx = rk_ahash_ctx_cast(rk_dev); + + memset(ctx->priv, 0x00, sizeof(struct rk_hash_mid_data)); + + ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_HASH_VALID, + reg_ctrl, + reg_ctrl & CRYPTO_HASH_IS_VALID, + RK_POLL_PERIOD_US, + RK_POLL_TIMEOUT_US); + if (ret) + goto exit; + + rk_crypto_read_regs(rk_dev, CRYPTO_HASH_DOUT_0, data, data_len); + + CRYPTO_WRITE(rk_dev, CRYPTO_HASH_VALID, CRYPTO_HASH_IS_VALID); + +exit: + clean_hash_setting(rk_dev); + + return ret; +} + +static int rk_cra_hash_init(struct crypto_tfm *tfm) +{ + struct rk_crypto_algt *algt = + rk_ahash_get_algt(__crypto_ahash_cast(tfm)); + const char *alg_name = crypto_tfm_alg_name(tfm); + struct rk_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + struct rk_crypto_dev *rk_dev = algt->rk_dev; + struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx; + + CRYPTO_TRACE(); + + memset(ctx, 0x00, sizeof(*ctx)); + + if (!rk_dev->request_crypto) + return -EFAULT; + + alg_ctx->align_size = RK_DMA_ALIGNMENT; + + alg_ctx->ops.start = rk_ahash_start; + alg_ctx->ops.update = rk_ahash_crypto_rx; + alg_ctx->ops.complete = rk_ahash_crypto_complete; + alg_ctx->ops.irq_handle = rk_crypto_irq_handle; + + alg_ctx->ops.hw_write_key = write_key_reg; + alg_ctx->ops.hw_init = rk_hw_hash_init; + alg_ctx->ops.hw_dma_start = rk_ahash_dma_start; + alg_ctx->ops.hw_get_result = rk_ahash_get_result; + + ctx->rk_dev = rk_dev; + ctx->hash_tmp = (u8 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); + if (!ctx->hash_tmp) { + dev_err(rk_dev->dev, "Can't get zeroed page for hash tmp.\n"); + return -ENOMEM; + } + + ctx->priv = kmalloc(sizeof(struct rk_hash_mid_data), GFP_KERNEL); + if (!ctx->priv) { + free_page((unsigned long)ctx->hash_tmp); + return -ENOMEM; + } + + memset(ctx->priv, 0x00, sizeof(struct rk_hash_mid_data)); + + rk_dev->request_crypto(rk_dev, alg_name); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct rk_ahash_rctx)); + + algt->alg.hash.halg.statesize = sizeof(struct rk_ahash_expt_ctx); + + return 0; +} + +static void rk_cra_hash_exit(struct crypto_tfm *tfm) +{ + struct rk_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + + CRYPTO_TRACE(); + + if (ctx->hash_tmp) + free_page((unsigned long)ctx->hash_tmp); + + kfree(ctx->priv); + + ctx->rk_dev->release_crypto(ctx->rk_dev, crypto_tfm_alg_name(tfm)); +} + +struct rk_crypto_algt rk_v3_ahash_md5 = RK_HASH_ALGO_INIT(MD5, md5); +struct rk_crypto_algt rk_v3_ahash_sha1 = RK_HASH_ALGO_INIT(SHA1, sha1); +struct rk_crypto_algt rk_v3_ahash_sha224 = RK_HASH_ALGO_INIT(SHA224, sha224); +struct rk_crypto_algt rk_v3_ahash_sha256 = RK_HASH_ALGO_INIT(SHA256, sha256); +struct rk_crypto_algt rk_v3_ahash_sha384 = RK_HASH_ALGO_INIT(SHA384, sha384); +struct rk_crypto_algt rk_v3_ahash_sha512 = RK_HASH_ALGO_INIT(SHA512, sha512); +struct rk_crypto_algt rk_v3_ahash_sm3 = RK_HASH_ALGO_INIT(SM3, sm3); + +struct rk_crypto_algt rk_v3_hmac_md5 = RK_HMAC_ALGO_INIT(MD5, md5); +struct rk_crypto_algt rk_v3_hmac_sha1 = RK_HMAC_ALGO_INIT(SHA1, sha1); +struct rk_crypto_algt rk_v3_hmac_sha256 = RK_HMAC_ALGO_INIT(SHA256, sha256); +struct rk_crypto_algt rk_v3_hmac_sha512 = RK_HMAC_ALGO_INIT(SHA512, sha512); +struct rk_crypto_algt rk_v3_hmac_sm3 = RK_HMAC_ALGO_INIT(SM3, sm3); + diff --git a/drivers/crypto/rockchip/rk_crypto_v3_reg.h b/drivers/crypto/rockchip/rk_crypto_v3_reg.h new file mode 100644 index 000000000000..1c4c45317a28 --- /dev/null +++ b/drivers/crypto/rockchip/rk_crypto_v3_reg.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2022 Rockchip Electronics Co. Ltd. */ + +#ifndef __RK_CRYPTO_V3_REG_H__ +#define __RK_CRYPTO_V3_REG_H__ + +#include "rk_crypto_v2_reg.h" + +#define CRYPTO_UNEQUAL_ERR_INT_EN BIT(9) +#define CRYPTO_ZERO_LEN_INT_EN BIT(6) + +/* DMA Destination Data Read Address Register */ +#define CRYPTO_DMA_DST_WADDR 0x0028 + +#define CRYPTO_BC_MID_IV_0 0x0060 +#define CRYPTO_BC_MID_WORD_SIZE 6 + +#define CRYPTO_MID_VALID 0x03e8 +#define CRYPTO_BC_MID_IS_VALID BIT(0) +#define CRYPTO_HASH_MID_IS_VALID BIT(1) + +#define CRYPTO_KEY_SEL 0x0610 + +#define CRYPTO_MID_VALID_SWITCH 0x0630 +#define CRYPTO_MID_VALID_ENABLE BIT(0) + +#define CRYPTO_AES_VERSION 0x0680 +#define CRYPTO_DES_VERSION 0x0684 +#define CRYPTO_SM4_VERSION 0x0688 + +#define CRYPTO_ECB_FLAG BIT(0) +#define CRYPTO_CBC_FLAG BIT(1) +#define CRYPTO_CTS_FLAG BIT(2) +#define CRYPTO_CTR_FLAG BIT(3) +#define CRYPTO_CFB_FLAG BIT(4) +#define CRYPTO_OFB_FLAG BIT(5) +#define CRYPTO_XTS_FLAG BIT(6) +#define CRYPTO_CCM_FLAG BIT(7) +#define CRYPTO_GCM_FLAG BIT(8) +#define CRYPTO_CMAC_FLAG BIT(9) +#define CRYPTO_CBCMAC_FLAG BIT(10) + +#define CRYPTO_AES128_FLAG BIT(16) +#define CRYPTO_AES192_FLAG BIT(17) +#define CRYPTO_AES256_FLAG BIT(18) + +#define CRYPTO_TDES_FLAG BIT(16) + +#define CRYPTO_LOCKSEP_FLAG BIT(20) +#define CRYPTO_SECURE_FLAG BIT(21) +#define CRYPTO_MULTI_CHN_FLAG BIT(22) + +#define CRYPTO_HASH_VERSION 0x068C +#define CRYPTO_HASH_SHA1_FLAG BIT(0) +#define CRYPTO_HASH_SHA224_FLAG BIT(1) +#define CRYPTO_HASH_SHA256_FLAG BIT(2) +#define CRYPTO_HASH_SHA384_FLAG BIT(3) +#define CRYPTO_HASH_SHA512_FLAG BIT(4) +#define CRYPTO_HASH_SHA512_224_FLAG BIT(5) +#define CRYPTO_HASH_SHA512_256_FLAG BIT(6) +#define CRYPTO_HASH_MD5_FLAG BIT(7) +#define CRYPTO_HASH_SM3_FLAG BIT(8) + +#define CRYPTO_HMAC_VERSION 0x0690 +#define CRYPTO_HMAC_SHA1_FLAG BIT(0) +#define CRYPTO_HMAC_SHA256_FLAG BIT(1) +#define CRYPTO_HMAC_SHA512_FLAG BIT(2) +#define CRYPTO_HMAC_MD5_FLAG BIT(3) +#define CRYPTO_HMAC_SM3_FLAG BIT(4) + +#define CRYPTO_RNG_VERSION 0x0694 +#define CRYPTO_PKA_VERSION 0x0698 +#define CRYPTO_CRYPTO_VERSION 0x06F0 + +#define CRYPTO_HASH_MID_DATA_0 0x0700 +#define CRYPTO_HASH_MID_WORD_SIZE 26 + +#endif + diff --git a/drivers/crypto/rockchip/rk_crypto_v3_skcipher.c b/drivers/crypto/rockchip/rk_crypto_v3_skcipher.c new file mode 100644 index 000000000000..9d8cde3e4014 --- /dev/null +++ b/drivers/crypto/rockchip/rk_crypto_v3_skcipher.c @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Crypto acceleration support for Rockchip Crypto V2 + * + * Copyright (c) 2022, Fuzhou Rockchip Electronics Co., Ltd + * + * Author: Lin Jinhan + * + */ + +#include +#include +#include + +#include "rk_crypto_core.h" +#include "rk_crypto_skcipher_utils.h" +#include "rk_crypto_v3.h" +#include "rk_crypto_v3_reg.h" + +static const u32 cipher_algo2bc[] = { + [CIPHER_ALGO_DES] = CRYPTO_BC_DES, + [CIPHER_ALGO_DES3_EDE] = CRYPTO_BC_TDES, + [CIPHER_ALGO_AES] = CRYPTO_BC_AES, + [CIPHER_ALGO_SM4] = CRYPTO_BC_SM4, +}; + +static const u32 cipher_mode2bc[] = { + [CIPHER_MODE_ECB] = CRYPTO_BC_ECB, + [CIPHER_MODE_CBC] = CRYPTO_BC_CBC, + [CIPHER_MODE_CFB] = CRYPTO_BC_CFB, + [CIPHER_MODE_OFB] = CRYPTO_BC_OFB, + [CIPHER_MODE_CTR] = CRYPTO_BC_CTR, + [CIPHER_MODE_XTS] = CRYPTO_BC_XTS, +}; + +static int rk_crypto_irq_handle(int irq, void *dev_id) +{ + struct rk_crypto_dev *rk_dev = platform_get_drvdata(dev_id); + u32 interrupt_status; + struct rk_hw_crypto_v3_info *hw_info = + (struct rk_hw_crypto_v3_info *)rk_dev->hw_info; + struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev); + + interrupt_status = CRYPTO_READ(rk_dev, CRYPTO_DMA_INT_ST); + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_ST, interrupt_status); + + interrupt_status &= CRYPTO_LOCKSTEP_MASK; + + if (interrupt_status != CRYPTO_DST_ITEM_DONE_INT_ST) { + dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->desc); + dev_err(rk_dev->dev, "DMA addr_in = %08x\n", + (u32)alg_ctx->addr_in); + dev_err(rk_dev->dev, "DMA addr_out = %08x\n", + (u32)alg_ctx->addr_out); + dev_err(rk_dev->dev, "DMA count = %08x\n", alg_ctx->count); + dev_err(rk_dev->dev, "DMA desc_dma = %08x\n", + (u32)hw_info->desc_dma); + dev_err(rk_dev->dev, "DMA Error status = %08x\n", + interrupt_status); + dev_err(rk_dev->dev, "DMA CRYPTO_DMA_LLI_ADDR status = %08x\n", + CRYPTO_READ(rk_dev, CRYPTO_DMA_LLI_ADDR)); + dev_err(rk_dev->dev, "DMA CRYPTO_DMA_ST status = %08x\n", + CRYPTO_READ(rk_dev, CRYPTO_DMA_ST)); + dev_err(rk_dev->dev, "DMA CRYPTO_DMA_STATE status = %08x\n", + CRYPTO_READ(rk_dev, CRYPTO_DMA_STATE)); + dev_err(rk_dev->dev, "DMA CRYPTO_DMA_LLI_RADDR status = %08x\n", + CRYPTO_READ(rk_dev, CRYPTO_DMA_LLI_RADDR)); + dev_err(rk_dev->dev, "DMA CRYPTO_DMA_SRC_RADDR status = %08x\n", + CRYPTO_READ(rk_dev, CRYPTO_DMA_SRC_RADDR)); + dev_err(rk_dev->dev, "DMA CRYPTO_DMA_DST_RADDR status = %08x\n", + CRYPTO_READ(rk_dev, CRYPTO_DMA_DST_RADDR)); + rk_dev->err = -EFAULT; + } + + return 0; +} + +static void set_iv_reg(struct rk_crypto_dev *rk_dev, const u8 *iv, u32 iv_len) +{ + if (!iv || iv_len == 0) + return; + + CRYPTO_DUMPHEX("set iv", iv, iv_len); + + rk_crypto_write_regs(rk_dev, CRYPTO_CH0_IV_0, iv, iv_len); + + CRYPTO_WRITE(rk_dev, CRYPTO_CH0_IV_LEN_0, iv_len); +} + +static void write_key_reg(struct rk_crypto_dev *rk_dev, const u8 *key, + u32 key_len) +{ + rk_crypto_write_regs(rk_dev, CRYPTO_CH0_KEY_0, key, key_len); +} + +static void write_tkey_reg(struct rk_crypto_dev *rk_dev, const u8 *key, + u32 key_len) +{ + rk_crypto_write_regs(rk_dev, CRYPTO_CH4_KEY_0, key, key_len); +} + +static bool is_force_fallback(struct rk_crypto_algt *algt, uint32_t key_len) +{ + if (algt->algo != CIPHER_ALGO_AES) + return false; + + /* crypto v2 not support xts with AES-192 */ + if (algt->mode == CIPHER_MODE_XTS && key_len == AES_KEYSIZE_192 * 2) + return true; + + if (algt->use_soft_aes192 && key_len == AES_KEYSIZE_192) + return true; + + return false; +} + +static bool is_calc_need_round_up(struct skcipher_request *req) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher); + + return (algt->mode == CIPHER_MODE_CFB || + algt->mode == CIPHER_MODE_OFB || + algt->mode == CIPHER_MODE_CTR) ? true : false; +} + +static void rk_cipher_reset(struct rk_crypto_dev *rk_dev) +{ + int ret; + u32 tmp = 0, tmp_mask = 0; + unsigned int pool_timeout_us = 1000; + + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x00); + + tmp = CRYPTO_SW_CC_RESET; + tmp_mask = tmp << CRYPTO_WRITE_MASK_SHIFT; + + CRYPTO_WRITE(rk_dev, CRYPTO_RST_CTL, tmp | tmp_mask); + + /* This is usually done in 20 clock cycles */ + ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_RST_CTL, + tmp, !tmp, 0, pool_timeout_us); + if (ret) + dev_err(rk_dev->dev, "cipher reset pool timeout %ums.", + pool_timeout_us); + + CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, 0xffff0000); +} + +static void rk_crypto_complete(struct crypto_async_request *base, int err) +{ + struct rk_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); + struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx; + struct rk_hw_crypto_v3_info *hw_info = ctx->rk_dev->hw_info; + struct crypto_lli_desc *lli_desc = hw_info->desc; + + CRYPTO_WRITE(ctx->rk_dev, CRYPTO_BC_CTL, 0xffff0000); + if (err) { + rk_cipher_reset(ctx->rk_dev); + pr_err("aligned = %u, align_size = %u\n", + alg_ctx->aligned, alg_ctx->align_size); + pr_err("total = %u, left = %u, count = %u\n", + alg_ctx->total, alg_ctx->left_bytes, alg_ctx->count); + pr_err("lli->src = %08x\n", lli_desc->src_addr); + pr_err("lli->src_len = %08x\n", lli_desc->src_len); + pr_err("lli->dst = %08x\n", lli_desc->dst_addr); + pr_err("lli->dst_len = %08x\n", lli_desc->dst_len); + pr_err("lli->dma_ctl = %08x\n", lli_desc->dma_ctrl); + pr_err("lli->usr_def = %08x\n", lli_desc->user_define); + pr_err("lli->next = %08x\n\n\n", lli_desc->next_addr); + } + + if (base->complete) + base->complete(base, err); +} + +static int rk_cipher_crypt(struct skcipher_request *req, bool encrypt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + struct rk_crypto_algt *algt = rk_cipher_get_algt(tfm); + + CRYPTO_TRACE("%s total = %u", + encrypt ? "encrypt" : "decrypt", req->cryptlen); + + if (!req->cryptlen) { + if (algt->mode == CIPHER_MODE_ECB || + algt->mode == CIPHER_MODE_CBC || + algt->mode == CIPHER_MODE_CTR || + algt->mode == CIPHER_MODE_CFB || + algt->mode == CIPHER_MODE_OFB) + return 0; + else + return -EINVAL; + } + + /* XTS data should >= chunksize */ + if (algt->mode == CIPHER_MODE_XTS) { + if (req->cryptlen < crypto_skcipher_chunksize(tfm)) + return -EINVAL; + + /* force use unalign branch */ + ctx->algs_ctx.align_size = ctx->rk_dev->vir_max; + + /* XTS can't pause when use hardware crypto */ + if (req->cryptlen > ctx->rk_dev->vir_max) + return rk_cipher_fallback(req, ctx, encrypt); + } + + if (is_force_fallback(algt, ctx->keylen)) + return rk_cipher_fallback(req, ctx, encrypt); + + ctx->mode = cipher_algo2bc[algt->algo] | + cipher_mode2bc[algt->mode]; + if (!encrypt) + ctx->mode |= CRYPTO_BC_DECRYPT; + + if (algt->algo == CIPHER_ALGO_AES) { + uint32_t key_factor; + + /* The key length of XTS is twice the normal length */ + key_factor = algt->mode == CIPHER_MODE_XTS ? 2 : 1; + + if (ctx->keylen == AES_KEYSIZE_128 * key_factor) + ctx->mode |= CRYPTO_BC_128_bit_key; + else if (ctx->keylen == AES_KEYSIZE_192 * key_factor) + ctx->mode |= CRYPTO_BC_192_bit_key; + else if (ctx->keylen == AES_KEYSIZE_256 * key_factor) + ctx->mode |= CRYPTO_BC_256_bit_key; + } + + if (req->iv) + memcpy(ctx->iv, req->iv, crypto_skcipher_ivsize(tfm)); + + ctx->is_enc = encrypt; + + CRYPTO_MSG("ctx->mode = %x\n", ctx->mode); + return rk_skcipher_handle_req(ctx->rk_dev, req); +} + +static int rk_cipher_encrypt(struct skcipher_request *req) +{ + return rk_cipher_crypt(req, true); +} + +static int rk_cipher_decrypt(struct skcipher_request *req) +{ + return rk_cipher_crypt(req, false); +} + +static int rk_ablk_hw_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 mode) +{ + struct skcipher_request *req = + skcipher_request_cast(rk_dev->async_req); + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher); + u32 ivsize; + + rk_cipher_reset(rk_dev); + + CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, 0x00010000); + + ivsize = crypto_skcipher_ivsize(cipher); + + if (mode == CIPHER_MODE_XTS) { + uint32_t tmp_len = ctx->keylen / 2; + + write_key_reg(ctx->rk_dev, ctx->key, tmp_len); + write_tkey_reg(ctx->rk_dev, ctx->key + tmp_len, tmp_len); + } else { + write_key_reg(ctx->rk_dev, ctx->key, ctx->keylen); + } + + if (mode != CIPHER_MODE_ECB) + set_iv_reg(rk_dev, req->iv, ivsize); + + ctx->mode |= CRYPTO_BC_ENABLE; + + CRYPTO_WRITE(rk_dev, CRYPTO_FIFO_CTL, 0x00030003); + + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x7f); + + CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, ctx->mode | CRYPTO_WRITE_MASK_ALL); + + return 0; +} + +static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) +{ + struct rk_hw_crypto_v3_info *hw_info = + (struct rk_hw_crypto_v3_info *)rk_dev->hw_info; + struct skcipher_request *req = + skcipher_request_cast(rk_dev->async_req); + struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev); + u32 calc_len = alg_ctx->count; + u32 start_flag = CRYPTO_DMA_START; + + memset(hw_info->desc, 0x00, sizeof(*hw_info->desc)); + + /* + * the data length is not aligned will use addr_vir to calculate, + * so crypto v2 could round up data length to chunk_size + */ + if (is_calc_need_round_up(req)) + calc_len = round_up(calc_len, alg_ctx->chunk_size); + + hw_info->desc->src_addr = alg_ctx->addr_in; + hw_info->desc->src_len = calc_len; + hw_info->desc->dst_addr = alg_ctx->addr_out; + hw_info->desc->dst_len = calc_len; + hw_info->desc->next_addr = 0; + hw_info->desc->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST; + hw_info->desc->user_define = LLI_USER_STRING_START | + LLI_USER_CIPHER_START | + LLI_USER_STRING_LAST; + + dma_wmb(); + + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma); + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, start_flag | (start_flag << WRITE_MASK)); + + return 0; +} + +static int rk_ablk_init_tfm(struct crypto_skcipher *tfm) +{ + struct rk_crypto_algt *algt = rk_cipher_get_algt(tfm); + struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + const char *alg_name = crypto_tfm_alg_name(crypto_skcipher_tfm(tfm)); + struct rk_crypto_dev *rk_dev = algt->rk_dev; + struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx; + + CRYPTO_TRACE(); + + memset(ctx, 0x00, sizeof(*ctx)); + + if (!rk_dev->request_crypto) + return -EFAULT; + + rk_dev->request_crypto(rk_dev, alg_name); + + /* always not aligned for crypto v2 cipher */ + alg_ctx->align_size = 64; + alg_ctx->chunk_size = crypto_skcipher_chunksize(tfm); + + alg_ctx->ops.start = rk_ablk_start; + alg_ctx->ops.update = rk_ablk_rx; + alg_ctx->ops.complete = rk_crypto_complete; + alg_ctx->ops.irq_handle = rk_crypto_irq_handle; + + alg_ctx->ops.hw_init = rk_ablk_hw_init; + alg_ctx->ops.hw_dma_start = crypto_dma_start; + alg_ctx->ops.hw_write_iv = set_iv_reg; + + ctx->rk_dev = rk_dev; + + if (algt->alg.crypto.base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) { + CRYPTO_MSG("alloc fallback tfm, name = %s", alg_name); + ctx->fallback_tfm = crypto_alloc_skcipher(alg_name, 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback_tfm)) { + CRYPTO_MSG("Could not load fallback driver %s : %ld.\n", + alg_name, PTR_ERR(ctx->fallback_tfm)); + ctx->fallback_tfm = NULL; + } + } + + return 0; +} + +static void rk_ablk_exit_tfm(struct crypto_skcipher *tfm) +{ + struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + const char *alg_name = crypto_tfm_alg_name(crypto_skcipher_tfm(tfm)); + + CRYPTO_TRACE(); + + if (ctx->fallback_tfm) { + CRYPTO_MSG("free fallback tfm"); + crypto_free_skcipher(ctx->fallback_tfm); + } + + ctx->rk_dev->release_crypto(ctx->rk_dev, alg_name); +} + +struct rk_crypto_algt rk_v3_ecb_sm4_alg = + RK_CIPHER_ALGO_INIT(SM4, ECB, ecb(sm4), ecb-sm4-rk); + +struct rk_crypto_algt rk_v3_cbc_sm4_alg = + RK_CIPHER_ALGO_INIT(SM4, CBC, cbc(sm4), cbc-sm4-rk); + +struct rk_crypto_algt rk_v3_xts_sm4_alg = + RK_CIPHER_ALGO_XTS_INIT(SM4, xts(sm4), xts-sm4-rk); + +struct rk_crypto_algt rk_v3_cfb_sm4_alg = + RK_CIPHER_ALGO_INIT(SM4, CFB, cfb(sm4), cfb-sm4-rk); + +struct rk_crypto_algt rk_v3_ofb_sm4_alg = + RK_CIPHER_ALGO_INIT(SM4, OFB, ofb(sm4), ofb-sm4-rk); + +struct rk_crypto_algt rk_v3_ctr_sm4_alg = + RK_CIPHER_ALGO_INIT(SM4, CTR, ctr(sm4), ctr-sm4-rk); + +struct rk_crypto_algt rk_v3_ecb_aes_alg = + RK_CIPHER_ALGO_INIT(AES, ECB, ecb(aes), ecb-aes-rk); + +struct rk_crypto_algt rk_v3_cbc_aes_alg = + RK_CIPHER_ALGO_INIT(AES, CBC, cbc(aes), cbc-aes-rk); + +struct rk_crypto_algt rk_v3_xts_aes_alg = + RK_CIPHER_ALGO_XTS_INIT(AES, xts(aes), xts-aes-rk); + +struct rk_crypto_algt rk_v3_cfb_aes_alg = + RK_CIPHER_ALGO_INIT(AES, CFB, cfb(aes), cfb-aes-rk); + +struct rk_crypto_algt rk_v3_ofb_aes_alg = + RK_CIPHER_ALGO_INIT(AES, OFB, ofb(aes), ofb-aes-rk); + +struct rk_crypto_algt rk_v3_ctr_aes_alg = + RK_CIPHER_ALGO_INIT(AES, CTR, ctr(aes), ctr-aes-rk); + +struct rk_crypto_algt rk_v3_ecb_des_alg = + RK_CIPHER_ALGO_INIT(DES, ECB, ecb(des), ecb-des-rk); + +struct rk_crypto_algt rk_v3_cbc_des_alg = + RK_CIPHER_ALGO_INIT(DES, CBC, cbc(des), cbc-des-rk); + +struct rk_crypto_algt rk_v3_cfb_des_alg = + RK_CIPHER_ALGO_INIT(DES, CFB, cfb(des), cfb-des-rk); + +struct rk_crypto_algt rk_v3_ofb_des_alg = + RK_CIPHER_ALGO_INIT(DES, OFB, ofb(des), ofb-des-rk); + +struct rk_crypto_algt rk_v3_ecb_des3_ede_alg = + RK_CIPHER_ALGO_INIT(DES3_EDE, ECB, ecb(des3_ede), ecb-des3_ede-rk); + +struct rk_crypto_algt rk_v3_cbc_des3_ede_alg = + RK_CIPHER_ALGO_INIT(DES3_EDE, CBC, cbc(des3_ede), cbc-des3_ede-rk); + +struct rk_crypto_algt rk_v3_cfb_des3_ede_alg = + RK_CIPHER_ALGO_INIT(DES3_EDE, CFB, cfb(des3_ede), cfb-des3_ede-rk); + +struct rk_crypto_algt rk_v3_ofb_des3_ede_alg = + RK_CIPHER_ALGO_INIT(DES3_EDE, OFB, ofb(des3_ede), ofb-des3_ede-rk); +