diff --git a/drivers/crypto/rockchip/Kconfig b/drivers/crypto/rockchip/Kconfig index c900e6cd5cef..842ff78e8bc1 100644 --- a/drivers/crypto/rockchip/Kconfig +++ b/drivers/crypto/rockchip/Kconfig @@ -21,6 +21,11 @@ config CRYPTO_DEV_ROCKCHIP_CRYPTO depends on CRYPTO_DEV_ROCKCHIP_V1 || CRYPTO_DEV_ROCKCHIP_V2 || CRYPTO_DEV_ROCKCHIP_V3 default y +config CRYPTO_DEV_ROCKCHIP_CE + bool "crypto engine for RV1126B" + depends on CPU_RV1126B + default y + endif config CRYPTO_DEV_ROCKCHIP_DEV diff --git a/drivers/crypto/rockchip/Makefile b/drivers/crypto/rockchip/Makefile index 6585fa3e7d4e..7bee5a6de098 100644 --- a/drivers/crypto/rockchip/Makefile +++ b/drivers/crypto/rockchip/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP_CRYPTO) += rkcrypto/ +obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP_CE) += rkce/ obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP_DEV) += cryptodev_linux/ diff --git a/drivers/crypto/rockchip/rkce/Makefile b/drivers/crypto/rockchip/rkce/Makefile new file mode 100644 index 000000000000..de65e66ffec7 --- /dev/null +++ b/drivers/crypto/rockchip/rkce/Makefile @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: GPL-2.0+ +obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rk_ce.o + +$(obj)/rkce_sm2signature.asn1.o: \ + $(obj)/rkce_sm2signature.asn1.c $(obj)/rkce_sm2signature.asn1.h +$(obj)/rkce_ecdsasignature.asn1.o: \ + $(obj)/rkce_ecdsasignature.asn1.c $(obj)/rkce_ecdsasignature.asn1.h +$(obj)/rkce_akcipher.o: \ + $(obj)/rkce_sm2signature.asn1.h $(obj)/rkce_ecdsasignature.asn1.h + +rk_ce-objs := rkce_core.o \ + rkce_dev.o \ + rkce_akcipher.o \ + rkce_skcipher.o \ + rkce_hash.o \ + rkce_buf.o \ + rkce_bignum.o \ + rkce_pka.o \ + rkce_ecc.o \ + rkce_sm2signature.asn1.o \ + rkce_ecdsasignature.asn1.o \ + rkce_debug.o \ + rkce_monitor.o diff --git a/drivers/crypto/rockchip/rkce/rkce_akcipher.c b/drivers/crypto/rockchip/rkce/rkce_akcipher.c new file mode 100644 index 000000000000..90066914ddad --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_akcipher.c @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Crypto acceleration support for Rockchip crypto engine + * + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + * Author: Lin Jinhan + * + */ +#define RKCE_MODULE_TAG "ASYM" +#define RKCE_MODULE_OFFSET 12 + +#include +#include + +#include "rkce_akcipher.h" +#include "rkce_core.h" +#include "rkce_dev.h" +#include "rkce_debug.h" +#include "rkce_pka.h" +#include "rkce_ecc.h" +#include "rkce_sm2signature.asn1.h" +#include "rkce_ecdsasignature.asn1.h" + +static void rkce_rsa_adjust_rsa_key(struct rsa_key *key) +{ + if (key->n_sz && key->n && !key->n[0]) { + key->n++; + key->n_sz--; + } + + if (key->e_sz && key->e && !key->e[0]) { + key->e++; + key->e_sz--; + } + + if (key->d_sz && key->d && !key->d[0]) { + key->d++; + key->d_sz--; + } +} + +static void rkce_rsa_clear_ctx(struct rkce_rsa_ctx *ctx) +{ + rk_trace("enter.\n"); + + /* Free the old key if any */ + rkce_bn_free(ctx->n); + ctx->n = NULL; + + rkce_bn_free(ctx->e); + ctx->e = NULL; + + rkce_bn_free(ctx->d); + ctx->d = NULL; + + rk_trace("exit.\n"); +} + +static int rkce_rsa_setkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen, bool private) +{ + struct rkce_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct rsa_key rsa_key; + int ret = -ENOMEM; + + rk_trace("enter.\n"); + + rk_debug("set %s keylen = %u\n", private ? "private" : "public", keylen); + + rkce_rsa_clear_ctx(ctx); + + memset(&rsa_key, 0x00, sizeof(rsa_key)); + + if (private) + ret = rsa_parse_priv_key(&rsa_key, key, keylen); + else + ret = rsa_parse_pub_key(&rsa_key, key, keylen); + + if (ret < 0) + goto error; + + rkce_rsa_adjust_rsa_key(&rsa_key); + + ctx->n = rkce_bn_alloc(rsa_key.n_sz); + if (!ctx->n) + goto error; + + ctx->e = rkce_bn_alloc(rsa_key.e_sz); + if (!ctx->e) + goto error; + + rkce_bn_set_data(ctx->n, rsa_key.n, rsa_key.n_sz, RK_BG_BIG_ENDIAN); + rkce_bn_set_data(ctx->e, rsa_key.e, rsa_key.e_sz, RK_BG_BIG_ENDIAN); + + if (private) { + ctx->d = rkce_bn_alloc(rsa_key.d_sz); + if (!ctx->d) + goto error; + + rkce_bn_set_data(ctx->d, rsa_key.d, rsa_key.d_sz, RK_BG_BIG_ENDIAN); + } + + rk_trace("exit.\n"); + + return 0; +error: + rkce_rsa_clear_ctx(ctx); + + rk_trace("exit.\n"); + + return ret; +} + +static unsigned int rkce_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct rkce_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + + return rkce_bn_get_size(ctx->n); +} + +static int rkce_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + return rkce_rsa_setkey(tfm, key, keylen, false); +} + +static int rkce_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + return rkce_rsa_setkey(tfm, key, keylen, true); +} + +static int rkce_rsa_handle_req(struct akcipher_request *req, bool encrypt) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct rkce_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct crypto_engine *engine = ctx->algt->rk_dev->asym_engine; + + rk_trace("enter.\n"); + + ctx->is_enc = encrypt; + + return crypto_transfer_akcipher_request_to_engine(engine, req); +} + +static int rkce_rsa_enc(struct akcipher_request *req) +{ + return rkce_rsa_handle_req(req, true); +} + +static int rkce_rsa_dec(struct akcipher_request *req) +{ + return rkce_rsa_handle_req(req, false); +} + +static int rkce_rsa_run_req(struct crypto_engine *engine, void *async_req) +{ + struct akcipher_request *req = container_of(async_req, struct akcipher_request, base); + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct rkce_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct rkce_bignum *in = NULL, *out = NULL; + uint32_t key_byte_size; + uint8_t *tmp_buf = NULL; + int ret = -EINVAL; + + rk_trace("enter.\n"); + + if (unlikely(!ctx->n || !ctx->e)) + goto exit; + + if (!ctx->is_enc && !ctx->d) + goto exit; + + key_byte_size = rkce_bn_get_size(ctx->n); + + if (req->dst_len < key_byte_size) { + req->dst_len = key_byte_size; + ret = -EOVERFLOW; + goto exit; + } + + if (req->src_len > key_byte_size) + goto exit; + + in = rkce_bn_alloc(key_byte_size); + if (!in) { + ret = -ENOMEM; + goto exit; + } + + out = rkce_bn_alloc(key_byte_size); + if (!out) { + ret = -ENOMEM; + goto exit; + } + + tmp_buf = kzalloc(key_byte_size, GFP_KERNEL); + if (!tmp_buf) { + ret = -ENOMEM; + goto exit; + } + + if (!sg_copy_to_buffer(req->src, sg_nents(req->src), tmp_buf, req->src_len)) { + rk_err("sg copy err\n"); + ret = -EINVAL; + goto exit; + } + + ret = rkce_bn_set_data(in, tmp_buf, req->src_len, RK_BG_BIG_ENDIAN); + if (ret) + goto exit; + + if (ctx->is_enc) + ret = rkce_pka_expt_mod(in, ctx->e, ctx->n, out); + else + ret = rkce_pka_expt_mod(in, ctx->d, ctx->n, out); + + if (ret) + goto exit; + + ret = rkce_bn_get_data(out, tmp_buf, key_byte_size, RK_BG_BIG_ENDIAN); + if (ret) + goto exit; + + if (!sg_copy_from_buffer(req->dst, sg_nents(req->dst), tmp_buf, key_byte_size)) { + rk_err("sg copy err\n"); + ret = -EINVAL; + goto exit; + } + + req->dst_len = key_byte_size; + +exit: + crypto_finalize_akcipher_request(ctx->algt->rk_dev->asym_engine, req, ret); + + kfree(tmp_buf); + + rkce_bn_free(in); + rkce_bn_free(out); + + rk_trace("exit.\n"); + + return ret; +} + +static int rkce_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct rkce_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct akcipher_alg *alg = __crypto_akcipher_alg(tfm->base.__crt_alg); + struct rkce_algt *algt = container_of(alg, struct rkce_algt, alg.asym); + + rk_trace("enter.\n"); + + rk_debug("alloc %s\n", algt->name); + + memzero_explicit(ctx, sizeof(*ctx)); + + ctx->algt = algt; + + ctx->enginectx.op.do_one_request = rkce_rsa_run_req; + + rkce_pka_set_crypto_base(algt->rk_dev->reg); + + rkce_enable_clk(algt->rk_dev); + + rk_trace("exit.\n"); + + return 0; +} + +static void rkce_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct rkce_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + + rk_trace("enter.\n"); + + rkce_rsa_clear_ctx(ctx); + + rkce_disable_clk(ctx->algt->rk_dev); + + memzero_explicit(ctx, sizeof(*ctx)); + + rk_trace("exit.\n"); +} + +struct rkce_algt rkce_asym_rsa = { + .name = "rsa", + .type = RKCE_ALGO_TYPE_ASYM, + .algo = RKCE_ASYM_ALGO_RSA, + .alg.asym = { + .encrypt = rkce_rsa_enc, + .decrypt = rkce_rsa_dec, + .set_pub_key = rkce_rsa_setpubkey, + .set_priv_key = rkce_rsa_setprivkey, + .max_size = rkce_rsa_max_size, + .init = rkce_rsa_init_tfm, + .exit = rkce_rsa_exit_tfm, + .reqsize = sizeof(struct rkce_asym_request_ctx), + .base = { + .cra_name = "rsa", + .cra_driver_name = "rsa-rk", + .cra_priority = RKCE_PRIORITY, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct rkce_rsa_ctx), + }, + }, +}; + +int rkce_ecc_get_signature_r(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct rkce_ecp_point *sig = context; + const uint8_t *tmp_value = value; + + if (!value || !vlen) + return -EINVAL; + + /* skip first zero */ + if (tmp_value[0] == 0x00) { + tmp_value += 1; + vlen -= 1; + } + + return rkce_bn_set_data(sig->x, tmp_value, vlen, RK_BG_BIG_ENDIAN); +} + +int rkce_ecc_get_signature_s(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct rkce_ecp_point *sig = context; + const uint8_t *tmp_value = value; + + if (!value || !vlen) + return -EINVAL; + + /* skip first zero */ + if (tmp_value[0] == 0x00) { + tmp_value += 1; + vlen -= 1; + } + + return rkce_bn_set_data(sig->y, tmp_value, vlen, RK_BG_BIG_ENDIAN); +} + +static int rkce_ecc_handle_req(struct akcipher_request *req, bool sign) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct rkce_ecc_ctx *ctx = akcipher_tfm_ctx(tfm); + struct crypto_engine *engine = ctx->algt->rk_dev->asym_engine; + + rk_trace("enter.\n"); + + ctx->is_sign = sign; + + return crypto_transfer_akcipher_request_to_engine(engine, req); +} + +static int rkce_ec_verify(struct akcipher_request *req) +{ + return rkce_ecc_handle_req(req, false); +} + +static int rkce_ecc_run_req(struct crypto_engine *engine, void *async_req) +{ + struct akcipher_request *req = container_of(async_req, struct akcipher_request, base); + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct rkce_ecc_ctx *ctx = akcipher_tfm_ctx(tfm); + size_t keylen = ctx->nbits / 8; + struct rkce_ecp_point *sig_point = NULL; + uint8_t rawhash[RK_ECP_MAX_BYTES]; + unsigned char *buffer; + ssize_t diff; + int ret; + + if (unlikely(!ctx->pub_key_set)) + return -EINVAL; + + buffer = kmalloc(req->src_len + req->dst_len, GFP_KERNEL); + if (!buffer) + return -ENOMEM; + + sig_point = rkce_ecc_alloc_point_zero(RK_ECP_MAX_BYTES); + if (!sig_point) { + ret = -ENOMEM; + goto exit; + } + + sg_pcopy_to_buffer(req->src, sg_nents_for_len(req->src, req->src_len + req->dst_len), + buffer, req->src_len + req->dst_len, 0); + + if (ctx->group_id == RK_ECP_DP_SM2P256V1) + ret = asn1_ber_decoder(&rkce_sm2signature_decoder, + sig_point, buffer, req->src_len); + else + ret = asn1_ber_decoder(&rkce_ecdsasignature_decoder, + sig_point, buffer, req->src_len); + if (ret < 0) + goto exit; + + /* if the hash is shorter then we will add leading zeros to fit to ndigits */ + memset(rawhash, 0x00, sizeof(rawhash)); + + diff = keylen - req->dst_len; + if (diff >= 0) { + if (diff) + memset(rawhash, 0, diff); + + memcpy(&rawhash[diff], buffer + req->src_len, req->dst_len); + } else if (diff < 0) { + /* given hash is longer, we take the left-most bytes */ + memcpy(&rawhash, buffer + req->src_len, keylen); + } + + ret = rkce_ecc_verify(ctx->group_id, rawhash, keylen, ctx->point_Q, sig_point); +exit: + crypto_finalize_akcipher_request(ctx->algt->rk_dev->asym_engine, req, ret); + + kfree(buffer); + rkce_ecc_free_point(sig_point); + + rk_trace("ret = %d\n", ret); + + return ret; +} + +/* + * Set the public key given the raw uncompressed key data from an X509 + * certificate. The key data contain the concatenated X and Y coordinates of + * the public key. + */ +static int rkce_ec_set_pub_key(struct crypto_akcipher *tfm, const void *key, unsigned int keylen) +{ + struct rkce_ecc_ctx *ctx = akcipher_tfm_ctx(tfm); + struct rkce_ecp_point *pub_Q = ctx->point_Q; + const unsigned char *d = key; + uint32_t nbytes; + + rk_trace("enter.\n"); + + if (keylen < 1 || (((keylen - 1) >> 1) % sizeof(uint32_t)) != 0) + return -EINVAL; + + /* we only accept uncompressed format indicated by '4' */ + if (d[0] != 4) + return -EINVAL; + + keylen--; + d++; + + nbytes = keylen / 2; + + rk_debug("keylen = %u, nbytes = %u, group_id = %u, curve_byte = %u\n", + keylen, nbytes, ctx->group_id, + rkce_ecc_get_curve_nbits(ctx->group_id) / 8); + + if (nbytes != rkce_ecc_get_curve_nbits(ctx->group_id) / 8) + return -EINVAL; + + rkce_bn_set_data(pub_Q->x, d, nbytes, RK_BG_BIG_ENDIAN); + rkce_bn_set_data(pub_Q->y, d + nbytes, nbytes, RK_BG_BIG_ENDIAN); + + if (rkce_ecp_point_is_zero(pub_Q)) + return -EINVAL; + + ctx->pub_key_set = true; + + return 0; +} + +static unsigned int rkce_ec_max_size(struct crypto_akcipher *tfm) +{ + rk_trace("enter.\n"); + + return rkce_ecc_get_max_size(); +} + +static int rkce_ec_init_tfm(struct crypto_akcipher *tfm) +{ + struct rkce_ecc_ctx *ctx = akcipher_tfm_ctx(tfm); + struct akcipher_alg *alg = __crypto_akcipher_alg(tfm->base.__crt_alg); + struct rkce_algt *algt = container_of(alg, struct rkce_algt, alg.asym); + + rk_trace("enter.\n"); + + rk_debug("alloc %s\n", algt->name); + + memzero_explicit(ctx, sizeof(*ctx)); + + ctx->algt = algt; + + ctx->enginectx.op.do_one_request = rkce_ecc_run_req; + + ctx->group_id = rkce_ecc_get_group_id(algt->algo); + ctx->nbits = rkce_ecc_get_curve_nbits(ctx->group_id); + ctx->point_Q = rkce_ecc_alloc_point_zero(RK_ECP_MAX_BYTES); + + rkce_enable_clk(ctx->algt->rk_dev); + + rkce_ecc_init(algt->rk_dev->reg); + + return 0; +} + +static void rkce_ec_exit_tfm(struct crypto_akcipher *tfm) +{ + struct rkce_ecc_ctx *ctx = akcipher_tfm_ctx(tfm); + + rk_trace("enter.\n"); + + if (!ctx) + return; + + rkce_ecc_free_point(ctx->point_Q); + + rkce_ecc_deinit(); + + rkce_disable_clk(ctx->algt->rk_dev); + + memzero_explicit(ctx, sizeof(*ctx)); + + rk_trace("exit.\n"); +} + +struct rkce_algt rkce_asym_ecc_p192 = RK_ASYM_ECC_INIT(192); +struct rkce_algt rkce_asym_ecc_p224 = RK_ASYM_ECC_INIT(224); +struct rkce_algt rkce_asym_ecc_p256 = RK_ASYM_ECC_INIT(256); + +struct rkce_algt rkce_asym_sm2 = { + .name = "sm2", + .type = RKCE_ALGO_TYPE_ASYM, + .algo = RKCE_ASYM_ALGO_SM2, + .alg.asym = { + .verify = rkce_ec_verify, + .set_pub_key = rkce_ec_set_pub_key, + .max_size = rkce_ec_max_size, + .init = rkce_ec_init_tfm, + .exit = rkce_ec_exit_tfm, + .reqsize = sizeof(struct rkce_asym_request_ctx), + .base = { + .cra_name = "sm2", + .cra_driver_name = "sm2-rk", + .cra_priority = RKCE_PRIORITY, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct rkce_ecc_ctx), + }, + } +}; diff --git a/drivers/crypto/rockchip/rkce/rkce_akcipher.h b/drivers/crypto/rockchip/rkce/rkce_akcipher.h new file mode 100644 index 000000000000..1f11abe0bf4e --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_akcipher.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2024 Rockchip Electronics Co., Ltd. */ + +#ifndef __RKCE_AKCIPHER_H__ +#define __RKCE_AKCIPHER_H__ + +#include + +#include "rkce_dev.h" + +extern struct rkce_algt rkce_asym_rsa; +extern struct rkce_algt rkce_asym_ecc_p192; +extern struct rkce_algt rkce_asym_ecc_p224; +extern struct rkce_algt rkce_asym_ecc_p256; +extern struct rkce_algt rkce_asym_sm2; + +#endif diff --git a/drivers/crypto/rockchip/rkce/rkce_bignum.c b/drivers/crypto/rockchip/rkce/rkce_bignum.c new file mode 100644 index 000000000000..061ee0218ccf --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_bignum.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * bignum support for Rockchip crypto + * + * Copyright (c) 2020 Rockchip Electronics Co., Ltd. + * + * Author: Lin Jinhan + * + */ +#include + +#include "rkce_bignum.h" + +#define DEFAULT_ENDIAN RK_BG_LITTILE_ENDIAN + +#define BYTES2WORDS(bytes) (round_up((bytes), sizeof(u32)) / sizeof(u32)) +#define WORDS2BYTES(words) ((words) * sizeof(u32)) +#define RK_WORD_SIZE 32 + +static void rk_reverse_memcpy(void *dst, const void *src, u32 size) +{ + char *_dst = (char *)dst, *_src = (char *)src; + u32 i; + + if (!dst || !src || !size) + return; + + for (i = 0; i < size; ++i) + _dst[size - i - 1] = _src[i]; +} + +struct rkce_bignum *rkce_bn_alloc(u32 max_size) +{ + struct rkce_bignum *bn; + + bn = kzalloc(sizeof(*bn), GFP_KERNEL); + if (!bn) + return NULL; + + bn->data = kzalloc(round_up(max_size, sizeof(u32)), GFP_KERNEL); + if (!bn->data) { + kfree(bn); + return NULL; + } + + bn->n_words = BYTES2WORDS(max_size); + + return bn; +} + +void rkce_bn_free(struct rkce_bignum *bn) +{ + if (!bn) + return; + + if (bn->data) { + memset(bn->data, 0x00, WORDS2BYTES(bn->n_words)); + kfree(bn->data); + } + + kfree(bn); +} + +int rkce_bn_set_data(struct rkce_bignum *bn, const u8 *data, u32 size, enum bignum_endian endian) +{ + if (!bn || !data) + return -EINVAL; + + if (BYTES2WORDS(size) > bn->n_words) + return -EINVAL; + + if (endian == DEFAULT_ENDIAN) + memcpy(bn->data, data, size); + else + rk_reverse_memcpy(bn->data, data, size); + + return 0; +} + +int rkce_bn_get_data(const struct rkce_bignum *bn, u8 *data, u32 size, enum bignum_endian endian) +{ + if (!bn || !data) + return -EINVAL; + + if (size < WORDS2BYTES(bn->n_words)) + return -EINVAL; + + memset(data, 0x00, size); + + if (endian == DEFAULT_ENDIAN) + memcpy(data + size - WORDS2BYTES(bn->n_words), bn->data, bn->n_words); + else + rk_reverse_memcpy(data + size - WORDS2BYTES(bn->n_words), + bn->data, WORDS2BYTES(bn->n_words)); + + return 0; +} + +u32 rkce_bn_get_size(const struct rkce_bignum *bn) +{ + if (!bn) + return 0; + + return WORDS2BYTES(bn->n_words); +} + +/* + * @brief Returns the index of the highest 1 in |bn|. + * @param bn: the point of input data bignum. + * @return The index starts at 0 for the least significant bit. + * If src == zero, it will return -1 + */ +int rkce_bn_highest_bit(const struct rkce_bignum *bn) +{ + u32 w; + u32 b; + + if (!bn || !bn->data || !bn->n_words) + return -1; + + w = bn->data[bn->n_words - 1]; + + for (b = 0; b < RK_WORD_SIZE; b++) { + w >>= 1; + if (w == 0) + break; + } + + return (int)(bn->n_words - 1) * RK_WORD_SIZE + b; +} + +void rkce_ecc_free_point(struct rkce_ecp_point *point) +{ + if (!point) + return; + + rkce_bn_free(point->x); + rkce_bn_free(point->y); + + kfree(point); +} + +struct rkce_ecp_point *rkce_ecc_alloc_point_zero(u32 max_size) +{ + struct rkce_ecp_point *point = NULL; + + point = kzalloc(sizeof(*point), GFP_KERNEL); + if (!point) + return NULL; + + point->x = rkce_bn_alloc(max_size); + if (!point->x) + goto error; + + point->y = rkce_bn_alloc(max_size); + if (!point->y) + goto error; + + return point; + +error: + rkce_ecc_free_point(point); + + return NULL; +} + +struct rkce_ecp_point *rkce_ecc_alloc_point(const uint8_t *x, uint32_t x_len, + const uint8_t *y, uint32_t y_len, + enum bignum_endian endian, u32 max_size) +{ + struct rkce_ecp_point *point = NULL; + + point = kzalloc(sizeof(*point), GFP_KERNEL); + if (!point) + return NULL; + + point->x = rkce_bn_alloc(max_size); + if (!point->x) + goto error; + + if (rkce_bn_set_data(point->x, x, x_len, endian) != 0) + goto error; + + point->y = rkce_bn_alloc(max_size); + if (!point->y) + goto error; + + if (rkce_bn_set_data(point->y, y, y_len, endian) != 0) + goto error; + + return point; + +error: + rkce_ecc_free_point(point); + + return NULL; +} + +bool rkce_ecp_point_is_zero(struct rkce_ecp_point *point) +{ + uint32_t i; + bool ret = true; + + if (!point || !point->x || !point->y) + return false; + + for (i = 0; i < point->x->n_words; i++) { + if (point->x->data[i] != 0) + ret = false; + } + + for (i = 0; i < point->y->n_words; i++) { + if (point->y->data[i] != 0) + ret = false; + } + + return ret; +} diff --git a/drivers/crypto/rockchip/rkce/rkce_bignum.h b/drivers/crypto/rockchip/rkce/rkce_bignum.h new file mode 100644 index 000000000000..607b79cf114e --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_bignum.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2020 Rockchip Electronics Co., Ltd. */ + +#ifndef __RKCE_BIGNUM_H__ +#define __RKCE_BIGNUM_H__ + +enum bignum_endian { + RK_BG_BIG_ENDIAN, + RK_BG_LITTILE_ENDIAN +}; + +/** + * struct rkce_bignum - crypto bignum struct. + */ +struct rkce_bignum { + u32 n_words; + u32 *data; +}; + +struct rkce_ecp_point { + struct rkce_bignum *x; /*!< the point's X coordinate */ + struct rkce_bignum *y; /*!< the point's Y coordinate */ +}; + +struct rkce_bignum *rkce_bn_alloc(u32 max_size); +void rkce_bn_free(struct rkce_bignum *bn); +int rkce_bn_set_data(struct rkce_bignum *bn, const u8 *data, u32 size, enum bignum_endian endian); +int rkce_bn_get_data(const struct rkce_bignum *bn, u8 *data, u32 size, enum bignum_endian endian); +u32 rkce_bn_get_size(const struct rkce_bignum *bn); +int rkce_bn_highest_bit(const struct rkce_bignum *src); + +struct rkce_ecp_point *rkce_ecc_alloc_point_zero(u32 max_size); +struct rkce_ecp_point *rkce_ecc_alloc_point(const uint8_t *x, uint32_t x_len, + const uint8_t *y, uint32_t y_len, + enum bignum_endian endian, u32 max_size); +void rkce_ecc_free_point(struct rkce_ecp_point *point); +bool rkce_ecp_point_is_zero(struct rkce_ecp_point *point); + +#endif diff --git a/drivers/crypto/rockchip/rkce/rkce_buf.c b/drivers/crypto/rockchip/rkce/rkce_buf.c new file mode 100644 index 000000000000..8b9fcb33cc0e --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_buf.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Crypto acceleration support for Rockchip crypto engine + * + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + * Author: Lin Jinhan + * + */ + +#define RKCE_MODULE_TAG "BUF" +#define RKCE_MODULE_OFFSET 10 + +#include +#include +#include +#include +#include +#include + +#include "rkce_buf.h" +#include "rkce_debug.h" +#include "rkce_error.h" + +struct rkce_cma_buf_data { + void *virt; + uint32_t phys; + uint32_t size; + uint32_t reserved; + struct list_head list; +}; + +static struct device *g_dev; +static LIST_HEAD(g_buf_list); +static DEFINE_MUTEX(g_buf_lock); + +int rkce_cma_init(void *device) +{ + int err; + + g_dev = device; + + err = dma_set_mask_and_coherent(g_dev, DMA_BIT_MASK(32)); + if (err) + dev_err(g_dev, "No suitable DMA available.\n"); + + return err; +} + +void rkce_cma_deinit(void *device) +{ + struct rkce_cma_buf_data *cma_buf = NULL; + struct list_head *pos = NULL, *q = NULL; + + if (!device || device != g_dev) + return; + + mutex_lock(&g_buf_lock); + + list_for_each_safe(pos, q, &g_buf_list) { + cma_buf = list_entry(pos, struct rkce_cma_buf_data, list); + + list_del(&cma_buf->list); + + dma_free_coherent(g_dev, cma_buf->size, cma_buf->virt, cma_buf->phys); + memset(cma_buf, 0x00, sizeof(*cma_buf)); + kfree(cma_buf); + } + + g_dev = NULL; + + mutex_unlock(&g_buf_lock); +} + +void *rkce_cma_alloc(uint32_t size) +{ + struct rkce_cma_buf_data *cma_buf; + dma_addr_t dma_handle = 0; + + mutex_lock(&g_buf_lock); + + cma_buf = kzalloc(sizeof(*cma_buf), GFP_KERNEL); + if (!cma_buf) + goto error; + + cma_buf->virt = dma_alloc_coherent(g_dev, size, &dma_handle, GFP_KERNEL); + if (!cma_buf->virt) { + kfree(cma_buf); + cma_buf = NULL; + goto error; + } + + cma_buf->phys = (uint32_t)dma_handle; + cma_buf->size = size; + + list_add(&cma_buf->list, &g_buf_list); + + mutex_unlock(&g_buf_lock); + + rk_debug("++++++ alloc cma buff: virt(%p), phys(%08x), size(%u)\n", + cma_buf->virt, cma_buf->phys, cma_buf->size); + + return cma_buf->virt; +error: + mutex_unlock(&g_buf_lock); + + return NULL; +} + +void rkce_cma_free(void *buf) +{ + struct rkce_cma_buf_data *cma_buf = NULL; + struct list_head *pos = NULL, *q = NULL; + bool virt_match = false; + + if (!buf) + return; + + mutex_lock(&g_buf_lock); + + list_for_each_safe(pos, q, &g_buf_list) { + cma_buf = list_entry(pos, struct rkce_cma_buf_data, list); + if (cma_buf->virt == buf) { + virt_match = true; + list_del(&cma_buf->list); + break; + } + } + + if (virt_match) { + rk_debug("------ free cma buff: virt(%p), phys(%08x), size(%u)\n", + cma_buf->virt, cma_buf->phys, cma_buf->size); + + dma_free_coherent(g_dev, cma_buf->size, cma_buf->virt, cma_buf->phys); + memset(cma_buf, 0x00, sizeof(*cma_buf)); + kfree(cma_buf); + } + + mutex_unlock(&g_buf_lock); +} + +uint32_t rkce_cma_virt2phys(void *buf) +{ + struct rkce_cma_buf_data *cma_buf = NULL; + struct list_head *pos = NULL, *q = NULL; + uint32_t phys = 0; + + if (!buf) + goto exit; + + mutex_lock(&g_buf_lock); + + list_for_each_safe(pos, q, &g_buf_list) { + cma_buf = list_entry(pos, struct rkce_cma_buf_data, list); + if (cma_buf->virt <= buf && buf <= cma_buf->virt + cma_buf->size) { + phys = cma_buf->phys + (buf - cma_buf->virt); + goto exit; + } + } +exit: + mutex_unlock(&g_buf_lock); + + rk_debug("virt(%p) -> phys(%08x)\n", buf, phys); + + return phys; +} + +void *rkce_cma_phys2virt(uint32_t phys) +{ + struct rkce_cma_buf_data *cma_buf = NULL; + struct list_head *pos = NULL, *q = NULL; + void *virt = NULL; + + if (!phys) + goto exit; + + mutex_lock(&g_buf_lock); + + list_for_each_safe(pos, q, &g_buf_list) { + cma_buf = list_entry(pos, struct rkce_cma_buf_data, list); + rk_debug("phys = %x, [%x, %x]\n ", + phys, cma_buf->phys, cma_buf->phys + cma_buf->size); + if (cma_buf->phys <= phys && phys <= cma_buf->phys + cma_buf->size) { + rk_debug("cma_buf->virt = %p phys = %x, cma_buf->phys = %x, diff = %x\n", + cma_buf->virt, phys, cma_buf->phys, (phys - cma_buf->phys)); + virt = cma_buf->virt + (phys - cma_buf->phys); + goto exit; + } + } + +exit: + mutex_unlock(&g_buf_lock); + + rk_debug("phys(%08x)-> virt(%p)\n", phys, virt); + + return virt; +} diff --git a/drivers/crypto/rockchip/rkce/rkce_buf.h b/drivers/crypto/rockchip/rkce/rkce_buf.h new file mode 100644 index 000000000000..f4ca77e6c097 --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_buf.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2024 Rockchip Electronics Co., Ltd. */ + +#ifndef __RKCE_BUF_H__ +#define __RKCE_BUF_H__ + +#include + +int rkce_cma_init(void *device); +void rkce_cma_deinit(void *device); +void *rkce_cma_alloc(uint32_t size); +void rkce_cma_free(void *buf); +uint32_t rkce_cma_virt2phys(void *buf); +void *rkce_cma_phys2virt(uint32_t phys); + +#endif diff --git a/drivers/crypto/rockchip/rkce/rkce_core.c b/drivers/crypto/rockchip/rkce/rkce_core.c new file mode 100644 index 000000000000..e2123d29aac5 --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_core.c @@ -0,0 +1,628 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Crypto acceleration support for Rockchip crypto engine + * + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + * Author: Lin Jinhan + * + */ + +#define RKCE_MODULE_TAG "CORE" +#define RKCE_MODULE_OFFSET 2 + +#include +#include +#include +#include + +#include "rkce_core.h" +#include "rkce_debug.h" +#include "rkce_error.h" +#include "rkce_reg.h" + +struct rkce_chn_info { + void *td_virt; + uint32_t int_st; + uint32_t td_id; + int result; + + request_cb_func cb_func; +}; + +struct rkce_hardware { + struct RKCE_REG *rkce_reg; + + struct rkce_chn_info chn[RKCE_TD_TYPE_MAX]; +}; + +#define RST_TIMEOUT_MS 100 +#define TD_PUSH_TIMEOUT_MS 3000 + +#define IP_VERSION_MASK (0xfU >> 28) +#define IP_VERSION_RKCE (0x1U >> 28) +#define GET_IP_VERSION(ver) ((ver) & IP_VERSION_MASK) + +#define IS_SYMM_TD(td_type) ((td_type) == RKCE_TD_TYPE_SYMM || \ + (td_type) == RKCE_TD_TYPE_SYMM_HASH_IN || \ + (td_type) == RKCE_TD_TYPE_SYMM_HASH_OUT) + +#define IS_HASH_TD(td_type) ((td_type) == RKCE_TD_TYPE_HASH) + +#define GET_RKCE_REG(hardware) (((struct rkce_hardware *)(hardware))->rkce_reg) +#define CHECK_RKCE_INITED(hardware) WARN_ON_ONCE(!(hardware) || \ + !(((struct rkce_hardware *)(hardware))->rkce_reg)) +#define WHILE_TIMEOUT(condition, timeout_ms) ({ \ + int timeout = timeout_ms; \ + while ((condition) && timeout--) { \ + usleep_range(1000, 2000); \ + } \ + if (timeout < 0) \ + rk_err("%s timeout!\n", #condition); \ + (timeout < 0) ? -RKCE_TIMEOUT : 0; \ + }) + +static const uint32_t cipher_mode2bit_mask[] = { + [RKCE_SYMM_MODE_ECB] = RKCE_AES_VER_ECB_FLAG_MASK, + [RKCE_SYMM_MODE_CBC] = RKCE_AES_VER_CBC_FLAG_MASK, + [RKCE_SYMM_MODE_CFB] = RKCE_AES_VER_CFB_FLAG_MASK, + [RKCE_SYMM_MODE_OFB] = RKCE_AES_VER_OFB_FLAG_MASK, + [RKCE_SYMM_MODE_CTR] = RKCE_AES_VER_CTR_FLAG_MASK, + [RKCE_SYMM_MODE_XTS] = RKCE_AES_VER_XTS_FLAG_MASK, + [RKCE_SYMM_MODE_CTS] = RKCE_AES_VER_CTS_FLAG_MASK, + [RKCE_SYMM_MODE_CCM] = RKCE_AES_VER_CCM_FLAG_MASK, + [RKCE_SYMM_MODE_GCM] = RKCE_AES_VER_GCM_FLAG_MASK, + [RKCE_SYMM_MODE_CMAC] = RKCE_AES_VER_CMAC_FLAG_MASK, + [RKCE_SYMM_MODE_CBC_MAC] = RKCE_AES_VER_CBC_MAC_FLAG_MASK, +}; + +static const uint32_t hash_algo2bit_mask[] = { + [RKCE_HASH_ALGO_SHA1] = RKCE_HASH_VER_SHA1_FLAG_MASK, + [RKCE_HASH_ALGO_SHA224] = RKCE_HASH_VER_SHA224_FLAG_MASK, + [RKCE_HASH_ALGO_SHA256] = RKCE_HASH_VER_SHA256_FLAG_MASK, + [RKCE_HASH_ALGO_SHA384] = RKCE_HASH_VER_SHA384_FLAG_MASK, + [RKCE_HASH_ALGO_SHA512] = RKCE_HASH_VER_SHA512_FLAG_MASK, + [RKCE_HASH_ALGO_SHA512_224] = RKCE_HASH_VER_SHA512_224_FLAG_MASK, + [RKCE_HASH_ALGO_SHA512_256] = RKCE_HASH_VER_SHA512_256_FLAG_MASK, + [RKCE_HASH_ALGO_MD5] = RKCE_HASH_VER_MD5_FLAG_MASK, + [RKCE_HASH_ALGO_SM3] = RKCE_HASH_VER_SM3_FLAG_MASK, +}; + +static const uint32_t hmac_algo2bit_mask[] = { + [RKCE_HASH_ALGO_SHA1] = RKCE_HMAC_VER_SHA1_FLAG_MASK, + [RKCE_HASH_ALGO_SHA256] = RKCE_HMAC_VER_SHA256_FLAG_MASK, + [RKCE_HASH_ALGO_SHA512] = RKCE_HMAC_VER_SHA512_FLAG_MASK, + [RKCE_HASH_ALGO_MD5] = RKCE_HMAC_VER_MD5_FLAG_MASK, + [RKCE_HASH_ALGO_SM3] = RKCE_HMAC_VER_SM3_FLAG_MASK, +}; + +static bool rk_is_cipher_support(struct RKCE_REG *rkce_reg, + uint32_t algo, uint32_t mode, uint32_t key_len) +{ + uint32_t version = 0; + uint32_t mask = 0; + bool key_len_valid = true; + + switch (algo) { + case RKCE_SYMM_ALGO_DES: + case RKCE_SYMM_ALGO_TDES: + version = rkce_reg->DES_VER; + + if (key_len == RKCE_DES_BLOCK_SIZE) + key_len_valid = true; + else if (key_len == 2 * RKCE_DES_BLOCK_SIZE || + key_len == 3 * RKCE_DES_BLOCK_SIZE) + key_len_valid = version & RKCE_DES_VER_TDES_FLAG_MASK; + else + key_len_valid = false; + break; + case RKCE_SYMM_ALGO_AES: + version = rkce_reg->AES_VER; + + if (key_len == RKCE_AES_KEYSIZE_128) + key_len_valid = version & RKCE_AES_VER_AES128_FLAG_MASK; + else if (key_len == RKCE_AES_KEYSIZE_192) + key_len_valid = version & RKCE_AES_VER_AES192_FLAG_MASK; + else if (key_len == RKCE_KEY_AES_256) + key_len_valid = version & RKCE_AES_VER_AES256_FLAG_MASK; + else + key_len_valid = false; + break; + case RKCE_SYMM_ALGO_SM4: + version = rkce_reg->SM4_VER; + + key_len_valid = (key_len == RKCE_SM4_KEYSIZE) ? true : false; + break; + default: + return false; + } + + mask = cipher_mode2bit_mask[mode]; + + if (key_len == 0) + key_len_valid = true; + + return (version & mask) && key_len_valid; +} + +static bool rk_is_hash_support(struct RKCE_REG *rkce_reg, uint32_t algo, uint32_t type) +{ + uint32_t version = 0; + uint32_t mask = 0; + + if (type == RKCE_ALGO_TYPE_HMAC) { + version = rkce_reg->HMAC_VER; + mask = hmac_algo2bit_mask[algo]; + } else if (type == RKCE_ALGO_TYPE_HASH) { + version = rkce_reg->HASH_VER; + mask = hash_algo2bit_mask[algo]; + } else { + return false; + } + + return version & mask; +} + +static bool rk_is_asym_support(struct RKCE_REG *rkce_reg, uint32_t algo) +{ + switch (algo) { + case RKCE_ASYM_ALGO_RSA: + return !!rkce_reg->PKA_VER; + case RKCE_ASYM_ALGO_ECC_P192: + case RKCE_ASYM_ALGO_ECC_P224: + case RKCE_ASYM_ALGO_ECC_P256: + case RKCE_ASYM_ALGO_SM2: + return !!rkce_reg->ECC_MAX_CURVE_WIDE; + default: + return false; + } +} + +bool rkce_hw_algo_valid(void *rkce_hw, uint32_t type, uint32_t algo, uint32_t mode) +{ + struct RKCE_REG *rkce_reg; + + CHECK_RKCE_INITED(rkce_hw); + + + rkce_reg = GET_RKCE_REG(rkce_hw); + + if (type == RKCE_ALGO_TYPE_CIPHER || type == RKCE_ALGO_TYPE_AEAD) { + rk_debug("CIPHER"); + return rk_is_cipher_support(rkce_reg, algo, mode, 0); + } else if (type == RKCE_ALGO_TYPE_HASH || type == RKCE_ALGO_TYPE_HMAC) { + rk_debug("HASH/HMAC"); + return rk_is_hash_support(rkce_reg, algo, type); + } else if (type == RKCE_ALGO_TYPE_ASYM) { + rk_debug("ASYM"); + return rk_is_asym_support(rkce_reg, algo); + } else { + return false; + } +} + +uint32_t rkce_get_td_type(void *td) +{ + if (!td) + return ~((uint32_t)0); + + return ((struct rkce_symm_td *)td)->ctrl.td_type; +} + +int rkce_soft_reset(void *rkce_hw, uint32_t reset_sel) +{ + struct RKCE_REG *rkce_reg; + uint32_t value = 0; + + CHECK_RKCE_INITED(rkce_hw); + + rkce_reg = GET_RKCE_REG(rkce_hw); + + if (reset_sel & RKCE_RESET_SYMM) + value |= RKCE_RST_CTL_SW_SYMM_RESET_SHIFT; + + if (reset_sel & RKCE_RESET_HASH) + value |= RKCE_RST_CTL_SW_HASH_RESET_SHIFT; + + if (reset_sel & RKCE_RESET_PKA) + value |= RKCE_RST_CTL_SW_PKA_RESET_SHIFT; + + rkce_reg->RST_CTL = value | RKCE_WRITE_MASK_ALL; + + return WHILE_TIMEOUT(rkce_reg->RST_CTL, RST_TIMEOUT_MS); +} + +static int rkce_check_version(struct RKCE_REG *rkce_reg) +{ + rk_debug("rkce_reg->CE_VER = %08x\n", rkce_reg->CE_VER); + + if (GET_IP_VERSION(rkce_reg->CE_VER) != IP_VERSION_RKCE) { + rk_err("IP version is %08x not a RKCE module.\n", rkce_reg->CE_VER); + return -RKCE_FAULT; + } + + return RKCE_SUCCESS; +} + +static int rkce_init(void *rkce_hw) +{ + struct RKCE_REG *rkce_reg = GET_RKCE_REG(rkce_hw); + uint32_t value = 0; + int ret; + + ret = rkce_check_version(rkce_hw); + if (ret) + goto exit; + + rkce_soft_reset(rkce_hw, RKCE_RESET_SYMM | RKCE_RESET_HASH | RKCE_RESET_PKA); + + /* clear symm interrupt register */ + rkce_reg->SYMM_INT_EN = 0; + value = rkce_reg->SYMM_INT_ST; + rkce_reg->SYMM_INT_ST = value; + + ret = WHILE_TIMEOUT(rkce_reg->SYMM_INT_ST, RST_TIMEOUT_MS); + if (ret) + goto exit; + + /* clear hash interrupt register */ + rkce_reg->HASH_INT_EN = 0; + value = rkce_reg->HASH_INT_ST; + rkce_reg->HASH_INT_ST = value; + + ret = WHILE_TIMEOUT(rkce_reg->HASH_INT_ST, RST_TIMEOUT_MS); + if (ret) + goto exit; + + if (rkce_reg->SYMM_CONTEXT_SIZE != RKCE_TD_SYMM_CTX_SIZE) { + rk_err("rkce symm context size (%u) != %u\n", + rkce_reg->SYMM_CONTEXT_SIZE, RKCE_TD_SYMM_CTX_SIZE); + return -RKCE_INVAL; + } + + if (rkce_reg->HASH_CONTEXT_SIZE != RKCE_TD_HASH_CTX_SIZE) { + rk_err("rkce hash context size (%u) != %u\n", + rkce_reg->HASH_CONTEXT_SIZE, RKCE_TD_HASH_CTX_SIZE); + return -RKCE_INVAL; + } + +exit: + return ret; +} + +void *rkce_hardware_alloc(void __iomem *reg_base) +{ + struct rkce_hardware *hardware; + + rk_debug("reg_base = %p", reg_base); + + if (!reg_base) + return NULL; + + hardware = kzalloc(sizeof(*hardware), GFP_KERNEL); + if (!hardware) + return NULL; + + hardware->rkce_reg = reg_base; + + if (rkce_init(hardware) != 0) { + kfree(hardware); + return NULL; + } + + rk_debug("hardware = %p", hardware); + + return hardware; +} + +void rkce_hardware_free(void *rkce_hw) +{ + if (!rkce_hw) + return; + + kfree(rkce_hw); +} + +void rkce_dump_reginfo(void *rkce_hw) +{ + struct RKCE_REG *rkce_reg; + + CHECK_RKCE_INITED(rkce_hw); + + rkce_reg = GET_RKCE_REG(rkce_hw); + + rk_info("\n============================== reg info ===========================\n"); + rk_info("FIFO_ST = %08x\n", rkce_reg->FIFO_ST); + rk_info("\n"); + rk_info("SYMM_INT_EN = %08x\n", rkce_reg->SYMM_INT_EN); + rk_info("SYMM_INT_ST = %08x\n", rkce_reg->SYMM_INT_ST); + rk_info("SYMM_TD_ST = %08x\n", rkce_reg->SYMM_TD_ST); + rk_info("SYMM_TD_ID = %08x\n", rkce_reg->SYMM_TD_ID); + rk_info("SYMM_ST_DBG = %08x\n", rkce_reg->SYMM_ST_DBG); + rk_info("SYMM_TD_ADDR_DBG = %08x\n", rkce_reg->SYMM_TD_ADDR_DBG); + rk_info("SYMM_TD_GRANT_DBG = %08x\n", rkce_reg->SYMM_TD_GRANT_DBG); + rk_info("\n"); + rk_info("HASH_INT_EN = %08x\n", rkce_reg->HASH_INT_EN); + rk_info("HASH_INT_ST = %08x\n", rkce_reg->HASH_INT_ST); + rk_info("HASH_TD_ST = %08x\n", rkce_reg->HASH_TD_ST); + rk_info("HASH_TD_ID = %08x\n", rkce_reg->HASH_TD_ID); + rk_info("HASH_ST_DBG = %08x\n", rkce_reg->HASH_ST_DBG); + rk_info("HASH_TD_ADDR_DBG = %08x\n", rkce_reg->HASH_TD_ADDR_DBG); + rk_info("HASH_TD_GRANT_DBG = %08x\n", rkce_reg->HASH_TD_GRANT_DBG); + rk_info("===================================================================\n"); +} + +int rkce_push_td(void *rkce_hw, void *td) +{ + int ret = RKCE_SUCCESS; + struct RKCE_REG *rkce_reg; + uint32_t td_type; + struct rkce_hardware *hardware = rkce_hw; + + CHECK_RKCE_INITED(rkce_hw); + + if (!td) + return -RKCE_INVAL; + + td_type = rkce_get_td_type(td); + rkce_reg = GET_RKCE_REG(rkce_hw); + + rkce_dump_td(td); + + if (IS_SYMM_TD(td_type)) { + rk_debug("rkce symm push td virt(%p), phys(%08x)\n", + td, rkce_cma_virt2phys(td)); + + WRITE_ONCE(rkce_reg->SYMM_INT_EN, 0x3f); + + /* wait symm fifo valid */ + ret = WHILE_TIMEOUT(rkce_reg->TD_LOAD_CTRL & RKCE_TD_LOAD_CTRL_SYMM_TLR_MASK, + TD_PUSH_TIMEOUT_MS); + if (ret) + goto exit; + + /* set task desc address */ + rkce_reg->TD_ADDR = rkce_cma_virt2phys(td); + hardware->chn[RKCE_TD_TYPE_SYMM].td_virt = td; + + /* tell rkce to load task desc address as symm td */ + rkce_reg->TD_LOAD_CTRL = 0xffff0000 | RKCE_TD_LOAD_CTRL_SYMM_TLR_MASK; + } else if (IS_HASH_TD(td_type)) { + rk_debug("rkce hash push td virt(%p), phys(%08x)\n", + td, rkce_cma_virt2phys(td)); + + WRITE_ONCE(rkce_reg->HASH_INT_EN, 0x3f); + + /* wait hash fifo valid */ + ret = WHILE_TIMEOUT(rkce_reg->TD_LOAD_CTRL & RKCE_TD_LOAD_CTRL_HASH_TLR_MASK, + TD_PUSH_TIMEOUT_MS); + if (ret) + goto exit; + + /* set task desc address */ + rkce_reg->TD_ADDR = rkce_cma_virt2phys(td); + hardware->chn[RKCE_TD_TYPE_HASH].td_virt = td; + + /* tell rkce to load task desc address as hash td */ + rkce_reg->TD_LOAD_CTRL = 0xffff0000 | RKCE_TD_LOAD_CTRL_HASH_TLR_MASK; + } else { + return -RKCE_INVAL; + } + +exit: + return ret; +} + +int rkce_push_td_sync(void *rkce_hw, void *td, uint32_t timeout_ms) +{ + int ret = RKCE_SUCCESS; + struct RKCE_REG *rkce_reg; + uint32_t td_type; + uint32_t value, mask; + + CHECK_RKCE_INITED(rkce_hw); + + if (!td) + return -RKCE_INVAL; + + td_type = rkce_get_td_type(td); + rkce_reg = GET_RKCE_REG(rkce_hw); + + rkce_dump_td(td); + + if (IS_SYMM_TD(td_type)) { + rk_debug("rkce symm push td virt(%p), phys(%08x)\n", + td, rkce_cma_virt2phys(td)); + + WRITE_ONCE(rkce_reg->SYMM_INT_EN, 0x00); + + /* wait symm fifo valid */ + ret = WHILE_TIMEOUT(rkce_reg->TD_LOAD_CTRL & RKCE_TD_LOAD_CTRL_SYMM_TLR_MASK, + timeout_ms); + if (ret) + goto exit; + + /* set task desc address */ + rkce_reg->TD_ADDR = rkce_cma_virt2phys(td); + + /* tell rkce to load task desc address as symm td */ + rkce_reg->TD_LOAD_CTRL = 0xffff0000 | RKCE_TD_LOAD_CTRL_SYMM_TLR_MASK; + + /* wait symm done */ + ret = WHILE_TIMEOUT(!(rkce_reg->SYMM_INT_ST), timeout_ms); + mask = RKCE_SYMM_INT_ST_TD_DONE_MASK; + value = READ_ONCE(rkce_reg->SYMM_INT_ST); + WRITE_ONCE(rkce_reg->SYMM_INT_ST, value); + rk_debug("symm ret = %d, value = %08x, IN_ST = %08x\n", + ret, value, READ_ONCE(rkce_reg->SYMM_INT_ST)); + } else if (IS_HASH_TD(td_type)) { + rk_debug("rkce hash push td virt(%p), phys(%08x)\n", + td, rkce_cma_virt2phys(td)); + + WRITE_ONCE(rkce_reg->HASH_INT_EN, 0x00); + + /* wait hash fifo valid */ + ret = WHILE_TIMEOUT(rkce_reg->TD_LOAD_CTRL & RKCE_TD_LOAD_CTRL_HASH_TLR_MASK, + timeout_ms); + if (ret) + goto exit; + + /* set task desc address */ + rkce_reg->TD_ADDR = rkce_cma_virt2phys(td); + + /* tell rkce to load task desc address as hash td */ + rkce_reg->TD_LOAD_CTRL = 0xffff0000 | RKCE_TD_LOAD_CTRL_HASH_TLR_MASK; + + /* wait hash done */ + ret = WHILE_TIMEOUT(!(rkce_reg->HASH_INT_ST), timeout_ms); + mask = RKCE_HASH_INT_ST_TD_DONE_MASK; + value = READ_ONCE(rkce_reg->HASH_INT_ST); + WRITE_ONCE(rkce_reg->HASH_INT_ST, value); + rk_debug("hash ret = %d, value = %08x, INT_ST = %08x\n", + ret, value, READ_ONCE(rkce_reg->HASH_INT_ST)); + } else { + rk_debug("unknown td_type = %u\n", td_type); + return -RKCE_INVAL; + } + + if (ret) + goto exit; + + ret = (value == mask) ? 0 : -RKCE_FAULT; +exit: + return ret; +} + +int rkce_init_symm_td(struct rkce_symm_td *td, struct rkce_symm_td_buf *buf) +{ + if (!td || + !buf || + !rkce_cma_virt2phys(td) || + !rkce_cma_virt2phys(buf)) { + rk_debug("td = %p buf = %p", td, buf); + return -RKCE_INVAL; + } + + memset(td, 0x00, sizeof(*td)); + + td->ctrl.td_type = RKCE_TD_TYPE_SYMM; + td->task_id = rkce_cma_virt2phys(buf); + td->key_addr = rkce_cma_virt2phys(buf->key1); + td->iv_addr = rkce_cma_virt2phys(buf->iv); + td->gcm_len_addr = rkce_cma_virt2phys(&buf->gcm_len); + td->tag_addr = rkce_cma_virt2phys(buf->tag); + td->symm_ctx_addr = rkce_cma_virt2phys(buf->ctx); + + return RKCE_SUCCESS; +} + +int rkce_init_hash_td(struct rkce_hash_td *td, struct rkce_hash_td_buf *buf) +{ + if (!td || + !buf || + !rkce_cma_virt2phys(td) || + !rkce_cma_virt2phys(buf)) { + rk_debug("td = %p buf = %p", td, buf); + return -RKCE_INVAL; + } + + memset(td, 0x00, sizeof(*td)); + + td->ctrl.td_type = RKCE_TD_TYPE_HASH; + td->task_id = rkce_cma_virt2phys(buf); + td->key_addr = rkce_cma_virt2phys(buf->key); + td->hash_addr = rkce_cma_virt2phys(buf->hash); + td->hash_ctx_addr = rkce_cma_virt2phys(buf->ctx); + + return RKCE_SUCCESS; +} + +int rkce_irq_callback_set(void *rkce_hw, enum rkce_td_type td_type, request_cb_func cb_func) +{ + struct rkce_hardware *hardware = rkce_hw; + + CHECK_RKCE_INITED(rkce_hw); + + if (!cb_func) + return -RKCE_INVAL; + + if (td_type == RKCE_TD_TYPE_SYMM) + hardware->chn[RKCE_TD_TYPE_SYMM].cb_func = cb_func; + else if (td_type == RKCE_TD_TYPE_HASH) + hardware->chn[RKCE_TD_TYPE_HASH].cb_func = cb_func; + else + return -RKCE_INVAL; + + return RKCE_SUCCESS; +} + +void rkce_irq_handler(void *rkce_hw) +{ + struct rkce_chn_info *cur_chn; + struct RKCE_REG *rkce_reg; + struct rkce_hardware *hardware = rkce_hw; + + CHECK_RKCE_INITED(rkce_hw); + + rkce_reg = GET_RKCE_REG(rkce_hw); + + if (rkce_reg->SYMM_INT_ST) { + cur_chn = &(hardware->chn[RKCE_TD_TYPE_SYMM]); + cur_chn->int_st = READ_ONCE(rkce_reg->SYMM_INT_ST); + cur_chn->td_id = rkce_reg->SYMM_TD_ID; + + /* clear symm int */ + WRITE_ONCE(rkce_reg->SYMM_INT_ST, cur_chn->int_st); + + cur_chn->result = (cur_chn->int_st == RKCE_SYMM_INT_ST_TD_DONE_MASK) ? + RKCE_SUCCESS : cur_chn->int_st; + } + + if (rkce_reg->HASH_INT_ST) { + cur_chn = &(hardware->chn[RKCE_TD_TYPE_HASH]); + cur_chn->int_st = READ_ONCE(rkce_reg->HASH_INT_ST); + cur_chn->td_id = rkce_reg->HASH_TD_ID; + + /* clear hash int */ + WRITE_ONCE(rkce_reg->HASH_INT_ST, cur_chn->int_st); + + cur_chn->result = (cur_chn->int_st == RKCE_HASH_INT_ST_TD_DONE_MASK) ? + RKCE_SUCCESS : cur_chn->int_st; + } +} + +void rkce_irq_thread(void *rkce_hw) +{ + uint32_t i; + bool is_fault = false; + struct rkce_hardware *hardware = rkce_hw; + + CHECK_RKCE_INITED(rkce_hw); + + for (i = 0; i < ARRAY_SIZE(hardware->chn); i++) { + struct rkce_chn_info *cur_chn = &(hardware->chn[i]); + + if (cur_chn->result) { + is_fault = true; + rk_err("td_type = %u, wrong SISR = %08x, td_id = %08x, td_virt = %p\n", + i, cur_chn->int_st, cur_chn->td_id, cur_chn->td_virt); + } + + if (cur_chn->int_st == 0 || !(cur_chn->cb_func)) + continue; + + rk_debug("##################### finalize td %p, result = %d\n", + cur_chn->td_virt, cur_chn->result); + + if (cur_chn->cb_func && cur_chn->td_virt) + cur_chn->cb_func(cur_chn->result, cur_chn->td_id, cur_chn->td_virt); + + cur_chn->result = 0; + cur_chn->int_st = 0; + cur_chn->td_id = 0; + cur_chn->td_virt = NULL; + } + + if (is_fault) + rkce_dump_reginfo(hardware); +} diff --git a/drivers/crypto/rockchip/rkce/rkce_core.h b/drivers/crypto/rockchip/rkce/rkce_core.h new file mode 100644 index 000000000000..8f98baee59e0 --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_core.h @@ -0,0 +1,338 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2024 Rockchip Electronics Co., Ltd. */ + +#ifndef __RKCE_CORE_H__ +#define __RKCE_CORE_H__ + +#include +#include + +#include "rkce_buf.h" +#include "rkce_error.h" +#include "rkce_reg.h" + +#define RKCE_TD_SG_NUM 8 + +#define RKCE_AES_BLOCK_SIZE 16 +#define RKCE_AES_KEYSIZE_128 16 +#define RKCE_AES_KEYSIZE_192 24 +#define RKCE_AES_KEYSIZE_256 32 + +#define RKCE_SM4_KEYSIZE 16 + +#define RKCE_DES_BLOCK_SIZE 8 +#define RKCE_DES_KEYSIZE 8 +#define RKCE_TDES_EDE_KEYSIZE 24 + +#define RKCE_TD_ALIGINMENT 16 +#define RKCE_TD_KEY_SIZE 128 +#define RKCE_TD_IV_SIZE 16 +#define RKCE_TD_GCM_LEN_SIZE 16 +#define RKCE_TD_HASH_CTX_SIZE RKCE_HASH_CONTEXT_SIZE +#define RKCE_TD_SYMM_CTX_SIZE RKCE_SYMM_CONTEXT_SIZE +#define RKCE_TD_TAG_SIZE 16 +#define RKCE_TD_TAG_SIZE_MIN 8 +#define RKCE_TD_TAG_SIZE_MAX RKCE_TD_TAG_SIZE +#define RKCE_TD_HASH_SIZE 64 +#define RKCE_TD_FIFO_DEPTH 8 + +#define RKCE_RESET_SYMM BIT(0) +#define RKCE_RESET_HASH BIT(1) +#define RKCE_RESET_PKA BIT(2) + +#define RKCE_WRITE_MASK_SHIFT (16) +#define RKCE_WRITE_MASK_ALL ((0xffffu << RKCE_WRITE_MASK_SHIFT)) + +enum rkce_expand_bit { + RKCE_EXPAND_BIT_4G = 0, + RKCE_EXPAND_BIT_8G, + RKCE_EXPAND_BIT_16G, + RKCE_EXPAND_BIT_32G, +}; + +enum rkce_td_type { + RKCE_TD_TYPE_SYMM = 0, + RKCE_TD_TYPE_HASH, + RKCE_TD_TYPE_SYMM_HASH_IN, + RKCE_TD_TYPE_SYMM_HASH_OUT, + RKCE_TD_TYPE_MAX, +}; + +enum rkce_algo_symm_type { + RKCE_SYMM_ALGO_AES = 0, + RKCE_SYMM_ALGO_SM4, + RKCE_SYMM_ALGO_DES, + RKCE_SYMM_ALGO_TDES, + RKCE_SYMM_ALGO_MAX, +}; + +enum rkce_algo_symm_mode { + RKCE_SYMM_MODE_ECB = 0, + RKCE_SYMM_MODE_CBC, + RKCE_SYMM_MODE_CTS, + RKCE_SYMM_MODE_CTR, + RKCE_SYMM_MODE_CFB, + RKCE_SYMM_MODE_OFB, + RKCE_SYMM_MODE_XTS, + RKCE_SYMM_MODE_CCM, + RKCE_SYMM_MODE_GCM, + RKCE_SYMM_MODE_CMAC, + RKCE_SYMM_MODE_CBC_MAC, + RKCE_SYMM_MODE_BYPASS = 0xf, + RKCE_SYMM_MODE_MAX, +}; + +enum { + RKCE_KEY_AES_128 = 0, + RKCE_KEY_AES_192, + RKCE_KEY_AES_256, +}; + +enum rkce_algo_hash_type { + RKCE_HASH_ALGO_SHA1 = 0, + RKCE_HASH_ALGO_MD5, + RKCE_HASH_ALGO_SHA256, + RKCE_HASH_ALGO_SHA224, + RKCE_HASH_ALGO_SM3 = 6, + RKCE_HASH_ALGO_SHA512 = 8, + RKCE_HASH_ALGO_SHA384 = 9, + RKCE_HASH_ALGO_SHA512_224, + RKCE_HASH_ALGO_SHA512_256, + RKCE_HASH_ALGO_MAX, +}; + +enum rkce_algo_asym_type { + RKCE_ASYM_ALGO_RSA = 0, + RKCE_ASYM_ALGO_ECC_P192, + RKCE_ASYM_ALGO_ECC_P224, + RKCE_ASYM_ALGO_ECC_P256, + RKCE_ASYM_ALGO_ECC_P384, + RKCE_ASYM_ALGO_ECC_P521, + RKCE_ASYM_ALGO_SM2, + RKCE_ASYM_ALGO_MAX, +}; + +enum rkce_algo_type { + RKCE_ALGO_TYPE_HASH, + RKCE_ALGO_TYPE_HMAC, + RKCE_ALGO_TYPE_CIPHER, + RKCE_ALGO_TYPE_ASYM, + RKCE_ALGO_TYPE_AEAD, + RKCE_ALGO_TYPE_MAX, +}; + +struct rkce_ip_info { + uint32_t aes_ver; + uint32_t des_ver; + uint32_t sm4_ver; + uint32_t hash_ver; + uint32_t hmac_ver; + uint32_t pka_ver; + uint32_t extra_feature; + uint32_t ce_ver; +}; + +struct rkce_gcm_len { + uint32_t pc_len_l; + uint32_t pc_len_h; + uint32_t aad_len_l; + uint32_t aad_len_h; +}; + +struct rkce_sg_info { + uint32_t src_size; + uint32_t src_addr_h; + uint32_t src_addr_l; + + uint32_t dst_size; + uint32_t dst_addr_h; + uint32_t dst_addr_l; +}; + +/* total = 64 + 16 + 16 + 16 + 32 = 114(Byte) */ +struct rkce_symm_td_buf { + uint8_t key1[RKCE_AES_KEYSIZE_256]; // offset 0x00 + uint8_t key2[RKCE_AES_KEYSIZE_256]; // offset 0x20 + uint8_t iv[RKCE_TD_IV_SIZE]; // offset 0x40 + struct rkce_gcm_len gcm_len; // offset 0x50 + uint8_t tag[RKCE_TD_TAG_SIZE]; // offset 0x60 + uint8_t ctx[RKCE_SYMM_CONTEXT_SIZE]; // offset 0x70 + void *user_data; +}; + +/* total = 128 + 64 + 208 = 360(Byte) */ +struct rkce_hash_td_buf { + uint8_t key[RKCE_TD_KEY_SIZE]; // offset 0x00 + uint8_t hash[RKCE_TD_HASH_SIZE]; // offset 0x80 + uint8_t ctx[RKCE_HASH_CONTEXT_SIZE]; // offset 0xB0 + void *user_data; +}; + +struct rkce_symm_hash_td_buf { + uint8_t key1[RKCE_AES_KEYSIZE_256]; // offset 0x00 + uint8_t key2[RKCE_AES_KEYSIZE_256]; // offset 0x20 + uint8_t key3[RKCE_AES_KEYSIZE_256 * 2]; // offset 0x40 + uint8_t iv[RKCE_TD_IV_SIZE]; // offset 0x80 + struct rkce_gcm_len gcm_len; // offset 0x90 + uint8_t tag[RKCE_TD_TAG_SIZE]; // offset 0xA0 + uint8_t hash[RKCE_TD_HASH_SIZE]; // offset 0xB0 + uint8_t symm_ctx[RKCE_SYMM_CONTEXT_SIZE]; // offset 0xF0 + uint8_t hash_ctx[RKCE_HASH_CONTEXT_SIZE]; // offset 0x110 + void *user_data; +}; + +struct rkce_symm_td_ctrl { + uint32_t td_type : 2; + uint32_t is_dec : 1; + uint32_t is_aad : 1; + uint32_t symm_algo : 2; + uint32_t: 2; + uint32_t symm_mode : 4; + uint32_t key_size : 2; + uint32_t first_pkg : 1; + uint32_t last_pkg : 1; + uint32_t key_sel : 3; + uint32_t iv_len : 5; + uint32_t: 4; + uint32_t is_key_inside : 1; + uint32_t: 1; + uint32_t is_preemptible : 1; + uint32_t int_en : 1; +}; + +struct rkce_hash_td_ctrl { + uint32_t td_type : 2; + uint32_t: 5; + uint32_t hw_pad_en : 1; + uint32_t: 6; + uint32_t first_pkg : 1; + uint32_t last_pkg : 1; + uint32_t: 8; + uint32_t hash_algo: 4; + uint32_t: 1; + uint32_t hmac_en : 1; + uint32_t is_preemptible : 1; + uint32_t int_en : 1; +}; + +struct rkce_symm_hash_td_ctrl { + uint32_t td_type : 2; + uint32_t is_dec : 1; + uint32_t is_aad : 1; + uint32_t symm_algo : 2; + uint32_t: 1; + uint32_t hw_pad_en : 1; + uint32_t symm_mode : 4; + uint32_t key_size : 2; + uint32_t first_pkg : 1; + uint32_t last_pkg : 1; + uint32_t key_sel : 3; + uint32_t iv_len : 5; + uint32_t hash_algo: 4; + uint32_t is_key_inside : 1; + uint32_t hmac_en : 1; + uint32_t is_preemptible : 1; + uint32_t int_en : 1; +}; + +struct rkce_symm_td { + uint32_t task_id; + struct rkce_symm_td_ctrl ctrl; + uint32_t reserve1; + uint32_t key_addr; + + uint32_t iv_addr; + uint32_t gcm_len_addr; + uint32_t reserve2; + uint32_t tag_addr; + + struct rkce_sg_info sg[RKCE_TD_SG_NUM]; + + uint32_t reserve3; + uint32_t symm_ctx_addr; + uint32_t reserve4[5]; + uint32_t next_task; +}; + +struct rkce_hash_td { + uint32_t task_id; + struct rkce_hash_td_ctrl ctrl; + uint32_t reserve1; + uint32_t key_addr; + + uint32_t reserve2[2]; + uint32_t hash_addr; + uint32_t reserve3; + + struct rkce_sg_info sg[RKCE_TD_SG_NUM]; + + uint32_t hash_ctx_addr; + uint32_t reserve4[6]; + uint32_t next_task; +}; + +struct rkce_symm_hash_td { + uint32_t task_id; + struct rkce_symm_hash_td_ctrl ctrl; + uint32_t reserve1; + uint32_t key_addr; + + uint32_t iv_addr; + uint32_t gcm_len_addr; + uint32_t hash_addr; + uint32_t tag_addr; + + struct rkce_sg_info sg[RKCE_TD_SG_NUM]; + + uint32_t hash_ctx_addr; + uint32_t symm_ctx_addr; + uint32_t reserve3[5]; + uint32_t next_task; +}; + +struct rkce_td { + union { + struct rkce_symm_td symm; + struct rkce_hash_td hash; + struct rkce_symm_hash_td symm_hash; + } td; +}; + +struct rkce_td_buf { + union { + struct rkce_symm_td_buf symm; + struct rkce_hash_td_buf hash; + struct rkce_symm_hash_td_buf symm_hash; + } td_buf; +}; + +typedef int (*request_cb_func)(int result, uint32_t td_id, void *td_virt); + +void rkce_dump_reginfo(void *rkce_hw); + +void *rkce_hardware_alloc(void __iomem *reg_base); + +void rkce_hardware_free(void *rkce_hw); + +void rkce_irq_handler(void *rkce_hw); + +void rkce_irq_thread(void *rkce_hw); + +int rkce_irq_callback_set(void *rkce_hw, enum rkce_td_type td_type, request_cb_func cb_func); + +int rkce_soft_reset(void *rkce_hw, uint32_t reset_sel); + +int rkce_push_td(void *rkce_hw, void *td); + +int rkce_push_td_sync(void *rkce_hw, void *td, uint32_t timeout_ms); + +uint32_t rkce_get_td_type(void *td_buf); + +int rkce_init_symm_td(struct rkce_symm_td *td, struct rkce_symm_td_buf *buf); + +int rkce_init_hash_td(struct rkce_hash_td *td, struct rkce_hash_td_buf *buf); + +bool rkce_hw_algo_valid(void *rkce_hw, uint32_t type, uint32_t algo, uint32_t mode); + +#endif diff --git a/drivers/crypto/rockchip/rkce/rkce_debug.c b/drivers/crypto/rockchip/rkce/rkce_debug.c new file mode 100644 index 000000000000..72ae7d9fb4e8 --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_debug.c @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Crypto acceleration support for Rockchip crypto engine + * + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + * Author: Lin Jinhan + * + */ + +#define RKCE_MODULE_TAG "DEBUG" +#define RKCE_MODULE_OFFSET 0 + +#include + +#include "rkce_core.h" +#include "rkce_buf.h" +#include "rkce_debug.h" + +ulong rkce_debug_level; + +typedef void (*rkce_dump_td_func)(void *td, uint32_t index); + +static void rkce_dump_symm_td(void *td, uint32_t index); +static void rkce_dump_hash_td(void *td, uint32_t index); + +static const rkce_dump_td_func g_dump_td_funcs[RKCE_TD_TYPE_MAX] = { + [RKCE_TD_TYPE_SYMM] = rkce_dump_symm_td, + [RKCE_TD_TYPE_HASH] = rkce_dump_hash_td, +}; + +static const char *rkce_td_type_str(void *td) +{ + struct rkce_symm_td *symm_td = (struct rkce_symm_td *)td; + + switch (symm_td->ctrl.td_type) { + case RKCE_TD_TYPE_SYMM: + return "SYMM"; + case RKCE_TD_TYPE_HASH: + return "HASH"; + case RKCE_TD_TYPE_SYMM_HASH_IN: + return "SYMM HASH IN"; + case RKCE_TD_TYPE_SYMM_HASH_OUT: + return "SYMM HASH OUT"; + default: + return "UNKNOWN"; + } +} + +static const char *rkce_td_symm_algo_str(void *td) +{ + struct rkce_symm_td *symm_td = (struct rkce_symm_td *)td; + + switch (symm_td->ctrl.symm_algo) { + case RKCE_SYMM_ALGO_AES: + return "AES"; + case RKCE_SYMM_ALGO_SM4: + return "SM4"; + case RKCE_SYMM_ALGO_DES: + return "DES"; + case RKCE_SYMM_ALGO_TDES: + return "TDES"; + default: + return "UNKNOWN"; + } +} + +static const char *rkce_td_hash_algo_str(void *td) +{ + struct rkce_hash_td *hash_td = (struct rkce_hash_td *)td; + + switch (hash_td->ctrl.hash_algo) { + case RKCE_HASH_ALGO_SHA1: + return "SHA1"; + case RKCE_HASH_ALGO_MD5: + return "MD5"; + case RKCE_HASH_ALGO_SHA256: + return "SHA256"; + case RKCE_HASH_ALGO_SHA224: + return "SHA224"; + case RKCE_HASH_ALGO_SM3: + return "SM3"; + case RKCE_HASH_ALGO_SHA512: + return "SHA512"; + case RKCE_HASH_ALGO_SHA384: + return "SHA384"; + case RKCE_HASH_ALGO_SHA512_224: + return "SHA512_224"; + case RKCE_HASH_ALGO_SHA512_256: + return "SHA512_256"; + default: + return "UNKNOWN"; + } +} + +static const char *rkce_td_symm_mode_str(void *td) +{ + struct rkce_symm_td *symm_td = (struct rkce_symm_td *)td; + + switch (symm_td->ctrl.symm_mode) { + case RKCE_SYMM_MODE_ECB: + return "ECB"; + case RKCE_SYMM_MODE_CBC: + return "CBC"; + case RKCE_SYMM_MODE_CTS: + return "CTS"; + case RKCE_SYMM_MODE_CTR: + return "CTR"; + case RKCE_SYMM_MODE_CFB: + return "CFB"; + case RKCE_SYMM_MODE_OFB: + return "OFB"; + case RKCE_SYMM_MODE_XTS: + return "XTS"; + case RKCE_SYMM_MODE_CCM: + return "CCM"; + case RKCE_SYMM_MODE_GCM: + return "GCM"; + case RKCE_SYMM_MODE_CMAC: + return "CMAC"; + case RKCE_SYMM_MODE_CBC_MAC: + return "CBC_MAC"; + case RKCE_SYMM_MODE_BYPASS: + return "BYP"; + default: + return "UNKNOWN"; + } +} + +static uint32_t rkce_td_symm_ks(void *td) +{ + struct rkce_symm_td *symm_td = (struct rkce_symm_td *)td; + uint32_t ks = 0; + + switch (symm_td->ctrl.symm_algo) { + case RKCE_SYMM_ALGO_AES: + if (symm_td->ctrl.key_size == RKCE_KEY_AES_128) + ks = RKCE_AES_KEYSIZE_128; + else if (symm_td->ctrl.key_size == RKCE_KEY_AES_192) + ks = RKCE_AES_KEYSIZE_192; + else if (symm_td->ctrl.key_size == RKCE_KEY_AES_256) + ks = RKCE_AES_KEYSIZE_256; + else + ks = 0; + break; + case RKCE_SYMM_ALGO_SM4: + ks = RKCE_SM4_KEYSIZE; + break; + case RKCE_SYMM_ALGO_DES: + ks = RKCE_DES_KEYSIZE; + break; + case RKCE_SYMM_ALGO_TDES: + ks = RKCE_DES_KEYSIZE * 3; + break; + default: + ks = 0; + } + + return ks * 8; +} + +static void rkce_get_single_td_len(void *td, uint32_t *src_total, uint32_t *dst_total) +{ + uint32_t i; + uint32_t tmp_src_len = 0; + uint32_t tmp_dst_len = 0; + struct rkce_symm_td *symm_td = (struct rkce_symm_td *)td; + + if (!td) + goto exit; + + for (i = 0; i < ARRAY_SIZE(symm_td->sg); i++) { + struct rkce_sg_info *sg = &symm_td->sg[i]; + + tmp_src_len += sg->src_size; + tmp_dst_len += sg->dst_size; + } + +exit: + if (src_total) + *src_total = tmp_src_len; + + if (dst_total) + *dst_total = tmp_dst_len; +} + +static void rkce_dump_symm_td(void *td, uint32_t index) +{ + uint32_t i; + struct rkce_symm_td *symm_td = (struct rkce_symm_td *)td; + uint32_t tmp_src_len = 0, tmp_dst_len = 0; + + rk_debug("\n"); + rk_debug("symm_td(%p) index[%u]:\n", td, index); + rk_debug("\ttask_id = %08x\n", symm_td->task_id); + rk_debug("\tkey_addr = %08x\n", symm_td->key_addr); + rk_debug("\tiv_addr = %08x\n", symm_td->iv_addr); + rk_debug("\tgcm_len_addr = %08x\n", symm_td->gcm_len_addr); + rk_debug("\ttag_addr = %08x\n", symm_td->tag_addr); + rk_debug("\tsymm_ctx_addr = %08x\n", symm_td->symm_ctx_addr); + + rk_debug("\tctrl: %s, %s-%u, %s, %s, %s, fpkg(%u), lpkg(%u), ksel(%u), ivl(%u), ki(%u), p(%u), int(%u)\n", + rkce_td_type_str(td), + rkce_td_symm_algo_str(td), + rkce_td_symm_ks(td), + rkce_td_symm_mode_str(td), + symm_td->ctrl.is_dec ? "DEC" : "ENC", + symm_td->ctrl.is_aad ? "AAD" : "PC", + symm_td->ctrl.first_pkg, symm_td->ctrl.last_pkg, + symm_td->ctrl.key_sel, symm_td->ctrl.iv_len, symm_td->ctrl.is_key_inside, + symm_td->ctrl.is_preemptible, symm_td->ctrl.int_en); + + rkce_get_single_td_len(td, &tmp_src_len, &tmp_dst_len); + + rk_debug("\tsg: src_len = %u, dst_len = %u\n", tmp_src_len, tmp_dst_len); + + for (i = 0; i < ARRAY_SIZE(symm_td->sg); i++) { + struct rkce_sg_info *sg = &symm_td->sg[i]; + + if (sg->src_addr_h || + sg->src_addr_l || + sg->src_size || + sg->dst_addr_h || + sg->dst_addr_l || + sg->dst_size) + rk_debug("\t\tsg[%u] = 0x%08x%08x(%8u) -> 0x%08x%08x(%8u)\n", + i, sg->src_addr_h, sg->src_addr_l, sg->src_size, + sg->dst_addr_h, sg->dst_addr_l, sg->dst_size); + } + rk_debug("\tnext_task = %08x\n", symm_td->next_task); +} + +static void rkce_dump_hash_td(void *td, uint32_t index) +{ + uint32_t i; + struct rkce_hash_td *hash_td = (struct rkce_hash_td *)td; + uint32_t tmp_src_len = 0; + + rk_debug("\n"); + rk_debug("hash_td(%p) index[%u]:\n", td, index); + rk_debug("\ttask_id = %08x\n", hash_td->task_id); + rk_debug("\tkey_addr = %08x\n", hash_td->key_addr); + rk_debug("\thash_addr = %08x\n", hash_td->hash_addr); + rk_debug("\thash_ctx_addr = %08x\n", hash_td->hash_ctx_addr); + + rk_debug("\tctrl: %s, %s:%s, hw_pad(%u), fpkg(%u), lpkg(%u), p(%u), int(%u)\n", + rkce_td_type_str(td), + hash_td->ctrl.hmac_en ? "HMAC" : "HASH", + rkce_td_hash_algo_str(td), + hash_td->ctrl.hw_pad_en, + hash_td->ctrl.first_pkg, hash_td->ctrl.last_pkg, + hash_td->ctrl.is_preemptible, hash_td->ctrl.int_en); + + rkce_get_single_td_len(td, &tmp_src_len, NULL); + + rk_debug("\tsg: src_len = %u\n", tmp_src_len); + + for (i = 0; i < ARRAY_SIZE(hash_td->sg); i++) { + struct rkce_sg_info *sg = &hash_td->sg[i]; + + if (sg->src_addr_h || + sg->src_addr_l || + sg->src_size) + rk_debug("sg[%u] = 0x%08x%08x(%-8u)\n", + i, sg->src_addr_h, sg->src_addr_l, sg->src_size); + } + + rk_debug("\tnext_task = %08x\n", hash_td->next_task); +} + +void rkce_dump_td(void *td) +{ + uint32_t i; + uint32_t td_type; + rkce_dump_td_func dump_func; + uint32_t src_total = 0, dst_total = 0; + struct rkce_symm_td *tmp_td = (struct rkce_symm_td *)td; + + if (!td) { + rk_info("empty td\n"); + goto exit; + } + + td_type = rkce_get_td_type(td); + if (td_type >= RKCE_TD_TYPE_MAX) { + rk_err("td_type(%u) >= %u\n", td_type, RKCE_TD_TYPE_MAX); + goto exit; + } + + dump_func = g_dump_td_funcs[td_type]; + if (!dump_func) + return; + + rk_info("==============================================================================\n"); + + for (i = 0; i < 1024; i++) { + uint32_t tmp_src_len, tmp_dst_len; + + rkce_get_single_td_len(tmp_td, &tmp_src_len, &tmp_dst_len); + + src_total += tmp_src_len; + dst_total += tmp_dst_len; + + dump_func(tmp_td, i); + + tmp_td = rkce_cma_phys2virt(tmp_td->next_task); + if (!tmp_td) + break; + } + + rk_info("=================== td chain src_total = %u, dst_total = %u ===================\n", + src_total, dst_total); +exit: + return; +} diff --git a/drivers/crypto/rockchip/rkce/rkce_debug.h b/drivers/crypto/rockchip/rkce/rkce_debug.h new file mode 100644 index 000000000000..87b6cc21e55b --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_debug.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2024 Rockchip Electronics Co., Ltd. */ + +#ifndef __RKCE_DEBUG_H__ +#define __RKCE_DEBUG_H__ + +extern ulong rkce_debug_level; + +/* must define marco before rkce_debug.h */ +//#define RKCE_MODULE_TAG +//#define RKCE_MODULE_OFFSET + +#define RKCE_LOG_LEVEL() (((rkce_debug_level) >> (RKCE_MODULE_OFFSET)) & 0x3) + +enum rkce_log_level { + RKCE_LOG_ERR = 0, + RKCE_LOG_INFO, + RKCE_LOG_DEBUG, + RKCE_LOG_VERBOSE, + RKCE_LOG_MAX, +}; + +#define rk_err(fmt, args...) \ + do { \ + if (RKCE_LOG_LEVEL() >= RKCE_LOG_ERR) \ + pr_err("RKCE-%s: E [%s %d]: " fmt "\n", \ + RKCE_MODULE_TAG, __func__, __LINE__, ##args); \ + } while (0) + +#define rk_info(fmt, args...) \ + do { \ + if (RKCE_LOG_LEVEL() >= RKCE_LOG_INFO) \ + pr_err(fmt, ##args); \ + } while (0) + +#define rk_debug(fmt, args...) \ + do { \ + if (RKCE_LOG_LEVEL() >= RKCE_LOG_DEBUG) \ + pr_err("RKCE-%s: D [%s %d]: " fmt "\n", \ + RKCE_MODULE_TAG, __func__, __LINE__, ##args); \ + } while (0) + +#define rk_trace(fmt, args...) \ + do { \ + if (RKCE_LOG_LEVEL() >= RKCE_LOG_VERBOSE) \ + pr_err("RKCE-%s: T [%s %d]: " fmt "\n", \ + RKCE_MODULE_TAG, __func__, __LINE__, ##args); \ + } while (0) + +void rkce_dump_td(void *td); + +#if defined(DEBUG) +#define rkce_dumphex(var_name, data, len) print_hex_dump(KERN_CONT, (var_name), \ + DUMP_PREFIX_OFFSET, \ + 16, 1, (data), (len), false) +#else +#define rkce_dumphex(var_name, data, len) \ + do { \ + if (0) \ + print_hex_dump(KERN_CONT, (var_name), \ + DUMP_PREFIX_OFFSET, \ + 16, 1, (data), (len), false); \ + } while (0) +#endif + +#endif diff --git a/drivers/crypto/rockchip/rkce/rkce_dev.c b/drivers/crypto/rockchip/rkce/rkce_dev.c new file mode 100644 index 000000000000..533bb9f303b2 --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_dev.c @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Crypto acceleration support for Rockchip RKCE crypto + * + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + * Author: Lin Jinhan + * + */ + +#define RKCE_MODULE_TAG "DEV" +#define RKCE_MODULE_OFFSET 4 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "rkce_akcipher.h" +#include "rkce_buf.h" +#include "rkce_core.h" +#include "rkce_debug.h" +#include "rkce_dev.h" +#include "rkce_hash.h" +#include "rkce_skcipher.h" +#include "rkce_monitor.h" +#include "../cryptodev_linux/rk_cryptodev.h" + +#define CRYPTO_NAME "RKCE" + +module_param_named(debug_level, rkce_debug_level, ulong, 0644); +MODULE_PARM_DESC(rkce_debug_level, "debug level | DBEUG | CORE | DEV | CIPHER | HASH | BUF | (0-3)"); + +static irqreturn_t rkce_dev_irq_handle(int irq, void *dev_id) +{ + struct rkce_dev *rk_dev = platform_get_drvdata(dev_id); + + rk_trace("enter.\n"); + + if (!rk_dev || !rk_dev->hardware) + return IRQ_HANDLED; + + rkce_irq_handler(rk_dev->hardware); + + return IRQ_WAKE_THREAD; +} + +static irqreturn_t rkce_dev_irq_thread(int irq, void *dev_id) +{ + struct rkce_dev *rk_dev = platform_get_drvdata(dev_id); + + rk_trace("enter.\n"); + + if (!rk_dev || !rk_dev->hardware) + goto exit; + + rkce_irq_thread(rk_dev->hardware); + +exit: + return IRQ_HANDLED; +} + +int rkce_enable_clk(struct rkce_dev *rk_dev) +{ + int ret; + + rk_trace("clk_bulk_prepare_enable.\n"); + + ret = clk_bulk_prepare_enable(rk_dev->clks_num, rk_dev->clk_bulks); + if (ret < 0) + rk_err("failed to enable clks %d\n", ret); + + return ret; +} + +void rkce_disable_clk(struct rkce_dev *rk_dev) +{ + rk_trace("clk_bulk_disable_unprepare.\n"); + + clk_bulk_disable_unprepare(rk_dev->clks_num, rk_dev->clk_bulks); +} + +static struct rkce_algt *g_rkce_algs[] = { + &rkce_ecb_sm4_alg, + &rkce_cbc_sm4_alg, + &rkce_xts_sm4_alg, + &rkce_cfb_sm4_alg, + &rkce_ofb_sm4_alg, + &rkce_ctr_sm4_alg, + &rkce_gcm_sm4_alg, + + &rkce_ecb_aes_alg, + &rkce_cbc_aes_alg, + &rkce_xts_aes_alg, + &rkce_cfb_aes_alg, + &rkce_ofb_aes_alg, + &rkce_ctr_aes_alg, + &rkce_gcm_aes_alg, + + &rkce_ecb_des_alg, + &rkce_cbc_des_alg, + &rkce_cfb_des_alg, + &rkce_ofb_des_alg, + + &rkce_ecb_des3_ede_alg, + &rkce_cbc_des3_ede_alg, + &rkce_cfb_des3_ede_alg, + &rkce_ofb_des3_ede_alg, + + &rkce_ahash_sha1, + &rkce_ahash_sha224, + &rkce_ahash_sha256, + &rkce_ahash_sha384, + &rkce_ahash_sha512, + &rkce_ahash_md5, + &rkce_ahash_sm3, + + &rkce_hmac_md5, + &rkce_hmac_sha1, + &rkce_hmac_sha256, + &rkce_hmac_sha512, + &rkce_hmac_sm3, + + &rkce_asym_rsa, + &rkce_asym_sm2, + &rkce_asym_ecc_p192, + &rkce_asym_ecc_p224, + &rkce_asym_ecc_p256, +}; + +static int rkce_crypto_register(struct rkce_dev *rk_dev, struct rkce_algt **algts, int algt_num) +{ + unsigned int i, k; + int err = 0; + + rkce_enable_clk(rk_dev); + + for (i = 0; i < algt_num; i++) { + struct rkce_algt *cur_algt = algts[i]; + + if (!rkce_hw_algo_valid(rk_dev->hardware, cur_algt->type, + cur_algt->algo, cur_algt->mode)) + continue; + + cur_algt->rk_dev = rk_dev; + cur_algt->valid_flag = true; + + if (cur_algt->type == RKCE_ALGO_TYPE_CIPHER) { + if (cur_algt->mode == RKCE_SYMM_MODE_CTR || + cur_algt->mode == RKCE_SYMM_MODE_CFB || + cur_algt->mode == RKCE_SYMM_MODE_OFB) + cur_algt->alg.cipher.base.cra_blocksize = 1; + + if (cur_algt->mode == RKCE_SYMM_MODE_ECB) + cur_algt->alg.cipher.ivsize = 0; + + err = crypto_register_skcipher(&cur_algt->alg.cipher); + } else if (cur_algt->type == RKCE_ALGO_TYPE_HASH || + cur_algt->type == RKCE_ALGO_TYPE_HMAC) { + err = crypto_register_ahash(&cur_algt->alg.hash); + } else if (cur_algt->type == RKCE_ALGO_TYPE_AEAD) { + err = crypto_register_aead(&cur_algt->alg.aead); + } else if (cur_algt->type == RKCE_ALGO_TYPE_ASYM) { + err = crypto_register_akcipher(&cur_algt->alg.asym); + } else { + continue; + } + + if (err) { + rk_err("crypto register %s failed.\n", cur_algt->name); + goto err_cipher_algs; + } + + rk_debug("register algo %s success.\n", cur_algt->name); + } + + rkce_disable_clk(rk_dev); + + return 0; + +err_cipher_algs: + for (k = 0; k < i; k++) { + struct rkce_algt *cur_algt = algts[i]; + + if (cur_algt->type == RKCE_ALGO_TYPE_CIPHER) + crypto_unregister_skcipher(&cur_algt->alg.cipher); + else if (cur_algt->type == RKCE_ALGO_TYPE_HASH || + cur_algt->type == RKCE_ALGO_TYPE_HMAC) + crypto_unregister_ahash(&cur_algt->alg.hash); + else if (cur_algt->type == RKCE_ALGO_TYPE_AEAD) + crypto_unregister_aead(&cur_algt->alg.aead); + else if (cur_algt->type == RKCE_ALGO_TYPE_ASYM) + crypto_unregister_akcipher(&cur_algt->alg.asym); + else + continue; + } + + rkce_disable_clk(rk_dev); + + return err; +} + +static void rkce_crypto_unregister(struct rkce_dev *rk_dev, struct rkce_algt **algts, int algt_num) +{ + int i; + + rkce_enable_clk(rk_dev); + + for (i = 0; i < algt_num; i++) { + struct rkce_algt *cur_algt = algts[i]; + + if (cur_algt->type == RKCE_ALGO_TYPE_CIPHER) + crypto_unregister_skcipher(&cur_algt->alg.cipher); + else if (cur_algt->type == RKCE_ALGO_TYPE_HASH || + cur_algt->type == RKCE_ALGO_TYPE_HMAC) + crypto_unregister_ahash(&cur_algt->alg.hash); + else if (cur_algt->type == RKCE_ALGO_TYPE_AEAD) + crypto_unregister_aead(&cur_algt->alg.aead); + else if (cur_algt->type == RKCE_ALGO_TYPE_ASYM) + crypto_unregister_akcipher(&cur_algt->alg.asym); + else + continue; + } + + rkce_disable_clk(rk_dev); +} + +static const struct of_device_id rkce_of_id_table[] = { + /* crypto rkce in below */ + { + .compatible = "rockchip,crypto-ce", + }, + + { /* sentinel */ } +}; + +MODULE_DEVICE_TABLE(of, rkce_of_id_table); + +static int rkce_probe(struct platform_device *pdev) +{ + struct resource *res; + struct device *dev = &pdev->dev; + struct rkce_dev *rk_dev; + int err = 0; + + rk_dev = devm_kzalloc(&pdev->dev, sizeof(*rk_dev), GFP_KERNEL); + if (!rk_dev) + return -ENOMEM; + + spin_lock_init(&rk_dev->lock); + + /* get crypto base */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + rk_dev->reg = devm_ioremap_resource(dev, res); + if (IS_ERR(rk_dev->reg)) { + err = PTR_ERR(rk_dev->reg); + goto err_crypto; + } + + rk_dev->clks_num = devm_clk_bulk_get_all(dev, &rk_dev->clk_bulks); + if (rk_dev->clks_num < 0) { + err = rk_dev->clks_num; + rk_err("failed to get clks property\n"); + goto err_crypto; + } + + rk_dev->irq = platform_get_irq(pdev, 0); + if (rk_dev->irq < 0) { + rk_err("control Interrupt is not available.\n"); + err = rk_dev->irq; + goto err_crypto; + } + + err = devm_request_threaded_irq(dev, rk_dev->irq, rkce_dev_irq_handle, + rkce_dev_irq_thread, IRQF_ONESHOT, + dev_name(dev), pdev); + if (err) { + rk_err("irq request failed.\n"); + goto err_crypto; + } + + disable_irq(rk_dev->irq); + + err = rkce_cma_init(dev); + if (err) { + rk_err("rkce_cma_init failed.\n"); + goto err_crypto; + } + + rk_dev->hardware = rkce_hardware_alloc(rk_dev->reg); + if (!rk_dev->hardware) { + err = -EFAULT; + rk_err("rkce_hardware_alloc failed.\n"); + goto err_crypto; + } + + rk_dev->dev = dev; + + err = rkce_crypto_register(rk_dev, g_rkce_algs, ARRAY_SIZE(g_rkce_algs)); + if (err) { + rk_err("rkce_crypto_register failed.\n"); + goto err_crypto; + } + + platform_set_drvdata(pdev, rk_dev); + + rkce_monitor_init(); + + rkce_irq_callback_set(rk_dev->hardware, RKCE_TD_TYPE_SYMM, rkce_cipher_request_callback); + rkce_irq_callback_set(rk_dev->hardware, RKCE_TD_TYPE_HASH, rkce_hash_request_callback); + + rk_dev->symm_engine = crypto_engine_alloc_init(&pdev->dev, true); + crypto_engine_start(rk_dev->symm_engine); + + rk_dev->hash_engine = crypto_engine_alloc_init(&pdev->dev, true); + crypto_engine_start(rk_dev->hash_engine); + + rk_dev->asym_engine = crypto_engine_alloc_init(&pdev->dev, true); + crypto_engine_start(rk_dev->asym_engine); + + rk_debug("symm_engine = %p hash_engine = %p asym_engine = %p", + rk_dev->symm_engine, rk_dev->hash_engine, rk_dev->asym_engine); + + rk_cryptodev_register_dev(dev, "RKCE multi"); + + enable_irq(rk_dev->irq); + + dev_info(dev, "%s Accelerator successfully registered\n", CRYPTO_NAME); + + return 0; + +err_crypto: + rkce_hardware_free(rk_dev->hardware); + rk_dev->hardware = NULL; + return err; +} + +static int rkce_remove(struct platform_device *pdev) +{ + struct rkce_dev *rk_dev = platform_get_drvdata(pdev); + + if (!rk_dev) + return 0; + + crypto_engine_exit(rk_dev->symm_engine); + crypto_engine_exit(rk_dev->hash_engine); + + rkce_monitor_deinit(); + + rkce_crypto_unregister(rk_dev, g_rkce_algs, ARRAY_SIZE(g_rkce_algs)); + + rkce_hardware_free(rk_dev->hardware); + + rkce_cma_deinit(rk_dev->dev); + + rkce_disable_clk(rk_dev); + + return 0; +} + +static struct platform_driver rkce_driver = { + .probe = rkce_probe, + .remove = rkce_remove, + .driver = { + .name = CRYPTO_NAME, + .of_match_table = rkce_of_id_table, + }, +}; + +module_platform_driver(rkce_driver); + +MODULE_AUTHOR("Lin Jinhan "); +MODULE_DESCRIPTION("Support for Rockchip's RKCE cryptographic engine"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/rockchip/rkce/rkce_dev.h b/drivers/crypto/rockchip/rkce/rkce_dev.h new file mode 100644 index 000000000000..743ed9daddd6 --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_dev.h @@ -0,0 +1,345 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2024 Rockchip Electronics Co., Ltd. */ + +#ifndef __RKCE_DEV_H__ +#define __RKCE_DEV_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "rkce_core.h" + +#define RKCE_PRIORITY 0 + +#define DES_MIN_KEY_SIZE DES_KEY_SIZE +#define DES_MAX_KEY_SIZE DES_KEY_SIZE +#define DES3_EDE_MIN_KEY_SIZE DES3_EDE_KEY_SIZE +#define DES3_EDE_MAX_KEY_SIZE DES3_EDE_KEY_SIZE +#define SM4_MIN_KEY_SIZE SM4_KEY_SIZE +#define SM4_MAX_KEY_SIZE SM4_KEY_SIZE + +#define MD5_BLOCK_SIZE SHA1_BLOCK_SIZE + +#define sha384_state sha512_state +#define sha224_state sha256_state + +#define RKCE_SYMM_ALGO_DES3_EDE RKCE_SYMM_ALGO_TDES +struct rkce_dev { + struct device *dev; + struct reset_control *rst; + void __iomem *reg; + int irq; + int clks_num; + struct clk_bulk_data *clk_bulks; + + /* device lock */ + spinlock_t lock; + struct crypto_engine *symm_engine; + struct crypto_engine *hash_engine; + struct crypto_engine *asym_engine; + + void *hardware; +}; + +struct rkce_cipher_ctx { + struct crypto_engine_ctx enginectx; + struct rkce_algt *algt; + struct rkce_symm_td_buf *td_buf; + void *req; + + uint8_t iv[AES_BLOCK_SIZE]; + uint32_t ivlen; + uint32_t keylen; + uint32_t authsize; +}; + +struct rkce_cipher_request_ctx { + struct rkce_symm_td *td_head; + struct rkce_symm_td *td_aad_head; + + struct device *dev; + struct scatterlist *sgs; + struct scatterlist *sgd; + uint32_t sgs_nents; + uint32_t sgd_nents; + uint32_t cryptlen; + + struct scatterlist src_sg[2]; + struct scatterlist dst_sg[2]; + struct scatterlist *sga; + uint32_t sga_nents; + uint32_t assoclen; + + uint32_t map_total; + uint8_t is_enc; + uint8_t is_aead; + uint8_t is_mapped; + uint8_t is_dma; +}; + +/* the private variable of hash */ +struct rkce_ahash_ctx { + struct crypto_engine_ctx enginectx; + struct rkce_algt *algt; + struct rkce_hash_td *key_td; + struct rkce_hash_td_buf *td_buf; + struct ahash_request *req; + + uint8_t *user_key; + uint32_t calculated; + uint8_t is_hmac; + uint8_t is_final; +}; + +struct rkce_ahash_request_ctx { + struct rkce_hash_td *td_head; + uint32_t sgs_nents; + uint8_t is_mapped; + uint8_t *hw_context; +}; + +struct rkce_rsa_ctx { + struct crypto_engine_ctx enginectx; + struct rkce_algt *algt; + + struct rkce_bignum *n; + struct rkce_bignum *e; + struct rkce_bignum *d; + + uint8_t is_enc; +}; + +struct rkce_ecc_ctx { + struct crypto_engine_ctx enginectx; + struct rkce_algt *algt; + + uint32_t group_id; + uint32_t nbits; + bool pub_key_set; + struct rkce_ecp_point *point_Q; + + uint8_t is_sign; +}; + +struct rkce_asym_request_ctx { + uint32_t reserved; +}; + +struct rkce_algt { + struct rkce_dev *rk_dev; + union { + struct skcipher_alg cipher; + struct ahash_alg hash; + struct akcipher_alg asym; + struct aead_alg aead; + } alg; + enum rkce_algo_type type; + uint32_t algo; + uint32_t mode; + char *name; + bool valid_flag; +}; + +#define RK_AEAD_ALGO_INIT(cipher_algo, cipher_mode, algo_name, driver_name) {\ + .name = #algo_name,\ + .type = RKCE_ALGO_TYPE_AEAD,\ + .algo = RKCE_SYMM_ALGO_##cipher_algo,\ + .mode = RKCE_SYMM_MODE_##cipher_mode,\ + .alg.aead = {\ + .base.cra_name = #algo_name,\ + .base.cra_driver_name = #driver_name,\ + .base.cra_priority = RKCE_PRIORITY,\ + .base.cra_flags = CRYPTO_ALG_TYPE_AEAD |\ + CRYPTO_ALG_KERN_DRIVER_ONLY |\ + CRYPTO_ALG_ASYNC |\ + CRYPTO_ALG_NEED_FALLBACK |\ + CRYPTO_ALG_INTERNAL,\ + .base.cra_blocksize = 1,\ + .base.cra_ctxsize = sizeof(struct rkce_cipher_ctx),\ + .base.cra_alignmask = 0x07,\ + .base.cra_module = THIS_MODULE,\ + .init = rkce_aead_init_tfm,\ + .exit = rkce_aead_exit_tfm,\ + .ivsize = GCM_AES_IV_SIZE,\ + .chunksize = cipher_algo##_BLOCK_SIZE,\ + .maxauthsize = AES_BLOCK_SIZE,\ + .setkey = rkce_aead_setkey,\ + .setauthsize = rkce_aead_setauthsize,\ + .encrypt = rkce_aead_encrypt,\ + .decrypt = rkce_aead_decrypt,\ + } \ +} + +#define RK_CIPHER_ALGO_INIT(cipher_algo, cipher_mode, algo_name, driver_name) {\ + .name = #algo_name,\ + .type = RKCE_ALGO_TYPE_CIPHER,\ + .algo = RKCE_SYMM_ALGO_##cipher_algo,\ + .mode = RKCE_SYMM_MODE_##cipher_mode,\ + .alg.cipher = {\ + .base.cra_name = #algo_name,\ + .base.cra_driver_name = #driver_name,\ + .base.cra_priority = RKCE_PRIORITY,\ + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |\ + CRYPTO_ALG_ASYNC |\ + CRYPTO_ALG_NEED_FALLBACK |\ + CRYPTO_ALG_INTERNAL,\ + .base.cra_blocksize = cipher_algo##_BLOCK_SIZE,\ + .base.cra_ctxsize = sizeof(struct rkce_cipher_ctx),\ + .base.cra_alignmask = 0x07,\ + .base.cra_module = THIS_MODULE,\ + .init = rkce_ablk_init_tfm,\ + .exit = rkce_ablk_exit_tfm,\ + .min_keysize = cipher_algo##_MIN_KEY_SIZE,\ + .max_keysize = cipher_algo##_MAX_KEY_SIZE,\ + .ivsize = cipher_algo##_BLOCK_SIZE,\ + .chunksize = cipher_algo##_BLOCK_SIZE,\ + .setkey = rkce_cipher_setkey,\ + .encrypt = rkce_cipher_encrypt,\ + .decrypt = rkce_cipher_decrypt,\ + } \ +} + +#define RK_CIPHER_ALGO_XTS_INIT(cipher_algo, algo_name, driver_name) {\ + .name = #algo_name,\ + .type = RKCE_ALGO_TYPE_CIPHER,\ + .algo = RKCE_SYMM_ALGO_##cipher_algo,\ + .mode = RKCE_SYMM_MODE_XTS,\ + .alg.cipher = {\ + .base.cra_name = #algo_name,\ + .base.cra_driver_name = #driver_name,\ + .base.cra_priority = RKCE_PRIORITY,\ + .base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |\ + CRYPTO_ALG_ASYNC |\ + CRYPTO_ALG_NEED_FALLBACK |\ + CRYPTO_ALG_INTERNAL,\ + .base.cra_blocksize = cipher_algo##_BLOCK_SIZE,\ + .base.cra_ctxsize = sizeof(struct rkce_cipher_ctx),\ + .base.cra_alignmask = 0x07,\ + .base.cra_module = THIS_MODULE,\ + .init = rkce_ablk_init_tfm,\ + .exit = rkce_ablk_exit_tfm,\ + .min_keysize = cipher_algo##_MAX_KEY_SIZE,\ + .max_keysize = cipher_algo##_MAX_KEY_SIZE * 2,\ + .ivsize = cipher_algo##_BLOCK_SIZE,\ + .chunksize = cipher_algo##_BLOCK_SIZE,\ + .setkey = rkce_cipher_setkey,\ + .encrypt = rkce_cipher_encrypt,\ + .decrypt = rkce_cipher_decrypt,\ + } \ +} + +#define RK_HASH_ALGO_INIT(hash_algo, algo_name) {\ + .name = #algo_name,\ + .type = RKCE_ALGO_TYPE_HASH,\ + .algo = RKCE_HASH_ALGO_##hash_algo,\ + .alg.hash = {\ + .init = rkce_ahash_init,\ + .update = rkce_ahash_update,\ + .final = rkce_ahash_final,\ + .finup = rkce_ahash_finup,\ + .export = rkce_ahash_export,\ + .import = rkce_ahash_import,\ + .digest = rkce_ahash_digest,\ + .halg = {\ + .digestsize = hash_algo##_DIGEST_SIZE,\ + .statesize = sizeof(struct rkce_ahash_request_ctx),\ + .base = {\ + .cra_name = #algo_name,\ + .cra_driver_name = #algo_name"-rk",\ + .cra_priority = RKCE_PRIORITY,\ + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |\ + CRYPTO_ALG_ASYNC |\ + CRYPTO_ALG_NEED_FALLBACK |\ + CRYPTO_ALG_INTERNAL,\ + .cra_blocksize = hash_algo##_BLOCK_SIZE,\ + .cra_ctxsize = sizeof(struct rkce_ahash_ctx),\ + .cra_alignmask = 0,\ + .cra_init = rkce_cra_hash_init,\ + .cra_exit = rkce_cra_hash_exit,\ + .cra_module = THIS_MODULE,\ + } \ + } \ + } \ +} + +#define RK_HMAC_ALGO_INIT(hash_algo, algo_name) {\ + .name = "hmac(" #algo_name ")",\ + .type = RKCE_ALGO_TYPE_HMAC,\ + .algo = RKCE_HASH_ALGO_##hash_algo,\ + .alg.hash = {\ + .init = rkce_ahash_init,\ + .update = rkce_ahash_update,\ + .final = rkce_ahash_final,\ + .finup = rkce_ahash_finup,\ + .export = rkce_ahash_export,\ + .import = rkce_ahash_import,\ + .digest = rkce_ahash_digest,\ + .setkey = rkce_ahash_hmac_setkey,\ + .halg = {\ + .digestsize = hash_algo##_DIGEST_SIZE,\ + .statesize = sizeof(struct rkce_ahash_request_ctx),\ + .base = {\ + .cra_name = "hmac(" #algo_name ")",\ + .cra_driver_name = "hmac-" #algo_name "-rk",\ + .cra_priority = RKCE_PRIORITY,\ + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |\ + CRYPTO_ALG_ASYNC |\ + CRYPTO_ALG_NEED_FALLBACK |\ + CRYPTO_ALG_INTERNAL,\ + .cra_blocksize = hash_algo##_BLOCK_SIZE,\ + .cra_ctxsize = sizeof(struct rkce_ahash_ctx),\ + .cra_alignmask = 0,\ + .cra_init = rkce_cra_hash_init,\ + .cra_exit = rkce_cra_hash_exit,\ + .cra_module = THIS_MODULE,\ + } \ + } \ + } \ +} + +#define RK_ASYM_ECC_INIT(key_bits) {\ + .name = "ecc-" #key_bits, \ + .type = RKCE_ALGO_TYPE_ASYM, \ + .algo = RKCE_ASYM_ALGO_ECC_P##key_bits, \ + .alg.asym = { \ + .verify = rkce_ec_verify, \ + .set_pub_key = rkce_ec_set_pub_key, \ + .max_size = rkce_ec_max_size, \ + .init = rkce_ec_init_tfm, \ + .exit = rkce_ec_exit_tfm, \ + .reqsize = sizeof(struct rkce_asym_request_ctx), \ + .base = { \ + .cra_name = "ecdsa-nist-p" #key_bits, \ + .cra_driver_name = "ecdsa-nist-p" #key_bits "-rk", \ + .cra_priority = RKCE_PRIORITY, \ + .cra_module = THIS_MODULE, \ + .cra_ctxsize = sizeof(struct rkce_ecc_ctx), \ + },\ + } \ +} + +int rkce_enable_clk(struct rkce_dev *rk_dev); +void rkce_disable_clk(struct rkce_dev *rk_dev); + +#endif diff --git a/drivers/crypto/rockchip/rkce/rkce_ecc.c b/drivers/crypto/rockchip/rkce/rkce_ecc.c new file mode 100644 index 000000000000..de17be0a18b2 --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_ecc.c @@ -0,0 +1,473 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + * Author: Lin Jinhan + * + */ + +#define RKCE_MODULE_TAG "ECC" +#define RKCE_MODULE_OFFSET 16 + +#include +#include + +#include "rkce_core.h" +#include "rkce_debug.h" +#include "rkce_reg.h" +#include "rkce_ecc.h" + +static void __iomem *ecc_base; + +#define WORDS2BYTES(words) ((words) * 4) + +#define RK_ECP_POLL_PERIOD_US 10000 +#define RK_ECP_POLL_TIMEOUT_US 500000 + +#define RK_LOAD_GROUP_A(G) do { \ + grp->curve_name = #G; \ + grp->wide = G ## _wide;\ + grp->p = G ## _p; \ + grp->p_len = sizeof(G ## _p); \ + grp->a = G ## _a; \ + grp->a_len = sizeof(G ## _a); \ + grp->n = G ## _n; \ + grp->n_len = sizeof(G ## _n); \ + grp->gx = G ## _gx; \ + grp->gx_len = sizeof(G ## _gx); \ + grp->gy = G ## _gy; \ + grp->gy_len = sizeof(G ## _gy); \ + } while (0) + +/* + * Domain parameters for secp192r1 + */ +static const uint32_t secp192r1_wide = RK_ECC_CURVE_WIDE_192; + +static const uint8_t secp192r1_p[] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, +}; + +static const uint8_t secp192r1_a[] = { + 0xFC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, +}; + +static const uint8_t secp192r1_gx[] = { + 0x12, 0x10, 0xFF, 0x82, 0xFD, 0x0A, 0xFF, 0xF4, + 0x00, 0x88, 0xA1, 0x43, 0xEB, 0x20, 0xBF, 0x7C, + 0xF6, 0x90, 0x30, 0xB0, 0x0E, 0xA8, 0x8D, 0x18, +}; + +static const uint8_t secp192r1_gy[] = { + 0x11, 0x48, 0x79, 0x1E, 0xA1, 0x77, 0xF9, 0x73, + 0xD5, 0xCD, 0x24, 0x6B, 0xED, 0x11, 0x10, 0x63, + 0x78, 0xDA, 0xC8, 0xFF, 0x95, 0x2B, 0x19, 0x07, +}; + +static const uint8_t secp192r1_n[] = { + 0x31, 0x28, 0xD2, 0xB4, 0xB1, 0xC9, 0x6B, 0x14, + 0x36, 0xF8, 0xDE, 0x99, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, +}; + +/* + * Domain parameters for secp224r1 + */ +static const uint32_t secp224r1_wide = RK_ECC_CURVE_WIDE_224; + +static const uint8_t secp224r1_p[] = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, +}; + +static const uint8_t secp224r1_a[] = { + 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, +}; + +static const uint8_t secp224r1_gx[] = { + 0x21, 0x1D, 0x5C, 0x11, 0xD6, 0x80, 0x32, 0x34, + 0x22, 0x11, 0xC2, 0x56, 0xD3, 0xC1, 0x03, 0x4A, + 0xB9, 0x90, 0x13, 0x32, 0x7F, 0xBF, 0xB4, 0x6B, + 0xBD, 0x0C, 0x0E, 0xB7, +}; + +static const uint8_t secp224r1_gy[] = { + 0x34, 0x7E, 0x00, 0x85, 0x99, 0x81, 0xD5, 0x44, + 0x64, 0x47, 0x07, 0x5A, 0xA0, 0x75, 0x43, 0xCD, + 0xE6, 0xDF, 0x22, 0x4C, 0xFB, 0x23, 0xF7, 0xB5, + 0x88, 0x63, 0x37, 0xBD, +}; + +static const uint8_t secp224r1_n[] = { + 0x3D, 0x2A, 0x5C, 0x5C, 0x45, 0x29, 0xDD, 0x13, + 0x3E, 0xF0, 0xB8, 0xE0, 0xA2, 0x16, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, +}; + +/* + * Domain parameters for secp256r1 + */ +static const uint32_t secp256r1_wide = RK_ECC_CURVE_WIDE_256; + +static const uint8_t secp256r1_p[] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, +}; + +static const uint8_t secp256r1_a[] = { + 0xFC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, +}; + +static const uint8_t secp256r1_gx[] = { + 0x96, 0xC2, 0x98, 0xD8, 0x45, 0x39, 0xA1, 0xF4, + 0xA0, 0x33, 0xEB, 0x2D, 0x81, 0x7D, 0x03, 0x77, + 0xF2, 0x40, 0xA4, 0x63, 0xE5, 0xE6, 0xBC, 0xF8, + 0x47, 0x42, 0x2C, 0xE1, 0xF2, 0xD1, 0x17, 0x6B, +}; + +static const uint8_t secp256r1_gy[] = { + 0xF5, 0x51, 0xBF, 0x37, 0x68, 0x40, 0xB6, 0xCB, + 0xCE, 0x5E, 0x31, 0x6B, 0x57, 0x33, 0xCE, 0x2B, + 0x16, 0x9E, 0x0F, 0x7C, 0x4A, 0xEB, 0xE7, 0x8E, + 0x9B, 0x7F, 0x1A, 0xFE, 0xE2, 0x42, 0xE3, 0x4F, +}; + +static const uint8_t secp256r1_n[] = { + 0x51, 0x25, 0x63, 0xFC, 0xC2, 0xCA, 0xB9, 0xF3, + 0x84, 0x9E, 0x17, 0xA7, 0xAD, 0xFA, 0xE6, 0xBC, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, +}; + +/* + * Domain parameters for sm2p256v1_p + */ +static const uint32_t sm2p256v1_wide = RK_ECC_CURVE_WIDE_256; + +static const uint8_t sm2p256v1_p[] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFF, +}; + +static const uint8_t sm2p256v1_a[] = { + 0xFC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFF, +}; + +static const uint8_t sm2p256v1_gx[] = { + 0xC7, 0x74, 0x4C, 0x33, 0x89, 0x45, 0x5A, 0x71, + 0xE1, 0x0B, 0x66, 0xF2, 0xBF, 0x0B, 0xE3, 0x8F, + 0x94, 0xC9, 0x39, 0x6A, 0x46, 0x04, 0x99, 0x5F, + 0x19, 0x81, 0x19, 0x1F, 0x2C, 0xAE, 0xC4, 0x32, +}; + +static const uint8_t sm2p256v1_gy[] = { + 0xA0, 0xF0, 0x39, 0x21, 0xE5, 0x32, 0xDF, 0x02, + 0x40, 0x47, 0x2A, 0xC6, 0x7C, 0x87, 0xA9, 0xD0, + 0x53, 0x21, 0x69, 0x6B, 0xE3, 0xCE, 0xBD, 0x59, + 0x9C, 0x77, 0xF6, 0xF4, 0xA2, 0x36, 0x37, 0xBC, +}; + +static const uint8_t sm2p256v1_n[] = { + 0x23, 0x41, 0xD5, 0x39, 0x09, 0xF4, 0xBB, 0x53, + 0x2B, 0x05, 0xC6, 0x21, 0x6B, 0xDF, 0x03, 0x72, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFE, 0xFF, 0xFF, 0xFF, +}; + +static void dump_ecc_sram(void) +{ +#ifdef DEBUG + int i; + uint32_t len = 0x300; + uint32_t *buffer; + + buffer = kzalloc(len, GFP_KERNEL); + + for (i = 0; i < len / 4; i++) + buffer[i] = ((uint32_t *)SM2_RAM_BASE)[i]; + +// CRYPTO_DUMPHEX("ECC SRAM ", buffer, len); + + kfree(buffer); +#endif +} + +static void ecc_word_memcpy(uint32_t *dst, uint32_t *src, uint32_t size) +{ + uint32_t i; + + for (i = 0; i < size; i++, dst++) + writel_relaxed(src[i], (void *)dst); +} + +static void ecc_word_memset(uint32_t *buff, uint32_t val, uint32_t size) +{ + uint32_t i; + + for (i = 0; i < size; i++, buff++) + writel_relaxed(val, (void *)buff); +} + +static void rk_reverse_buf(uint8_t *buff, uint32_t size) +{ + uint8_t *buf_h_swap, *buf_l_swap; + uint32_t i; + uint32_t temp; + + buf_h_swap = buff + size - 1; + buf_l_swap = buff; + + for (i = 0 ; i < (size / 2) ; i++) { + temp = *buf_h_swap; + *(buf_h_swap--) = *buf_l_swap; + *(buf_l_swap++) = temp; + } +} + +static int rk_word_cmp_zero(uint32_t *buf1, uint32_t n_words) +{ + int ret = 0; + uint32_t i; + + for (i = 0 ; i < n_words; i++) { + if (buf1[i] != 0) + ret = -EINVAL; + } + + return ret; +} + +static int rk_load_hash_bn(struct rk_ecp_group *grp, struct rkce_bignum *bn, + uint8_t *hash, uint32_t hash_len) +{ + if (grp == NULL || RK_ECP_IS_BIGNUM_INVALID(bn)) + return -EINVAL; + + hash_len = hash_len > grp->p_len ? grp->p_len : hash_len; + + memset(bn->data, 0x00, WORDS2BYTES(bn->n_words)); + memcpy(bn->data, hash, hash_len); + + rk_reverse_buf((void *)bn->data, hash_len); + + return 0; +} + +/* + * Set a group using well-known domain parameters + */ +static int rk_ecp_group_load(struct rk_ecp_group *grp, enum rk_ecp_group_id id) +{ + memset(grp, 0x00, sizeof(*grp)); + + grp->id = id; + grp->endian = RK_BG_LITTILE_ENDIAN; + + switch (id) { + case RK_ECP_DP_SECP192R1: + RK_LOAD_GROUP_A(secp192r1); + return 0; + + case RK_ECP_DP_SECP224R1: + RK_LOAD_GROUP_A(secp224r1); + return 0; + + case RK_ECP_DP_SECP256R1: + RK_LOAD_GROUP_A(secp256r1); + return 0; + + case RK_ECP_DP_SM2P256V1: + RK_LOAD_GROUP_A(sm2p256v1); + return 0; + + default: + return -EINVAL; + } +} + +static int rockchip_ecc_request_set(uint32_t ecc_ctl, uint32_t wide) +{ + RK_ECP_WRITE_REG(RK_ECC_CURVE_WIDE, wide); + + RK_ECP_WRITE_REG(RK_ECC_INT_EN, 0); + RK_ECP_WRITE_REG(RK_ECC_INT_ST, RK_ECP_READ_REG(RK_ECC_INT_ST)); + RK_ECP_WRITE_REG(RK_ECC_CTL, ecc_ctl); + + return 0; +} + +static int rockchip_ecc_request_wait_done(void) +{ + int ret; + uint32_t reg_val = 0; + + ret = readx_poll_timeout(RK_ECP_READ_REG, RK_ECC_INT_ST, reg_val, + reg_val != 0, RK_ECP_POLL_PERIOD_US, RK_ECP_POLL_TIMEOUT_US); + if (ret) { + rk_err("rk ecp poll RK_ECC_INT_ST timeout.\ns"); + goto exit; + } + + if (RK_ECP_READ_REG(RK_ECC_ABN_ST)) { + ret = -EFAULT; + goto exit; + } + +exit: + if (ret) { + rk_err("RK_ECC_CTL = %08x\n", RK_ECP_READ_REG(RK_ECC_CTL)); + rk_err("RK_ECC_INT_EN = %08x\n", RK_ECP_READ_REG(RK_ECC_INT_EN)); + rk_err("RK_ECC_CURVE_WIDE = %08x\n", RK_ECP_READ_REG(RK_ECC_CURVE_WIDE)); + rk_err("RK_ECC_RAM_CTL = %08x\n", RK_ECP_READ_REG(RK_ECC_RAM_CTL)); + rk_err("RK_ECC_INT_ST = %08x\n", RK_ECP_READ_REG(RK_ECC_INT_ST)); + rk_err("RK_ECC_ABN_ST = %08x\n", RK_ECP_READ_REG(RK_ECC_ABN_ST)); + rk_err("RK_ECC_MAX_CURVE_WIDE = %08x\n", RK_ECP_READ_REG(RK_ECC_MAX_CURVE_WIDE)); + + } + + RK_ECP_WRITE_REG(RK_ECC_CTL, 0); + RK_ECP_RAM_FOR_CPU(); + + return ret; +} + +static int rockchip_ecc_request_trigger(void) +{ + uint32_t ecc_ctl = RK_ECP_READ_REG(RK_ECC_CTL); + + RK_ECP_RAM_FOR_ECC(); + + RK_ECP_WRITE_REG(RK_ECC_CTL, ecc_ctl | RK_ECC_CTL_REQ_ECC); + + return rockchip_ecc_request_wait_done(); +} + +int rkce_ecc_verify(int group_id, uint8_t *hash, uint32_t hash_len, + struct rkce_ecp_point *point_P, struct rkce_ecp_point *point_sign) +{ + int ret; + uint32_t curve_sel = 0; + struct rkce_bignum *bn_hash = NULL; + struct rk_ecp_group grp; + struct rk_ecc_verify *ecc_st = (struct rk_ecc_verify *)SM2_RAM_BASE; + + if (!hash || + RK_ECP_IS_POINT_INVALID(point_P) || + RK_ECP_IS_POINT_INVALID(point_sign)) { + ret = -EINVAL; + goto exit; + } + + ret = rk_ecp_group_load(&grp, group_id); + if (ret) + goto exit; + + bn_hash = rkce_bn_alloc(RK_ECP_MAX_BYTES); + if (!bn_hash) { + ret = -ENOMEM; + goto exit; + } + + curve_sel = group_id == RK_ECP_DP_SM2P256V1 ? + RK_ECC_CTL_FUNC_SM2_CURVER : RK_ECC_CTL_FUNC_ECC_CURVER; + + rk_load_hash_bn(&grp, bn_hash, hash, hash_len); + + RK_ECP_LOAD_DATA(ecc_st->e, bn_hash); + RK_ECP_LOAD_DATA(ecc_st->r_, point_sign->x); + RK_ECP_LOAD_DATA(ecc_st->s_, point_sign->y); + RK_ECP_LOAD_DATA(ecc_st->p_x, point_P->x); + RK_ECP_LOAD_DATA(ecc_st->p_y, point_P->y); + + RK_ECP_LOAD_DATA_EXT(ecc_st->A, grp.a, grp.a_len); + RK_ECP_LOAD_DATA_EXT(ecc_st->P, grp.p, grp.p_len); + RK_ECP_LOAD_DATA_EXT(ecc_st->N, grp.n, grp.n_len); + + RK_ECP_LOAD_DATA_EXT(ecc_st->G_x, grp.gx, grp.gx_len); + RK_ECP_LOAD_DATA_EXT(ecc_st->G_y, grp.gy, grp.gy_len); + + rockchip_ecc_request_set(curve_sel | RK_ECC_CTL_FUNC_SEL_VERIFY, grp.wide); + + ret = rockchip_ecc_request_trigger(); +exit: + if (ret || rk_word_cmp_zero(ecc_st->v, RK_ECP_MAX_WORDS)) { + ret = -EKEYREJECTED; + dump_ecc_sram(); + } + + rkce_bn_free(bn_hash); + + return ret; +} + +void rkce_ecc_init(void __iomem *base) +{ + ecc_base = base; + + RK_ECP_WRITE_REG(RK_ECC_DATA_ENDIAN, RK_ECC_DATA_ENDIAN_LITTLE); +} + +void rkce_ecc_deinit(void) +{ + +} + +uint32_t rkce_ecc_get_max_size(void) +{ + return RK_ECP_MAX_BYTES; +} + +uint32_t rkce_ecc_get_curve_nbits(uint32_t group_id) +{ + switch (group_id) { + case RK_ECP_DP_SECP192R1: + return 192; + + case RK_ECP_DP_SECP224R1: + return 224; + + case RK_ECP_DP_SECP256R1: + case RK_ECP_DP_SM2P256V1: + return 256; + + default: + return 0; + } +} + +uint32_t rkce_ecc_get_group_id(uint32_t asym_algo) +{ + + switch (asym_algo) { + case RKCE_ASYM_ALGO_ECC_P192: + return RK_ECP_DP_SECP192R1; + case RKCE_ASYM_ALGO_ECC_P224: + return RK_ECP_DP_SECP224R1; + case RKCE_ASYM_ALGO_ECC_P256: + return RK_ECP_DP_SECP256R1; + case RKCE_ASYM_ALGO_SM2: + return RK_ECP_DP_SM2P256V1; + default: + return RK_ECP_DP_NONE; + } +} diff --git a/drivers/crypto/rockchip/rkce/rkce_ecc.h b/drivers/crypto/rockchip/rkce/rkce_ecc.h new file mode 100644 index 000000000000..1e4853d54f0a --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_ecc.h @@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2024 Rockchip Electronics Co., Ltd. */ + +#ifndef __RKCE_ECC_H__ +#define __RKCE_ECC_H__ + +#include "rkce_bignum.h" + +#define _SBF(s, v) ((v) << (s)) + +#define RK_ECP_MAX_BITS 256 +#define RK_ECP_MAX_BYTES ((RK_ECP_MAX_BITS) / 8) +#define RK_ECP_MAX_WORDS ((RK_ECP_MAX_BITS) / 32) +#define RK_ECP_MAX_WORDS_ALL (512 / 32) + +/*************************************************************/ +/* Macros for waiting PKA machine ready states */ +/*************************************************************/ +#define RK_ECP_WRITE_REG(offset, val) writel_relaxed((val), (ecc_base) + (offset)) +#define RK_ECP_READ_REG(offset) readl_relaxed((ecc_base) + (offset)) + +#define RK_ECP_RAM_FOR_ECC() \ + RK_ECP_WRITE_REG(RK_ECC_RAM_CTL, RK_ECC_RAM_CTL_SEL_MASK | RK_ECC_RAM_CTL_ECC) + +#define RK_ECP_RAM_FOR_CPU() \ + RK_ECP_WRITE_REG(RK_ECC_RAM_CTL, RK_ECC_RAM_CTL_SEL_MASK | RK_ECC_RAM_CTL_CPU) + +#define RK_ECP_LOAD_DATA(dst, big_src) \ + do { \ + ecc_word_memset((dst), 0, RK_ECP_MAX_WORDS);\ + ecc_word_memcpy((dst), (big_src->data), (big_src->n_words)); \ + } while (0) + +#define RK_ECP_LOAD_DATA_EXT(dst, src, n_bytes) \ + do { \ + ecc_word_memset((void *)(dst), 0, RK_ECP_MAX_WORDS);\ + ecc_word_memcpy((void *)(dst), (void *)(src), (n_bytes) / 4); \ + } while (0) + +#define RK_ECC_BASE_OFFSET 0x0480 + +#define RK_ECC_CTL 0x03F0 +#define RK_ECC_CTL_RAND_K_SRC_SOFT _SBF(12, 0) +#define RK_ECC_CTL_RAND_K_SRC_HARD _SBF(12, 1) +#define RK_ECC_CTL_FUNC_SM2_CURVER _SBF(8, 0x0) +#define RK_ECC_CTL_FUNC_ECC_CURVER _SBF(8, 0x1) + +#define RK_ECC_CTL_FUNC_SEL_MUL _SBF(4, 0x0) +#define RK_ECC_CTL_FUNC_SEL_KG _SBF(4, 0x1) +#define RK_ECC_CTL_FUNC_SEL_SIGN _SBF(4, 0x2) +#define RK_ECC_CTL_FUNC_SEL_VERIFY _SBF(4, 0x3) +#define RK_ECC_CTL_FUNC_SEL_MUL_MOD _SBF(4, 0x4) +#define RK_ECC_CTL_FUNC_SEL_ADD_MOD _SBF(4, 0x5) +#define RK_ECC_CTL_FUNC_SEL_DOUBLE_POINT _SBF(4, 0x6) +#define RK_ECC_CTL_FUNC_SEL_ADD_POINT _SBF(4, 0x7) +#define RK_ECC_CTL_FUNC_SEL_KP _SBF(4, 0x8) +#define RK_ECC_CTL_FUNC_SEL_KP_KG _SBF(4, 0xa) +#define RK_ECC_CTL_REQ_ECC _SBF(0, 1) + +#define RK_ECC_INT_EN 0x03F4 +#define RK_ECC_INT_EN_DONE _BIT(0) + +#define RK_ECC_INT_ST 0x03F8 +#define RK_ECC_INT_ST_DONE _BIT(0) + +#define RK_ECC_ABN_ST 0x03FC +#define RK_ECC_ABN_ST_BAD_INV_OUT _BIT(8) +#define RK_ECC_ABN_ST_BAD_K_IN _BIT(7) +#define RK_ECC_ABN_ST_BAD_R_IN _BIT(6) +#define RK_ECC_ABN_ST_BAD_S_IN _BIT(5) +#define RK_ECC_ABN_ST_BAD_R_K_MID _BIT(4) +#define RK_ECC_ABN_ST_BAD_R_OUT _BIT(3) +#define RK_ECC_ABN_ST_BAD_S_OUT _BIT(2) +#define RK_ECC_ABN_ST_BAD_T_OUT _BIT(1) +#define RK_ECC_ABN_ST_BAD_POINT_OUT _BIT(0) + +#define RK_ECC_CURVE_WIDE 0x0400 +#define RK_ECC_CURVE_WIDE_192 192 +#define RK_ECC_CURVE_WIDE_224 224 +#define RK_ECC_CURVE_WIDE_256 256 + +#define RK_ECC_MAX_CURVE_WIDE 0x0404 + +#define RK_ECC_DATA_ENDIAN 0x0408 +#define RK_ECC_DATA_ENDIAN_LITTLE 0x0 +#define RK_ECC_DATA_ENDIAN_BIG 0x1 + +#define RK_ECC_RAM_CTL 0x0480 +#define RK_ECC_RAM_CTL_SEL_MASK _SBF(16, 3) +#define RK_ECC_RAM_CTL_CPU _SBF(0, 0) +#define RK_ECC_RAM_CTL_PKA _SBF(0, 1) +#define RK_ECC_RAM_CTL_ECC _SBF(0, 2) + +#define SM2_RAM_BASE ((ecc_base) + 0x1000) + +enum rk_ecp_group_id { + RK_ECP_DP_NONE = 0, + RK_ECP_DP_SECP192R1, /*!< 192-bits NIST curve */ + RK_ECP_DP_SECP224R1, /*!< 224-bits NIST curve */ + RK_ECP_DP_SECP256R1, /*!< 256-bits NIST curve */ + RK_ECP_DP_SM2P256V1, /*!< */ +}; + +#define RK_ECP_IS_BIGNUM_INVALID(b) (!b || !b->data || b->n_words < RK_ECP_MAX_WORDS) +#define RK_ECP_IS_POINT_INVALID(p) (RK_ECP_IS_BIGNUM_INVALID(p->x) && \ + RK_ECP_IS_BIGNUM_INVALID(p->y)) + +struct rk_ecp_group { + enum rk_ecp_group_id id; /*!< internal group identifier */ + const char *curve_name; + uint32_t wide; + const uint8_t *p; /*!< prime modulus of the base field */ + const uint8_t *a; /*!< 1. A in the equation, or 2. (A + 2) / 4 */ + const uint8_t *n; + const uint8_t *gx; + const uint8_t *gy; + size_t p_len; + size_t a_len; + size_t n_len; + size_t gx_len; + size_t gy_len; + enum bignum_endian endian; +}; + +struct rk_ecc_verify { + uint32_t e[RK_ECP_MAX_WORDS_ALL]; // 0x00 + uint32_t r_[RK_ECP_MAX_WORDS_ALL]; // 0x40 + uint32_t s_[RK_ECP_MAX_WORDS_ALL]; // 0x80 + uint32_t p_x[RK_ECP_MAX_WORDS_ALL]; // 0xC0 + uint32_t p_y[RK_ECP_MAX_WORDS_ALL]; // 0x100 + uint32_t A[RK_ECP_MAX_WORDS_ALL]; // 0x140 + uint32_t P[RK_ECP_MAX_WORDS_ALL]; // 0x180 + uint32_t N[RK_ECP_MAX_WORDS_ALL]; // 0x1C0 + uint32_t G_x[RK_ECP_MAX_WORDS_ALL]; // 0x200 + uint32_t G_y[RK_ECP_MAX_WORDS_ALL]; // 0x240 + uint32_t r[RK_ECP_MAX_WORDS_ALL]; // 0x280 + uint32_t v[RK_ECP_MAX_WORDS_ALL]; // 0x2C0 +}; + +int rkce_ecc_verify(int group_id, uint8_t *hash, uint32_t hash_len, + struct rkce_ecp_point *point_P, struct rkce_ecp_point *point_sign); + +void rkce_ecc_init(void __iomem *base); + +void rkce_ecc_deinit(void); + +uint32_t rkce_ecc_get_max_size(void); + +uint32_t rkce_ecc_get_curve_nbits(uint32_t group_id); + +uint32_t rkce_ecc_get_group_id(uint32_t asym_algo); + +#endif diff --git a/drivers/crypto/rockchip/rkce/rkce_ecdsasignature.asn1 b/drivers/crypto/rockchip/rkce/rkce_ecdsasignature.asn1 new file mode 100644 index 000000000000..ff0e14ebbdb0 --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_ecdsasignature.asn1 @@ -0,0 +1,8 @@ +-- SPDX-License-Identifier: BSD-3-Clause +-- +-- Copyright (c) 2024 Rockchip Electronics Co., Ltd. + +ECDSASignature ::= SEQUENCE { + x INTEGER ({ rkce_ecc_get_signature_r }), + y INTEGER ({ rkce_ecc_get_signature_s }) +} diff --git a/drivers/crypto/rockchip/rkce/rkce_error.h b/drivers/crypto/rockchip/rkce/rkce_error.h new file mode 100644 index 000000000000..3f24c3202a18 --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_error.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2024 Rockchip Electronics Co., Ltd. */ + +#ifndef __RKCE_ERROR_H__ +#define __RKCE_ERROR_H__ + +#include + +#define RKCE_SUCCESS 0 +#define RKCE_NOMEM ENOMEM +#define RKCE_FAULT EFAULT +#define RKCE_INVAL EINVAL +#define RKCE_TIMEOUT ETIMEDOUT + +#endif diff --git a/drivers/crypto/rockchip/rkce/rkce_hash.c b/drivers/crypto/rockchip/rkce/rkce_hash.c new file mode 100644 index 000000000000..f154e57860bd --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_hash.c @@ -0,0 +1,520 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Crypto acceleration support for Rockchip crypto engine + * + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + * Author: Lin Jinhan + * + */ +#define RKCE_MODULE_TAG "HASH" +#define RKCE_MODULE_OFFSET 8 + +#include +#include + +#include "rkce_core.h" +#include "rkce_dev.h" +#include "rkce_debug.h" +#include "rkce_monitor.h" +#include "rkce_hash.h" + +static inline struct rkce_ahash_ctx *hash_req2ctx(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + + return crypto_ahash_ctx(tfm); +} + +static inline struct rkce_ahash_ctx *asyn_req2ctx(void *async_req) +{ + struct ahash_request *req = container_of(async_req, struct ahash_request, base); + + return hash_req2ctx(req); +} + +static struct rkce_hash_td *rkce_hash_td_chain_alloc(uint32_t sgs_nents, + struct rkce_hash_td_buf *td_buf) +{ + int ret = -ENOMEM; + struct rkce_hash_td *td_head = NULL; + uint32_t i, td_nums; + + td_nums = DIV_ROUND_UP(sgs_nents, RKCE_TD_SG_NUM); + + rk_debug("sgs_nents = %u, td_nums = %u\n", sgs_nents, td_nums); + + td_head = rkce_cma_alloc(sizeof(*td_head) * td_nums); + if (!td_head) { + rk_debug("rkce_cma_alloc %u td failed.\n", td_nums); + goto error; + } + + for (i = 0; i < td_nums; i++) { + ret = rkce_init_hash_td(&td_head[i], td_buf); + if (ret) { + rk_debug("rkce_init_symm_td td[%u] failed.\n", i); + goto error; + } + + if (i < td_nums - 1) + td_head[i].next_task = rkce_cma_virt2phys(&td_head[i + 1]); + + } + + return td_head; +error: + rkce_cma_free(td_head); + + return NULL; +} + +static void rkce_hash_td_chain_free(struct rkce_hash_td *td_head) +{ + rkce_cma_free(td_head); +} + +static void rkce_set_hash_td_sg(struct rkce_hash_td *td_head, + uint32_t index, uint32_t len, + const dma_addr_t in) +{ + struct rkce_hash_td *cur_td = &td_head[index / RKCE_TD_SG_NUM]; + uint32_t sg_idx = index % 8; + + memset(&(cur_td->sg[sg_idx]), 0x00, sizeof(struct rkce_sg_info)); + +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + cur_td->sg[sg_idx].src_addr_h = in >> 32; +#endif + + cur_td->sg[sg_idx].src_addr_l = in & 0xffffffff; + cur_td->sg[sg_idx].src_size = len; +} + +static int rkce_hash_set_td_chain(struct scatterlist *sgs, + uint32_t data_len, struct rkce_hash_td *td_head, + struct rkce_hash_td_ctrl ctrl) +{ + uint32_t last_td_idx = 0; + uint32_t td_sg_idx = 0; + uint32_t src_len; + dma_addr_t src_dma; + int first_pkg, last_pkg, int_en; + + rk_debug("sgs = %p data_len = %u, td_head = %p", sgs, data_len, td_head); + + /* only set first td*/ + first_pkg = ctrl.first_pkg; + ctrl.first_pkg = 0; + last_pkg = ctrl.last_pkg; + ctrl.last_pkg = 0; + + /* only set last td */ + int_en = ctrl.int_en; + ctrl.int_en = 0; + + while (data_len) { + if (!sgs) { + rk_err("sgs is empty\n"); + goto error; + } + + src_len = sg_dma_len(sgs); + src_dma = sg_dma_address(sgs); + + rk_debug("data_len = %u, src_len = %u, src_dma = %pad, td_sg_idx = %u\n", + data_len, src_len, &src_dma, td_sg_idx); + + /* in case src length too long */ + if (src_len > data_len) + src_len = data_len; + + /* set one td and point to next */ + rkce_set_hash_td_sg(td_head, td_sg_idx, src_len, src_dma); + + if (td_sg_idx % RKCE_TD_SG_NUM == 0) + memcpy(&(td_head[td_sg_idx / RKCE_TD_SG_NUM].ctrl), &ctrl, sizeof(ctrl)); + + td_sg_idx++; + data_len -= src_len; + sgs = sg_next(sgs); + } + + /* for data_len = 0 */ + if (td_sg_idx == 0) { + memcpy(&td_head[0].ctrl, &ctrl, sizeof(ctrl)); + td_sg_idx = 1; + } + + td_head[0].ctrl.first_pkg = first_pkg; + + /* clear last td */ + last_td_idx = DIV_ROUND_UP(td_sg_idx, RKCE_TD_SG_NUM) - 1; + + td_head[last_td_idx].next_task = 0; + td_head[last_td_idx].ctrl.last_pkg = last_pkg; + td_head[last_td_idx].ctrl.int_en = int_en; + + return 0; +error: + return -1; +} + +int rkce_hash_request_callback(int result, uint32_t td_id, void *td_addr) +{ + struct rkce_hash_td *td = (struct rkce_hash_td *)td_addr; + struct rkce_hash_td_buf *td_buf = + container_of(rkce_cma_phys2virt(td->hash_ctx_addr), struct rkce_hash_td_buf, ctx); + struct rkce_ahash_ctx *ctx = (struct rkce_ahash_ctx *)td_buf->user_data; + struct rkce_ahash_request_ctx *rctx = ahash_request_ctx(ctx->req); + + rk_trace("enter.\n"); + + ctx->calculated += ctx->req->nbytes; + + if (result != -ETIMEDOUT) + rkce_monitor_del(rctx->td_head); + + if (result) { + rkce_dump_reginfo(ctx->algt->rk_dev->hardware); + rkce_soft_reset(ctx->algt->rk_dev->hardware, RKCE_RESET_HASH); + } else { + if (ctx->is_final && ctx->req->result) { + memcpy(ctx->req->result, td_buf->hash, ctx->algt->alg.hash.halg.digestsize); + rkce_dumphex("req->result", + ctx->req->result, ctx->algt->alg.hash.halg.digestsize); + } + } + + crypto_finalize_hash_request(ctx->algt->rk_dev->hash_engine, ctx->req, result); + + rk_trace("exit.\n"); + + return 0; +} + +static int rkce_hash_prepare(struct crypto_engine *engine, void *breq) +{ + struct ahash_request *req = container_of(breq, struct ahash_request, base); + struct rkce_ahash_request_ctx *rctx = ahash_request_ctx(req); + struct rkce_ahash_ctx *ctx = hash_req2ctx(req); + struct device *dev = ctx->algt->rk_dev->dev; + struct rkce_hash_td_ctrl ctrl; + uint32_t sgs_nents; + int ret; + + rk_trace("enter.\n"); + + memzero_explicit(rctx, sizeof(*rctx)); + + if (req->nbytes) { + sgs_nents = sg_nents_for_len(req->src, req->nbytes); + + if (!sg_dma_address(req->src)) { + ret = dma_map_sg(dev, req->src, sgs_nents, DMA_TO_DEVICE); + if (ret <= 0) + return -EINVAL; + + rctx->is_mapped = true; + } + } else { + sgs_nents = 1; + } + + rctx->td_head = rkce_hash_td_chain_alloc(sgs_nents, ctx->td_buf); + if (!rctx->td_head) { + ret = -ENOMEM; + rk_err("rkce_hash_td_chain_alloc failed ret = %d\n", ret); + goto exit; + } + + ctx->req = req; + ctx->td_buf->user_data = ctx; + rctx->sgs_nents = sgs_nents; + + memset(&ctrl, 0x00, sizeof(ctrl)); + + ctrl.td_type = RKCE_TD_TYPE_HASH; + ctrl.hw_pad_en = 1; + ctrl.first_pkg = !ctx->calculated; + ctrl.last_pkg = ctx->is_final; + ctrl.hash_algo = ctx->algt->algo; + ctrl.hmac_en = ctx->is_hmac; + ctrl.is_preemptible = 0; + ctrl.int_en = 1; + + /* translate scatter list to td chain */ + ret = rkce_hash_set_td_chain(req->src, req->nbytes, rctx->td_head, ctrl); + if (ret) + goto exit; + +exit: + rk_trace("exit.\n"); + + return ret; +} + +static int rkce_hash_unprepare(struct crypto_engine *engine, void *breq) +{ + struct ahash_request *req = container_of(breq, struct ahash_request, base); + struct rkce_ahash_request_ctx *rctx = ahash_request_ctx(req); + struct rkce_ahash_ctx *ctx = hash_req2ctx(req); + struct device *dev = ctx->algt->rk_dev->dev; + + rk_trace("enter.\n"); + + if (rctx->is_mapped) + dma_unmap_sg(dev, req->src, rctx->sgs_nents, DMA_TO_DEVICE); + + rkce_hash_td_chain_free(rctx->td_head); + + memzero_explicit(rctx, sizeof(*rctx)); + + rk_trace("exit.\n"); + + return 0; +} + +static int rkce_hash_handle_req(struct ahash_request *req, bool is_final) +{ + struct rkce_ahash_ctx *ctx = hash_req2ctx(req); + struct crypto_engine *engine = ctx->algt->rk_dev->hash_engine; + + rk_trace("enter.\n"); + + ctx->is_final = is_final; + + rk_debug("handle req %u bytes, %s\n", req->nbytes, is_final ? "final" : "update"); + + return crypto_transfer_hash_request_to_engine(engine, req); +} + +static int rkce_hash_run(struct crypto_engine *engine, void *breq) +{ + struct ahash_request *req = container_of(breq, struct ahash_request, base); + struct rkce_ahash_request_ctx *rctx = ahash_request_ctx(req); + struct rkce_ahash_ctx *ctx = hash_req2ctx(req); + + rk_trace("enter.\n"); + + rkce_monitor_add(rctx->td_head, rkce_hash_request_callback); + + return rkce_push_td(ctx->algt->rk_dev->hardware, rctx->td_head); +} + +static int rkce_ahash_hmac_setkey(struct crypto_ahash *tfm, const uint8_t *key, unsigned int keylen) +{ + unsigned int blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + struct rkce_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + struct rkce_algt *algt = ctx->algt; + int ret = 0; + + rk_trace("enter.\n"); + + if (algt->algo >= RKCE_HASH_ALGO_MAX) { + rk_err("hash algo %d invalid\n", algt->algo); + return -EINVAL; + } + + if (keylen > PAGE_SIZE) { + rk_err("keylen %u > %lu invalid\n", keylen, PAGE_SIZE); + return -EINVAL; + } + + if (keylen <= blocksize) { + memcpy(ctx->td_buf->key, key, keylen); + goto exit; + } + + /* calculate key hash as hmac key */ + ctx->user_key = rkce_cma_alloc(keylen); + if (!ctx->user_key) { + rk_err("rkce_cma_alloc key_td failed.\n"); + ret = -ENOMEM; + goto exit; + } + + memcpy(ctx->user_key, key, keylen); + + ctx->key_td = rkce_cma_alloc(sizeof(*(ctx->key_td))); + if (!ctx->key_td) { + rk_err("rkce_cma_alloc key_td failed.\n"); + ret = -ENOMEM; + goto exit; + } + + ret = rkce_init_hash_td(ctx->key_td, ctx->td_buf); + if (ret) { + rk_debug("rkce_init_symm_td key_td failed.\n"); + goto exit; + } + + /* key hash as hmac key */ + ctx->key_td->hash_addr = ctx->key_td->key_addr; + + rkce_set_hash_td_sg(ctx->key_td, 0, keylen, rkce_cma_virt2phys(ctx->user_key)); + + ctx->key_td->ctrl.td_type = RKCE_TD_TYPE_HASH; + ctx->key_td->ctrl.hw_pad_en = 1; + ctx->key_td->ctrl.first_pkg = 1; + ctx->key_td->ctrl.last_pkg = 1; + ctx->key_td->ctrl.hash_algo = ctx->algt->algo; + + ret = rkce_push_td(ctx->algt->rk_dev->hardware, ctx->key_td); +exit: + rk_trace("exit.\n"); + + return ret; +} + +static int rkce_ahash_init(struct ahash_request *req) +{ + struct rkce_ahash_ctx *ctx = hash_req2ctx(req); + + rk_trace("enter.\n"); + + ctx->req = NULL; + ctx->calculated = 0; + ctx->is_final = 0; + + return 0; +} + +static int rkce_ahash_update(struct ahash_request *req) +{ + rk_trace("enter.\n"); + + return rkce_hash_handle_req(req, false); +} + +static int rkce_ahash_final(struct ahash_request *req) +{ + rk_trace("enter.\n"); + + return rkce_hash_handle_req(req, true); +} + +static int rkce_ahash_finup(struct ahash_request *req) +{ + rk_trace("enter.\n"); + + return rkce_ahash_final(req); +} + +static int rkce_ahash_digest(struct ahash_request *req) +{ + rk_trace("enter.\n"); + + return rkce_ahash_final(req); +} + +static int rkce_ahash_import(struct ahash_request *req, const void *in) +{ + struct rkce_ahash_request_ctx *rctx = ahash_request_ctx(req); + struct rkce_ahash_ctx *ctx = hash_req2ctx(req); + + rk_trace("enter.\n"); + + if (!ctx->td_buf) + return -EFAULT; + + memcpy(rctx, in, sizeof(*rctx)); + + memcpy(ctx->td_buf->ctx, rctx->hw_context, RKCE_TD_HASH_CTX_SIZE); + + kfree(rctx->hw_context); + + rk_trace("exit.\n"); + + return 0; +} + +static int rkce_ahash_export(struct ahash_request *req, void *out) +{ + struct rkce_ahash_request_ctx *rctx = ahash_request_ctx(req); + struct rkce_ahash_ctx *ctx = hash_req2ctx(req); + + rk_trace("enter.\n"); + + if (!ctx->td_buf) + return -EFAULT; + + rctx->hw_context = kmalloc(RKCE_TD_HASH_CTX_SIZE, GFP_KERNEL); + + memcpy(rctx->hw_context, ctx->td_buf->ctx, RKCE_TD_HASH_CTX_SIZE); + + memcpy(out, rctx, sizeof(*rctx)); + + rk_trace("exit.\n"); + + return 0; +} + +static int rkce_cra_hash_init(struct crypto_tfm *tfm) +{ + struct rkce_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + struct rkce_algt *algt; + struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); + + rk_trace("enter.\n"); + + algt = container_of(alg, struct rkce_algt, alg.hash); + + rk_debug("alloc %s\n", algt->name); + + memzero_explicit(ctx, sizeof(*ctx)); + + ctx->algt = algt; + + ctx->enginectx.op.do_one_request = rkce_hash_run; + ctx->enginectx.op.prepare_request = rkce_hash_prepare; + ctx->enginectx.op.unprepare_request = rkce_hash_unprepare; + + ctx->td_buf = rkce_cma_alloc(sizeof(*(ctx->td_buf))); + if (!ctx->td_buf) { + rk_err("rkce_cma_alloc td_buf failed.\n"); + return -ENOMEM; + } + + ctx->is_hmac = algt->type == RKCE_ALGO_TYPE_HMAC; + + rkce_enable_clk(algt->rk_dev); + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct rkce_ahash_request_ctx)); + + rk_trace("exit.\n"); + + return 0; +} + +static void rkce_cra_hash_exit(struct crypto_tfm *tfm) +{ + struct rkce_ahash_ctx *ctx = crypto_tfm_ctx(tfm); + + rk_trace("enter.\n"); + + rkce_cma_free(ctx->td_buf); + rkce_cma_free(ctx->key_td); + rkce_cma_free(ctx->user_key); + + rkce_disable_clk(ctx->algt->rk_dev); + + memzero_explicit(ctx, sizeof(*ctx)); + + rk_trace("exit.\n"); +} + +struct rkce_algt rkce_ahash_md5 = RK_HASH_ALGO_INIT(MD5, md5); +struct rkce_algt rkce_ahash_sha1 = RK_HASH_ALGO_INIT(SHA1, sha1); +struct rkce_algt rkce_ahash_sha224 = RK_HASH_ALGO_INIT(SHA224, sha224); +struct rkce_algt rkce_ahash_sha256 = RK_HASH_ALGO_INIT(SHA256, sha256); +struct rkce_algt rkce_ahash_sha384 = RK_HASH_ALGO_INIT(SHA384, sha384); +struct rkce_algt rkce_ahash_sha512 = RK_HASH_ALGO_INIT(SHA512, sha512); +struct rkce_algt rkce_ahash_sm3 = RK_HASH_ALGO_INIT(SM3, sm3); + +struct rkce_algt rkce_hmac_md5 = RK_HMAC_ALGO_INIT(MD5, md5); +struct rkce_algt rkce_hmac_sha1 = RK_HMAC_ALGO_INIT(SHA1, sha1); +struct rkce_algt rkce_hmac_sha256 = RK_HMAC_ALGO_INIT(SHA256, sha256); +struct rkce_algt rkce_hmac_sha512 = RK_HMAC_ALGO_INIT(SHA512, sha512); +struct rkce_algt rkce_hmac_sm3 = RK_HMAC_ALGO_INIT(SM3, sm3); diff --git a/drivers/crypto/rockchip/rkce/rkce_hash.h b/drivers/crypto/rockchip/rkce/rkce_hash.h new file mode 100644 index 000000000000..74a642528fd8 --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_hash.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2024 Rockchip Electronics Co., Ltd. */ + +#ifndef __RKCE_HASH_H__ +#define __RKCE_HASH_H__ + +#include + +#include "rkce_dev.h" + +extern struct rkce_algt rkce_ahash_sha1; +extern struct rkce_algt rkce_ahash_sha224; +extern struct rkce_algt rkce_ahash_sha256; +extern struct rkce_algt rkce_ahash_sha384; +extern struct rkce_algt rkce_ahash_sha512; +extern struct rkce_algt rkce_ahash_md5; +extern struct rkce_algt rkce_ahash_sm3; + +extern struct rkce_algt rkce_hmac_md5; +extern struct rkce_algt rkce_hmac_sha1; +extern struct rkce_algt rkce_hmac_sha256; +extern struct rkce_algt rkce_hmac_sha512; +extern struct rkce_algt rkce_hmac_sm3; + +int rkce_hash_request_callback(int result, uint32_t td_id, void *td_addr); + +#endif diff --git a/drivers/crypto/rockchip/rkce/rkce_monitor.c b/drivers/crypto/rockchip/rkce/rkce_monitor.c new file mode 100644 index 000000000000..6ec87e80743c --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_monitor.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Crypto acceleration support for Rockchip crypto engine + * + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + * Author: Lin Jinhan + * + */ + +#define RKCE_MODULE_TAG "MONITOR" +#define RKCE_MODULE_OFFSET 18 + +#include +#include +#include +#include +#include +#include +#include + +#include "rkce_debug.h" +#include "rkce_monitor.h" + +#define TIMER_INTERVAL_MS 100 + +struct rkce_monitor_data { + void *td; + request_cb_func callback; + unsigned long timeout; + + struct list_head list; +}; + +static int g_active; +static struct timer_list g_timer; +static struct work_struct g_timeout_work; +static DEFINE_MUTEX(g_monitor_lock); +static LIST_HEAD(g_monitor_list); + +static void timeout_work_handler(struct work_struct *work) +{ + struct rkce_monitor_data *monitor_data = NULL; + struct list_head *pos = NULL, *q = NULL; + + mutex_lock(&g_monitor_lock); + + list_for_each_safe(pos, q, &g_monitor_list) { + monitor_data = list_entry(pos, struct rkce_monitor_data, list); + + if (monitor_data && + monitor_data->callback && + time_after(jiffies, monitor_data->timeout)) { + rk_debug("!!!!!!!!!!!!!!!!!! trigger timeout for (%p)\n", monitor_data->td); + monitor_data->callback(-ETIMEDOUT, 0, monitor_data->td); + list_del(&monitor_data->list); + } + } + + mutex_unlock(&g_monitor_lock); +} + +static void timer_callback(struct timer_list *t) +{ + mutex_lock(&g_monitor_lock); + + if (g_active) + mod_timer(t, jiffies + msecs_to_jiffies(TIMER_INTERVAL_MS)); + + mutex_unlock(&g_monitor_lock); + + schedule_work(&g_timeout_work); +} + +static void start_timer(void) +{ + rk_trace("enter.\n"); + + g_active = 1; + + if (g_active) { + rk_debug("reload timer.\n"); + + mod_timer(&g_timer, jiffies + msecs_to_jiffies(TIMER_INTERVAL_MS)); + } + + rk_trace("exit.\n"); +} + +static void stop_timer(void) +{ + rk_trace("enter.\n"); + + if (g_active) { + del_timer_sync(&g_timer); + g_active = 0; + rk_debug("Timer stopped\n"); + } + + rk_trace("exit.\n"); +} + +int rkce_monitor_add(void *td, request_cb_func callback) +{ + struct rkce_monitor_data *monitor_data; + + rk_trace("enter.\n"); + + if (!td) + return -EINVAL; + + monitor_data = kzalloc(sizeof(*monitor_data), GFP_KERNEL); + if (!monitor_data) + return -ENOMEM; + + monitor_data->td = td; + monitor_data->callback = callback; + monitor_data->timeout = jiffies + 3 * HZ; + + rk_debug("add %p to monitor, timeout = %u.\n", + td, jiffies_to_msecs(monitor_data->timeout)); + + mutex_lock(&g_monitor_lock); + + list_add(&monitor_data->list, &g_monitor_list); + + start_timer(); + + mutex_unlock(&g_monitor_lock); + + rk_trace("exit.\n"); + + return 0; +} + +void rkce_monitor_del(void *td) +{ + struct rkce_monitor_data *monitor_data = NULL; + struct list_head *pos = NULL, *q = NULL; + + rk_trace("enter.\n"); + + mutex_lock(&g_monitor_lock); + + list_for_each_safe(pos, q, &g_monitor_list) { + monitor_data = list_entry(pos, struct rkce_monitor_data, list); + if (monitor_data->td == td) { + list_del(&monitor_data->list); + kfree(monitor_data); + break; + } + } + + if (list_empty(&g_monitor_list)) + stop_timer(); + + mutex_unlock(&g_monitor_lock); + + rk_trace("exit.\n"); +} + +int rkce_monitor_init(void) +{ + rk_debug("Initializing timer module\n"); + + INIT_WORK(&g_timeout_work, timeout_work_handler); + + timer_setup(&g_timer, timer_callback, 0); + + return 0; +} + +void rkce_monitor_deinit(void) +{ + struct rkce_monitor_data *monitor_data = NULL; + struct list_head *pos = NULL, *q = NULL; + + rk_debug("Exiting timer module\n"); + + mutex_lock(&g_monitor_lock); + + list_for_each_safe(pos, q, &g_monitor_list) { + monitor_data = list_entry(pos, struct rkce_monitor_data, list); + list_del(&monitor_data->list); + kfree(monitor_data); + } + + stop_timer(); + + mutex_unlock(&g_monitor_lock); +} diff --git a/drivers/crypto/rockchip/rkce/rkce_monitor.h b/drivers/crypto/rockchip/rkce/rkce_monitor.h new file mode 100644 index 000000000000..5a9755e74b64 --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_monitor.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2024 Rockchip Electronics Co., Ltd. */ + +#ifndef __RKCE_MONITOR_H__ +#define __RKCE_MONITOR_H__ + +#include + +#include "rkce_core.h" + +int rkce_monitor_add(void *td, request_cb_func callback); + +void rkce_monitor_del(void *td); + +int rkce_monitor_init(void); + +void rkce_monitor_deinit(void); + +#endif diff --git a/drivers/crypto/rockchip/rkce/rkce_pka.c b/drivers/crypto/rockchip/rkce/rkce_pka.c new file mode 100644 index 000000000000..86b3d798b12b --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_pka.c @@ -0,0 +1,691 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + */ + +#define RKCE_MODULE_TAG "PKA" +#define RKCE_MODULE_OFFSET 14 + +#include + +#include "rkce_core.h" +#include "rkce_debug.h" +#include "rkce_reg.h" +#include "rkce_pka.h" + +#define PKA_WORDS2BITS(words) ((words) * 32) +#define PKA_BITS2WORDS(bits) (((bits) + 31) / 32) + +#define PKA_WORDS2BYTES(words) ((words) * 4) +#define PKA_BYTES2BITS(bytes) ((bytes) * 8) + +/* PKA length set */ +enum { + PKA_EXACT_LEN_ID = 0, + PKA_CALC_LEN_ID, + PKA_USED_LEN_MAX, +}; + +/********************* Private MACRO Definition ******************************/ +#define PKA_POLL_PERIOD_US 1000 +#define PKA_POLL_TIMEOUT_US 50000 + +/* for private key EXP_MOD operation */ +#define PKA_MAX_POLL_PERIOD_US 20000 +#define PKA_MAX_POLL_TIMEOUT_US 2000000 + +#define PKA_MAX_CALC_BITS 4096 +#define PKA_MAX_CALC_WORDS PKA_BITS2WORDS(PKA_MAX_CALC_BITS) + +/* PKA N_NP_T0_T1 register default (reset) value: N=0, NP=1, T0=30, T1=31 */ +#define PKA_N 0UL +#define PKA_NP 1UL +#define PKA_T0 30UL /*tmp reg */ +#define PKA_T1 31UL /*tmp reg */ +#define PKA_TMP_REG_CNT 2 + +#define PKA_N_NP_T0_T1_REG_DEFAULT \ + (PKA_N << RKCE_N_NP_T0_T1_ADDR_REG_N_SHIFT | \ + PKA_NP << RKCE_N_NP_T0_T1_ADDR_REG_NP_SHIFT | \ + PKA_T0 << RKCE_N_NP_T0_T1_ADDR_REG_T0_SHIFT | \ + PKA_T1 << RKCE_N_NP_T0_T1_ADDR_REG_T1_SHIFT) + +#define RES_DISCARD 0x3F + +/* values for defining, that PKA entry is not in use */ +#define PKA_ADDR_NOT_USED 0xFFC + +/* Machine Opcodes definitions (according to HW CRS ) */ + +enum pka_opcode { + PKA_OPCODE_ADD = 0x04, + PKA_OPCODE_SUB, + PKA_OPCODE_MOD_ADD, + PKA_OPCODE_MOD_SUB, + PKA_OPCODE_AND, + PKA_OPCODE_OR, + PKA_OPCODE_XOR, + PKA_OPCODE_SHR0 = 0x0C, + PKA_OPCODE_SHR1, + PKA_OPCODE_SHL0, + PKA_OPCODE_SHL1, + PKA_OPCODE_LMUL, + PKA_OPCODE_MOD_MUL, + PKA_OPCODE_MOD_MUL_NR, + PKA_OPCODE_MOD_EXP, + PKA_OPCODE_DIV, + PKA_OPCODE_MOD_INV, + PKA_OPCODE_MOD_DIV, + PKA_OPCODE_HMUL, + PKA_OPCODE_TERMINATE, +}; + +#define PKA_CLK_ENABLE() +#define PKA_CLK_DISABLE() + +#define PKA_READ(offset) readl_relaxed((pka_base) + (offset)) +#define PKA_WRITE(val, offset) writel_relaxed((val), (pka_base) + (offset)) + +#define PKA_BIGNUM_WORDS(x) (rkce_bn_get_size(x) / sizeof(u32)) + +#define PKA_RAM_FOR_PKA() PKA_WRITE((RKCE_PKA_RAM_CTL_PKA_RDY << RKCE_WRITE_MASK_SHIFT) | \ + RKCE_PKA_RAM_CTL_PKA_RDY, RKCE_PKA_RAM_CTL_OFFSET) + +#define PKA_RAM_FOR_CPU() do { \ + PKA_WRITE((RKCE_PKA_RAM_CTL_PKA_RDY << RKCE_WRITE_MASK_SHIFT), RKCE_PKA_RAM_CTL_OFFSET); \ + while ((PKA_READ(RKCE_PKA_RAM_ST_OFFSET) & 0x01) != RKCE_PKA_RAM_CTL_PKA_RDY) \ + cpu_relax(); \ +} while (0) + +#define PKA_GET_SRAM_ADDR(addr) ((void *)(pka_base + RKCE_SRAM_ADDR_OFFSET + (addr))) + +/************************************************************************* + * Macros for calling PKA operations (names according to operation issue * + *************************************************************************/ + +/*--------------------------------------*/ +/* 1. ADD - SUBTRACT operations */ +/*--------------------------------------*/ +/* Add: res = op_a + op_b */ +#define RK_PKA_ADD(op_a, op_b, res) pka_exec_op(PKA_OPCODE_ADD, PKA_CALC_LEN_ID, \ + 0, (op_a), 0, (op_b), 0, (res), 0) + +/* Clr: res = op_a & 0 - clears the operand A. */ +#define RK_PKA_CLR(op_a) pka_exec_op(PKA_OPCODE_AND, PKA_CALC_LEN_ID, \ + 0, (op_a), 1, 0x00, 0, (op_a), 0) + +/* Copy: OpDest = OpSrc || 0 */ +#define RK_PKA_COPY(op_dest, op_src) pka_exec_op(PKA_OPCODE_OR, PKA_CALC_LEN_ID, \ + 0, (op_src), 1, 0x00, 0, (op_dest), 0) + +/* Set0: res = op_a || 1 : set bit0 = 1, other bits are not changed */ +#define RK_PKA_SET_0(op_a, res) pka_exec_op(PKA_OPCODE_OR, PKA_CALC_LEN_ID, \ + 0, (op_a), 1, 0x01, 0, (res), 0) + +/*----------------------------------------------*/ +/* 3. SHIFT operations */ +/*----------------------------------------------*/ +/* SHL0: res = op_a << (S+1) : + * shifts left operand A by S+1 bits, insert 0 to right most bits + */ +#define RK_PKA_SHL0(op_a, S, res) pka_exec_op(PKA_OPCODE_SHL0, PKA_CALC_LEN_ID, \ + 0, (op_a), 0, (S), 0, (res), 0) + +/* SHL1: res = op_a << (S+1) : + * shifts left operand A by S+1 bits, insert 1 to right most bits + */ +#define RK_PKA_SHL1(op_a, S, res) pka_exec_op(PKA_OPCODE_SHL1, PKA_CALC_LEN_ID, \ + 0, (op_a), 0, (S), 0, (res), 0) + +/*--------------------------------------------------------------*/ +/* 2. Multiplication and other operations */ +/* Note: See notes to RK_PKAExecOperation */ +/*--------------------------------------------------------------*/ + +/* ModExp: res = op_a ** op_b mod N - modular exponentiation */ +#define RK_PKA_MOD_EXP(op_a, op_b, res) \ + pka_exec_op(PKA_OPCODE_MOD_EXP, PKA_EXACT_LEN_ID, 0, (op_a), \ + 0, (op_b), 0, (res), 0) + +/* Divide: res = op_a / op_b , op_a = op_a mod op_b - division, */ +#define RK_PKA_DIV(op_a, op_b, res) pka_exec_op(PKA_OPCODE_DIV, PKA_CALC_LEN_ID, \ + 0, (op_a), 0, (op_b), 0, (res), 0) + +/* Terminate - special operation, which allows HOST access */ +/* to PKA data memory registers after end of PKA operations */ +#define RK_PKA_TERMINATE() pka_exec_op(PKA_OPCODE_TERMINATE, 0, 0, 0, 0, 0, 0, 0, 0) + +/********************* Private Variable Definition ***************************/ +static void __iomem *pka_base; + +static void pka_word_memcpy(u32 *dst, u32 *src, u32 size) +{ + u32 i; + + for (i = 0; i < size; i++, dst++) + writel_relaxed(src[i], (void *)dst); +} + +static void pka_word_memset(u32 *buff, u32 val, u32 size) +{ + u32 i; + + for (i = 0; i < size; i++, buff++) + writel_relaxed(val, (void *)buff); +} + +static int pka_wait_pipe_rdy(void) +{ + u32 reg_val = 0; + + return readx_poll_timeout(PKA_READ, RKCE_PKA_PIPE_RDY_OFFSET, reg_val, + reg_val, PKA_POLL_PERIOD_US, PKA_POLL_TIMEOUT_US); +} + +static int pka_wait_done(void) +{ + u32 reg_val = 0; + + return readx_poll_timeout(PKA_READ, RKCE_PKA_DONE_OFFSET, reg_val, + reg_val, PKA_POLL_PERIOD_US, PKA_POLL_TIMEOUT_US); +} + +static int pka_max_wait_done(void) +{ + u32 reg_val = 0; + + return readx_poll_timeout(PKA_READ, RKCE_PKA_DONE_OFFSET, reg_val, + reg_val, PKA_MAX_POLL_PERIOD_US, PKA_MAX_POLL_TIMEOUT_US); +} + +static u32 pka_check_status(u32 mask) +{ + u32 status; + + pka_wait_done(); + status = PKA_READ(RKCE_PKA_STATUS_OFFSET); + status = status & mask; + + return !!status; +} +static void pka_set_len_words(u32 words, u32 index) +{ + PKA_WRITE(PKA_WORDS2BITS(words), RKCE_PKA_L0_OFFSET + index * sizeof(u32)); +} + +static u32 pka_get_len_words(u32 index) +{ + pka_wait_done(); + return PKA_BITS2WORDS(PKA_READ(RKCE_PKA_L0_OFFSET + (index) * sizeof(u32))); +} + +static void pka_set_map_addr(u32 addr, u32 index) +{ + PKA_WRITE(addr, RKCE_PKA_MEM_MAP0_OFFSET + sizeof(u32) * index); +} + +static u32 pka_get_map_addr(u32 index) +{ + pka_wait_done(); + return PKA_READ(RKCE_PKA_MEM_MAP0_OFFSET + sizeof(u32) * (index)); +} + +static u32 pka_make_full_opcode(u32 opcode, u32 len_id, + u32 is_a_immed, u32 op_a, + u32 is_b_immed, u32 op_b, + u32 res_discard, u32 res, + u32 tag) +{ + u32 full_opcode; + + full_opcode = ((opcode & 31) << RKCE_PKA_OPCODE_OPCODE_SHIFT | + (len_id & 7) << RKCE_PKA_OPCODE_LEN_SHIFT | + (is_a_immed & 1) << RKCE_PKA_OPCODE_A_IMMED_SHIFT | + (op_a & 31) << RKCE_PKA_OPCODE_REG_A_SHIFT | + (is_b_immed & 1) << RKCE_PKA_OPCODE_B_IMMED_SHIFT | + (op_b & 31) << RKCE_PKA_OPCODE_REG_B_SHIFT | + (res_discard & 1) << RKCE_PKA_OPCODE_R_DIS_SHIFT | + (res & 31) << RKCE_PKA_OPCODE_REG_R_SHIFT | + (tag & 31) << RKCE_PKA_OPCODE_TAG_SHIFT); + + return full_opcode; +} + +static void pka_load_data(u32 addr, u32 *data, u32 size_words) +{ + pka_wait_done(); + + PKA_RAM_FOR_CPU(); + pka_word_memcpy(PKA_GET_SRAM_ADDR(addr), data, size_words); + PKA_RAM_FOR_PKA(); +} + +static void pka_clr_mem(u32 addr, u32 size_words) +{ + pka_wait_done(); + + PKA_RAM_FOR_CPU(); + pka_word_memset(PKA_GET_SRAM_ADDR(addr), 0x00, size_words); + PKA_RAM_FOR_PKA(); +} + +static void pka_read_data(u32 addr, u32 *data, u32 size_words) +{ + pka_wait_done(); + + PKA_RAM_FOR_CPU(); + pka_word_memcpy(data, PKA_GET_SRAM_ADDR(addr), size_words); + PKA_RAM_FOR_PKA(); +} + +static int pka_exec_op(enum pka_opcode opcode, u8 len_id, + u8 is_a_immed, u8 op_a, u8 is_b_immed, u8 op_b, + u8 res_discard, u8 res, u8 tag) +{ + int ret = 0; + u32 full_opcode; + + if (res == RES_DISCARD) { + res_discard = 1; + res = 0; + } + + full_opcode = pka_make_full_opcode(opcode, len_id, + is_a_immed, op_a, + is_b_immed, op_b, + res_discard, res, tag); + + /* write full opcode into PKA CRYPTO_OPCODE register */ + PKA_WRITE(full_opcode, RKCE_PKA_OPCODE_OFFSET); + + /*************************************************/ + /* finishing operations for different cases */ + /*************************************************/ + switch (opcode) { + case PKA_OPCODE_DIV: + /* for Div operation check, that op_b != 0*/ + if (pka_check_status(RKCE_PKA_STATUS_DIV_BY_ZERO_MASK)) + goto end; + break; + case PKA_OPCODE_TERMINATE: + /* wait for PKA done bit */ + ret = pka_wait_done(); + break; + default: + /* wait for PKA pipe ready bit */ + ret = pka_wait_pipe_rdy(); + } +end: + return ret; +} + +static int pk_int_len_tbl(u32 exact_size_words, u32 calc_size_words) +{ + u32 i; + + /* clear all length reg */ + for (i = 0; i < RKCE_LEN_REG_NUM; i++) + pka_set_len_words(0, i); + + /* Case of default settings */ + /* write exact size into first table entry */ + pka_set_len_words(exact_size_words, PKA_EXACT_LEN_ID); + + /* write size with extra word into tab[1] = tab[0] + 32 */ + pka_set_len_words(calc_size_words, PKA_CALC_LEN_ID); + + return 0; +} + +static int pka_int_map_tbl(u32 *regs_cnt, u32 max_size_words) +{ + u32 i; + u32 cur_addr = 0; + u32 max_size_bytes, default_regs_cnt; + + max_size_bytes = PKA_WORDS2BYTES(max_size_words); + default_regs_cnt = + min_t(u32, RKCE_MAP_REG_NUM, RKCE_SRAM_SIZE / max_size_bytes); + + /* clear all address */ + for (i = 0; i < RKCE_MAP_REG_NUM; i++) + pka_set_map_addr(PKA_ADDR_NOT_USED, i); + + /* set addresses of N,NP and user requested registers (excluding 2 temp registers T0,T1) */ + for (i = 0; i < default_regs_cnt - PKA_TMP_REG_CNT; i++, cur_addr += max_size_bytes) + pka_set_map_addr(cur_addr, i); + + /* set addresses of 2 temp registers: T0=30, T1=31 */ + pka_set_map_addr(cur_addr, PKA_T0); + cur_addr += max_size_bytes; + pka_set_map_addr(cur_addr, PKA_T1); + + /* output maximal count of allowed registers */ + *regs_cnt = default_regs_cnt; + + /* set default virtual addresses of N,NP,T0,T1 registers into N_NP_T0_T1_Reg */ + PKA_WRITE((u32)PKA_N_NP_T0_T1_REG_DEFAULT, RKCE_N_NP_T0_T1_ADDR_OFFSET); + + return 0; +} + +static int pka_clear_regs_block(u8 first_reg, u8 regs_cnt) +{ + u32 i; + u32 size_words; + int cnt_tmps = 0; + u32 user_reg_num = RKCE_MAP_REG_NUM - PKA_TMP_REG_CNT; + + /* calculate size_words of register in words */ + size_words = pka_get_len_words(PKA_CALC_LEN_ID); + + if (first_reg + regs_cnt > user_reg_num) { + cnt_tmps = min_t(u8, (regs_cnt + first_reg - user_reg_num), PKA_TMP_REG_CNT); + regs_cnt = user_reg_num; + } else { + cnt_tmps = PKA_TMP_REG_CNT; + } + + /* clear ordinary registers */ + for (i = first_reg; i < regs_cnt; i++) + RK_PKA_CLR(i); + + pka_wait_done(); + + /* clear PKA temp registers (without PKA operations) */ + if (cnt_tmps > 0) { + pka_clr_mem(pka_get_map_addr(PKA_T0), size_words); + if (cnt_tmps > 1) + pka_clr_mem(pka_get_map_addr(PKA_T1), size_words); + + } + + return 0; +} + +static int pka_init(u32 exact_size_words) +{ + int ret; + u32 regs_cnt = 0; + u32 calc_size_words = exact_size_words + 1; + + PKA_CLK_ENABLE(); + PKA_RAM_FOR_PKA(); + + if (exact_size_words > PKA_MAX_CALC_WORDS) + return -1; + + ret = pk_int_len_tbl(exact_size_words, calc_size_words); + if (ret) + goto exit; + + ret = pka_int_map_tbl(®s_cnt, calc_size_words); + if (ret) + goto exit; + + /* clean PKA data memory */ + pka_clear_regs_block(0, regs_cnt - PKA_TMP_REG_CNT); + + /* clean temp PKA registers 30,31 */ + pka_clr_mem(pka_get_map_addr(PKA_T0), calc_size_words); + pka_clr_mem(pka_get_map_addr(PKA_T1), calc_size_words); + +exit: + return ret; +} + +static void pka_finish(void) +{ + RK_PKA_TERMINATE(); + PKA_RAM_FOR_CPU(); + PKA_CLK_DISABLE(); +} + +static void pka_copy_bn_into_reg(u8 dst_reg, struct rkce_bignum *bn) +{ + u32 cur_addr; + u32 size_words, bn_words; + + RK_PKA_TERMINATE(); + + bn_words = PKA_BIGNUM_WORDS(bn); + size_words = pka_get_len_words(PKA_CALC_LEN_ID); + cur_addr = pka_get_map_addr(dst_reg); + + pka_load_data(cur_addr, bn->data, bn_words); + cur_addr += PKA_WORDS2BYTES(bn_words); + + pka_clr_mem(cur_addr, size_words - bn_words); +} + +static int pka_copy_bn_from_reg(struct rkce_bignum *bn, u32 size_words, + u8 src_reg, bool is_max_poll) +{ + int ret; + + PKA_WRITE(0, RKCE_PKA_OPCODE_OFFSET); + + ret = is_max_poll ? pka_max_wait_done() : pka_wait_done(); + if (ret) + return ret; + + pka_read_data(pka_get_map_addr(src_reg), bn->data, size_words); + + return 0; +} + +/*********** pka_div_bignum function **********************/ +/** + * @brief The function divides long number A*(2^S) by B: + * res = A*(2^S) / B, remainder A = A*(2^S) % B. + * where: A,B - are numbers of size, which is not grate than, + * maximal operands size, + * and B > 2^S; + * S - exponent of binary factor of A. + * ^ - exponentiation operator. + * + * The function algorithm: + * + * 1. Let nWords = S/32; nBits = S % 32; + * 2. Set res = 0, r_t1 = op_a; + * 3. for(i=0; i<=nWords; i++) do: + * 3.1. if(i < nWords ) + * s1 = 32; + * else + * s1 = nBits; + * 3.2. r_t1 = r_t1 << s1; + * 3.3. call PKA_div for calculating the quotient and remainder: + * r_t2 = floor(r_t1/op_b) //quotient; + * r_t1 = r_t1 % op_b //remainder (is in r_t1 register); + * 3.4. res = (res << s1) + r_t2; + * end do; + * 4. Exit. + * + * Assuming: + * - 5 PKA registers are used: op_a, op_b, res, r_t1, r_t2. + * - The registers sizes and mapping tables are set on + * default mode according to operands size. + * - The PKA clocks are initialized. + * NOTE ! Operand op_a shall be overwritten by remainder. + * + * @param[in] len_id - ID of operation size (modSize+32). + * @param[in] op_a - Operand A: virtual register pointer of A. + * @param[in] S - exponent of binary factor of A. + * @param[in] op_b - Operand B: virtual register pointer of B. + * @param[in] res - Virtual register pointer for result quotient. + * @param[in] r_t1 - Virtual pointer to remainder. + * @param[in] r_t2 - Virtual pointer of temp register. + * + * @return int - On success 0 is returned: + * + */ +static int pka_div_bignum(u8 op_a, u32 s, u8 op_b, u8 res, u8 r_t1, u8 r_t2) +{ + u8 s1; + u32 i; + u32 n_bits, n_words; + + /* calculate shifting parameters (words and bits ) */ + n_words = ((u32)s + 31) / 32; + n_bits = (u32)s % 32; + + /* copy operand op_a (including extra word) into temp reg r_t1 */ + RK_PKA_COPY(r_t1, op_a); + + /* set res = 0 (including extra word) */ + RK_PKA_CLR(res); + + /*----------------------------------------------------*/ + /* Step 1. Shifting and dividing loop */ + /*----------------------------------------------------*/ + for (i = 0; i < n_words; i++) { + /* 3.1 set shift value s1 */ + s1 = i > 0 ? 32 : n_bits; + + /* 3.2. shift: r_t1 = r_t1 * 2**s1 (in code (s1-1), + * because PKA performs s+1 shifts) + */ + if (s1 > 0) + RK_PKA_SHL0(r_t1 /*op_a*/, (s1 - 1) /*s*/, r_t1 /*res*/); + + /* 3.3. perform PKA_OPCODE_MOD_DIV for calculating a quotient + * r_t2 = floor(r_t1 / N) + * and remainder r_t1 = r_t1 % op_b + */ + RK_PKA_DIV(r_t1 /*op_a*/, op_b /*B*/, r_t2 /*res*/); + + /* 3.4. res = res * 2**s1 + res; */ + if (s1 > 0) + RK_PKA_SHL0(res /*op_a*/, (s1 - 1) /*s*/, res /*res*/); + + RK_PKA_ADD(res /*op_a*/, r_t2 /*op_b*/, res /*res*/); + } + + pka_wait_done(); + + return 0; +} /* END OF pka_div_bignum */ + +static u32 pka_calc_and_init_np(struct rkce_bignum *bn, u8 r_t0, u8 r_t1, u8 r_t2) +{ + int ret; + u32 i; + u32 s; + u32 mod_size_bits; + u32 num_bits, num_words; + + /* Set s = 132 */ + s = 132; + + mod_size_bits = PKA_BYTES2BITS(rkce_bn_get_size(bn)); + + rk_debug("size_bits = %u", mod_size_bits); + + /* copy modulus N into r0 register */ + pka_copy_bn_into_reg(PKA_N, bn); + + /*--------------------------------------------------------------*/ + /* Step 1,2. Set registers: Set op_a = 2^(sizeN+32) */ + /* Registers using: 0 - N (is set in register 0, */ + /* 1 - NP, temp regs: r_t0 (A), r_t1, r_t2. */ + /* len_id: 0 - exact size, 1 - exact+32 bit */ + /*--------------------------------------------------------------*/ + + /* set register r_t0 = 0 */ + RK_PKA_CLR(r_t0); + + /* calculate bit position of said bit in the word */ + num_bits = mod_size_bits % 32; + num_words = mod_size_bits / 32; + + rk_debug("num_bits = %u, num_words = %u, size_bits = %u", + num_bits, num_words, mod_size_bits); + + /* set 1 into register r_t0 */ + RK_PKA_SET_0(r_t0 /*op_a*/, r_t0 /*res*/); + + /* shift 1 to num_bits+31 position */ + if (num_bits > 0) + RK_PKA_SHL0(r_t0 /*op_a*/, num_bits - 1 /*s*/, r_t0 /*res*/); + + /* shift to word position */ + for (i = 0; i < num_words; i++) + RK_PKA_SHL0(r_t0 /*op_a*/, 31 /*s*/, r_t0 /*res*/); + + /*--------------------------------------------------------------*/ + /* Step 3. Dividing: PKA_NP = (r_t0 * 2**s) / N */ + /*--------------------------------------------------------------*/ + ret = pka_div_bignum(r_t0, s, PKA_N, PKA_NP, r_t1, r_t2); + + return ret; +} /* END OF pka_calc_and_init_np */ + +/********************* Public Function Definition ****************************/ + +void rkce_pka_set_crypto_base(void __iomem *base) +{ + pka_base = base; +} + +/** + * @brief calculate exp mod. out = in ^ e mod n + * @param in: the point of input data bignum. + * @param e: the point of exponent bignum. + * @param n: the point of modulus bignum. + * @param out: the point of outputs bignum. + * @param pTmp: the point of tmpdata bignum. + * @return 0 for success + */ +int rkce_pka_expt_mod(struct rkce_bignum *in, + struct rkce_bignum *e, + struct rkce_bignum *n, + struct rkce_bignum *out) +{ + int ret = -1; + u32 max_word_size; + bool is_max_poll; + u8 r_in = 2, r_e = 3, r_out = 4; + u8 r_t0 = 2, r_t1 = 3, r_t2 = 4; + + if (!in || !e || !n || !out || PKA_BIGNUM_WORDS(n) == 0) + return -1; + + max_word_size = PKA_BIGNUM_WORDS(n); + + ret = pka_init(max_word_size); + if (ret) { + rk_debug("pka_init error\n"); + goto exit; + } + + /* calculate NP by initialization PKA for modular operations */ + ret = pka_calc_and_init_np(n, r_t0, r_t1, r_t2); + if (ret) { + rk_debug("pka_calc_and_init_np error\n"); + goto exit; + } + + pka_clear_regs_block(r_in, 3); + + pka_copy_bn_into_reg(r_in, in); + pka_copy_bn_into_reg(r_e, e); + pka_copy_bn_into_reg(PKA_N, n); + + ret = RK_PKA_MOD_EXP(r_in, r_e, r_out); + if (ret) { + rk_debug("RK_PKA_MOD_EXP error\n"); + goto exit; + } + + /* e is usually 0x10001 in public key EXP_MOD operation */ + is_max_poll = rkce_bn_highest_bit(e) * 2 > rkce_bn_highest_bit(n) ? true : false; + + ret = pka_copy_bn_from_reg(out, max_word_size, r_out, is_max_poll); + +exit: + pka_clear_regs_block(0, 5); + pka_clear_regs_block(30, 2); + pka_finish(); + + return ret; +} diff --git a/drivers/crypto/rockchip/rkce/rkce_pka.h b/drivers/crypto/rockchip/rkce/rkce_pka.h new file mode 100644 index 000000000000..ab7c02549f7f --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_pka.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2022 Rockchip Electronics Co., Ltd. */ + +#ifndef __RKCE_PKA_H__ +#define __RKCE_PKA_H__ + +#include "rkce_bignum.h" + +void rkce_pka_set_crypto_base(void __iomem *base); + +int rkce_pka_expt_mod(struct rkce_bignum *in, + struct rkce_bignum *e, + struct rkce_bignum *n, + struct rkce_bignum *out); + +#endif diff --git a/drivers/crypto/rockchip/rkce/rkce_reg.h b/drivers/crypto/rockchip/rkce/rkce_reg.h new file mode 100644 index 000000000000..c7943c4e7b45 --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_reg.h @@ -0,0 +1,533 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2024 Rockchip Electronics Co., Ltd. */ + +#ifndef __RKCE_REG_H +#define __RKCE_REG_H + +#include + +/****************************************************************************************/ +/* */ +/* Module Structure Section */ +/* */ +/****************************************************************************************/ +/* RKCE Register Structure Define */ +struct RKCE_REG { + uint32_t CLK_CTL; /* Address Offset: 0x0000 */ + uint32_t RST_CTL; /* Address Offset: 0x0004 */ + uint32_t RESERVED0008[126]; /* Address Offset: 0x0008 */ + uint32_t TD_ADDR; /* Address Offset: 0x0200 */ + uint32_t TD_LOAD_CTRL; /* Address Offset: 0x0204 */ + uint32_t FIFO_ST; /* Address Offset: 0x0208 */ + uint32_t RESERVED020C; /* Address Offset: 0x020C */ + uint32_t SYMM_INT_EN; /* Address Offset: 0x0210 */ + uint32_t SYMM_INT_ST; /* Address Offset: 0x0214 */ + uint32_t SYMM_TD_ID; /* Address Offset: 0x0218 */ + uint32_t SYMM_TD_ST; /* Address Offset: 0x021C */ + uint32_t SYMM_ST_DBG; /* Address Offset: 0x0220 */ + uint32_t SYMM_CONTEXT_SIZE; /* Address Offset: 0x0224 */ + uint32_t SYMM_TD_ADDR_DBG; /* Address Offset: 0x0228 */ + uint32_t SYMM_TD_GRANT_DBG; /* Address Offset: 0x022C */ + uint32_t HASH_INT_EN; /* Address Offset: 0x0230 */ + uint32_t HASH_INT_ST; /* Address Offset: 0x0234 */ + uint32_t HASH_TD_ID; /* Address Offset: 0x0238 */ + uint32_t HASH_TD_ST; /* Address Offset: 0x023C */ + uint32_t HASH_ST_DBG; /* Address Offset: 0x0240 */ + uint32_t HASH_CONTEXT_SIZE; /* Address Offset: 0x0244 */ + uint32_t HASH_TD_ADDR_DBG; /* Address Offset: 0x0248 */ + uint32_t HASH_TD_GRANT_DBG; /* Address Offset: 0x024C */ + uint32_t SYMM_TD_POP_ADDR; /* Address Offset: 0x0250 */ + uint32_t HASH_TD_POP_ADDR; /* Address Offset: 0x0254 */ + uint32_t TD_POP_CTRL; /* Address Offset: 0x0258 */ + uint32_t RESERVED025C[5]; /* Address Offset: 0x025C */ + uint32_t KL_TO_CE_PADDR; /* Address Offset: 0x0270 */ + uint32_t KL_KD_ADDR; /* Address Offset: 0x0274 */ + uint32_t RESERVED0278[94]; /* Address Offset: 0x0278 */ + uint32_t ECC_CTL; /* Address Offset: 0x03F0 */ + uint32_t ECC_INT_EN; /* Address Offset: 0x03F4 */ + uint32_t ECC_INT_ST; /* Address Offset: 0x03F8 */ + uint32_t ECC_ABN_ST; /* Address Offset: 0x03FC */ + uint32_t ECC_CURVE_WIDE; /* Address Offset: 0x0400 */ + uint32_t ECC_MAX_CURVE_WIDE; /* Address Offset: 0x0404 */ + uint32_t ECC_DATA_ENDIAN; /* Address Offset: 0x0408 */ + uint32_t RESERVED040C[17]; /* Address Offset: 0x040C */ + uint32_t KL_APB_CMD; /* Address Offset: 0x0450 */ + uint32_t KL_APB_PADDR; /* Address Offset: 0x0454 */ + uint32_t KL_APB_PWDATA; /* Address Offset: 0x0458 */ + uint32_t RESERVED045C[2]; /* Address Offset: 0x045C */ + uint32_t KL_KD_VID; /* Address Offset: 0x0464 */ + uint32_t KL_KD_MODE; /* Address Offset: 0x0468 */ + uint32_t RESERVED046C[5]; /* Address Offset: 0x046C */ + uint32_t PKA_RAM_CTL; /* Address Offset: 0x0480 */ + uint32_t PKA_RAM_ST; /* Address Offset: 0x0484 */ + uint32_t RESERVED0488[6]; /* Address Offset: 0x0488 */ + uint32_t PKA_DEBUG_CTL; /* Address Offset: 0x04A0 */ + uint32_t PKA_DEBUG_ST; /* Address Offset: 0x04A4 */ + uint32_t PKA_DEBUG_MONITOR; /* Address Offset: 0x04A8 */ + uint32_t RESERVED04AC[85]; /* Address Offset: 0x04AC */ + uint32_t KT_ST; /* Address Offset: 0x0600 */ + uint32_t RESERVED0604; /* Address Offset: 0x0604 */ + uint32_t KL_INTER_COPY; /* Address Offset: 0x0608 */ + uint32_t RESERVED060C[4]; /* Address Offset: 0x060C */ + uint32_t LOCKSTEP_EN; /* Address Offset: 0x061C */ + uint32_t RESERVED0620[2]; /* Address Offset: 0x0620 */ + uint32_t LOCKSTEP_IJERR; /* Address Offset: 0x0628 */ + uint32_t RESERVED062C[5]; /* Address Offset: 0x062C */ + uint32_t KL_OTP_KEY_REQ; /* Address Offset: 0x0640 */ + uint32_t KL_KEY_CLEAR; /* Address Offset: 0x0644 */ + uint32_t KL_OTP_KEY_LEN; /* Address Offset: 0x0648 */ + uint32_t KL_HW_DRNG_REQ; /* Address Offset: 0x064C */ + uint32_t RESERVED0650[12]; /* Address Offset: 0x0650 */ + uint32_t AES_VER; /* Address Offset: 0x0680 */ + uint32_t DES_VER; /* Address Offset: 0x0684 */ + uint32_t SM4_VER; /* Address Offset: 0x0688 */ + uint32_t HASH_VER; /* Address Offset: 0x068C */ + uint32_t HMAC_VER; /* Address Offset: 0x0690 */ + uint32_t RESERVED0694; /* Address Offset: 0x0694 */ + uint32_t PKA_VER; /* Address Offset: 0x0698 */ + uint32_t EXTRA_FEATURE; /* Address Offset: 0x069C */ + uint32_t RESERVED06A0[20]; /* Address Offset: 0x06A0 */ + uint32_t CE_VER; /* Address Offset: 0x06F0 */ + uint32_t RESERVED06F4[67]; /* Address Offset: 0x06F4 */ + uint32_t PKA_MEM_MAP0; /* Address Offset: 0x0800 */ + uint32_t PKA_MEM_MAP1; /* Address Offset: 0x0804 */ + uint32_t PKA_MEM_MAP2; /* Address Offset: 0x0808 */ + uint32_t PKA_MEM_MAP3; /* Address Offset: 0x080C */ + uint32_t PKA_MEM_MAP4; /* Address Offset: 0x0810 */ + uint32_t PKA_MEM_MAP5; /* Address Offset: 0x0814 */ + uint32_t PKA_MEM_MAP6; /* Address Offset: 0x0818 */ + uint32_t PKA_MEM_MAP7; /* Address Offset: 0x081C */ + uint32_t PKA_MEM_MAP8; /* Address Offset: 0x0820 */ + uint32_t PKA_MEM_MAP9; /* Address Offset: 0x0824 */ + uint32_t PKA_MEM_MAP10; /* Address Offset: 0x0828 */ + uint32_t PKA_MEM_MAP11; /* Address Offset: 0x082C */ + uint32_t PKA_MEM_MAP12; /* Address Offset: 0x0830 */ + uint32_t PKA_MEM_MAP13; /* Address Offset: 0x0834 */ + uint32_t PKA_MEM_MAP14; /* Address Offset: 0x0838 */ + uint32_t PKA_MEM_MAP15; /* Address Offset: 0x083C */ + uint32_t PKA_MEM_MAP16; /* Address Offset: 0x0840 */ + uint32_t PKA_MEM_MAP17; /* Address Offset: 0x0844 */ + uint32_t PKA_MEM_MAP18; /* Address Offset: 0x0848 */ + uint32_t PKA_MEM_MAP19; /* Address Offset: 0x084C */ + uint32_t PKA_MEM_MAP20; /* Address Offset: 0x0850 */ + uint32_t PKA_MEM_MAP21; /* Address Offset: 0x0854 */ + uint32_t PKA_MEM_MAP22; /* Address Offset: 0x0858 */ + uint32_t PKA_MEM_MAP23; /* Address Offset: 0x085C */ + uint32_t PKA_MEM_MAP24; /* Address Offset: 0x0860 */ + uint32_t PKA_MEM_MAP25; /* Address Offset: 0x0864 */ + uint32_t PKA_MEM_MAP26; /* Address Offset: 0x0868 */ + uint32_t PKA_MEM_MAP27; /* Address Offset: 0x086C */ + uint32_t PKA_MEM_MAP28; /* Address Offset: 0x0870 */ + uint32_t PKA_MEM_MAP29; /* Address Offset: 0x0874 */ + uint32_t PKA_MEM_MAP30; /* Address Offset: 0x0878 */ + uint32_t PKA_MEM_MAP31; /* Address Offset: 0x087C */ + uint32_t PKA_OPCODE; /* Address Offset: 0x0880 */ + uint32_t N_NP_T0_T1_ADDR; /* Address Offset: 0x0884 */ + uint32_t PKA_STATUS; /* Address Offset: 0x0888 */ + uint32_t RESERVED088C; /* Address Offset: 0x088C */ + uint32_t PKA_L0; /* Address Offset: 0x0890 */ + uint32_t PKA_L1; /* Address Offset: 0x0894 */ + uint32_t PKA_L2; /* Address Offset: 0x0898 */ + uint32_t PKA_L3; /* Address Offset: 0x089C */ + uint32_t PKA_L4; /* Address Offset: 0x08A0 */ + uint32_t PKA_L5; /* Address Offset: 0x08A4 */ + uint32_t PKA_L6; /* Address Offset: 0x08A8 */ + uint32_t PKA_L7; /* Address Offset: 0x08AC */ + uint32_t PKA_PIPE_RDY; /* Address Offset: 0x08B0 */ + uint32_t PKA_DONE; /* Address Offset: 0x08B4 */ + uint32_t PKA_MON_SELECT; /* Address Offset: 0x08B8 */ + uint32_t PKA_DEBUG_REG_EN; /* Address Offset: 0x08BC */ + uint32_t DEBUG_CNT_ADDR; /* Address Offset: 0x08C0 */ + uint32_t DEBUG_EXT_ADDR; /* Address Offset: 0x08C4 */ + uint32_t PKA_DEBUG_HALT; /* Address Offset: 0x08C8 */ + uint32_t RESERVED08CC; /* Address Offset: 0x08CC */ + uint32_t PKA_MON_READ; /* Address Offset: 0x08D0 */ + uint32_t PKA_INT_ENA; /* Address Offset: 0x08D4 */ + uint32_t PKA_INT_ST; /* Address Offset: 0x08D8 */ + uint32_t TD0_TD1_TX_ADDR; /* Address Offset: 0x08DC */ + uint32_t RESERVED08E0[456]; /* Address Offset: 0x08E0 */ + uint32_t SRAM_ADDR; /* Address Offset: 0x1000 */ +}; + +/****************************************************************************************/ +/* */ +/* Register Bitmap Section */ +/* */ +/****************************************************************************************/ +/******************************************RKCE******************************************/ +/* CLK_CTL */ +#define RKCE_CLK_CTL_OFFSET (0x0U) +#define RKCE_CLK_CTL_AUTO_CLKGATE_EN_SHIFT (0U) +#define RKCE_CLK_CTL_AUTO_CLKGATE_EN_MASK (0x1U << RKCE_CLK_CTL_AUTO_CLKGATE_EN_SHIFT) +/* RST_CTL */ +#define RKCE_RST_CTL_OFFSET (0x4U) +#define RKCE_RST_CTL_SW_SYMM_RESET_SHIFT (0U) +#define RKCE_RST_CTL_SW_SYMM_RESET_MASK (0x1U << RKCE_RST_CTL_SW_SYMM_RESET_SHIFT) +#define RKCE_RST_CTL_SW_HASH_RESET_SHIFT (1U) +#define RKCE_RST_CTL_SW_HASH_RESET_MASK (0x1U << RKCE_RST_CTL_SW_HASH_RESET_SHIFT) +#define RKCE_RST_CTL_SW_PKA_RESET_SHIFT (2U) +#define RKCE_RST_CTL_SW_PKA_RESET_MASK (0x1U << RKCE_RST_CTL_SW_PKA_RESET_SHIFT) +/* TD_ADDR */ +#define RKCE_TD_ADDR_OFFSET (0x200U) +#define RKCE_TD_ADDR_TD_ADDR_SHIFT (0U) +#define RKCE_TD_ADDR_TD_ADDR_MASK (0xFFFFFFFFU << RKCE_TD_ADDR_TD_ADDR_SHIFT) +/* TD_LOAD_CTRL */ +#define RKCE_TD_LOAD_CTRL_OFFSET (0x204U) +#define RKCE_TD_LOAD_CTRL_SYMM_TLR_SHIFT (0U) +#define RKCE_TD_LOAD_CTRL_SYMM_TLR_MASK (0x1U << RKCE_TD_LOAD_CTRL_SYMM_TLR_SHIFT) +#define RKCE_TD_LOAD_CTRL_HASH_TLR_SHIFT (1U) +#define RKCE_TD_LOAD_CTRL_HASH_TLR_MASK (0x1U << RKCE_TD_LOAD_CTRL_HASH_TLR_SHIFT) +/* SYMM_INT_ST */ +#define RKCE_SYMM_INT_ST_OFFSET (0x214U) +#define RKCE_SYMM_INT_ST_TD_DONE_SHIFT (0U) +#define RKCE_SYMM_INT_ST_TD_DONE_MASK (0x1U << RKCE_SYMM_INT_ST_TD_DONE_SHIFT) +#define RKCE_SYMM_INT_ST_DST_ERROR_SHIFT (1U) +#define RKCE_SYMM_INT_ST_DST_ERROR_MASK (0x1U << RKCE_SYMM_INT_ST_DST_ERROR_SHIFT) +#define RKCE_SYMM_INT_ST_SRC_ERROR_SHIFT (2U) +#define RKCE_SYMM_INT_ST_SRC_ERROR_MASK (0x1U << RKCE_SYMM_INT_ST_SRC_ERROR_SHIFT) +#define RKCE_SYMM_INT_ST_TD_ERROR_SHIFT (3U) +#define RKCE_SYMM_INT_ST_TD_ERROR_MASK (0x1U << RKCE_SYMM_INT_ST_TD_ERROR_SHIFT) +#define RKCE_SYMM_INT_ST_NE_LEN_SHIFT (4U) +#define RKCE_SYMM_INT_ST_NE_LEN_MASK (0x1U << RKCE_SYMM_INT_ST_NE_LEN_SHIFT) +#define RKCE_SYMM_INT_ST_LOCKSTEP_ERROR_SHIFT (5U) +#define RKCE_SYMM_INT_ST_LOCKSTEP_ERROR_MASK (0x1U << RKCE_SYMM_INT_ST_LOCKSTEP_ERROR_SHIFT) +/* SYMM_TD_ID */ +#define RKCE_SYMM_TD_ID_OFFSET (0x218U) +#define RKCE_SYMM_TD_ID (0x0U) +#define RKCE_SYMM_TD_ID_STDID_SHIFT (0U) +#define RKCE_SYMM_TD_ID_STDID_MASK (0xFFFFFFFFU << RKCE_SYMM_TD_ID_STDID_SHIFT) +/* SYMM_TD_ST */ +#define RKCE_SYMM_TD_ST_OFFSET (0x21CU) +#define RKCE_SYMM_TD_ST (0x0U) +#define RKCE_SYMM_TD_ST_FIRST_PKG_FLAG_SHIFT (0U) +#define RKCE_SYMM_TD_ST_FIRST_PKG_FLAG_MASK (0x1U << RKCE_SYMM_TD_ST_FIRST_PKG_FLAG_SHIFT) +#define RKCE_SYMM_TD_ST_LAST_PKG_FLAG_SHIFT (1U) +#define RKCE_SYMM_TD_ST_LAST_PKG_FLAG_MASK (0x1U << RKCE_SYMM_TD_ST_LAST_PKG_FLAG_SHIFT) +#define RKCE_SYMM_TD_ST_DMA_START_FLAG_SHIFT (2U) +#define RKCE_SYMM_TD_ST_DMA_START_FLAG_MASK (0x1U << RKCE_SYMM_TD_ST_DMA_START_FLAG_SHIFT) +#define RKCE_SYMM_TD_ST_PRMPT_PKG_FLGA_SHIFT (3U) +#define RKCE_SYMM_TD_ST_PRMPT_PKG_FLGA_MASK (0x1U << RKCE_SYMM_TD_ST_PRMPT_PKG_FLGA_SHIFT) +/* SYMM_CONTEXT_SIZE */ +#define RKCE_SYMM_CONTEXT_SIZE_OFFSET (0x224U) +#define RKCE_SYMM_CONTEXT_SIZE (0x20U) +/* HASH_INT_ST */ +#define RKCE_HASH_INT_ST_OFFSET (0x234U) +#define RKCE_HASH_INT_ST_TD_DONE_SHIFT (0U) +#define RKCE_HASH_INT_ST_TD_DONE_MASK (0x1U << RKCE_HASH_INT_ST_TD_DONE_SHIFT) +#define RKCE_HASH_INT_ST_DST_ERROR_SHIFT (1U) +#define RKCE_HASH_INT_ST_DST_ERROR_MASK (0x1U << RKCE_HASH_INT_ST_DST_ERROR_SHIFT) +#define RKCE_HASH_INT_ST_SRC_ERROR_SHIFT (2U) +#define RKCE_HASH_INT_ST_SRC_ERROR_MASK (0x1U << RKCE_HASH_INT_ST_SRC_ERROR_SHIFT) +#define RKCE_HASH_INT_ST_TD_ERROR_SHIFT (3U) +#define RKCE_HASH_INT_ST_TD_ERROR_MASK (0x1U << RKCE_HASH_INT_ST_TD_ERROR_SHIFT) +#define RKCE_HASH_INT_ST_NE_LEN_SHIFT (4U) +#define RKCE_HASH_INT_ST_NE_LEN_MASK (0x1U << RKCE_HASH_INT_ST_NE_LEN_SHIFT) +#define RKCE_HASH_INT_ST_LOCKSTEP_ERROR_SHIFT (5U) +#define RKCE_HASH_INT_ST_LOCKSTEP_ERROR_MASK (0x1U << RKCE_HASH_INT_ST_LOCKSTEP_ERROR_SHIFT) +/* HASH_TD_ID */ +#define RKCE_HASH_TD_ID_OFFSET (0x238U) +#define RKCE_HASH_TD_ID (0x0U) +#define RKCE_HASH_TD_ID_HTDID_SHIFT (0U) +#define RKCE_HASH_TD_ID_HTDID_MASK (0xFFFFFFFFU << RKCE_HASH_TD_ID_HTDID_SHIFT) +/* HASH_TD_ST */ +#define RKCE_HASH_TD_ST_OFFSET (0x23CU) +#define RKCE_HASH_TD_ST (0x0U) +#define RKCE_HASH_TD_ST_FIRST_PKG_FLAG_SHIFT (0U) +#define RKCE_HASH_TD_ST_FIRST_PKG_FLAG_MASK (0x1U << RKCE_HASH_TD_ST_FIRST_PKG_FLAG_SHIFT) +#define RKCE_HASH_TD_ST_LAST_PKG_FLAG_SHIFT (1U) +#define RKCE_HASH_TD_ST_LAST_PKG_FLAG_MASK (0x1U << RKCE_HASH_TD_ST_LAST_PKG_FLAG_SHIFT) +#define RKCE_HASH_TD_ST_DMA_START_FLAG_SHIFT (2U) +#define RKCE_HASH_TD_ST_DMA_START_FLAG_MASK (0x1U << RKCE_HASH_TD_ST_DMA_START_FLAG_SHIFT) +#define RKCE_HASH_TD_ST_PRMPT_PKG_FLGA_SHIFT (3U) +#define RKCE_HASH_TD_ST_PRMPT_PKG_FLGA_MASK (0x1U << RKCE_HASH_TD_ST_PRMPT_PKG_FLGA_SHIFT) +/* HASH_CONTEXT_SIZE */ +#define RKCE_HASH_CONTEXT_SIZE_OFFSET (0x244U) +#define RKCE_HASH_CONTEXT_SIZE (0xD0U) +/* TD_POP_CTRL */ +#define RKCE_TD_POP_CTRL_OFFSET (0x258U) +#define RKCE_TD_POP_CTRL (0x0U) +#define RKCE_TD_POP_CTRL_SYMM_TPR_SHIFT (0U) +#define RKCE_TD_POP_CTRL_SYMM_TPR_MASK (0x1U << RKCE_TD_POP_CTRL_SYMM_TPR_SHIFT) +#define RKCE_TD_POP_CTRL_HASH_TPR_SHIFT (1U) +#define RKCE_TD_POP_CTRL_HASH_TPR_MASK (0x1U << RKCE_TD_POP_CTRL_HASH_TPR_SHIFT) +/* ECC_CTL */ +#define RKCE_ECC_CTL_OFFSET (0x3F0U) +#define RKCE_ECC_CTL_ECC_REQ_SHIFT (0U) +#define RKCE_ECC_CTL_ECC_REQ_MASK (0x1U << RKCE_ECC_CTL_ECC_REQ_SHIFT) +#define RKCE_ECC_CTL_FUNC_SEL_SHIFT (4U) +#define RKCE_ECC_CTL_FUNC_SEL_MASK (0xFU << RKCE_ECC_CTL_FUNC_SEL_SHIFT) +#define RKCE_ECC_CTL_CURVE_MODE_SEL_SHIFT (8U) +#define RKCE_ECC_CTL_CURVE_MODE_SEL_MASK (0x1U << RKCE_ECC_CTL_CURVE_MODE_SEL_SHIFT) +#define RKCE_ECC_CTL_RAND_K_SRC_SHIFT (12U) +#define RKCE_ECC_CTL_RAND_K_SRC_MASK (0x1U << RKCE_ECC_CTL_RAND_K_SRC_SHIFT) +/* ECC_INT_EN */ +#define RKCE_ECC_INT_EN_OFFSET (0x3F4U) +#define RKCE_ECC_INT_EN_DONE_INT_EN_SHIFT (0U) +#define RKCE_ECC_INT_EN_DONE_INT_EN_MASK (0x1U << RKCE_ECC_INT_EN_DONE_INT_EN_SHIFT) +/* ECC_INT_ST */ +#define RKCE_ECC_INT_ST_OFFSET (0x3F8U) +#define RKCE_ECC_INT_ST_DONE_INT_ST_SHIFT (0U) +#define RKCE_ECC_INT_ST_DONE_INT_ST_MASK (0x1U << RKCE_ECC_INT_ST_DONE_INT_ST_SHIFT) +/* ECC_ABN_ST */ +#define RKCE_ECC_ABN_ST_OFFSET (0x3FCU) +#define RKCE_ECC_ABN_ST (0x0U) +#define RKCE_ECC_ABN_ST_BAD_POINT_OUT_SHIFT (0U) +#define RKCE_ECC_ABN_ST_BAD_POINT_OUT_MASK (0x1U << RKCE_ECC_ABN_ST_BAD_POINT_OUT_SHIFT) +#define RKCE_ECC_ABN_ST_BAD_T_OUT_SHIFT (1U) +#define RKCE_ECC_ABN_ST_BAD_T_OUT_MASK (0x1U << RKCE_ECC_ABN_ST_BAD_T_OUT_SHIFT) +#define RKCE_ECC_ABN_ST_BAD_S_OUT_SHIFT (2U) +#define RKCE_ECC_ABN_ST_BAD_S_OUT_MASK (0x1U << RKCE_ECC_ABN_ST_BAD_S_OUT_SHIFT) +#define RKCE_ECC_ABN_ST_BAD_R_OUT_SHIFT (3U) +#define RKCE_ECC_ABN_ST_BAD_R_OUT_MASK (0x1U << RKCE_ECC_ABN_ST_BAD_R_OUT_SHIFT) +#define RKCE_ECC_ABN_ST_BAD_R_K_MID_SHIFT (4U) +#define RKCE_ECC_ABN_ST_BAD_R_K_MID_MASK (0x1U << RKCE_ECC_ABN_ST_BAD_R_K_MID_SHIFT) +#define RKCE_ECC_ABN_ST_BAD_S_IN_SHIFT (5U) +#define RKCE_ECC_ABN_ST_BAD_S_IN_MASK (0x1U << RKCE_ECC_ABN_ST_BAD_S_IN_SHIFT) +#define RKCE_ECC_ABN_ST_BAD_R_IN_SHIFT (6U) +#define RKCE_ECC_ABN_ST_BAD_R_IN_MASK (0x1U << RKCE_ECC_ABN_ST_BAD_R_IN_SHIFT) +#define RKCE_ECC_ABN_ST_BAD_K_IN_SHIFT (7U) +#define RKCE_ECC_ABN_ST_BAD_K_IN_MASK (0x1U << RKCE_ECC_ABN_ST_BAD_K_IN_SHIFT) +#define RKCE_ECC_ABN_ST_BAD_INV_OT_SHIFT (8U) +#define RKCE_ECC_ABN_ST_BAD_INV_OT_MASK (0x1U << RKCE_ECC_ABN_ST_BAD_INV_OT_SHIFT) +/* ECC_CURVE_WIDE */ +#define RKCE_ECC_CURVE_WIDE_OFFSET (0x400U) +#define RKCE_ECC_CURVE_WIDE_CURVE_WIDE_SHIFT (0U) +#define RKCE_ECC_CURVE_WIDE_CURVE_WIDE_MASK (0x3FFU << RKCE_ECC_CURVE_WIDE_CURVE_WIDE_SHIFT) +/* ECC_MAX_CURVE_WIDE */ +#define RKCE_ECC_MAX_CURVE_WIDE_OFFSET (0x404U) +#define RKCE_ECC_MAX_CURVE_WIDE (0x100U) +/* ECC_DATA_ENDIAN */ +#define RKCE_ECC_DATA_ENDIAN_OFFSET (0x408U) +/* PKA_RAM_CTL */ +#define RKCE_PKA_RAM_CTL_OFFSET (0x480U) +#define RKCE_PKA_RAM_CTL_PKA_RDY BIT(0) +#define RKCE_PKA_RAM_CTL_RAM_PKA_RDY_SHIFT (0U) +#define RKCE_PKA_RAM_CTL_RAM_PKA_RDY_MASK (0x3U << RKCE_PKA_RAM_CTL_RAM_PKA_RDY_SHIFT) +/* PKA_RAM_ST */ +#define RKCE_PKA_RAM_ST_OFFSET (0x484U) +#define RKCE_PKA_RAM_ST (0x1U) +#define RKCE_PKA_RAM_ST_CLK_RAM_RDY_SHIFT (0U) +#define RKCE_PKA_RAM_ST_CLK_RAM_RDY_MASK (0x1U << RKCE_PKA_RAM_ST_CLK_RAM_RDY_SHIFT) +/* AES_VER */ +#define RKCE_AES_VER_OFFSET (0x680U) +#define RKCE_AES_VER_ECB_FLAG_SHIFT (0U) +#define RKCE_AES_VER_ECB_FLAG_MASK (0x1U << RKCE_AES_VER_ECB_FLAG_SHIFT) +#define RKCE_AES_VER_CBC_FLAG_SHIFT (1U) +#define RKCE_AES_VER_CBC_FLAG_MASK (0x1U << RKCE_AES_VER_CBC_FLAG_SHIFT) +#define RKCE_AES_VER_CTS_FLAG_SHIFT (2U) +#define RKCE_AES_VER_CTS_FLAG_MASK (0x1U << RKCE_AES_VER_CTS_FLAG_SHIFT) +#define RKCE_AES_VER_CTR_FLAG_SHIFT (3U) +#define RKCE_AES_VER_CTR_FLAG_MASK (0x1U << RKCE_AES_VER_CTR_FLAG_SHIFT) +#define RKCE_AES_VER_CFB_FLAG_SHIFT (4U) +#define RKCE_AES_VER_CFB_FLAG_MASK (0x1U << RKCE_AES_VER_CFB_FLAG_SHIFT) +#define RKCE_AES_VER_OFB_FLAG_SHIFT (5U) +#define RKCE_AES_VER_OFB_FLAG_MASK (0x1U << RKCE_AES_VER_OFB_FLAG_SHIFT) +#define RKCE_AES_VER_XTS_FLAG_SHIFT (6U) +#define RKCE_AES_VER_XTS_FLAG_MASK (0x1U << RKCE_AES_VER_XTS_FLAG_SHIFT) +#define RKCE_AES_VER_CCM_FLAG_SHIFT (7U) +#define RKCE_AES_VER_CCM_FLAG_MASK (0x1U << RKCE_AES_VER_CCM_FLAG_SHIFT) +#define RKCE_AES_VER_GCM_FLAG_SHIFT (8U) +#define RKCE_AES_VER_GCM_FLAG_MASK (0x1U << RKCE_AES_VER_GCM_FLAG_SHIFT) +#define RKCE_AES_VER_CMAC_FLAG_SHIFT (9U) +#define RKCE_AES_VER_CMAC_FLAG_MASK (0x1U << RKCE_AES_VER_CMAC_FLAG_SHIFT) +#define RKCE_AES_VER_CBC_MAC_FLAG_SHIFT (10U) +#define RKCE_AES_VER_CBC_MAC_FLAG_MASK (0x1U << RKCE_AES_VER_CBC_MAC_FLAG_SHIFT) +#define RKCE_AES_VER_BYPASS_SHIFT (12U) +#define RKCE_AES_VER_BYPASS_MASK (0x1U << RKCE_AES_VER_BYPASS_SHIFT) +#define RKCE_AES_VER_AES128_FLAG_SHIFT (16U) +#define RKCE_AES_VER_AES128_FLAG_MASK (0x1U << RKCE_AES_VER_AES128_FLAG_SHIFT) +#define RKCE_AES_VER_AES192_FLAG_SHIFT (17U) +#define RKCE_AES_VER_AES192_FLAG_MASK (0x1U << RKCE_AES_VER_AES192_FLAG_SHIFT) +#define RKCE_AES_VER_AES256_FLAG_SHIFT (18U) +#define RKCE_AES_VER_AES256_FLAG_MASK (0x1U << RKCE_AES_VER_AES256_FLAG_SHIFT) +#define RKCE_AES_VER_LOCKSTEP_FLAG_SHIFT (20U) +#define RKCE_AES_VER_LOCKSTEP_FLAG_MASK (0x1U << RKCE_AES_VER_LOCKSTEP_FLAG_SHIFT) +#define RKCE_AES_VER_SECURE_FLAG_SHIFT (21U) +#define RKCE_AES_VER_SECURE_FLAG_MASK (0x1U << RKCE_AES_VER_SECURE_FLAG_SHIFT) +/* DES_VER */ +#define RKCE_DES_VER_OFFSET (0x684U) +#define RKCE_DES_VER_ECB_FLAG_SHIFT (0U) +#define RKCE_DES_VER_ECB_FLAG_MASK (0x1U << RKCE_DES_VER_ECB_FLAG_SHIFT) +#define RKCE_DES_VER_CBC_FLAG_SHIFT (1U) +#define RKCE_DES_VER_CBC_FLAG_MASK (0x1U << RKCE_DES_VER_CBC_FLAG_SHIFT) +#define RKCE_DES_VER_CFB_FLAG_SHIFT (4U) +#define RKCE_DES_VER_CFB_FLAG_MASK (0x1U << RKCE_DES_VER_CFB_FLAG_SHIFT) +#define RKCE_DES_VER_OFB_FLAG_SHIFT (5U) +#define RKCE_DES_VER_OFB_FLAG_MASK (0x1U << RKCE_DES_VER_OFB_FLAG_SHIFT) +#define RKCE_DES_VER_TDES_FLAG_SHIFT (16U) +#define RKCE_DES_VER_TDES_FLAG_MASK (0x1U << RKCE_DES_VER_TDES_FLAG_SHIFT) +#define RKCE_DES_VER_EEE_FLAG_SHIFT (17U) +#define RKCE_DES_VER_EEE_FLAG_MASK (0x1U << RKCE_DES_VER_EEE_FLAG_SHIFT) +#define RKCE_DES_VER_EDE_FLAG_SHIFT (18U) +#define RKCE_DES_VER_EDE_FLAG_MASK (0x1U << RKCE_DES_VER_EDE_FLAG_SHIFT) +#define RKCE_DES_VER_LOCKSTEP_FLAG_SHIFT (20U) +#define RKCE_DES_VER_LOCKSTEP_FLAG_MASK (0x1U << RKCE_DES_VER_LOCKSTEP_FLAG_SHIFT) +#define RKCE_DES_VER_SECURE_FLAG_SHIFT (21U) +#define RKCE_DES_VER_SECURE_FLAG_MASK (0x1U << RKCE_DES_VER_SECURE_FLAG_SHIFT) +/* SM4_VER */ +#define RKCE_SM4_VER_OFFSET (0x688U) +#define RKCE_SM4_VER_ECB_FLAG_SHIFT (0U) +#define RKCE_SM4_VER_ECB_FLAG_MASK (0x1U << RKCE_SM4_VER_ECB_FLAG_SHIFT) +#define RKCE_SM4_VER_CBC_FLAG_SHIFT (1U) +#define RKCE_SM4_VER_CBC_FLAG_MASK (0x1U << RKCE_SM4_VER_CBC_FLAG_SHIFT) +#define RKCE_SM4_VER_CTS_FLAG_SHIFT (2U) +#define RKCE_SM4_VER_CTS_FLAG_MASK (0x1U << RKCE_SM4_VER_CTS_FLAG_SHIFT) +#define RKCE_SM4_VER_CTR_FLAG_SHIFT (3U) +#define RKCE_SM4_VER_CTR_FLAG_MASK (0x1U << RKCE_SM4_VER_CTR_FLAG_SHIFT) +#define RKCE_SM4_VER_CFB_FLAG_SHIFT (4U) +#define RKCE_SM4_VER_CFB_FLAG_MASK (0x1U << RKCE_SM4_VER_CFB_FLAG_SHIFT) +#define RKCE_SM4_VER_OFB_FLAG_SHIFT (5U) +#define RKCE_SM4_VER_OFB_FLAG_MASK (0x1U << RKCE_SM4_VER_OFB_FLAG_SHIFT) +#define RKCE_SM4_VER_XTS_FLAG_SHIFT (6U) +#define RKCE_SM4_VER_XTS_FLAG_MASK (0x1U << RKCE_SM4_VER_XTS_FLAG_SHIFT) +#define RKCE_SM4_VER_CCM_FLAG_SHIFT (7U) +#define RKCE_SM4_VER_CCM_FLAG_MASK (0x1U << RKCE_SM4_VER_CCM_FLAG_SHIFT) +#define RKCE_SM4_VER_GCM_FLAG_SHIFT (8U) +#define RKCE_SM4_VER_GCM_FLAG_MASK (0x1U << RKCE_SM4_VER_GCM_FLAG_SHIFT) +#define RKCE_SM4_VER_CMAC_FLAG_SHIFT (9U) +#define RKCE_SM4_VER_CMAC_FLAG_MASK (0x1U << RKCE_SM4_VER_CMAC_FLAG_SHIFT) +#define RKCE_SM4_VER_CBC_MAC_FLAG_SHIFT (10U) +#define RKCE_SM4_VER_CBC_MAC_FLAG_MASK (0x1U << RKCE_SM4_VER_CBC_MAC_FLAG_SHIFT) +#define RKCE_SM4_VER_LOCKSTEP_FLAG_SHIFT (20U) +#define RKCE_SM4_VER_LOCKSTEP_FLAG_MASK (0x1U << RKCE_SM4_VER_LOCKSTEP_FLAG_SHIFT) +#define RKCE_SM4_VER_SECURE_FLAG_SHIFT (21U) +#define RKCE_SM4_VER_SECURE_FLAG_MASK (0x1U << RKCE_SM4_VER_SECURE_FLAG_SHIFT) +/* HASH_VER */ +#define RKCE_HASH_VER_OFFSET (0x68CU) +#define RKCE_HASH_VER_SHA1_FLAG_SHIFT (0U) +#define RKCE_HASH_VER_SHA1_FLAG_MASK (0x1U << RKCE_HASH_VER_SHA1_FLAG_SHIFT) +#define RKCE_HASH_VER_SHA224_FLAG_SHIFT (1U) +#define RKCE_HASH_VER_SHA224_FLAG_MASK (0x1U << RKCE_HASH_VER_SHA224_FLAG_SHIFT) +#define RKCE_HASH_VER_SHA256_FLAG_SHIFT (2U) +#define RKCE_HASH_VER_SHA256_FLAG_MASK (0x1U << RKCE_HASH_VER_SHA256_FLAG_SHIFT) +#define RKCE_HASH_VER_SHA384_FLAG_SHIFT (3U) +#define RKCE_HASH_VER_SHA384_FLAG_MASK (0x1U << RKCE_HASH_VER_SHA384_FLAG_SHIFT) +#define RKCE_HASH_VER_SHA512_FLAG_SHIFT (4U) +#define RKCE_HASH_VER_SHA512_FLAG_MASK (0x1U << RKCE_HASH_VER_SHA512_FLAG_SHIFT) +#define RKCE_HASH_VER_SHA512_224_FLAG_SHIFT (5U) +#define RKCE_HASH_VER_SHA512_224_FLAG_MASK (0x1U << RKCE_HASH_VER_SHA512_224_FLAG_SHIFT) +#define RKCE_HASH_VER_SHA512_256_FLAG_SHIFT (6U) +#define RKCE_HASH_VER_SHA512_256_FLAG_MASK (0x1U << RKCE_HASH_VER_SHA512_256_FLAG_SHIFT) +#define RKCE_HASH_VER_MD5_FLAG_SHIFT (7U) +#define RKCE_HASH_VER_MD5_FLAG_MASK (0x1U << RKCE_HASH_VER_MD5_FLAG_SHIFT) +#define RKCE_HASH_VER_SM3_FLAG_SHIFT (8U) +#define RKCE_HASH_VER_SM3_FLAG_MASK (0x1U << RKCE_HASH_VER_SM3_FLAG_SHIFT) +#define RKCE_HASH_VER_LOCKSTEP_FLAG_SHIFT (20U) +#define RKCE_HASH_VER_LOCKSTEP_FLAG_MASK (0x1U << RKCE_HASH_VER_LOCKSTEP_FLAG_SHIFT) +/* HMAC_VER */ +#define RKCE_HMAC_VER_OFFSET (0x690U) +#define RKCE_HMAC_VER_SHA1_FLAG_SHIFT (0U) +#define RKCE_HMAC_VER_SHA1_FLAG_MASK (0x1U << RKCE_HMAC_VER_SHA1_FLAG_SHIFT) +#define RKCE_HMAC_VER_SHA256_FLAG_SHIFT (1U) +#define RKCE_HMAC_VER_SHA256_FLAG_MASK (0x1U << RKCE_HMAC_VER_SHA256_FLAG_SHIFT) +#define RKCE_HMAC_VER_SHA512_FLAG_SHIFT (2U) +#define RKCE_HMAC_VER_SHA512_FLAG_MASK (0x1U << RKCE_HMAC_VER_SHA512_FLAG_SHIFT) +#define RKCE_HMAC_VER_MD5_FLAG_SHIFT (3U) +#define RKCE_HMAC_VER_MD5_FLAG_MASK (0x1U << RKCE_HMAC_VER_MD5_FLAG_SHIFT) +#define RKCE_HMAC_VER_SM3_FLAG_SHIFT (4U) +#define RKCE_HMAC_VER_SM3_FLAG_MASK (0x1U << RKCE_HMAC_VER_SM3_FLAG_SHIFT) +#define RKCE_HMAC_VER_LOCKSTEP_FLAG_SHIFT (20U) +#define RKCE_HMAC_VER_LOCKSTEP_FLAG_MASK (0x1U << RKCE_HMAC_VER_LOCKSTEP_FLAG_SHIFT) +/* PKA_VER */ +#define RKCE_PKA_VER_OFFSET (0x698U) +/* EXTRA_FEATURE */ +#define RKCE_EXTRA_FEATURE_OFFSET (0x69CU) +#define RKCE_EXTRA_FEATURE_AXI_EXPAND_BIT_SHIFT (0U) +#define RKCE_EXTRA_FEATURE_AXI_EXPAND_BIT_MASK (0xFU << RKCE_EXTRA_FEATURE_AXI_EXPAND_BIT_SHIFT) +/* CE_VER */ +#define RKCE_CE_VER_OFFSET (0x6F0U) +/* PKA_MEM_MAP0 */ +#define RKCE_PKA_MEM_MAP0_OFFSET (0x800U) +#define RKCE_MAP_REG_NUM (32) +/* PKA_OPCODE */ +#define RKCE_PKA_OPCODE_OFFSET (0x880U) +#define RKCE_PKA_OPCODE_TAG_SHIFT (0U) +#define RKCE_PKA_OPCODE_TAG_MASK (0x3FU << RKCE_PKA_OPCODE_TAG_SHIFT) +#define RKCE_PKA_OPCODE_REG_R_SHIFT (6U) +#define RKCE_PKA_OPCODE_REG_R_MASK (0x3FU << RKCE_PKA_OPCODE_REG_R_SHIFT) +#define RKCE_PKA_OPCODE_R_DIS_SHIFT (11U) +#define RKCE_PKA_OPCODE_REG_B_SHIFT (12U) +#define RKCE_PKA_OPCODE_REG_B_MASK (0x3FU << RKCE_PKA_OPCODE_REG_B_SHIFT) +#define RKCE_PKA_OPCODE_B_IMMED_SHIFT (17U) +#define RKCE_PKA_OPCODE_REG_A_SHIFT (18U) +#define RKCE_PKA_OPCODE_REG_A_MASK (0x3FU << RKCE_PKA_OPCODE_REG_A_SHIFT) +#define RKCE_PKA_OPCODE_A_IMMED_SHIFT (23U) +#define RKCE_PKA_OPCODE_LEN_SHIFT (24U) +#define RKCE_PKA_OPCODE_LEN_MASK (0x7U << RKCE_PKA_OPCODE_LEN_SHIFT) +#define RKCE_PKA_OPCODE_OPCODE_SHIFT (27U) +#define RKCE_PKA_OPCODE_OPCODE_MASK (0x1FU << RKCE_PKA_OPCODE_OPCODE_SHIFT) +/* N_NP_T0_T1_ADDR */ +#define RKCE_N_NP_T0_T1_ADDR_OFFSET (0x884U) +#define RKCE_N_NP_T0_T1_ADDR_REG_N_SHIFT (0U) +#define RKCE_N_NP_T0_T1_ADDR_REG_N_MASK (0x1FU << RKCE_N_NP_T0_T1_ADDR_REG_N_SHIFT) +#define RKCE_N_NP_T0_T1_ADDR_REG_NP_SHIFT (5U) +#define RKCE_N_NP_T0_T1_ADDR_REG_NP_MASK (0x1FU << RKCE_N_NP_T0_T1_ADDR_REG_NP_SHIFT) +#define RKCE_N_NP_T0_T1_ADDR_REG_T0_SHIFT (10U) +#define RKCE_N_NP_T0_T1_ADDR_REG_T0_MASK (0x1FU << RKCE_N_NP_T0_T1_ADDR_REG_T0_SHIFT) +#define RKCE_N_NP_T0_T1_ADDR_REG_T1_SHIFT (15U) +#define RKCE_N_NP_T0_T1_ADDR_REG_T1_MASK (0x1FU << RKCE_N_NP_T0_T1_ADDR_REG_T1_SHIFT) +/* PKA_STATUS */ +#define RKCE_PKA_STATUS_OFFSET (0x888U) +#define RKCE_PKA_STATUS (0x1U) +#define RKCE_PKA_STATUS_PIPE_IS_BUSY_SHIFT (0U) +#define RKCE_PKA_STATUS_PIPE_IS_BUSY_MASK (0x1U << RKCE_PKA_STATUS_PIPE_IS_BUSY_SHIFT) +#define RKCE_PKA_STATUS_PKA_BUSY_SHIFT (1U) +#define RKCE_PKA_STATUS_PKA_BUSY_MASK (0x1U << RKCE_PKA_STATUS_PKA_BUSY_SHIFT) +#define RKCE_PKA_STATUS_ALU_OUT_ZERO_SHIFT (2U) +#define RKCE_PKA_STATUS_ALU_OUT_ZERO_MASK (0x1U << RKCE_PKA_STATUS_ALU_OUT_ZERO_SHIFT) +#define RKCE_PKA_STATUS_ALU_MOD_OVFLW_SHIFT (3U) +#define RKCE_PKA_STATUS_ALU_MOD_OVFLW_MASK (0x1U << RKCE_PKA_STATUS_ALU_MOD_OVFLW_SHIFT) +#define RKCE_PKA_STATUS_DIV_BY_ZERO_SHIFT (4U) +#define RKCE_PKA_STATUS_DIV_BY_ZERO_MASK (0x1U << RKCE_PKA_STATUS_DIV_BY_ZERO_SHIFT) +#define RKCE_PKA_STATUS_ALU_CARRY_SHIFT (5U) +#define RKCE_PKA_STATUS_ALU_CARRY_MASK (0x1U << RKCE_PKA_STATUS_ALU_CARRY_SHIFT) +#define RKCE_PKA_STATUS_ALU_SIGN_OUT_SHIFT (6U) +#define RKCE_PKA_STATUS_ALU_SIGN_OUT_MASK (0x1U << RKCE_PKA_STATUS_ALU_SIGN_OUT_SHIFT) +#define RKCE_PKA_STATUS_MODINV_OF_ZERO_SHIFT (7U) +#define RKCE_PKA_STATUS_MODINV_OF_ZERO_MASK (0x1U << RKCE_PKA_STATUS_MODINV_OF_ZERO_SHIFT) +#define RKCE_PKA_STATUS_PKA_CPU_BUSY_SHIFT (8U) +#define RKCE_PKA_STATUS_PKA_CPU_BUSY_MASK (0x1U << RKCE_PKA_STATUS_PKA_CPU_BUSY_SHIFT) +#define RKCE_PKA_STATUS_OPCODE_SHIFT (9U) +#define RKCE_PKA_STATUS_OPCODE_MASK (0x1FU << RKCE_PKA_STATUS_OPCODE_SHIFT) +#define RKCE_PKA_STATUS_TAG_SHIFT (14U) +#define RKCE_PKA_STATUS_TAG_MASK (0x3FU << RKCE_PKA_STATUS_TAG_SHIFT) +/* PKA_L0 */ +#define RKCE_PKA_L0_OFFSET (0x890U) +#define RKCE_LEN_REG_NUM (8) +/* PKA_PIPE_RDY */ +#define RKCE_PKA_PIPE_RDY_OFFSET (0x8B0U) +#define RKCE_PKA_PIPE_RDY (0x1U) +#define RKCE_PKA_PIPE_RDY_PKA_PIPE_RDY_SHIFT (0U) +#define RKCE_PKA_PIPE_RDY_PKA_PIPE_RDY_MASK (0x1U << RKCE_PKA_PIPE_RDY_PKA_PIPE_RDY_SHIFT) +/* PKA_DONE */ +#define RKCE_PKA_DONE_OFFSET (0x8B4U) +#define RKCE_PKA_DONE (0x1U) +#define RKCE_PKA_DONE_PKA_DONE_SHIFT (0U) +#define RKCE_PKA_DONE_PKA_DONE_MASK (0x1U << RKCE_PKA_DONE_PKA_DONE_SHIFT) +/* PKA_INT_ENA */ +#define RKCE_PKA_INT_ENA_OFFSET (0x8D4U) +#define RKCE_PKA_INT_ENA_PKA_INT_ENA_SHIFT (0U) +#define RKCE_PKA_INT_ENA_PKA_INT_ENA_MASK (0x1U << RKCE_PKA_INT_ENA_PKA_INT_ENA_SHIFT) +/* PKA_INT_ST */ +#define RKCE_PKA_INT_ST_OFFSET (0x8D8U) +#define RKCE_PKA_INT_ST_PKA_INT_ST_SHIFT (0U) +#define RKCE_PKA_INT_ST_PKA_INT_ST_MASK (0x1U << RKCE_PKA_INT_ST_PKA_INT_ST_SHIFT) +/* TD0_TD1_TX_ADDR */ +#define RKCE_TD0_TD1_TX_ADDR_OFFSET (0x8DCU) +#define RKCE_TD0_TD1_TX_ADDR_REG_TD0_SHIFT (0U) +#define RKCE_TD0_TD1_TX_ADDR_REG_TD0_MASK (0x1FU << RKCE_TD0_TD1_TX_ADDR_REG_TD0_SHIFT) +#define RKCE_TD0_TD1_TX_ADDR_REG_TD1_SHIFT (5U) +#define RKCE_TD0_TD1_TX_ADDR_REG_TD1_MASK (0x1FU << RKCE_TD0_TD1_TX_ADDR_REG_TD1_SHIFT) +#define RKCE_TD0_TD1_TX_ADDR_REG_TX_SHIFT (10U) +#define RKCE_TD0_TD1_TX_ADDR_REG_TX_MASK (0x1FU << RKCE_TD0_TD1_TX_ADDR_REG_TX_SHIFT) +#define RKCE_TD0_TD1_TX_ADDR_PKA_ASCA_EN_SHIFT (31U) +#define RKCE_TD0_TD1_TX_ADDR_PKA_ASCA_EN_MASK (0x1U << RKCE_TD0_TD1_TX_ADDR_PKA_ASCA_EN_SHIFT) +/* SRAM_ADDR */ +#define RKCE_SRAM_ADDR_OFFSET (0x1000U) +#define RKCE_SRAM_ADDR_SRAM_ADDR_SHIFT (0U) +#define RKCE_SRAM_ADDR_SRAM_ADDR_MASK (0xFFFFFFFFU << RKCE_SRAM_ADDR_SRAM_ADDR_SHIFT) + +#define RKCE_SRAM_SIZE (0x1000U) + +#endif /* __RKCE_REG_H */ diff --git a/drivers/crypto/rockchip/rkce/rkce_skcipher.c b/drivers/crypto/rockchip/rkce/rkce_skcipher.c new file mode 100644 index 000000000000..bfcc80b7c20b --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_skcipher.c @@ -0,0 +1,1076 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Crypto acceleration support for Rockchip crypto engine + * + * Copyright (c) 2024 Rockchip Electronics Co., Ltd. + * + * Author: Lin Jinhan + * + */ + +#define RKCE_MODULE_TAG "CIPHER" +#define RKCE_MODULE_OFFSET 6 + +#include +#include +#include + +#include "rkce_core.h" +#include "rkce_dev.h" +#include "rkce_debug.h" +#include "rkce_monitor.h" +#include "rkce_skcipher.h" + +#define TD_SYNC_TIMEOUT_MS 3000 + +static void rkce_set_symm_td_sg(struct rkce_symm_td *td_head, + uint32_t index, uint32_t len, + const dma_addr_t in, + const dma_addr_t out); + +static inline bool is_algt_aead(struct rkce_algt *algt) +{ + return algt->type == RKCE_ALGO_TYPE_AEAD; +} + +static inline struct rkce_cipher_ctx *sk_req2cipher_ctx(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + + return crypto_skcipher_ctx(tfm); +} + +static inline struct rkce_cipher_ctx *asyn_req2cipher_ctx(void *async_req) +{ + struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base); + + return sk_req2cipher_ctx(areq); +} + +static uint32_t rkce_get_td_keysize(uint32_t algo, uint32_t mode, uint32_t key_len) +{ + uint32_t key_size = 0; + + if (mode == RKCE_SYMM_MODE_XTS) + key_len /= 2; + + if (algo == RKCE_SYMM_ALGO_AES) { + if (key_len == AES_KEYSIZE_128) + key_size = RKCE_KEY_AES_128; + else if (key_len == AES_KEYSIZE_192) + key_size = RKCE_KEY_AES_192; + else if (key_len == AES_KEYSIZE_256) + key_size = RKCE_KEY_AES_256; + else + key_size = 0; + } + + return key_size; +} + +static void rkce_reverse_memcpy(void *dst, const void *src, uint32_t size) +{ + char *_dst = (char *)dst, *_src = (char *)src; + uint32_t i; + + if (!dst || !src || !size) + return; + + for (i = 0; i < size; ++i) + _dst[size - i - 1] = _src[i]; +} + +static int rkce_decrypt_block(void *hardware, uint32_t algo, + const uint8_t *key, uint32_t key_len, + const uint8_t *input, uint8_t *output) +{ + int ret; + uint32_t block_size = AES_BLOCK_SIZE; + uint32_t mode = RKCE_SYMM_MODE_ECB; + uint8_t *data_buf = NULL; + struct rkce_symm_td *td = NULL; + struct rkce_symm_td_buf *td_buf = NULL; + + if (!hardware || + !key || + !input || + !output || + key_len == 0) + return -EINVAL; + + td_buf = rkce_cma_alloc(sizeof(*td_buf)); + if (!td_buf) { + rk_debug("rkce_cma_alloc td_buf failed.\n"); + ret = -ENOMEM; + goto exit; + } + + td = rkce_cma_alloc(sizeof(*td)); + if (!td) { + rk_debug("rkce_cma_alloc td failed.\n"); + ret = -ENOMEM; + goto exit; + } + + data_buf = rkce_cma_alloc(block_size); + if (!data_buf) { + rk_debug("rkce_cma_alloc block failed.\n"); + ret = -ENOMEM; + goto exit; + } + + memcpy(td_buf->key1, key, key_len); + memcpy(data_buf, input, block_size); + + rkce_init_symm_td(td, td_buf); + rkce_set_symm_td_sg(td, 0, block_size, + rkce_cma_virt2phys(data_buf), + rkce_cma_virt2phys(data_buf)); + + td->ctrl.td_type = RKCE_TD_TYPE_SYMM; + td->ctrl.is_dec = 1; + td->ctrl.symm_algo = algo; + td->ctrl.symm_mode = mode; + td->ctrl.key_size = rkce_get_td_keysize(algo, mode, key_len); + td->ctrl.first_pkg = 1; + td->ctrl.last_pkg = 1; + td->ctrl.int_en = 1; + + ret = rkce_push_td_sync(hardware, td, TD_SYNC_TIMEOUT_MS); + if (ret == 0) + memcpy(output, data_buf, block_size); +exit: + rkce_cma_free(data_buf); + rkce_cma_free(td); + rkce_cma_free(td_buf); + + return ret; +} + +static void rkce_update_iv(struct rkce_cipher_ctx *ctx, uint8_t *iv) +{ + struct rkce_algt *algt; + struct rkce_symm_td_buf *td_buf; + + if (!ctx || !ctx->algt || !ctx->td_buf || !iv || + ctx->algt->mode == RKCE_SYMM_MODE_ECB) + return; + + algt = ctx->algt; + td_buf = ctx->td_buf; + + rkce_reverse_memcpy(iv + 0, td_buf->ctx + 24, 4); + rkce_reverse_memcpy(iv + 4, td_buf->ctx + 28, 4); + rkce_reverse_memcpy(iv + 8, td_buf->ctx + 0, 4); + rkce_reverse_memcpy(iv + 12, td_buf->ctx + 4, 4); + + if (algt->mode == RKCE_SYMM_MODE_XTS) + rkce_decrypt_block(algt->rk_dev->hardware, algt->algo, + td_buf->key2, ctx->keylen / 2, iv, iv); + + rkce_dumphex("td_buf->ctx", td_buf->ctx, sizeof(td_buf->ctx)); + rkce_dumphex("updated iv", iv, 16); + + memset(td_buf->ctx, 0x00, sizeof(td_buf->ctx)); +} + +int rkce_cipher_request_callback(int result, uint32_t td_id, void *td_addr) +{ + struct rkce_symm_td *td = (struct rkce_symm_td *)td_addr; + struct rkce_symm_td_buf *td_buf = + container_of(rkce_cma_phys2virt(td->symm_ctx_addr), struct rkce_symm_td_buf, ctx); + struct rkce_cipher_ctx *ctx = (struct rkce_cipher_ctx *)td_buf->user_data; + struct crypto_engine *engine = ctx->algt->rk_dev->symm_engine; + + rk_trace("enter.\n"); + + if (is_algt_aead(ctx->algt)) { + struct aead_request *tmp_req = (struct aead_request *)ctx->req; + struct rkce_cipher_request_ctx *rctx = aead_request_ctx(tmp_req); + + if (result != -ETIMEDOUT) + rkce_monitor_del(rctx->td_head); + + if (result) + crypto_finalize_aead_request(engine, ctx->req, result); + + rk_debug("dst = %p, nents %u, tag = %p, authsize = %u,offset = %u\n", + tmp_req->dst, + sg_nents(tmp_req->dst), + td_buf->tag, + ctx->authsize, + rctx->assoclen + rctx->cryptlen); + + if (rctx->is_enc) { + if (!sg_pcopy_from_buffer(tmp_req->dst, + sg_nents(tmp_req->dst), + td_buf->tag, + ctx->authsize, + rctx->assoclen + rctx->cryptlen)) + result = -EBADMSG; + + } else { + uint8_t auth_data[RKCE_TD_TAG_SIZE]; + + if (!sg_pcopy_to_buffer(tmp_req->src, + sg_nents(tmp_req->src), + auth_data, ctx->authsize, + rctx->assoclen + rctx->cryptlen) || + crypto_memneq(auth_data, td_buf->tag, ctx->authsize)) + result = -EBADMSG; + } + + crypto_finalize_aead_request(engine, ctx->req, result); + } else { + struct skcipher_request *tmp_req = (struct skcipher_request *)ctx->req; + struct rkce_cipher_request_ctx *rctx = skcipher_request_ctx(tmp_req); + + if (result != -ETIMEDOUT) + rkce_monitor_del(rctx->td_head); + + if (result) + crypto_finalize_skcipher_request(engine, tmp_req, result); + + /* update iv */ + rkce_update_iv(ctx, tmp_req->iv); + + crypto_finalize_skcipher_request(engine, tmp_req, result); + } + + if (result) { + rkce_dump_reginfo(ctx->algt->rk_dev->hardware); + rkce_soft_reset(ctx->algt->rk_dev->hardware, RKCE_RESET_SYMM); + } + + rk_trace("exit.\n"); + + return 0; +} + +static int rkce_set_symm_td_buf_key(struct rkce_symm_td_buf *td_buf, + uint32_t algo, uint32_t mode, + const uint8_t *key, uint32_t key_len) +{ + rk_trace("enter.\n"); + + memset(td_buf->key1, 0x00, sizeof(td_buf->key1)); + memset(td_buf->key2, 0x00, sizeof(td_buf->key2)); + + if (mode == RKCE_SYMM_MODE_XTS) { + memcpy(td_buf->key1, key, key_len / 2); + memcpy(td_buf->key2, key + key_len / 2, key_len / 2); + rkce_dumphex("key1", td_buf->key1, sizeof(td_buf->key1)); + rkce_dumphex("key2", td_buf->key2, sizeof(td_buf->key2)); + } else { + memcpy(td_buf->key1, key, key_len); + } + + if (key_len == DES_KEY_SIZE * 2 && + (algo == RKCE_SYMM_ALGO_DES || algo == RKCE_SYMM_ALGO_TDES)) + memcpy(td_buf->key1 + DES_KEY_SIZE * 2, td_buf->key1, DES_KEY_SIZE); + + rk_trace("exit.\n"); + + return 0; +} + +static struct rkce_symm_td *rkce_cipher_td_chain_alloc(uint32_t sg_nents, + struct rkce_symm_td_buf *td_buf) +{ + int ret = -ENOMEM; + uint32_t i, td_nums; + struct rkce_symm_td *td_head = NULL; + + rk_trace("enter.\n"); + + td_nums = DIV_ROUND_UP(sg_nents, RKCE_TD_SG_NUM); + + rk_debug("sg_nents = %u, td_nums = %u\n", sg_nents, td_nums); + + td_head = rkce_cma_alloc(sizeof(*td_head) * td_nums); + if (!td_head) { + rk_debug("rkce_cma_alloc %u td failed.\n", td_nums); + goto error; + } + + for (i = 0; i < td_nums; i++) { + ret = rkce_init_symm_td(&td_head[i], td_buf); + if (ret) { + rk_debug("rkce_init_symm_td td[%u] failed.\n", i); + goto error; + } + + if (i < td_nums - 1) + td_head[i].next_task = rkce_cma_virt2phys(&td_head[i + 1]); + + } + + rk_trace("exit.\n"); + + return td_head; +error: + rkce_cma_free(td_head); + + rk_trace("exit.\n"); + + return NULL; +} + +static void rkce_cipher_td_chain_free(struct rkce_symm_td *td_head) +{ + rk_trace("enter.\n"); + + rkce_cma_free(td_head); + + rk_trace("exit.\n"); +} + +static void rkce_set_symm_td_sg(struct rkce_symm_td *td_head, + uint32_t index, uint32_t len, + const dma_addr_t in, + const dma_addr_t out) +{ + struct rkce_symm_td *cur_td = &td_head[index / RKCE_TD_SG_NUM]; + uint32_t sg_idx = index % 8; + + rk_trace("enter.\n"); + + memset(&(cur_td->sg[sg_idx]), 0x00, sizeof(struct rkce_sg_info)); + +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + cur_td->sg[sg_idx].src_addr_h = in >> 32; +#endif + + cur_td->sg[sg_idx].src_addr_l = in & 0xffffffff; + cur_td->sg[sg_idx].src_size = len; + + if (out) { +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + cur_td->sg[sg_idx].dst_addr_h = out >> 32; +#endif + + cur_td->sg[sg_idx].dst_addr_l = out & 0xffffffff; + cur_td->sg[sg_idx].dst_size = len; + } + + rk_trace("exit.\n"); +} + +static int rkce_cipher_set_td_chain(struct scatterlist *sgs, struct scatterlist *sgd, + uint32_t cryptlen, struct rkce_symm_td *td_head, + struct rkce_symm_td_ctrl ctrl) +{ + uint32_t last_td_idx = 0; + uint32_t td_sg_idx = 0; + uint32_t split_len = 0; + uint32_t src_left = 0, dst_left = 0; + dma_addr_t src_dma = 0, dst_dma = 0; + int first_pkg, last_pkg, int_en; + + rk_trace("enter.\n"); + + if (cryptlen == 0) + return -1; + + /* only set first td*/ + first_pkg = ctrl.first_pkg; + ctrl.first_pkg = 0; + last_pkg = ctrl.last_pkg; + ctrl.last_pkg = 0; + + /* only set last td */ + int_en = ctrl.int_en; + ctrl.int_en = 0; + + /* first sg */ + src_left = sg_dma_len(sgs); + src_dma = sg_dma_address(sgs); + + if (sgd) { + dst_left = sg_dma_len(sgd); + dst_dma = sg_dma_address(sgd); + } + + while (cryptlen) { + rk_debug("cryptlen = %u, src_left = %u, src_dma = %pad, dst_left = %u, dst_dma = %pad\n", + cryptlen, src_left, &src_dma, dst_left, &dst_dma); + if (src_left == 0) { + sgs = sg_next(sgs); + if (!sgs) { + rk_debug("sgs not enough.\n"); + goto error; + } + src_left = sg_dma_len(sgs); + src_dma = sg_dma_address(sgs); + } + + if (sgd && dst_left == 0) { + sgd = sg_next(sgd); + if (!sgd) { + rk_debug("sgd not enough.\n"); + goto error; + } + + dst_left = sg_dma_len(sgd); + dst_dma = sg_dma_address(sgd); + } + + split_len = src_left >= dst_left && sgd ? dst_left : src_left; + + /* in case split length too long */ + if (split_len > cryptlen) + split_len = cryptlen; + + rk_debug("split_len = %u\n", split_len); + /* set one td and point to next */ + rkce_set_symm_td_sg(td_head, td_sg_idx, split_len, src_dma, dst_dma); + + if (td_sg_idx % RKCE_TD_SG_NUM == 0) + memcpy(&td_head[td_sg_idx / RKCE_TD_SG_NUM].ctrl, &ctrl, sizeof(ctrl)); + + td_sg_idx++; + + cryptlen -= split_len; + src_dma += split_len; + src_left -= split_len; + + if (sgd) { + dst_dma += split_len; + dst_left -= split_len; + } + } + + td_head[0].ctrl.first_pkg = first_pkg; + + /* clear last td */ + last_td_idx = DIV_ROUND_UP(td_sg_idx, RKCE_TD_SG_NUM) - 1; + + td_head[last_td_idx].next_task = 0; + td_head[last_td_idx].ctrl.last_pkg = last_pkg; + td_head[last_td_idx].ctrl.int_en = int_en; + + rk_trace("exit.\n"); + + return 0; +error: + rk_trace("exit.\n"); + + return -EINVAL; +} + +static int rkce_cipher_sg_map(struct device *dev, + struct scatterlist *sgs, uint32_t sgs_nents, + struct scatterlist *sgd, uint32_t sgd_nents) +{ + int ret = 0; + + rk_trace("enter.\n"); + + /* flush src & invalid dst */ + if (sgs == sgd) { + ret = dma_map_sg(dev, sgs, sgs_nents, DMA_BIDIRECTIONAL); + if (ret <= 0) { + rk_err("dma_map_sg src dst DMA_BIDIRECTIONAL failed, ret = %d.\n", ret); + goto exit; + } + } else { + ret = dma_map_sg(dev, sgs, sgs_nents, DMA_TO_DEVICE); + if (ret <= 0) { + rk_err("dma_map_sg src DMA_TO_DEVICE failed, ret = %d.\n", ret); + goto exit; + } + + ret = dma_map_sg(dev, sgd, sgd_nents, DMA_FROM_DEVICE); + if (ret <= 0) { + dma_unmap_sg(dev, sgs, sgs_nents, DMA_TO_DEVICE); + rk_err("dma_map_sg dst DMA_FROM_DEVICE failed, ret = %d.\n", ret); + goto exit; + } + } + + ret = 0; + +exit: + rk_trace("exit.\n"); + + return ret; +} + +static void rkce_cipher_sg_unmap(struct device *dev, + struct scatterlist *sgs, uint32_t sgs_nents, + struct scatterlist *sgd, uint32_t sgd_nents) +{ + rk_trace("enter.\n"); + + if (sgs == sgd) { + dma_unmap_sg(dev, sgs, sgs_nents, DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(dev, sgs, sgs_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, sgd, sgd_nents, DMA_FROM_DEVICE); + } + + rk_trace("exit.\n"); +} + +static int rkce_common_prepare_req(struct rkce_cipher_ctx *ctx, + struct rkce_cipher_request_ctx *rctx, + void *req) +{ + int ret = 0; + struct rkce_symm_td_ctrl ctrl; + struct device *dev = rctx->dev; + struct rkce_algt *algt = ctx->algt; + + rk_trace("enter.\n"); + + rk_debug("rctx = %p, sgs = %p, sgd = %p\n", + rctx, rctx->sgs, rctx->sgd); + + /* check key length */ + if (ctx->keylen == 0) { + rk_err("Key should set before calculating.\n"); + return -EINVAL; + } + + rctx->td_head = rkce_cipher_td_chain_alloc(rctx->sgs_nents + rctx->sgd_nents, ctx->td_buf); + if (!rctx->td_head) { + ret = -ENOMEM; + rk_err("rkce_cipher_td_chain_alloc td_head failed ret = %d\n", ret); + goto exit; + } + + rk_debug("rctx = %p, sgs = %p, sgd = %p\n", + rctx, rctx->sgs, rctx->sgd); + + memset(&ctrl, 0x00, sizeof(ctrl)); + + ctrl.td_type = RKCE_TD_TYPE_SYMM; + ctrl.is_dec = !rctx->is_enc; + ctrl.symm_algo = algt->algo; + ctrl.symm_mode = algt->mode; + ctrl.key_size = rkce_get_td_keysize(algt->algo, algt->mode, ctx->keylen); + ctrl.iv_len = ctx->ivlen; + ctrl.first_pkg = 1; + ctrl.last_pkg = 1; + ctrl.int_en = 1; + ctrl.is_aad = 0; + + rk_debug("rctx = %p, sgs = %p, sgd = %p, sga = %p, is_aead = %d\n", + rctx, rctx->sgs, rctx->sgd, rctx->sga, rctx->is_aead); + + if (rctx->is_aead) { + struct aead_request *tmp_req = (struct aead_request *)req; + + memcpy(ctx->td_buf->iv, tmp_req->iv, ctrl.iv_len); + + if (!rctx->is_dma) { + ret = rkce_cipher_sg_map(dev, + tmp_req->src, + sg_nents_for_len(tmp_req->src, rctx->map_total), + tmp_req->dst, + sg_nents_for_len(tmp_req->dst, rctx->map_total)); + if (ret) + goto exit; + } + + rctx->td_aad_head = rkce_cipher_td_chain_alloc(rctx->sga_nents, ctx->td_buf); + if (!rctx->td_aad_head) { + ret = -ENOMEM; + rk_err("rkce_cipher_td_chain_alloc td_aad_head failed ret = %d\n", ret); + goto exit; + } + + ctrl.is_aad = 1; + + ctx->td_buf->gcm_len.aad_len_h = 0; + ctx->td_buf->gcm_len.aad_len_l = rctx->assoclen; + ctx->td_buf->gcm_len.pc_len_h = 0; + ctx->td_buf->gcm_len.pc_len_l = rctx->cryptlen; + + /* translate scatter list to td chain */ + ret = rkce_cipher_set_td_chain(rctx->sga, NULL, rctx->assoclen, + rctx->td_aad_head, ctrl); + if (ret) + goto exit; + } else { + struct skcipher_request *tmp_req = (struct skcipher_request *)req; + + memcpy(ctx->td_buf->iv, tmp_req->iv, ctrl.iv_len); + + if (!rctx->is_dma) { + ret = rkce_cipher_sg_map(dev, + rctx->sgs, rctx->sgs_nents, + rctx->sgd, rctx->sgd_nents); + if (ret) + return ret; + } + } + + ctrl.is_aad = 0; + + /* translate scatter list to td chain */ + ret = rkce_cipher_set_td_chain(rctx->sgs, rctx->sgd, rctx->cryptlen, rctx->td_head, ctrl); + if (ret) + goto exit; + + rctx->is_mapped = true; + +exit: + rk_trace("exit.\n"); + + return ret; +} + +static int rkce_common_unprepare_req(struct rkce_cipher_request_ctx *rctx) +{ + struct device *dev = rctx->dev; + + rk_trace("enter.\n"); + + if (!rctx->is_dma && rctx->is_mapped) + rkce_cipher_sg_unmap(dev, rctx->sgs, rctx->sgs_nents, rctx->sgd, rctx->sgd_nents); + + rkce_cipher_td_chain_free(rctx->td_aad_head); + rkce_cipher_td_chain_free(rctx->td_head); + + memzero_explicit(rctx, sizeof(*rctx)); + + rk_trace("exit.\n"); + + return 0; +} + +static int rkce_cipher_prepare_req(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = container_of(areq, struct skcipher_request, base); + struct rkce_cipher_request_ctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct rkce_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + struct device *dev = ctx->algt->rk_dev->dev; + + rk_trace("enter.\n"); + + ctx->req = req; + + rctx->dev = dev; + rctx->cryptlen = req->cryptlen; + rctx->sgs = req->src; + rctx->sgs_nents = sg_nents_for_len(req->src, req->cryptlen); + rctx->sgd = req->dst; + rctx->sgd_nents = sg_nents_for_len(req->dst, req->cryptlen); + rctx->map_total = rctx->cryptlen; + rctx->is_dma = sg_dma_address(req->src) && sg_dma_address(req->dst); + + rk_debug("rctx = %p, sgs = %p, sgd = %p\n", + rctx, rctx->sgs, rctx->sgd); + + return rkce_common_prepare_req(ctx, rctx, req); +} + +static int rkce_cipher_unprepare_req(struct crypto_engine *engine, void *areq) +{ + struct skcipher_request *req = container_of(areq, struct skcipher_request, base); + struct rkce_cipher_request_ctx *rctx = skcipher_request_ctx(req); + + rk_trace("enter.\n"); + + return rkce_common_unprepare_req(rctx); +} + +static int rkce_cipher_run_req(struct crypto_engine *engine, void *async_req) +{ + struct skcipher_request *req = container_of(async_req, struct skcipher_request, base); + struct rkce_cipher_request_ctx *rctx = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct rkce_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + + rk_trace("enter.\n"); + + rkce_monitor_add(rctx->td_head, rkce_cipher_request_callback); + + return rkce_push_td(ctx->algt->rk_dev->hardware, rctx->td_head); +} + +static int rkce_ablk_init_tfm(struct crypto_skcipher *tfm) +{ + struct rkce_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct rkce_algt *algt = container_of(alg, struct rkce_algt, alg.cipher); + + rk_trace("enter.\n"); + + rk_debug("alloc %s\n", algt->name); + + memzero_explicit(ctx, sizeof(*ctx)); + + ctx->algt = algt; + ctx->ivlen = algt->mode == RKCE_SYMM_MODE_ECB ? 0 : crypto_skcipher_ivsize(tfm); + + ctx->enginectx.op.prepare_request = rkce_cipher_prepare_req; + ctx->enginectx.op.do_one_request = rkce_cipher_run_req; + ctx->enginectx.op.unprepare_request = rkce_cipher_unprepare_req; + + ctx->td_buf = rkce_cma_alloc(sizeof(*(ctx->td_buf))); + if (!ctx->td_buf) { + rk_debug("rkce_cma_alloc td_buf failed.\n"); + return -ENOMEM; + } + + ctx->td_buf->user_data = ctx; + + crypto_skcipher_set_reqsize(tfm, sizeof(struct rkce_cipher_request_ctx)); + + rkce_enable_clk(algt->rk_dev); + + rk_trace("exit.\n"); + + return 0; +} + +static void rkce_ablk_exit_tfm(struct crypto_skcipher *tfm) +{ + struct rkce_cipher_ctx *ctx = crypto_skcipher_ctx(tfm); + + rk_trace("enter.\n"); + + rkce_cma_free(ctx->td_buf); + + rkce_disable_clk(ctx->algt->rk_dev); + + memzero_explicit(ctx, sizeof(*ctx)); + + rk_trace("exit.\n"); +} + +static int rkce_common_setkey(struct rkce_cipher_ctx *ctx, const uint8_t *key, unsigned int keylen) +{ + struct rkce_algt *algt = ctx->algt; + uint32_t key_factor; + + /* The key length of XTS is twice the normal length */ + key_factor = algt->mode == RKCE_SYMM_MODE_XTS ? 2 : 1; + + rk_debug("algo = %x, mode = %x, key_len = %d\n", algt->algo, algt->mode, keylen); + + switch (algt->algo) { + case RKCE_SYMM_ALGO_DES: + if (keylen != DES_KEY_SIZE) + goto error; + break; + case RKCE_SYMM_ALGO_TDES: + if (keylen != DES_KEY_SIZE * 2 && + keylen != DES_KEY_SIZE * 3) + goto error; + break; + case RKCE_SYMM_ALGO_AES: + if (keylen != (AES_KEYSIZE_128 * key_factor) && + keylen != (AES_KEYSIZE_192 * key_factor) && + keylen != (AES_KEYSIZE_256 * key_factor)) + goto error; + break; + case RKCE_SYMM_ALGO_SM4: + if (keylen != SM4_KEY_SIZE * key_factor) + goto error; + break; + default: + goto error; + } + + ctx->keylen = keylen; + + /* set td buf info */ + return rkce_set_symm_td_buf_key(ctx->td_buf, algt->algo, algt->mode, key, keylen); + +error: + return -EINVAL; +} + +static int rkce_cipher_setkey(struct crypto_skcipher *cipher, + const uint8_t *key, unsigned int keylen) +{ + rk_trace("enter.\n"); + + return rkce_common_setkey(crypto_skcipher_ctx(cipher), key, keylen); +} + +static int rkce_cipher_handle_req(struct skcipher_request *req, bool is_enc) +{ + struct rkce_cipher_request_ctx *rctx = skcipher_request_ctx(req); + struct rkce_cipher_ctx *ctx = sk_req2cipher_ctx(req); + struct rkce_dev *rk_dev = ctx->algt->rk_dev; + struct crypto_engine *engine = rk_dev->symm_engine; + + rk_trace("enter.\n"); + + memzero_explicit(rctx, sizeof(*rctx)); + + rctx->is_enc = is_enc; + rctx->is_aead = false; + + rk_debug("cryptlen = %u, %s\n", req->cryptlen, is_enc ? "encrypt" : "decrypt"); + + return crypto_transfer_skcipher_request_to_engine(engine, req); +} + +static int rkce_cipher_encrypt(struct skcipher_request *req) +{ + rk_trace("enter.\n"); + + return rkce_cipher_handle_req(req, true); +} + +static int rkce_cipher_decrypt(struct skcipher_request *req) +{ + rk_trace("enter.\n"); + + return rkce_cipher_handle_req(req, false); +} + +static int rkce_aead_prepare_req(struct crypto_engine *engine, void *areq) +{ + struct aead_request *req = container_of(areq, struct aead_request, base); + struct rkce_cipher_request_ctx *rctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct rkce_cipher_ctx *ctx = crypto_aead_ctx(tfm); + struct device *dev = ctx->algt->rk_dev->dev; + + rk_trace("enter.\n"); + + ctx->req = req; + ctx->authsize = crypto_aead_authsize(tfm); + + rctx->dev = dev; + rctx->sga = req->src; + rctx->sga_nents = sg_nents_for_len(req->src, req->assoclen); + rctx->assoclen = req->assoclen; + rctx->cryptlen = rctx->is_enc ? req->cryptlen : req->cryptlen - ctx->authsize; + rctx->map_total = rctx->assoclen + rctx->cryptlen; + rctx->is_dma = sg_dma_address(req->src) && sg_dma_address(req->dst); + + rk_debug("assoclen = %u, cryptlen = %u, authsize = %u, is_dma = %u\n", + req->assoclen, req->cryptlen, ctx->authsize, rctx->is_dma); + + /* point sg_src and sg_dst skip assoc data */ + rctx->sgs = scatterwalk_ffwd(rctx->src_sg, req->src, req->assoclen); + rctx->sgs_nents = sg_nents_for_len(rctx->sgs, rctx->cryptlen); + + if (req->src == req->dst) { + rctx->sgd = rctx->sgs; + rctx->sgd_nents = rctx->sgs_nents; + } else { + rctx->sgd = scatterwalk_ffwd(rctx->dst_sg, req->dst, req->assoclen); + rctx->sgd_nents = sg_nents_for_len(rctx->sgd, rctx->cryptlen); + } + + return rkce_common_prepare_req(ctx, rctx, req); +} + +static int rkce_aead_unprepare_req(struct crypto_engine *engine, void *areq) +{ + struct aead_request *req = container_of(areq, struct aead_request, base); + struct rkce_cipher_request_ctx *rctx = aead_request_ctx(req); + + rk_trace("enter.\n"); + + return rkce_common_unprepare_req(rctx); +} + +static int rkce_aead_run_req(struct crypto_engine *engine, void *async_req) +{ + int ret = 0; + struct aead_request *req = container_of(async_req, struct aead_request, base); + struct rkce_cipher_request_ctx *rctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct rkce_cipher_ctx *ctx = crypto_aead_ctx(tfm); + + rk_trace("enter.\n"); + + ret = rkce_push_td_sync(ctx->algt->rk_dev->hardware, rctx->td_aad_head, TD_SYNC_TIMEOUT_MS); + if (ret) { + rk_debug("calc aad data error.\n"); + goto exit; + } + + ret = rkce_push_td(ctx->algt->rk_dev->hardware, rctx->td_head); + if (ret) { + rk_debug("calc data error.\n"); + goto exit; + } + + rkce_monitor_add(rctx->td_head, rkce_cipher_request_callback); +exit: + return ret; +} + +static int rkce_aead_handle_req(struct aead_request *req, bool is_enc) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct rkce_cipher_ctx *ctx = crypto_aead_ctx(tfm); + struct rkce_cipher_request_ctx *rctx = aead_request_ctx(req); + struct rkce_dev *rk_dev = ctx->algt->rk_dev; + struct crypto_engine *engine = rk_dev->symm_engine; + + rk_trace("enter.\n"); + + memzero_explicit(rctx, sizeof(*rctx)); + + rctx->is_enc = is_enc; + rctx->is_aead = true; + + rk_debug("assoclen = %u, cryptlen = %u, %s\n", + req->assoclen, req->cryptlen, + is_enc ? "encrypt" : "decrypt"); + + return crypto_transfer_aead_request_to_engine(engine, req); +} + +static int rkce_aead_init_tfm(struct crypto_aead *tfm) +{ + struct rkce_cipher_ctx *ctx = crypto_aead_ctx(tfm); + struct aead_alg *alg = crypto_aead_alg(tfm); + struct rkce_algt *algt = container_of(alg, struct rkce_algt, alg.aead); + + rk_trace("enter.\n"); + + rk_debug("alloc %s\n", algt->name); + + memzero_explicit(ctx, sizeof(*ctx)); + + ctx->algt = algt; + ctx->ivlen = crypto_aead_ivsize(tfm); + + ctx->enginectx.op.prepare_request = rkce_aead_prepare_req; + ctx->enginectx.op.do_one_request = rkce_aead_run_req; + ctx->enginectx.op.unprepare_request = rkce_aead_unprepare_req; + + ctx->td_buf = rkce_cma_alloc(sizeof(*(ctx->td_buf))); + if (!ctx->td_buf) { + rk_debug("rkce_cma_alloc td_buf failed.\n"); + return -ENOMEM; + } + + crypto_aead_set_reqsize(tfm, sizeof(struct rkce_cipher_request_ctx)); + + ctx->td_buf->user_data = ctx; + + rkce_enable_clk(algt->rk_dev); + + rk_trace("exit.\n"); + + return 0; +} + +static void rkce_aead_exit_tfm(struct crypto_aead *tfm) +{ + struct rkce_cipher_ctx *ctx = crypto_aead_ctx(tfm); + + rk_trace("enter.\n"); + + rkce_cma_free(ctx->td_buf); + + rkce_disable_clk(ctx->algt->rk_dev); + + memzero_explicit(ctx, sizeof(*ctx)); + + rk_trace("exit.\n"); +} + +static int rkce_aead_setkey(struct crypto_aead *cipher, const uint8_t *key, unsigned int keylen) +{ + rk_trace("enter.\n"); + + return rkce_common_setkey(crypto_aead_ctx(cipher), key, keylen); +} + +static int rkce_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + struct rkce_cipher_ctx *ctx = crypto_aead_ctx(tfm); + + if (authsize < RKCE_TD_TAG_SIZE_MIN || + authsize > RKCE_TD_TAG_SIZE_MAX) + return -EINVAL; + + ctx->authsize = authsize; + + return 0; +} + +static int rkce_aead_encrypt(struct aead_request *req) +{ + rk_trace("enter.\n"); + + return rkce_aead_handle_req(req, true); +} + +static int rkce_aead_decrypt(struct aead_request *req) +{ + rk_trace("enter.\n"); + + return rkce_aead_handle_req(req, false); +} + +struct rkce_algt rkce_ecb_sm4_alg = + RK_CIPHER_ALGO_INIT(SM4, ECB, ecb(sm4), ecb-sm4-rk); + +struct rkce_algt rkce_cbc_sm4_alg = + RK_CIPHER_ALGO_INIT(SM4, CBC, cbc(sm4), cbc-sm4-rk); + +struct rkce_algt rkce_xts_sm4_alg = + RK_CIPHER_ALGO_XTS_INIT(SM4, xts(sm4), xts-sm4-rk); + +struct rkce_algt rkce_cfb_sm4_alg = + RK_CIPHER_ALGO_INIT(SM4, CFB, cfb(sm4), cfb-sm4-rk); + +struct rkce_algt rkce_ofb_sm4_alg = + RK_CIPHER_ALGO_INIT(SM4, OFB, ofb(sm4), ofb-sm4-rk); + +struct rkce_algt rkce_ctr_sm4_alg = + RK_CIPHER_ALGO_INIT(SM4, CTR, ctr(sm4), ctr-sm4-rk); + +struct rkce_algt rkce_gcm_sm4_alg = + RK_AEAD_ALGO_INIT(SM4, GCM, gcm(sm4), gcm-sm4-rk); + +struct rkce_algt rkce_ecb_aes_alg = + RK_CIPHER_ALGO_INIT(AES, ECB, ecb(aes), ecb-aes-rk); + +struct rkce_algt rkce_cbc_aes_alg = + RK_CIPHER_ALGO_INIT(AES, CBC, cbc(aes), cbc-aes-rk); + +struct rkce_algt rkce_xts_aes_alg = + RK_CIPHER_ALGO_XTS_INIT(AES, xts(aes), xts-aes-rk); + +struct rkce_algt rkce_cfb_aes_alg = + RK_CIPHER_ALGO_INIT(AES, CFB, cfb(aes), cfb-aes-rk); + +struct rkce_algt rkce_ofb_aes_alg = + RK_CIPHER_ALGO_INIT(AES, OFB, ofb(aes), ofb-aes-rk); + +struct rkce_algt rkce_ctr_aes_alg = + RK_CIPHER_ALGO_INIT(AES, CTR, ctr(aes), ctr-aes-rk); + +struct rkce_algt rkce_gcm_aes_alg = + RK_AEAD_ALGO_INIT(AES, GCM, gcm(aes), gcm-aes-rk); + +struct rkce_algt rkce_ecb_des_alg = + RK_CIPHER_ALGO_INIT(DES, ECB, ecb(des), ecb-des-rk); + +struct rkce_algt rkce_cbc_des_alg = + RK_CIPHER_ALGO_INIT(DES, CBC, cbc(des), cbc-des-rk); + +struct rkce_algt rkce_cfb_des_alg = + RK_CIPHER_ALGO_INIT(DES, CFB, cfb(des), cfb-des-rk); + +struct rkce_algt rkce_ofb_des_alg = + RK_CIPHER_ALGO_INIT(DES, OFB, ofb(des), ofb-des-rk); + +struct rkce_algt rkce_ecb_des3_ede_alg = + RK_CIPHER_ALGO_INIT(DES3_EDE, ECB, ecb(des3_ede), ecb-des3_ede-rk); + +struct rkce_algt rkce_cbc_des3_ede_alg = + RK_CIPHER_ALGO_INIT(DES3_EDE, CBC, cbc(des3_ede), cbc-des3_ede-rk); + +struct rkce_algt rkce_cfb_des3_ede_alg = + RK_CIPHER_ALGO_INIT(DES3_EDE, CFB, cfb(des3_ede), cfb-des3_ede-rk); + +struct rkce_algt rkce_ofb_des3_ede_alg = + RK_CIPHER_ALGO_INIT(DES3_EDE, OFB, ofb(des3_ede), ofb-des3_ede-rk); diff --git a/drivers/crypto/rockchip/rkce/rkce_skcipher.h b/drivers/crypto/rockchip/rkce/rkce_skcipher.h new file mode 100644 index 000000000000..daa96b868bac --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_skcipher.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Copyright (c) 2024 Rockchip Electronics Co., Ltd. */ + +#ifndef __RKCE_SCKIPHER_H__ +#define __RKCE_SCKIPHER_H__ + +#include + +#include "rkce_dev.h" + +extern struct rkce_algt rkce_ecb_sm4_alg; +extern struct rkce_algt rkce_cbc_sm4_alg; +extern struct rkce_algt rkce_xts_sm4_alg; +extern struct rkce_algt rkce_cfb_sm4_alg; +extern struct rkce_algt rkce_ofb_sm4_alg; +extern struct rkce_algt rkce_ctr_sm4_alg; +extern struct rkce_algt rkce_gcm_sm4_alg; + +extern struct rkce_algt rkce_ecb_aes_alg; +extern struct rkce_algt rkce_cbc_aes_alg; +extern struct rkce_algt rkce_xts_aes_alg; +extern struct rkce_algt rkce_cfb_aes_alg; +extern struct rkce_algt rkce_ofb_aes_alg; +extern struct rkce_algt rkce_ctr_aes_alg; +extern struct rkce_algt rkce_gcm_aes_alg; + +extern struct rkce_algt rkce_ecb_des_alg; +extern struct rkce_algt rkce_cbc_des_alg; +extern struct rkce_algt rkce_cfb_des_alg; +extern struct rkce_algt rkce_ofb_des_alg; + +extern struct rkce_algt rkce_ecb_des3_ede_alg; +extern struct rkce_algt rkce_cbc_des3_ede_alg; +extern struct rkce_algt rkce_cfb_des3_ede_alg; +extern struct rkce_algt rkce_ofb_des3_ede_alg; + +int rkce_cipher_request_callback(int result, uint32_t td_id, void *td_addr); + +#endif diff --git a/drivers/crypto/rockchip/rkce/rkce_sm2signature.asn1 b/drivers/crypto/rockchip/rkce/rkce_sm2signature.asn1 new file mode 100644 index 000000000000..c585eff50954 --- /dev/null +++ b/drivers/crypto/rockchip/rkce/rkce_sm2signature.asn1 @@ -0,0 +1,8 @@ +-- SPDX-License-Identifier: BSD-3-Clause +-- +-- Copyright (c) 2024 Rockchip Electronics Co., Ltd. + +Sm2Signature ::= SEQUENCE { + x INTEGER ({ rkce_ecc_get_signature_r }), + y INTEGER ({ rkce_ecc_get_signature_s }) +}