shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-07 03:15:31 +09:00
Code Issues Packages Projects Releases Wiki Activity

crypto: rockchip: v2&v3 add support gcm

Browse Source 
Signed-off-by: Lin Jinhan <troy.lin@rock-chips.com>
Change-Id: I7e63440c6f652b8939ade56d51957279310bec51
This commit is contained in:
Lin Jinhan
2022-04-12 17:08:59 +08:00
committed by Tao Huang
parent ba7429f655
commit b71bd8763d
14 changed files with 902 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
drivers/crypto/Kconfig
View File

@@ -757,6 +757,7 @@ config CRYPTO_DEV_ROCKCHIP
select CRYPTO_CFB
select CRYPTO_OFB
select CRYPTO_CTR
select CRYPTO_GCM
select CRYPTO_LIB_DES
select CRYPTO_MD5
select CRYPTO_SHA1

92
drivers/crypto/rockchip/rk_crypto_core.c
View File

@@ -10,6 +10,7 @@
* Some ideas are from marvell-cesa.c and s5p-sss.c driver.
*/
#include <crypto/scatterwalk.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include <linux/of.h>
@@ -60,6 +61,7 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev,
{
int ret = -EINVAL;
unsigned int count;
u32 src_nents, dst_nents;
struct device *dev = rk_dev->dev;
struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev->async_req);
@@ -69,13 +71,58 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev,
if (alg_ctx->total == 0)
return 0;
src_nents = alg_ctx->src_nents;
dst_nents = alg_ctx->dst_nents;
/* skip assoclen data */
if (alg_ctx->assoclen && alg_ctx->left_bytes == alg_ctx->total) {
CRYPTO_TRACE("have assoclen...");
if (alg_ctx->assoclen > rk_dev->aad_max) {
ret = -ENOMEM;
goto error;
}
if (!sg_pcopy_to_buffer(alg_ctx->req_src, alg_ctx->src_nents,
rk_dev->addr_aad, alg_ctx->assoclen, 0)) {
dev_err(dev, "[%s:%d] assoc pcopy err\n",
__func__, __LINE__);
ret = -EINVAL;
goto error;
}
sg_init_one(&alg_ctx->sg_aad, rk_dev->addr_aad, alg_ctx->assoclen);
if (!dma_map_sg(dev, &alg_ctx->sg_aad, 1, DMA_TO_DEVICE)) {
dev_err(dev, "[%s:%d] dma_map_sg(sg_aad) error\n",
__func__, __LINE__);
ret = -ENOMEM;
goto error;
}
alg_ctx->addr_aad_in = sg_dma_address(&alg_ctx->sg_aad);
/* point sg_src and sg_dst skip assoc data */
sg_src = scatterwalk_ffwd(rk_dev->src, alg_ctx->req_src,
alg_ctx->assoclen);
sg_dst = (alg_ctx->req_src == alg_ctx->req_dst) ? sg_src :
scatterwalk_ffwd(rk_dev->dst, alg_ctx->req_dst,
alg_ctx->assoclen);
alg_ctx->sg_src = sg_src;
alg_ctx->sg_dst = sg_dst;
src_nents = sg_nents_for_len(sg_src, alg_ctx->total);
dst_nents = sg_nents_for_len(sg_dst, alg_ctx->total);
CRYPTO_TRACE("src_nents = %u, dst_nents = %u", src_nents, dst_nents);
}
if (alg_ctx->left_bytes == alg_ctx->total)
alg_ctx->aligned = rk_crypto_check_align(alg_ctx->req_src, alg_ctx->src_nents,
alg_ctx->req_dst, alg_ctx->dst_nents,
alg_ctx->aligned = rk_crypto_check_align(sg_src, src_nents, sg_dst, dst_nents,
alg_ctx->align_size);
CRYPTO_TRACE("aligned = %d, total = %u, left_bytes = %u\n",
alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes);
CRYPTO_TRACE("aligned = %d, total = %u, left_bytes = %u, assoclen = %u\n",
alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes, alg_ctx->assoclen);
if (alg_ctx->aligned) {
u32 nents;
@@ -118,7 +165,7 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev,
if (!sg_pcopy_to_buffer(alg_ctx->req_src, alg_ctx->src_nents,
rk_dev->addr_vir, count,
alg_ctx->total - alg_ctx->left_bytes)) {
alg_ctx->assoclen + alg_ctx->total - alg_ctx->left_bytes)) {
dev_err(dev, "[%s:%d] pcopy err\n",
__func__, __LINE__);
ret = -EINVAL;
@@ -183,12 +230,22 @@ static int rk_unload_data(struct rk_crypto_dev *rk_dev)
if (!sg_pcopy_from_buffer(alg_ctx->req_dst, alg_ctx->dst_nents,
rk_dev->addr_vir, alg_ctx->count,
alg_ctx->total - alg_ctx->left_bytes -
alg_ctx->count)) {
alg_ctx->count + alg_ctx->assoclen)) {
ret = -EINVAL;
goto exit;
}
}
if (alg_ctx->assoclen) {
dma_unmap_sg(rk_dev->dev, &alg_ctx->sg_aad, 1, DMA_TO_DEVICE);
/* copy assoc data to dst */
if (!sg_pcopy_from_buffer(alg_ctx->req_dst, sg_nents(alg_ctx->req_dst),
rk_dev->addr_aad, alg_ctx->assoclen, 0)) {
ret = -EINVAL;
goto exit;
}
}
exit:
return ret;
}
@@ -421,6 +478,11 @@ static int rk_crypto_register(struct rk_crypto_dev *rk_dev)
err = crypto_register_ahash(&tmp_algs->alg.hash);
} else if (tmp_algs->type == ALG_TYPE_ASYM) {
err = crypto_register_akcipher(&tmp_algs->alg.asym);
} else if (tmp_algs->type == ALG_TYPE_AEAD) {
if (soc_data->use_soft_aes192 &&
tmp_algs->algo == CIPHER_ALGO_AES)
tmp_algs->use_soft_aes192 = true;
err = crypto_register_aead(&tmp_algs->alg.aead);
} else {
continue;
}
@@ -449,6 +511,8 @@ err_cipher_algs:
crypto_unregister_ahash(&tmp_algs->alg.hash);
else if (tmp_algs->type == ALG_TYPE_ASYM)
crypto_unregister_akcipher(&tmp_algs->alg.asym);
else if (tmp_algs->type == ALG_TYPE_AEAD)
crypto_unregister_aead(&tmp_algs->alg.aead);
}
rk_dev->release_crypto(rk_dev, __func__);
@@ -524,8 +588,8 @@ static char *crypto_rv1126_algs_name[] = {
};
static char *crypto_full_algs_name[] = {
"ecb(sm4)", "cbc(sm4)", "cfb(sm4)", "ofb(sm4)", "ctr(sm4)",
"ecb(aes)", "cbc(aes)", "cfb(aes)", "ofb(aes)", "ctr(aes)",
"ecb(sm4)", "cbc(sm4)", "cfb(sm4)", "ofb(sm4)", "ctr(sm4)", "gcm(sm4)",
"ecb(aes)", "cbc(aes)", "cfb(aes)", "ofb(aes)", "ctr(aes)", "gcm(aes)",
"ecb(des)", "cbc(des)", "cfb(des)", "ofb(des)",
"ecb(des3_ede)", "cbc(des3_ede)", "cfb(des3_ede)", "ofb(des3_ede)",
"sha1", "sha224", "sha256", "sha384", "sha512", "md5", "sm3",
@@ -710,6 +774,15 @@ static int rk_crypto_probe(struct platform_device *pdev)
rk_dev->vir_max = RK_BUFFER_SIZE;
rk_dev->addr_aad = (void *)__get_free_page(GFP_KERNEL);
if (!rk_dev->addr_aad) {
err = -ENOMEM;
dev_err(dev, "__get_free_page failed.\n");
goto err_crypto;
}
rk_dev->aad_max = RK_BUFFER_SIZE;
platform_set_drvdata(pdev, rk_dev);
tasklet_init(&rk_dev->queue_task,
@@ -760,6 +833,9 @@ static int rk_crypto_remove(struct platform_device *pdev)
if (rk_dev->addr_vir)
free_pages((unsigned long)rk_dev->addr_vir, RK_BUFFER_ORDER);
if (rk_dev->addr_aad)
free_page((unsigned long)rk_dev->addr_aad);
rk_dev->soc_data->hw_deinit(&pdev->dev, rk_dev->hw_info);
return 0;

43
drivers/crypto/rockchip/rk_crypto_core.h
View File

@@ -12,7 +12,9 @@
#include <crypto/sha.h>
#include <crypto/sm3.h>
#include <crypto/sm4.h>
#include <crypto/gcm.h>
#include <crypto/skcipher.h>
#include <crypto/internal/aead.h>
#include <crypto/internal/akcipher.h>
#include <crypto/internal/hash.h>
#include <crypto/internal/rsa.h>
@@ -68,6 +70,10 @@ struct rk_crypto_dev {
struct crypto_async_request *async_req;
void *addr_vir;
u32 vir_max;
void *addr_aad;
int aad_max;
struct scatterlist src[2];
struct scatterlist dst[2];
struct timer_list timer;
bool busy;
@@ -116,18 +122,22 @@ struct rk_alg_ctx {
struct scatterlist *sg_src;
struct scatterlist *sg_dst;
struct scatterlist sg_tmp;
struct scatterlist sg_aad;
struct scatterlist *req_src;
struct scatterlist *req_dst;
size_t src_nents;
size_t dst_nents;
size_t map_nents;
int is_aead;
unsigned int total;
unsigned int assoclen;
unsigned int count;
unsigned int left_bytes;
dma_addr_t addr_in;
dma_addr_t addr_out;
dma_addr_t addr_aad_in;
bool aligned;
int align_size;
@@ -169,6 +179,7 @@ struct rk_cipher_ctx {
unsigned int keylen;
u32 mode;
u8 iv[AES_BLOCK_SIZE];
u32 iv_len;
u8 lastc[AES_BLOCK_SIZE];
bool is_enc;
void *priv;
@@ -177,6 +188,7 @@ struct rk_cipher_ctx {
bool fallback_key_inited;
struct crypto_skcipher *fallback_tfm;
struct skcipher_request fallback_req; // keep at the end
struct crypto_aead *fallback_aead;
};
struct rk_rsa_ctx {
@@ -193,6 +205,7 @@ enum alg_type {
ALG_TYPE_HMAC,
ALG_TYPE_CIPHER,
ALG_TYPE_ASYM,
ALG_TYPE_AEAD,
};
struct rk_crypto_algt {
@@ -201,6 +214,7 @@ struct rk_crypto_algt {
struct skcipher_alg crypto;
struct ahash_alg hash;
struct akcipher_alg asym;
struct aead_alg aead;
} alg;
enum alg_type type;
u32 algo;
@@ -251,6 +265,35 @@ enum rk_cipher_mode {
#define MD5_BLOCK_SIZE SHA1_BLOCK_SIZE
#define RK_AEAD_ALGO_INIT(cipher_algo, cipher_mode, algo_name, driver_name) {\
.name = #algo_name,\
.type = ALG_TYPE_AEAD,\
.algo = CIPHER_ALGO_##cipher_algo,\
.mode = CIPHER_MODE_##cipher_mode,\
.alg.aead = {\
.base.cra_name = #algo_name,\
.base.cra_driver_name = #driver_name,\
.base.cra_priority = RK_CRYPTO_PRIORITY,\
.base.cra_flags = CRYPTO_ALG_TYPE_AEAD |\
CRYPTO_ALG_KERN_DRIVER_ONLY |\
CRYPTO_ALG_ASYNC |\
CRYPTO_ALG_NEED_FALLBACK,\
.base.cra_blocksize = 1,\
.base.cra_ctxsize = sizeof(struct rk_cipher_ctx),\
.base.cra_alignmask = 0x07,\
.base.cra_module = THIS_MODULE,\
.init = rk_aead_init_tfm,\
.exit = rk_aead_exit_tfm,\
.ivsize = GCM_AES_IV_SIZE,\
.chunksize = cipher_algo##_BLOCK_SIZE,\
.maxauthsize = AES_BLOCK_SIZE,\
.setkey = rk_aead_setkey,\
.setauthsize = rk_aead_gcm_setauthsize,\
.encrypt = rk_aead_encrypt,\
.decrypt = rk_aead_decrypt,\
} \
}
#define RK_CIPHER_ALGO_INIT(cipher_algo, cipher_mode, algo_name, driver_name) {\
.name = #algo_name,\
.type = ALG_TYPE_CIPHER,\

191
drivers/crypto/rockchip/rk_crypto_skcipher_utils.c
View File

@@ -17,11 +17,16 @@ struct rk_crypto_algt *rk_cipher_get_algt(struct crypto_skcipher *tfm)
return container_of(alg, struct rk_crypto_algt, alg.crypto);
}
static struct rk_cipher_ctx *rk_cipher_ctx_cast(struct rk_crypto_dev *rk_dev)
struct rk_crypto_algt *rk_aead_get_algt(struct crypto_aead *tfm)
{
struct skcipher_request *req = skcipher_request_cast(rk_dev->async_req);
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(tfm);
struct aead_alg *alg = crypto_aead_alg(tfm);
return container_of(alg, struct rk_crypto_algt, alg.aead);
}
struct rk_cipher_ctx *rk_cipher_ctx_cast(struct rk_crypto_dev *rk_dev)
{
struct rk_cipher_ctx *ctx = crypto_tfm_ctx(rk_dev->async_req->tfm);
return ctx;
}
@@ -31,15 +36,13 @@ struct rk_alg_ctx *rk_cipher_alg_ctx(struct rk_crypto_dev *rk_dev)
return &(rk_cipher_ctx_cast(rk_dev)->algs_ctx);
}
static bool is_no_multi_blocksize(struct skcipher_request *req)
static bool is_no_multi_blocksize(uint32_t mode)
{
struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher);
return (algt->mode == CIPHER_MODE_CFB ||
algt->mode == CIPHER_MODE_OFB ||
algt->mode == CIPHER_MODE_CTR ||
algt->mode == CIPHER_MODE_XTS) ? true : false;
return (mode == CIPHER_MODE_CFB ||
mode == CIPHER_MODE_OFB ||
mode == CIPHER_MODE_CTR ||
mode == CIPHER_MODE_XTS ||
mode == CIPHER_MODE_GCM) ? true : false;
}
int rk_cipher_fallback(struct skcipher_request *req, struct rk_cipher_ctx *ctx, bool encrypt)
@@ -251,6 +254,7 @@ exit:
int rk_ablk_rx(struct rk_crypto_dev *rk_dev)
{
int err = 0;
struct rk_cipher_ctx *ctx = rk_cipher_ctx_cast(rk_dev);
struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev);
CRYPTO_TRACE("left_bytes = %u\n", alg_ctx->left_bytes);
@@ -273,7 +277,48 @@ int rk_ablk_rx(struct rk_crypto_dev *rk_dev)
}
err = rk_set_data_start(rk_dev);
} else {
rk_iv_copyback(rk_dev);
if (alg_ctx->is_aead) {
u8 hard_tag[RK_MAX_TAG_SIZE];
u8 user_tag[RK_MAX_TAG_SIZE];
struct aead_request *req =
aead_request_cast(rk_dev->async_req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
unsigned int authsize = crypto_aead_authsize(tfm);
CRYPTO_TRACE("cryptlen = %u, assoclen = %u, aead authsize = %u",
alg_ctx->total, alg_ctx->assoclen, authsize);
err = alg_ctx->ops.hw_get_result(rk_dev, hard_tag, authsize);
if (err)
goto out_rx;
CRYPTO_DUMPHEX("hard_tag", hard_tag, authsize);
if (!ctx->is_enc) {
if (!sg_pcopy_to_buffer(alg_ctx->req_src,
sg_nents(alg_ctx->req_src),
user_tag, authsize,
alg_ctx->total +
alg_ctx->assoclen)) {
err = -EINVAL;
goto out_rx;
}
CRYPTO_DUMPHEX("user_tag", user_tag, authsize);
err = crypto_memneq(user_tag, hard_tag, authsize) ? -EBADMSG : 0;
} else {
if (!sg_pcopy_from_buffer(alg_ctx->req_dst,
sg_nents(alg_ctx->req_dst),
hard_tag, authsize,
alg_ctx->total +
alg_ctx->assoclen)) {
err = -EINVAL;
goto out_rx;
}
}
} else {
rk_iv_copyback(rk_dev);
}
}
out_rx:
return err;
@@ -310,10 +355,128 @@ int rk_ablk_start(struct rk_crypto_dev *rk_dev)
int rk_skcipher_handle_req(struct rk_crypto_dev *rk_dev, struct skcipher_request *req)
{
struct rk_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher);
if (!IS_ALIGNED(req->cryptlen, ctx->algs_ctx.chunk_size) &&
!is_no_multi_blocksize(req))
!is_no_multi_blocksize(algt->mode))
return -EINVAL;
else
return rk_dev->enqueue(rk_dev, &req->base);
}
int rk_aead_fallback(struct aead_request *req, struct rk_cipher_ctx *ctx, bool encrypt)
{
int ret;
struct aead_request *subreq = aead_request_ctx(req);
if (!ctx->fallback_aead) {
CRYPTO_TRACE("fallback_tfm is empty");
return -EINVAL;
}
CRYPTO_MSG("use fallback tfm");
if (!ctx->fallback_key_inited) {
ret = crypto_aead_setkey(ctx->fallback_aead, ctx->key, ctx->keylen);
if (ret) {
CRYPTO_MSG("fallback crypto_skcipher_setkey err = %d\n", ret);
goto exit;
}
ctx->fallback_key_inited = true;
}
aead_request_set_tfm(subreq, ctx->fallback_aead);
aead_request_set_callback(subreq, req->base.flags, req->base.complete, req->base.data);
aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, req->iv);
aead_request_set_ad(subreq, req->assoclen);
ret = encrypt ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq);
exit:
return ret;
}
int rk_aead_setkey(struct crypto_aead *cipher, const u8 *key, unsigned int keylen)
{
struct crypto_tfm *tfm = crypto_aead_tfm(cipher);
struct rk_crypto_algt *algt = rk_aead_get_algt(cipher);
struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm);
int ret = -EINVAL;
CRYPTO_MSG("algo = %x, mode = %x, key_len = %d\n", algt->algo, algt->mode, keylen);
switch (algt->algo) {
case CIPHER_ALGO_AES:
if (keylen != AES_KEYSIZE_128 &&
keylen != AES_KEYSIZE_192 &&
keylen != AES_KEYSIZE_256)
goto error;
break;
case CIPHER_ALGO_SM4:
if (keylen != SM4_KEY_SIZE)
goto error;
break;
default:
CRYPTO_TRACE();
goto error;
}
memcpy(ctx->key, key, keylen);
ctx->keylen = keylen;
ctx->fallback_key_inited = false;
return 0;
error:
return ret;
}
int rk_aead_start(struct rk_crypto_dev *rk_dev)
{
struct aead_request *req = aead_request_cast(rk_dev->async_req);
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_aead_ctx(tfm);
struct rk_crypto_algt *algt = rk_aead_get_algt(tfm);
struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev);
unsigned int total = 0, authsize;
unsigned long flags;
int err = 0;
total = req->cryptlen + req->assoclen;
authsize = ctx->is_enc ? 0 : crypto_aead_authsize(tfm);
alg_ctx->total = req->cryptlen - authsize;
alg_ctx->assoclen = req->assoclen;
alg_ctx->sg_src = req->src;
alg_ctx->req_src = req->src;
alg_ctx->src_nents = sg_nents_for_len(req->src, total);
alg_ctx->sg_dst = req->dst;
alg_ctx->req_dst = req->dst;
alg_ctx->dst_nents = sg_nents_for_len(req->dst, total - authsize);
alg_ctx->left_bytes = alg_ctx->total;
CRYPTO_TRACE("src_nents = %zu, dst_nents = %zu", alg_ctx->src_nents, alg_ctx->dst_nents);
CRYPTO_TRACE("is_enc = %d, authsize = %u, cryptlen = %u, total = %u, assoclen = %u",
ctx->is_enc, authsize, req->cryptlen, alg_ctx->total, alg_ctx->assoclen);
spin_lock_irqsave(&rk_dev->lock, flags);
alg_ctx->ops.hw_init(rk_dev, algt->algo, algt->mode);
err = rk_set_data_start(rk_dev);
spin_unlock_irqrestore(&rk_dev->lock, flags);
return err;
}
int rk_aead_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
{
return crypto_gcm_check_authsize(authsize);
}
int rk_aead_handle_req(struct rk_crypto_dev *rk_dev, struct aead_request *req)
{
return rk_dev->enqueue(rk_dev, &req->base);
}

17
drivers/crypto/rockchip/rk_crypto_skcipher_utils.h
View File

@@ -5,16 +5,23 @@
#ifndef __RK_CRYPTO_SKCIPHER_UTILS_H__
#define __RK_CRYPTO_SKCIPHER_UTILS_H__
#include <crypto/aead.h>
#include <crypto/skcipher.h>
#include <crypto/internal/skcipher.h>
#include "rk_crypto_core.h"
#include "rk_crypto_utils.h"
#define RK_MAX_TAG_SIZE 32
struct rk_crypto_algt *rk_cipher_get_algt(struct crypto_skcipher *tfm);
struct rk_crypto_algt *rk_aead_get_algt(struct crypto_aead *tfm);
struct rk_alg_ctx *rk_cipher_alg_ctx(struct rk_crypto_dev *rk_dev);
struct rk_cipher_ctx *rk_cipher_ctx_cast(struct rk_crypto_dev *rk_dev);
int rk_cipher_fallback(struct skcipher_request *req, struct rk_cipher_ctx *ctx, bool encrypt);
int rk_cipher_setkey(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen);
@@ -25,5 +32,15 @@ int rk_ablk_start(struct rk_crypto_dev *rk_dev);
int rk_skcipher_handle_req(struct rk_crypto_dev *rk_dev, struct skcipher_request *req);
int rk_aead_fallback(struct aead_request *req, struct rk_cipher_ctx *ctx, bool encrypt);
int rk_aead_setkey(struct crypto_aead *cipher, const u8 *key, unsigned int keylen);
int rk_aead_start(struct rk_crypto_dev *rk_dev);
int rk_aead_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize);
int rk_aead_handle_req(struct rk_crypto_dev *rk_dev, struct aead_request *req);
#endif

25
drivers/crypto/rockchip/rk_crypto_utils.c
View File

@@ -8,6 +8,7 @@
*
*/
#include <crypto/scatterwalk.h>
#include <linux/scatterlist.h>
#include "rk_crypto_core.h"
@@ -174,10 +175,18 @@ int rk_crypto_hw_desc_alloc(struct device *dev, struct rk_hw_desc *hw_desc)
memset(hw_desc, 0x00, sizeof(*hw_desc));
hw_desc->lli_aad = dma_alloc_coherent(dev, sizeof(struct crypto_lli_desc),
&hw_desc->lli_aad_dma, GFP_KERNEL);
if (!hw_desc->lli_aad)
return -ENOMEM;
///TODO: cma
hw_desc->lli_head = dma_alloc_coherent(dev, lli_len, &hw_desc->lli_head_dma, GFP_KERNEL);
if (!hw_desc->lli_head)
if (!hw_desc->lli_head) {
dma_free_coherent(dev, sizeof(struct crypto_lli_desc),
hw_desc->lli_aad, hw_desc->lli_aad_dma);
return -ENOMEM;
}
hw_desc->lli_tail = hw_desc->lli_head;
hw_desc->total = lli_cnt;
@@ -185,9 +194,9 @@ int rk_crypto_hw_desc_alloc(struct device *dev, struct rk_hw_desc *hw_desc)
memset(hw_desc->lli_head, 0x00, lli_len);
CRYPTO_TRACE("dev = %lx, buffer_len = %u, lli_head = %lx, lli_head_dma = %x",
CRYPTO_TRACE("dev = %lx, buffer_len = %u, lli_head = %lx, lli_head_dma = %lx",
(unsigned long)hw_desc->dev, lli_len,
(unsigned long)hw_desc->lli_head, hw_desc->lli_head_dma);
(unsigned long)hw_desc->lli_head, (unsigned long)hw_desc->lli_head_dma);
return 0;
}
@@ -197,9 +206,13 @@ void rk_crypto_hw_desc_free(struct rk_hw_desc *hw_desc)
if (!hw_desc || !hw_desc->dev || !hw_desc->lli_head)
return;
CRYPTO_TRACE("dev = %lx, buffer_len = %u, lli_head = %lx, lli_head_dma = %x",
(unsigned long)hw_desc->dev, hw_desc->total * sizeof(struct crypto_lli_desc),
(unsigned long)hw_desc->lli_head, hw_desc->lli_head_dma);
CRYPTO_TRACE("dev = %lx, buffer_len = %lu, lli_head = %lx, lli_head_dma = %lx",
(unsigned long)hw_desc->dev,
(unsigned long)hw_desc->total * sizeof(struct crypto_lli_desc),
(unsigned long)hw_desc->lli_head, (unsigned long)hw_desc->lli_head_dma);
dma_free_coherent(hw_desc->dev, sizeof(struct crypto_lli_desc),
hw_desc->lli_aad, hw_desc->lli_aad_dma);
dma_free_coherent(hw_desc->dev, hw_desc->total * sizeof(struct crypto_lli_desc),
hw_desc->lli_head, hw_desc->lli_head_dma);

2
drivers/crypto/rockchip/rk_crypto_utils.h
View File

@@ -26,9 +26,11 @@ struct crypto_lli_desc {
struct rk_hw_desc {
struct device *dev;
struct crypto_lli_desc *lli_aad;
struct crypto_lli_desc *lli_head;
struct crypto_lli_desc *lli_tail;
dma_addr_t lli_head_dma;
dma_addr_t lli_aad_dma;
u32 total;
};

2
drivers/crypto/rockchip/rk_crypto_v2.c
View File

@@ -22,6 +22,7 @@ static struct rk_crypto_algt *crypto_v2_algs[] = {
&rk_v2_cfb_sm4_alg, /* cfb(sm4) */
&rk_v2_ofb_sm4_alg, /* ofb(sm4) */
&rk_v2_ctr_sm4_alg, /* ctr(sm4) */
&rk_v2_gcm_sm4_alg, /* gcm(sm4) */
&rk_v2_ecb_aes_alg, /* ecb(aes) */
&rk_v2_cbc_aes_alg, /* cbc(aes) */
@@ -29,6 +30,7 @@ static struct rk_crypto_algt *crypto_v2_algs[] = {
&rk_v2_cfb_aes_alg, /* cfb(aes) */
&rk_v2_ofb_aes_alg, /* ofb(aes) */
&rk_v2_ctr_aes_alg, /* ctr(aes) */
&rk_v2_gcm_aes_alg, /* gcm(aes) */
&rk_v2_ecb_des_alg, /* ecb(des) */
&rk_v2_cbc_des_alg, /* cbc(des) */

2
drivers/crypto/rockchip/rk_crypto_v2.h
View File

@@ -36,6 +36,7 @@ extern struct rk_crypto_algt rk_v2_xts_sm4_alg;
extern struct rk_crypto_algt rk_v2_cfb_sm4_alg;
extern struct rk_crypto_algt rk_v2_ofb_sm4_alg;
extern struct rk_crypto_algt rk_v2_ctr_sm4_alg;
extern struct rk_crypto_algt rk_v2_gcm_sm4_alg;
extern struct rk_crypto_algt rk_v2_ecb_aes_alg;
extern struct rk_crypto_algt rk_v2_cbc_aes_alg;
@@ -43,6 +44,7 @@ extern struct rk_crypto_algt rk_v2_xts_aes_alg;
extern struct rk_crypto_algt rk_v2_cfb_aes_alg;
extern struct rk_crypto_algt rk_v2_ofb_aes_alg;
extern struct rk_crypto_algt rk_v2_ctr_aes_alg;
extern struct rk_crypto_algt rk_v2_gcm_aes_alg;
extern struct rk_crypto_algt rk_v2_ecb_des_alg;
extern struct rk_crypto_algt rk_v2_cbc_des_alg;

42
drivers/crypto/rockchip/rk_crypto_v2_reg.h
View File

@@ -214,6 +214,40 @@
#define CRYPTO_CH7_KEY_3 0x01fc
#define CRYPTO_KEY_CHANNEL_NUM 8
#define CRYPTO_CH0_PC_LEN_0 0x0280
#define CRYPTO_CH0_PC_LEN_1 0x0284
#define CRYPTO_CH1_PC_LEN_0 0x0288
#define CRYPTO_CH1_PC_LEN_1 0x028c
#define CRYPTO_CH2_PC_LEN_0 0x0290
#define CRYPTO_CH2_PC_LEN_1 0x0294
#define CRYPTO_CH3_PC_LEN_0 0x0298
#define CRYPTO_CH3_PC_LEN_1 0x029c
#define CRYPTO_CH4_PC_LEN_0 0x02a0
#define CRYPTO_CH4_PC_LEN_1 0x02a4
#define CRYPTO_CH5_PC_LEN_0 0x02a8
#define CRYPTO_CH5_PC_LEN_1 0x02ac
#define CRYPTO_CH6_PC_LEN_0 0x02b0
#define CRYPTO_CH6_PC_LEN_1 0x02b4
#define CRYPTO_CH7_PC_LEN_0 0x02b8
#define CRYPTO_CH7_PC_LEN_1 0x02bc
#define CRYPTO_CH0_AAD_LEN_0 0x02c0
#define CRYPTO_CH0_AAD_LEN_1 0x02c4
#define CRYPTO_CH1_AAD_LEN_0 0x02c8
#define CRYPTO_CH1_AAD_LEN_1 0x02cc
#define CRYPTO_CH2_AAD_LEN_0 0x02d0
#define CRYPTO_CH2_AAD_LEN_1 0x02d4
#define CRYPTO_CH3_AAD_LEN_0 0x02d8
#define CRYPTO_CH3_AAD_LEN_1 0x02dc
#define CRYPTO_CH4_AAD_LEN_0 0x02e0
#define CRYPTO_CH4_AAD_LEN_1 0x02e4
#define CRYPTO_CH5_AAD_LEN_0 0x02e8
#define CRYPTO_CH5_AAD_LEN_1 0x02ec
#define CRYPTO_CH6_AAD_LEN_0 0x02f0
#define CRYPTO_CH6_AAD_LEN_1 0x02f4
#define CRYPTO_CH7_AAD_LEN_0 0x02f8
#define CRYPTO_CH7_AAD_LEN_1 0x02fc
#define CRYPTO_CH0_IV_LEN_0 0x0300
#define CRYPTO_CH1_IV_LEN_0 0x0304
#define CRYPTO_CH2_IV_LEN_0 0x0308
@@ -223,6 +257,11 @@
#define CRYPTO_CH6_IV_LEN_0 0x0318
#define CRYPTO_CH7_IV_LEN_0 0x031c
#define CRYPTO_CH0_TAG_0 0x0320
#define CRYPTO_CH0_TAG_1 0x0324
#define CRYPTO_CH0_TAG_2 0x0328
#define CRYPTO_CH0_TAG_3 0x032c
#define CRYPTO_HASH_DOUT_0 0x03a0
#define CRYPTO_HASH_DOUT_1 0x03a4
#define CRYPTO_HASH_DOUT_2 0x03a8
@@ -240,6 +279,9 @@
#define CRYPTO_HASH_DOUT_14 0x03d8
#define CRYPTO_HASH_DOUT_15 0x03dc
#define CRYPTO_TAG_VALID 0x03e0
#define CRYPTO_CH0_TAG_VALID BIT(0)
#define CRYPTO_HASH_VALID 0x03e4
#define CRYPTO_HASH_IS_VALID BIT(0)

252
drivers/crypto/rockchip/rk_crypto_v2_skcipher.c
View File

@@ -9,15 +9,20 @@
* Some ideas are from marvell-cesa.c and s5p-sss.c driver.
*/
#include <crypto/scatterwalk.h>
#include <linux/iopoll.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include "rk_crypto_core.h"
#include "rk_crypto_utils.h"
#include "rk_crypto_skcipher_utils.h"
#include "rk_crypto_v2.h"
#include "rk_crypto_v2_reg.h"
#define RK_POLL_PERIOD_US 100
#define RK_POLL_TIMEOUT_US 50000
static const u32 cipher_algo2bc[] = {
[CIPHER_ALGO_DES] = CRYPTO_BC_DES,
[CIPHER_ALGO_DES3_EDE] = CRYPTO_BC_TDES,
@@ -32,6 +37,7 @@ static const u32 cipher_mode2bc[] = {
[CIPHER_MODE_OFB] = CRYPTO_BC_OFB,
[CIPHER_MODE_CTR] = CRYPTO_BC_CTR,
[CIPHER_MODE_XTS] = CRYPTO_BC_XTS,
[CIPHER_MODE_GCM] = CRYPTO_BC_GCM,
};
static int rk_crypto_irq_handle(int irq, void *dev_id)
@@ -76,6 +82,26 @@ static int rk_crypto_irq_handle(int irq, void *dev_id)
return 0;
}
static inline void set_pc_len_reg(struct rk_crypto_dev *rk_dev, u64 pc_len)
{
u32 chn_base = CRYPTO_CH0_PC_LEN_0;
CRYPTO_TRACE("PC length = %lu\n", (unsigned long)pc_len);
CRYPTO_WRITE(rk_dev, chn_base, pc_len & 0xffffffff);
CRYPTO_WRITE(rk_dev, chn_base + 4, pc_len >> 32);
}
static inline void set_aad_len_reg(struct rk_crypto_dev *rk_dev, u64 aad_len)
{
u32 chn_base = CRYPTO_CH0_AAD_LEN_0;
CRYPTO_TRACE("AAD length = %lu\n", (unsigned long)aad_len);
CRYPTO_WRITE(rk_dev, chn_base, aad_len & 0xffffffff);
CRYPTO_WRITE(rk_dev, chn_base + 4, aad_len >> 32);
}
static void set_iv_reg(struct rk_crypto_dev *rk_dev, const u8 *iv, u32 iv_len)
{
if (!iv || iv_len == 0)
@@ -100,6 +126,29 @@ static void write_tkey_reg(struct rk_crypto_dev *rk_dev, const u8 *key,
rk_crypto_write_regs(rk_dev, CRYPTO_CH4_KEY_0, key, key_len);
}
static int get_tag_reg(struct rk_crypto_dev *rk_dev, u8 *tag, u32 tag_len)
{
int ret;
u32 reg_ctrl = 0;
CRYPTO_TRACE("tag_len = %u", tag_len);
if (tag_len > RK_MAX_TAG_SIZE)
return -EINVAL;
ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_TAG_VALID,
reg_ctrl,
reg_ctrl & CRYPTO_CH0_TAG_VALID,
RK_POLL_PERIOD_US,
RK_POLL_TIMEOUT_US);
if (ret)
goto exit;
rk_crypto_read_regs(rk_dev, CRYPTO_CH0_TAG_0, tag, tag_len);
exit:
return ret;
}
static bool is_force_fallback(struct rk_crypto_algt *algt, uint32_t key_len)
{
if (algt->algo != CIPHER_ALGO_AES)
@@ -230,8 +279,10 @@ static int rk_cipher_crypt(struct skcipher_request *req, bool encrypt)
ctx->mode |= CRYPTO_BC_256_bit_key;
}
if (req->iv)
memcpy(ctx->iv, req->iv, crypto_skcipher_ivsize(tfm));
ctx->iv_len = crypto_skcipher_ivsize(tfm);
memset(ctx->iv, 0x00, sizeof(ctx->iv));
memcpy(ctx->iv, req->iv, ctx->iv_len);
ctx->is_enc = encrypt;
@@ -251,18 +302,12 @@ static int rk_cipher_decrypt(struct skcipher_request *req)
static int rk_ablk_hw_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 mode)
{
struct skcipher_request *req =
skcipher_request_cast(rk_dev->async_req);
struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
u32 ivsize;
struct rk_cipher_ctx *ctx = rk_cipher_ctx_cast(rk_dev);
rk_cipher_reset(rk_dev);
CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, 0x00010000);
ivsize = crypto_skcipher_ivsize(cipher);
if (mode == CIPHER_MODE_XTS) {
uint32_t tmp_len = ctx->keylen / 2;
@@ -273,7 +318,7 @@ static int rk_ablk_hw_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 mode)
}
if (mode != CIPHER_MODE_ECB)
set_iv_reg(rk_dev, req->iv, ivsize);
set_iv_reg(rk_dev, ctx->iv, ctx->iv_len);
ctx->mode |= CRYPTO_BC_ENABLE;
@@ -293,7 +338,7 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag)
struct skcipher_request *req =
skcipher_request_cast(rk_dev->async_req);
struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev);
struct crypto_lli_desc *lli_head, *lli_tail;
struct crypto_lli_desc *lli_head, *lli_tail, *lli_aad;
u32 calc_len = alg_ctx->count;
u32 start_flag = CRYPTO_DMA_START;
int ret;
@@ -309,26 +354,50 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag)
lli_head = hw_info->hw_desc.lli_head;
lli_tail = hw_info->hw_desc.lli_tail;
lli_aad = hw_info->hw_desc.lli_aad;
/*
* the data length is not aligned will use addr_vir to calculate,
* so crypto v2 could round up data length to chunk_size
*/
if (is_calc_need_round_up(req))
if (!alg_ctx->is_aead && is_calc_need_round_up(req))
calc_len = round_up(calc_len, alg_ctx->chunk_size);
CRYPTO_TRACE("calc_len = %u, cryptlen = %u, assoclen= %u, is_aead = %d",
calc_len, alg_ctx->total, alg_ctx->assoclen, alg_ctx->is_aead);
lli_head->user_define = LLI_USER_STRING_START | LLI_USER_CIPHER_START;
lli_tail->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST;
lli_tail->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST;
lli_tail->user_define |= LLI_USER_STRING_LAST;
lli_tail->src_len += (calc_len - alg_ctx->count);
lli_tail->dst_len += (calc_len - alg_ctx->count);
lli_tail->src_len += (calc_len - alg_ctx->count);
lli_tail->dst_len += (calc_len - alg_ctx->count);
if (alg_ctx->is_aead) {
lli_aad->src_addr = alg_ctx->addr_aad_in;
lli_aad->src_len = alg_ctx->assoclen;
lli_aad->user_define = LLI_USER_CIPHER_START |
LLI_USER_STRING_START |
LLI_USER_STRING_LAST |
LLI_USER_STRING_AAD;
lli_aad->next_addr = hw_info->hw_desc.lli_head_dma;
/* clear cipher start */
lli_head->user_define &= (~((u32)LLI_USER_CIPHER_START));
set_pc_len_reg(rk_dev, alg_ctx->total);
set_aad_len_reg(rk_dev, alg_ctx->assoclen);
}
rk_crypto_dump_hw_desc(&hw_info->hw_desc);
dma_wmb();
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma);
if (alg_ctx->is_aead)
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_aad_dma);
else
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma);
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, start_flag | (start_flag << WRITE_MASK));
return 0;
@@ -396,6 +465,151 @@ static void rk_ablk_exit_tfm(struct crypto_skcipher *tfm)
ctx->rk_dev->release_crypto(ctx->rk_dev, alg_name);
}
static int rk_aead_init_tfm(struct crypto_aead *tfm)
{
struct aead_alg *alg = crypto_aead_alg(tfm);
struct rk_crypto_algt *algt =
container_of(alg, struct rk_crypto_algt, alg.aead);
struct rk_cipher_ctx *ctx = crypto_tfm_ctx(&tfm->base);
const char *alg_name = crypto_tfm_alg_name(&tfm->base);
struct rk_crypto_dev *rk_dev = algt->rk_dev;
struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx;
CRYPTO_TRACE();
if (!rk_dev->request_crypto)
return -EFAULT;
rk_dev->request_crypto(rk_dev, alg_name);
alg_ctx->align_size = 64;
alg_ctx->chunk_size = crypto_aead_chunksize(tfm);
alg_ctx->ops.start = rk_aead_start;
alg_ctx->ops.update = rk_ablk_rx;
alg_ctx->ops.complete = rk_crypto_complete;
alg_ctx->ops.irq_handle = rk_crypto_irq_handle;
alg_ctx->ops.hw_init = rk_ablk_hw_init;
alg_ctx->ops.hw_dma_start = crypto_dma_start;
alg_ctx->ops.hw_write_iv = set_iv_reg;
alg_ctx->ops.hw_get_result = get_tag_reg;
ctx->rk_dev = rk_dev;
alg_ctx->is_aead = 1;
if (algt->alg.crypto.base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
CRYPTO_MSG("alloc fallback tfm, name = %s", alg_name);
ctx->fallback_aead =
crypto_alloc_aead(alg_name, 0,
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(ctx->fallback_aead)) {
dev_err(rk_dev->dev,
"Load fallback driver %s err: %ld.\n",
alg_name, PTR_ERR(ctx->fallback_aead));
ctx->fallback_aead = NULL;
}
crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) +
crypto_aead_reqsize(ctx->fallback_aead));
}
return 0;
}
static void rk_aead_exit_tfm(struct crypto_aead *tfm)
{
struct rk_cipher_ctx *ctx = crypto_tfm_ctx(&tfm->base);
CRYPTO_TRACE();
if (ctx->fallback_aead) {
CRYPTO_MSG("free fallback tfm");
crypto_free_aead(ctx->fallback_aead);
}
ctx->rk_dev->release_crypto(ctx->rk_dev, crypto_tfm_alg_name(&tfm->base));
}
static int rk_aead_crypt(struct aead_request *req, bool encrypt)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_aead_ctx(tfm);
struct rk_crypto_algt *algt = rk_aead_get_algt(tfm);
struct scatterlist *sg_src, *sg_dst;
struct scatterlist src[2], dst[2];
u64 data_len;
bool aligned;
int ret = -EINVAL;
CRYPTO_TRACE("%s cryptlen = %u, assoclen = %u",
encrypt ? "encrypt" : "decrypt",
req->cryptlen, req->assoclen);
data_len = encrypt ? req->cryptlen : (req->cryptlen - crypto_aead_authsize(tfm));
if (req->assoclen == 0 ||
req->cryptlen == 0 ||
data_len == 0 ||
is_force_fallback(algt, ctx->keylen))
return rk_aead_fallback(req, ctx, encrypt);
/* point sg_src and sg_dst skip assoc data */
sg_src = scatterwalk_ffwd(src, req->src, req->assoclen);
sg_dst = (req->src == req->dst) ? sg_src : scatterwalk_ffwd(dst, req->dst, req->assoclen);
aligned = rk_crypto_check_align(sg_src, sg_nents_for_len(sg_src, data_len),
sg_dst, sg_nents_for_len(sg_dst, data_len),
64);
if (sg_nents_for_len(sg_src, data_len) > RK_DEFAULT_LLI_CNT ||
sg_nents_for_len(sg_dst, data_len) > RK_DEFAULT_LLI_CNT)
return rk_aead_fallback(req, ctx, encrypt);
if (!aligned) {
if (req->assoclen > ctx->rk_dev->aad_max ||
data_len > ctx->rk_dev->vir_max)
return rk_aead_fallback(req, ctx, encrypt);
}
ctx->mode = cipher_algo2bc[algt->algo] |
cipher_mode2bc[algt->mode];
if (!encrypt)
ctx->mode |= CRYPTO_BC_DECRYPT;
if (algt->algo == CIPHER_ALGO_AES) {
if (ctx->keylen == AES_KEYSIZE_128)
ctx->mode |= CRYPTO_BC_128_bit_key;
else if (ctx->keylen == AES_KEYSIZE_192)
ctx->mode |= CRYPTO_BC_192_bit_key;
else if (ctx->keylen == AES_KEYSIZE_256)
ctx->mode |= CRYPTO_BC_256_bit_key;
}
ctx->iv_len = crypto_aead_ivsize(tfm);
memset(ctx->iv, 0x00, sizeof(ctx->iv));
memcpy(ctx->iv, req->iv, ctx->iv_len);
ctx->is_enc = encrypt;
CRYPTO_MSG("ctx->mode = %x\n", ctx->mode);
ret = rk_aead_handle_req(ctx->rk_dev, req);
return ret;
}
static int rk_aead_encrypt(struct aead_request *req)
{
return rk_aead_crypt(req, true);
}
static int rk_aead_decrypt(struct aead_request *req)
{
return rk_aead_crypt(req, false);
}
struct rk_crypto_algt rk_v2_ecb_sm4_alg =
RK_CIPHER_ALGO_INIT(SM4, ECB, ecb(sm4), ecb-sm4-rk);
@@ -414,6 +628,9 @@ struct rk_crypto_algt rk_v2_ofb_sm4_alg =
struct rk_crypto_algt rk_v2_ctr_sm4_alg =
RK_CIPHER_ALGO_INIT(SM4, CTR, ctr(sm4), ctr-sm4-rk);
struct rk_crypto_algt rk_v2_gcm_sm4_alg =
RK_AEAD_ALGO_INIT(SM4, GCM, gcm(sm4), gcm-sm4-rk);
struct rk_crypto_algt rk_v2_ecb_aes_alg =
RK_CIPHER_ALGO_INIT(AES, ECB, ecb(aes), ecb-aes-rk);
@@ -432,6 +649,9 @@ struct rk_crypto_algt rk_v2_ofb_aes_alg =
struct rk_crypto_algt rk_v2_ctr_aes_alg =
RK_CIPHER_ALGO_INIT(AES, CTR, ctr(aes), ctr-aes-rk);
struct rk_crypto_algt rk_v2_gcm_aes_alg =
RK_AEAD_ALGO_INIT(AES, GCM, gcm(aes), gcm-aes-rk);
struct rk_crypto_algt rk_v2_ecb_des_alg =
RK_CIPHER_ALGO_INIT(DES, ECB, ecb(des), ecb-des-rk);

72
drivers/crypto/rockchip/rk_crypto_v3.c
View File

@@ -52,46 +52,48 @@ static const char * const crypto_v3_rsts[] = {
};
static struct rk_crypto_algt *crypto_v3_algs[] = {
&rk_v3_ecb_sm4_alg, /* ecb(sm4) */
&rk_v3_cbc_sm4_alg, /* cbc(sm4) */
&rk_v3_xts_sm4_alg, /* xts(sm4) */
&rk_v3_cfb_sm4_alg, /* cfb(sm4) */
&rk_v3_ofb_sm4_alg, /* ofb(sm4) */
&rk_v3_ctr_sm4_alg, /* ctr(sm4) */
&rk_v3_ecb_sm4_alg, /* ecb(sm4) */
&rk_v3_cbc_sm4_alg, /* cbc(sm4) */
&rk_v3_xts_sm4_alg, /* xts(sm4) */
&rk_v3_cfb_sm4_alg, /* cfb(sm4) */
&rk_v3_ofb_sm4_alg, /* ofb(sm4) */
&rk_v3_ctr_sm4_alg, /* ctr(sm4) */
&rk_v3_gcm_sm4_alg, /* ctr(sm4) */
&rk_v3_ecb_aes_alg, /* ecb(aes) */
&rk_v3_cbc_aes_alg, /* cbc(aes) */
&rk_v3_xts_aes_alg, /* xts(aes) */
&rk_v3_cfb_aes_alg, /* cfb(aes) */
&rk_v3_ofb_aes_alg, /* ofb(aes) */
&rk_v3_ctr_aes_alg, /* ctr(aes) */
&rk_v3_ecb_aes_alg, /* ecb(aes) */
&rk_v3_cbc_aes_alg, /* cbc(aes) */
&rk_v3_xts_aes_alg, /* xts(aes) */
&rk_v3_cfb_aes_alg, /* cfb(aes) */
&rk_v3_ofb_aes_alg, /* ofb(aes) */
&rk_v3_ctr_aes_alg, /* ctr(aes) */
&rk_v3_gcm_aes_alg, /* gcm(aes) */
&rk_v3_ecb_des_alg, /* ecb(des) */
&rk_v3_cbc_des_alg, /* cbc(des) */
&rk_v3_cfb_des_alg, /* cfb(des) */
&rk_v3_ofb_des_alg, /* ofb(des) */
&rk_v3_ecb_des_alg, /* ecb(des) */
&rk_v3_cbc_des_alg, /* cbc(des) */
&rk_v3_cfb_des_alg, /* cfb(des) */
&rk_v3_ofb_des_alg, /* ofb(des) */
&rk_v3_ecb_des3_ede_alg, /* ecb(des3_ede) */
&rk_v3_cbc_des3_ede_alg, /* cbc(des3_ede) */
&rk_v3_cfb_des3_ede_alg, /* cfb(des3_ede) */
&rk_v3_ofb_des3_ede_alg, /* ofb(des3_ede) */
&rk_v3_ecb_des3_ede_alg, /* ecb(des3_ede) */
&rk_v3_cbc_des3_ede_alg, /* cbc(des3_ede) */
&rk_v3_cfb_des3_ede_alg, /* cfb(des3_ede) */
&rk_v3_ofb_des3_ede_alg, /* ofb(des3_ede) */
&rk_v3_ahash_sha1, /* sha1 */
&rk_v3_ahash_sha224, /* sha224 */
&rk_v3_ahash_sha256, /* sha256 */
&rk_v3_ahash_sha384, /* sha384 */
&rk_v3_ahash_sha512, /* sha512 */
&rk_v3_ahash_md5, /* md5 */
&rk_v3_ahash_sm3, /* sm3 */
&rk_v3_ahash_sha1, /* sha1 */
&rk_v3_ahash_sha224, /* sha224 */
&rk_v3_ahash_sha256, /* sha256 */
&rk_v3_ahash_sha384, /* sha384 */
&rk_v3_ahash_sha512, /* sha512 */
&rk_v3_ahash_md5, /* md5 */
&rk_v3_ahash_sm3, /* sm3 */
&rk_v3_hmac_sha1, /* hmac(sha1) */
&rk_v3_hmac_sha256, /* hmac(sha256) */
&rk_v3_hmac_sha512, /* hmac(sha512) */
&rk_v3_hmac_md5, /* hmac(md5) */
&rk_v3_hmac_sm3, /* hmac(sm3) */
&rk_v3_hmac_sha1, /* hmac(sha1) */
&rk_v3_hmac_sha256, /* hmac(sha256) */
&rk_v3_hmac_sha512, /* hmac(sha512) */
&rk_v3_hmac_md5, /* hmac(md5) */
&rk_v3_hmac_sm3, /* hmac(sm3) */
/* Shared v2 version implementation */
&rk_v2_asym_rsa, /* rsa */
/* Shared v2 version implementation */
&rk_v2_asym_rsa, /* rsa */
};
static bool rk_is_cipher_support(struct rk_crypto_dev *rk_dev, u32 algo, u32 mode, u32 key_len)
@@ -199,7 +201,7 @@ struct rk_crypto_algt **rk_hw_crypto_v3_get_algts(uint32_t *num)
bool rk_hw_crypto_v3_algo_valid(struct rk_crypto_dev *rk_dev, struct rk_crypto_algt *aglt)
{
if (aglt->type == ALG_TYPE_CIPHER) {
if (aglt->type == ALG_TYPE_CIPHER || aglt->type == ALG_TYPE_AEAD) {
CRYPTO_TRACE("CIPHER");
return rk_is_cipher_support(rk_dev, aglt->algo, aglt->mode, 0);
} else if (aglt->type == ALG_TYPE_HASH || aglt->type == ALG_TYPE_HMAC) {

2
drivers/crypto/rockchip/rk_crypto_v3.h
View File

@@ -36,6 +36,7 @@ extern struct rk_crypto_algt rk_v3_xts_sm4_alg;
extern struct rk_crypto_algt rk_v3_cfb_sm4_alg;
extern struct rk_crypto_algt rk_v3_ofb_sm4_alg;
extern struct rk_crypto_algt rk_v3_ctr_sm4_alg;
extern struct rk_crypto_algt rk_v3_gcm_sm4_alg;
extern struct rk_crypto_algt rk_v3_ecb_aes_alg;
extern struct rk_crypto_algt rk_v3_cbc_aes_alg;
@@ -43,6 +44,7 @@ extern struct rk_crypto_algt rk_v3_xts_aes_alg;
extern struct rk_crypto_algt rk_v3_cfb_aes_alg;
extern struct rk_crypto_algt rk_v3_ofb_aes_alg;
extern struct rk_crypto_algt rk_v3_ctr_aes_alg;
extern struct rk_crypto_algt rk_v3_gcm_aes_alg;
extern struct rk_crypto_algt rk_v3_ecb_des_alg;
extern struct rk_crypto_algt rk_v3_cbc_des_alg;

256
drivers/crypto/rockchip/rk_crypto_v3_skcipher.c
View File

@@ -8,15 +8,20 @@
*
*/
#include <crypto/scatterwalk.h>
#include <linux/iopoll.h>
#include <linux/module.h>
#include <linux/platform_device.h>
#include "rk_crypto_core.h"
#include "rk_crypto_utils.h"
#include "rk_crypto_skcipher_utils.h"
#include "rk_crypto_v3.h"
#include "rk_crypto_v3_reg.h"
#define RK_POLL_PERIOD_US 100
#define RK_POLL_TIMEOUT_US 50000
static const u32 cipher_algo2bc[] = {
[CIPHER_ALGO_DES] = CRYPTO_BC_DES,
[CIPHER_ALGO_DES3_EDE] = CRYPTO_BC_TDES,
@@ -31,6 +36,7 @@ static const u32 cipher_mode2bc[] = {
[CIPHER_MODE_OFB] = CRYPTO_BC_OFB,
[CIPHER_MODE_CTR] = CRYPTO_BC_CTR,
[CIPHER_MODE_XTS] = CRYPTO_BC_XTS,
[CIPHER_MODE_GCM] = CRYPTO_BC_GCM,
};
static int rk_crypto_irq_handle(int irq, void *dev_id)
@@ -75,6 +81,26 @@ static int rk_crypto_irq_handle(int irq, void *dev_id)
return 0;
}
static inline void set_pc_len_reg(struct rk_crypto_dev *rk_dev, u64 pc_len)
{
u32 chn_base = CRYPTO_CH0_PC_LEN_0;
CRYPTO_TRACE("PC length = %lu\n", (unsigned long)pc_len);
CRYPTO_WRITE(rk_dev, chn_base, pc_len & 0xffffffff);
CRYPTO_WRITE(rk_dev, chn_base + 4, pc_len >> 32);
}
static inline void set_aad_len_reg(struct rk_crypto_dev *rk_dev, u64 aad_len)
{
u32 chn_base = CRYPTO_CH0_AAD_LEN_0;
CRYPTO_TRACE("AAD length = %lu\n", (unsigned long)aad_len);
CRYPTO_WRITE(rk_dev, chn_base, aad_len & 0xffffffff);
CRYPTO_WRITE(rk_dev, chn_base + 4, aad_len >> 32);
}
static void set_iv_reg(struct rk_crypto_dev *rk_dev, const u8 *iv, u32 iv_len)
{
if (!iv || iv_len == 0)
@@ -99,6 +125,29 @@ static void write_tkey_reg(struct rk_crypto_dev *rk_dev, const u8 *key,
rk_crypto_write_regs(rk_dev, CRYPTO_CH4_KEY_0, key, key_len);
}
static int get_tag_reg(struct rk_crypto_dev *rk_dev, u8 *tag, u32 tag_len)
{
int ret;
u32 reg_ctrl = 0;
CRYPTO_TRACE("tag_len = %u", tag_len);
if (tag_len > RK_MAX_TAG_SIZE)
return -EINVAL;
ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_TAG_VALID,
reg_ctrl,
reg_ctrl & CRYPTO_CH0_TAG_VALID,
RK_POLL_PERIOD_US,
RK_POLL_TIMEOUT_US);
if (ret)
goto exit;
rk_crypto_read_regs(rk_dev, CRYPTO_CH0_TAG_0, tag, tag_len);
exit:
return ret;
}
static bool is_force_fallback(struct rk_crypto_algt *algt, uint32_t key_len)
{
if (algt->algo != CIPHER_ALGO_AES)
@@ -120,8 +169,8 @@ static bool is_calc_need_round_up(struct skcipher_request *req)
struct rk_crypto_algt *algt = rk_cipher_get_algt(cipher);
return (algt->mode == CIPHER_MODE_CFB ||
algt->mode == CIPHER_MODE_OFB ||
algt->mode == CIPHER_MODE_CTR) ? true : false;
algt->mode == CIPHER_MODE_OFB ||
algt->mode == CIPHER_MODE_CTR) ? true : false;
}
static void rk_cipher_reset(struct rk_crypto_dev *rk_dev)
@@ -229,8 +278,10 @@ static int rk_cipher_crypt(struct skcipher_request *req, bool encrypt)
ctx->mode |= CRYPTO_BC_256_bit_key;
}
if (req->iv)
memcpy(ctx->iv, req->iv, crypto_skcipher_ivsize(tfm));
ctx->iv_len = crypto_skcipher_ivsize(tfm);
memset(ctx->iv, 0x00, sizeof(ctx->iv));
memcpy(ctx->iv, req->iv, ctx->iv_len);
ctx->is_enc = encrypt;
@@ -250,18 +301,12 @@ static int rk_cipher_decrypt(struct skcipher_request *req)
static int rk_ablk_hw_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 mode)
{
struct skcipher_request *req =
skcipher_request_cast(rk_dev->async_req);
struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
u32 ivsize;
struct rk_cipher_ctx *ctx = rk_cipher_ctx_cast(rk_dev);
rk_cipher_reset(rk_dev);
CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, 0x00010000);
ivsize = crypto_skcipher_ivsize(cipher);
if (mode == CIPHER_MODE_XTS) {
uint32_t tmp_len = ctx->keylen / 2;
@@ -272,7 +317,7 @@ static int rk_ablk_hw_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 mode)
}
if (mode != CIPHER_MODE_ECB)
set_iv_reg(rk_dev, req->iv, ivsize);
set_iv_reg(rk_dev, ctx->iv, ctx->iv_len);
ctx->mode |= CRYPTO_BC_ENABLE;
@@ -292,7 +337,7 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag)
struct skcipher_request *req =
skcipher_request_cast(rk_dev->async_req);
struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev);
struct crypto_lli_desc *lli_head, *lli_tail;
struct crypto_lli_desc *lli_head, *lli_tail, *lli_aad;
u32 calc_len = alg_ctx->count;
u32 start_flag = CRYPTO_DMA_START;
int ret;
@@ -308,26 +353,50 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag)
lli_head = hw_info->hw_desc.lli_head;
lli_tail = hw_info->hw_desc.lli_tail;
lli_aad = hw_info->hw_desc.lli_aad;
/*
* the data length is not aligned will use addr_vir to calculate,
* so crypto v2 could round up data length to chunk_size
*/
if (is_calc_need_round_up(req))
if (!alg_ctx->is_aead && is_calc_need_round_up(req))
calc_len = round_up(calc_len, alg_ctx->chunk_size);
CRYPTO_TRACE("calc_len = %u, cryptlen = %u, assoclen= %u, is_aead = %d",
calc_len, alg_ctx->total, alg_ctx->assoclen, alg_ctx->is_aead);
lli_head->user_define = LLI_USER_STRING_START | LLI_USER_CIPHER_START;
lli_tail->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST;
lli_tail->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST;
lli_tail->user_define |= LLI_USER_STRING_LAST;
lli_tail->src_len += (calc_len - alg_ctx->count);
lli_tail->dst_len += (calc_len - alg_ctx->count);
lli_tail->src_len += (calc_len - alg_ctx->count);
lli_tail->dst_len += (calc_len - alg_ctx->count);
if (alg_ctx->is_aead) {
lli_aad->src_addr = alg_ctx->addr_aad_in;
lli_aad->src_len = alg_ctx->assoclen;
lli_aad->user_define = LLI_USER_CIPHER_START |
LLI_USER_STRING_START |
LLI_USER_STRING_LAST |
LLI_USER_STRING_AAD;
lli_aad->next_addr = hw_info->hw_desc.lli_head_dma;
/* clear cipher start */
lli_head->user_define &= (~((u32)LLI_USER_CIPHER_START));
set_pc_len_reg(rk_dev, alg_ctx->total);
set_aad_len_reg(rk_dev, alg_ctx->assoclen);
}
rk_crypto_dump_hw_desc(&hw_info->hw_desc);
dma_wmb();
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma);
if (alg_ctx->is_aead)
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_aad_dma);
else
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma);
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, start_flag | (start_flag << WRITE_MASK));
return 0;
@@ -395,6 +464,151 @@ static void rk_ablk_exit_tfm(struct crypto_skcipher *tfm)
ctx->rk_dev->release_crypto(ctx->rk_dev, alg_name);
}
static int rk_aead_init_tfm(struct crypto_aead *tfm)
{
struct aead_alg *alg = crypto_aead_alg(tfm);
struct rk_crypto_algt *algt =
container_of(alg, struct rk_crypto_algt, alg.aead);
struct rk_cipher_ctx *ctx = crypto_tfm_ctx(&tfm->base);
const char *alg_name = crypto_tfm_alg_name(&tfm->base);
struct rk_crypto_dev *rk_dev = algt->rk_dev;
struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx;
CRYPTO_TRACE();
if (!rk_dev->request_crypto)
return -EFAULT;
rk_dev->request_crypto(rk_dev, alg_name);
alg_ctx->align_size = 64;
alg_ctx->chunk_size = crypto_aead_chunksize(tfm);
alg_ctx->ops.start = rk_aead_start;
alg_ctx->ops.update = rk_ablk_rx;
alg_ctx->ops.complete = rk_crypto_complete;
alg_ctx->ops.irq_handle = rk_crypto_irq_handle;
alg_ctx->ops.hw_init = rk_ablk_hw_init;
alg_ctx->ops.hw_dma_start = crypto_dma_start;
alg_ctx->ops.hw_write_iv = set_iv_reg;
alg_ctx->ops.hw_get_result = get_tag_reg;
ctx->rk_dev = rk_dev;
alg_ctx->is_aead = 1;
if (algt->alg.crypto.base.cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
CRYPTO_MSG("alloc fallback tfm, name = %s", alg_name);
ctx->fallback_aead =
crypto_alloc_aead(alg_name, 0,
CRYPTO_ALG_ASYNC |
CRYPTO_ALG_NEED_FALLBACK);
if (IS_ERR(ctx->fallback_aead)) {
dev_err(rk_dev->dev,
"Load fallback driver %s err: %ld.\n",
alg_name, PTR_ERR(ctx->fallback_aead));
ctx->fallback_aead = NULL;
}
crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) +
crypto_aead_reqsize(ctx->fallback_aead));
}
return 0;
}
static void rk_aead_exit_tfm(struct crypto_aead *tfm)
{
struct rk_cipher_ctx *ctx = crypto_tfm_ctx(&tfm->base);
CRYPTO_TRACE();
if (ctx->fallback_aead) {
CRYPTO_MSG("free fallback tfm");
crypto_free_aead(ctx->fallback_aead);
}
ctx->rk_dev->release_crypto(ctx->rk_dev, crypto_tfm_alg_name(&tfm->base));
}
static int rk_aead_crypt(struct aead_request *req, bool encrypt)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct rk_cipher_ctx *ctx = crypto_aead_ctx(tfm);
struct rk_crypto_algt *algt = rk_aead_get_algt(tfm);
struct scatterlist *sg_src, *sg_dst;
struct scatterlist src[2], dst[2];
u64 data_len;
bool aligned;
int ret = -EINVAL;
CRYPTO_TRACE("%s cryptlen = %u, assoclen = %u",
encrypt ? "encrypt" : "decrypt",
req->cryptlen, req->assoclen);
data_len = encrypt ? req->cryptlen : (req->cryptlen - crypto_aead_authsize(tfm));
if (req->assoclen == 0 ||
req->cryptlen == 0 ||
data_len == 0 ||
is_force_fallback(algt, ctx->keylen))
return rk_aead_fallback(req, ctx, encrypt);
/* point sg_src and sg_dst skip assoc data */
sg_src = scatterwalk_ffwd(src, req->src, req->assoclen);
sg_dst = (req->src == req->dst) ? sg_src : scatterwalk_ffwd(dst, req->dst, req->assoclen);
aligned = rk_crypto_check_align(sg_src, sg_nents_for_len(sg_src, data_len),
sg_dst, sg_nents_for_len(sg_dst, data_len),
64);
if (sg_nents_for_len(sg_src, data_len) > RK_DEFAULT_LLI_CNT ||
sg_nents_for_len(sg_dst, data_len) > RK_DEFAULT_LLI_CNT)
return rk_aead_fallback(req, ctx, encrypt);
if (!aligned) {
if (req->assoclen > ctx->rk_dev->aad_max ||
data_len > ctx->rk_dev->vir_max)
return rk_aead_fallback(req, ctx, encrypt);
}
ctx->mode = cipher_algo2bc[algt->algo] |
cipher_mode2bc[algt->mode];
if (!encrypt)
ctx->mode |= CRYPTO_BC_DECRYPT;
if (algt->algo == CIPHER_ALGO_AES) {
if (ctx->keylen == AES_KEYSIZE_128)
ctx->mode |= CRYPTO_BC_128_bit_key;
else if (ctx->keylen == AES_KEYSIZE_192)
ctx->mode |= CRYPTO_BC_192_bit_key;
else if (ctx->keylen == AES_KEYSIZE_256)
ctx->mode |= CRYPTO_BC_256_bit_key;
}
ctx->iv_len = crypto_aead_ivsize(tfm);
memset(ctx->iv, 0x00, sizeof(ctx->iv));
memcpy(ctx->iv, req->iv, ctx->iv_len);
ctx->is_enc = encrypt;
CRYPTO_MSG("ctx->mode = %x\n", ctx->mode);
ret = rk_aead_handle_req(ctx->rk_dev, req);
return ret;
}
static int rk_aead_encrypt(struct aead_request *req)
{
return rk_aead_crypt(req, true);
}
static int rk_aead_decrypt(struct aead_request *req)
{
return rk_aead_crypt(req, false);
}
struct rk_crypto_algt rk_v3_ecb_sm4_alg =
RK_CIPHER_ALGO_INIT(SM4, ECB, ecb(sm4), ecb-sm4-rk);
@@ -413,6 +627,9 @@ struct rk_crypto_algt rk_v3_ofb_sm4_alg =
struct rk_crypto_algt rk_v3_ctr_sm4_alg =
RK_CIPHER_ALGO_INIT(SM4, CTR, ctr(sm4), ctr-sm4-rk);
struct rk_crypto_algt rk_v3_gcm_sm4_alg =
RK_AEAD_ALGO_INIT(SM4, GCM, gcm(sm4), gcm-sm4-rk);
struct rk_crypto_algt rk_v3_ecb_aes_alg =
RK_CIPHER_ALGO_INIT(AES, ECB, ecb(aes), ecb-aes-rk);
@@ -431,6 +648,9 @@ struct rk_crypto_algt rk_v3_ofb_aes_alg =
struct rk_crypto_algt rk_v3_ctr_aes_alg =
RK_CIPHER_ALGO_INIT(AES, CTR, ctr(aes), ctr-aes-rk);
struct rk_crypto_algt rk_v3_gcm_aes_alg =
RK_AEAD_ALGO_INIT(AES, GCM, gcm(aes), gcm-aes-rk);
struct rk_crypto_algt rk_v3_ecb_des_alg =
RK_CIPHER_ALGO_INIT(DES, ECB, ecb(des), ecb-des-rk);