crypto: rockchip: v2: ahash init/update/final use hardware crypto

1. disable crypto interrupt when crypto is idle
2. ahash init/update/final use hardware crypto
3. reset crypto cc before cipher/hash hardware init

Signed-off-by: Lin Jinhan <troy.lin@rock-chips.com>
Change-Id: Id2c85c36c9f2fbe34af553b3ba12239e3cd281b4

drivers/crypto/rockchip/rk_crypto_core.c

@@ -103,17 +103,21 @@ static int check_scatter_align(struct scatterlist *sg_src,
 static bool check_from_dmafd(struct crypto_async_request *async_req)
 {
 	struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(async_req);
+	bool in, out;
 
-	if (alg_ctx->src_nents == 1 &&
-	    sg_virt(alg_ctx->req_src) &&
-	    sg_dma_address(alg_ctx->req_src) &&
-	    alg_ctx->dst_nents == 1 &&
-	    sg_virt(alg_ctx->req_dst) &&
-	    sg_dma_address(alg_ctx->req_dst) &&
-	    sg_dma_len(alg_ctx->req_src) == sg_dma_len(alg_ctx->req_dst))
-		return true;
+	in = alg_ctx->src_nents == 1 &&
+	     sg_virt(alg_ctx->req_src) &&
+	     sg_dma_address(alg_ctx->req_src);
 
-	return false;
+	if (!alg_ctx->req_dst)
+		return in;
+
+	out = alg_ctx->dst_nents == 1 &&
+	      sg_virt(alg_ctx->req_dst) &&
+	      sg_dma_address(alg_ctx->req_dst) &&
+	      sg_dma_len(alg_ctx->req_src) == sg_dma_len(alg_ctx->req_dst);
+
+	return in && out;
 }
 
 static bool check_scatterlist_align(struct crypto_async_request *async_req,
@@ -124,21 +128,23 @@ static bool check_scatterlist_align(struct crypto_async_request *async_req,
 	struct scatterlist *dst_tmp = NULL;
 	unsigned int i;
 
-	if (alg_ctx->src_nents != alg_ctx->dst_nents)
+	if (alg_ctx->req_dst && alg_ctx->src_nents != alg_ctx->dst_nents)
 		return false;
 
 	src_tmp = alg_ctx->req_src;
 	dst_tmp = alg_ctx->req_dst;
 
 	for (i = 0; i < alg_ctx->src_nents; i++) {
-		if (!src_tmp || !dst_tmp)
+		if (!src_tmp)
 			return false;
 
 		if (!check_scatter_align(src_tmp, dst_tmp, align_mask))
 			return false;
 
 		src_tmp = sg_next(src_tmp);
-		dst_tmp = sg_next(dst_tmp);
+
+		if (alg_ctx->req_dst)
+			dst_tmp = sg_next(dst_tmp);
 	}
 
 	return true;
@@ -153,6 +159,10 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev,
 	struct device *dev = rk_dev->dev;
 	struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev->async_req);
 
+	/* 0 data input just do nothing */
+	if (alg_ctx->total == 0)
+		return 0;
+
 	if (alg_ctx->left_bytes == alg_ctx->total) {
 		alg_ctx->is_dma = check_from_dmafd(rk_dev->async_req);
 		alg_ctx->aligned = check_scatterlist_align(rk_dev->async_req,
@@ -239,7 +249,8 @@ static int rk_unload_data(struct rk_crypto_dev *rk_dev)
 	CRYPTO_TRACE("aligned = %d, total = %u, left_bytes = %u\n",
 		     alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes);
 
-	if (alg_ctx->count == 0)
+	/* 0 data input just do nothing */
+	if (alg_ctx->total == 0 || alg_ctx->count == 0)
 		return 0;
 
 	sg_in = alg_ctx->aligned ? alg_ctx->sg_src : &alg_ctx->sg_tmp;
@@ -306,12 +317,19 @@ static int rk_start_op(struct rk_crypto_dev *rk_dev)
 
 	alg_ctx->aligned = false;
 
+	enable_irq(rk_dev->irq);
 	start_irq_timer(rk_dev);
 
 	ret = alg_ctx->ops.start(rk_dev);
 	if (ret)
 		return ret;
 
+	/* fake calculations are used to trigger the Done Task */
+	if (alg_ctx->total == 0) {
+		CRYPTO_TRACE("fake done_task");
+		tasklet_schedule(&rk_dev->done_task);
+	}
+
 	return 0;
 }
 
@@ -329,6 +347,7 @@ static void rk_complete_op(struct rk_crypto_dev *rk_dev, int err)
 {
 	struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev->async_req);
 
+	disable_irq(rk_dev->irq);
 	del_timer(&rk_dev->timer);
 
 	if (!alg_ctx || !alg_ctx->ops.complete)
@@ -336,6 +355,9 @@ static void rk_complete_op(struct rk_crypto_dev *rk_dev, int err)
 
 	alg_ctx->ops.complete(rk_dev->async_req, err);
 
+	if (err)
+		dev_err(rk_dev->dev, "complete_op err = %d\n", err);
+
 	tasklet_schedule(&rk_dev->queue_task);
 }
 
@@ -395,7 +417,8 @@ static void rk_crypto_done_task_cb(unsigned long data)
 	if (rk_dev->err)
 		goto exit;
 
-	if (alg_ctx->total && alg_ctx->left_bytes == 0) {
+	if (alg_ctx->left_bytes == 0) {
+		CRYPTO_TRACE("done task cb last calc");
 		/* unload data for last calculation */
 		rk_dev->err = rk_update_op(rk_dev);
 		goto exit;
@@ -756,6 +779,8 @@ static int rk_crypto_probe(struct platform_device *pdev)
 		goto err_crypto;
 	}
 
+	disable_irq(rk_dev->irq);
+
 	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
 	if (err) {
 		dev_err(dev, "crypto: No suitable DMA available.\n");

drivers/crypto/rockchip/rk_crypto_core.h

@@ -132,6 +132,13 @@ struct rk_ahash_ctx {
 	struct rk_alg_ctx		algs_ctx;
 	struct rk_crypto_dev		*rk_dev;
 	u8				authkey[SHA512_BLOCK_SIZE];
+	u32				authkey_len;
+	struct scatterlist		hash_sg[2];
+	u8				*hash_tmp;
+	u32				hash_tmp_len;
+
+	u8				lastc[RK_DMA_ALIGNMENT];
+	u32				lastc_len;
 
 	/* for fallback */
 	struct crypto_ahash		*fallback_tfm;
@@ -139,9 +146,9 @@ struct rk_ahash_ctx {
 
 /* the privete variable of hash for fallback */
 struct rk_ahash_rctx {
-	struct rk_alg_ctx		algs_ctx;
 	struct ahash_request		fallback_req;
 	u32				mode;
+	bool				is_final;
 };
 
 /* the private variable of cipher */
@@ -305,7 +312,7 @@ enum rk_cipher_mode {
 					     CRYPTO_ALG_NEED_FALLBACK,\
 				.cra_blocksize = hash_algo##_BLOCK_SIZE,\
 				.cra_ctxsize = sizeof(struct rk_ahash_ctx),\
-				.cra_alignmask = 3,\
+				.cra_alignmask = 0,\
 				.cra_init = rk_cra_hash_init,\
 				.cra_exit = rk_cra_hash_exit,\
 				.cra_module = THIS_MODULE,\
@@ -339,7 +346,7 @@ enum rk_cipher_mode {
 					     CRYPTO_ALG_NEED_FALLBACK,\
 				.cra_blocksize = hash_algo##_BLOCK_SIZE,\
 				.cra_ctxsize = sizeof(struct rk_ahash_ctx),\
-				.cra_alignmask = 3,\
+				.cra_alignmask = 0,\
 				.cra_init = rk_cra_hash_init,\
 				.cra_exit = rk_cra_hash_exit,\
 				.cra_module = THIS_MODULE,\

drivers/crypto/rockchip/rk_crypto_v2.h

@@ -21,6 +21,9 @@ struct crypto_lli_desc {
 struct rk_hw_crypto_v2_info {
 	struct crypto_lli_desc		*desc;
 	dma_addr_t			desc_dma;
+	bool				is_started;
+	u32				hash_calc_cnt;
+	u32				ciher_calc_cnt;
 };
 
 extern struct rk_crypto_algt rk_v2_ecb_sm4_alg;

drivers/crypto/rockchip/rk_crypto_v2_ahash.c

@@ -20,6 +20,11 @@
 #define RK_POLL_PERIOD_US	100
 #define RK_POLL_TIMEOUT_US	50000
 
+struct rk_ahash_expt_ctx {
+	struct rk_ahash_ctx	ctx;
+	u8			lastc[RK_DMA_ALIGNMENT];
+};
+
 static const u32 hash_algo2bc[] = {
 	[HASH_ALGO_MD5]    = CRYPTO_MD5,
 	[HASH_ALGO_SHA1]   = CRYPTO_SHA1,
@@ -40,16 +45,31 @@ const char *hash_algo2name[] = {
 	[HASH_ALGO_SM3]    = "sm3",
 };
 
-static struct rk_alg_ctx *rk_alg_ctx_cast(
+static struct rk_ahash_ctx *rk_ahash_ctx_cast(
 	struct rk_crypto_dev *rk_dev)
 {
 	struct ahash_request *req =
 		ahash_request_cast(rk_dev->async_req);
 
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-	struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
 
-	return &ctx->algs_ctx;
+	return crypto_ahash_ctx(tfm);
+}
+
+static struct rk_alg_ctx *rk_alg_ctx_cast(
+	struct rk_crypto_dev *rk_dev)
+{
+	return &(rk_ahash_ctx_cast(rk_dev))->algs_ctx;
+}
+
+static void rk_alg_ctx_clear(struct rk_alg_ctx *alg_ctx)
+{
+	alg_ctx->total	    = 0;
+	alg_ctx->left_bytes = 0;
+	alg_ctx->count      = 0;
+	alg_ctx->sg_src     = 0;
+	alg_ctx->req_src    = 0;
+	alg_ctx->src_nents  = 0;
 }
 
 static inline void word2byte_be(u32 word, u8 *ch)
@@ -73,6 +93,29 @@ static struct rk_crypto_algt *rk_ahash_get_algt(struct crypto_ahash *tfm)
 	return container_of(alg, struct rk_crypto_algt, alg.hash);
 }
 
+static void rk_hash_reset(struct rk_crypto_dev *rk_dev)
+{
+	int ret;
+	u32 tmp = 0, tmp_mask = 0;
+	unsigned int pool_timeout_us = 1000;
+
+	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x00);
+
+	tmp = CRYPTO_SW_CC_RESET;
+	tmp_mask = tmp << CRYPTO_WRITE_MASK_SHIFT;
+
+	CRYPTO_WRITE(rk_dev, CRYPTO_RST_CTL, tmp | tmp_mask);
+
+	/* This is usually done in 20 clock cycles */
+	ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_RST_CTL,
+					tmp, !tmp, 0, pool_timeout_us);
+	if (ret)
+		dev_err(rk_dev->dev, "cipher reset pool timeout %ums.",
+			pool_timeout_us);
+
+	CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, 0xffff0000);
+}
+
 static int rk_crypto_irq_handle(int irq, void *dev_id)
 {
 	struct rk_crypto_dev *rk_dev  = platform_get_drvdata(dev_id);
@@ -81,6 +124,9 @@ static int rk_crypto_irq_handle(int irq, void *dev_id)
 			(struct rk_hw_crypto_v2_info *)rk_dev->hw_info;
 	struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev);
 
+	/* disable crypto irq */
+	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0);
+
 	interrupt_status = CRYPTO_READ(rk_dev, CRYPTO_DMA_INT_ST);
 	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_ST, interrupt_status);
 
@@ -117,9 +163,28 @@ static int rk_crypto_irq_handle(int irq, void *dev_id)
 
 static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err)
 {
-	struct rk_ahash_ctx *ctx = crypto_tfm_ctx(base->tfm);
+	struct ahash_request *req = ahash_request_cast(base);
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
+	struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(ctx->rk_dev);
 
-	CRYPTO_WRITE(ctx->rk_dev, CRYPTO_HASH_CTL, 0xffff0000);
+	struct rk_hw_crypto_v2_info *hw_info = ctx->rk_dev->hw_info;
+	struct crypto_lli_desc *lli_desc = hw_info->desc;
+
+	if (err) {
+		rk_hash_reset(ctx->rk_dev);
+		pr_err("aligned = %u, align_size = %u\n",
+		       alg_ctx->aligned, alg_ctx->align_size);
+		pr_err("total = %u, left = %u, count = %u\n",
+		       alg_ctx->total, alg_ctx->left_bytes, alg_ctx->count);
+		pr_err("lli->src     = %08x\n", lli_desc->src_addr);
+		pr_err("lli->src_len = %08x\n", lli_desc->src_len);
+		pr_err("lli->dst     = %08x\n", lli_desc->dst_addr);
+		pr_err("lli->dst_len = %08x\n", lli_desc->dst_len);
+		pr_err("lli->dma_ctl = %08x\n", lli_desc->dma_ctrl);
+		pr_err("lli->usr_def = %08x\n", lli_desc->user_define);
+		pr_err("lli->next    = %08x\n\n\n", lli_desc->next_addr);
+	}
 
 	if (base->complete)
 		base->complete(base, err);
@@ -147,128 +212,152 @@ static void write_key_reg(struct rk_crypto_dev *rk_dev, const u8 *key,
 			     byte2word_be(key + i * 4));
 }
 
+static int rk_hw_hash_init(struct rk_crypto_dev *rk_dev, u32 algo, u32 type)
+{
+	u32 reg_ctrl = 0;
+
+	if (algo >= ARRAY_SIZE(hash_algo2bc))
+		goto exit;
+
+	rk_hash_reset(rk_dev);
+
+	clear_hash_out_reg(rk_dev);
+
+	reg_ctrl = hash_algo2bc[algo] | CRYPTO_HW_PAD_ENABLE;
+
+	if (IS_TYPE_HMAC(type)) {
+		CRYPTO_TRACE("this is hmac");
+		reg_ctrl |= CRYPTO_HMAC_ENABLE;
+	}
+
+	CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, reg_ctrl | CRYPTO_WRITE_MASK_ALL);
+	CRYPTO_WRITE(rk_dev, CRYPTO_FIFO_CTL, 0x00030003);
+
+	return 0;
+exit:
+	CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, 0 | CRYPTO_WRITE_MASK_ALL);
+
+	return -EINVAL;
+}
+
+static void clean_hash_setting(struct rk_crypto_dev *rk_dev)
+{
+	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0);
+	CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, 0 | CRYPTO_WRITE_MASK_ALL);
+}
+
 static int rk_ahash_init(struct ahash_request *req)
 {
-	struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+	struct rk_crypto_algt *algt = rk_ahash_get_algt(tfm);
+	struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 	struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
 
 	CRYPTO_TRACE();
 
-	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
-	rctx->fallback_req.base.flags = req->base.flags &
-					CRYPTO_TFM_REQ_MAY_SLEEP;
+	memset(rctx, 0x00, sizeof(*rctx));
 
-	return crypto_ahash_init(&rctx->fallback_req);
+	return rk_hw_hash_init(ctx->rk_dev, algt->algo, algt->type);
 }
 
 static int rk_ahash_update(struct ahash_request *req)
 {
-	struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
+	struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
+	struct rk_crypto_dev *rk_dev = ctx->rk_dev;
 
-	CRYPTO_TRACE();
+	CRYPTO_TRACE("nbytes = %u", req->nbytes);
 
-	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
-	rctx->fallback_req.base.flags = req->base.flags &
-					CRYPTO_TFM_REQ_MAY_SLEEP;
-	rctx->fallback_req.nbytes = req->nbytes;
-	rctx->fallback_req.src = req->src;
+	memset(rctx, 0x00, sizeof(*rctx));
 
-	return crypto_ahash_update(&rctx->fallback_req);
+	return rk_dev->enqueue(rk_dev, &req->base);
 }
 
 static int rk_ahash_final(struct ahash_request *req)
 {
-	struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
+	struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
+	struct rk_crypto_dev *rk_dev = ctx->rk_dev;
 
 	CRYPTO_TRACE();
 
-	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
-	rctx->fallback_req.base.flags = req->base.flags &
-					CRYPTO_TFM_REQ_MAY_SLEEP;
-	rctx->fallback_req.result = req->result;
+	memset(rctx, 0x00, sizeof(*rctx));
 
-	return crypto_ahash_final(&rctx->fallback_req);
+	rctx->is_final = true;
+
+	return rk_dev->enqueue(rk_dev, &req->base);
 }
 
 static int rk_ahash_finup(struct ahash_request *req)
 {
-	struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
-	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-	struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
+	int ret = 0;
+	crypto_completion_t complete_bak;
+	void *data_bak;
 
-	CRYPTO_TRACE();
+	DECLARE_CRYPTO_WAIT(wait);
 
-	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
-	rctx->fallback_req.base.flags = req->base.flags &
-					CRYPTO_TFM_REQ_MAY_SLEEP;
+	/* update not trigger user complete */
+	complete_bak = req->base.complete;
+	data_bak     = req->base.data;
 
-	rctx->fallback_req.nbytes = req->nbytes;
-	rctx->fallback_req.src = req->src;
-	rctx->fallback_req.result = req->result;
+	req->base.complete = crypto_req_done;
+	req->base.data     = &wait;
 
-	return crypto_ahash_finup(&rctx->fallback_req);
+	ret = crypto_wait_req(rk_ahash_update(req), &wait);
+	if (ret) {
+		CRYPTO_MSG("rk_ahash_update failed, ret = %d", ret);
+		goto exit;
+	}
+
+	/* final will trigger user complete */
+	req->base.complete = complete_bak;
+	req->base.data     = data_bak;
+
+	ret = rk_ahash_final(req);
+
+exit:
+	return ret;
 }
 
 static int rk_ahash_import(struct ahash_request *req, const void *in)
 {
-	struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
-	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-	struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
+	struct rk_ahash_expt_ctx state;
 
-	CRYPTO_TRACE();
+	/* 'in' may not be aligned so memcpy to local variable */
+	memcpy(&state, in, sizeof(state));
 
-	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
-	rctx->fallback_req.base.flags = req->base.flags &
-					CRYPTO_TFM_REQ_MAY_SLEEP;
+	///TODO:  deal with import
 
-	return crypto_ahash_import(&rctx->fallback_req, in);
+	return 0;
 }
 
 static int rk_ahash_export(struct ahash_request *req, void *out)
 {
-	struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
-	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-	struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
+	struct rk_ahash_expt_ctx state;
 
-	CRYPTO_TRACE();
+	/* Don't let anything leak to 'out' */
+	memset(&state, 0, sizeof(state));
 
-	ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
-	rctx->fallback_req.base.flags = req->base.flags &
-					CRYPTO_TFM_REQ_MAY_SLEEP;
+	///TODO:  deal with import
 
-	return crypto_ahash_export(&rctx->fallback_req, out);
+	memcpy(out, &state, sizeof(state));
+
+	return 0;
 }
 
 static int rk_ahash_digest(struct ahash_request *req)
 {
-	struct rk_ahash_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
-	struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
-	struct rk_crypto_dev *rk_dev = ctx->rk_dev;
-
 	CRYPTO_TRACE("calc data %u bytes.", req->nbytes);
 
-	if (!req->nbytes || req->nbytes > rk_dev->vir_max) {
-		ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
-		rctx->fallback_req.base.flags = req->base.flags &
-						CRYPTO_TFM_REQ_MAY_SLEEP;
-
-		rctx->fallback_req.nbytes = req->nbytes;
-		rctx->fallback_req.src    = req->src;
-		rctx->fallback_req.result = req->result;
-
-		return crypto_ahash_digest(&rctx->fallback_req);
-	} else {
-		return rk_dev->enqueue(rk_dev, &req->base);
-	}
+	return rk_ahash_init(req) ?: rk_ahash_finup(req);
 }
 
-static int rk_ahash_calc_digest(const char *alg_name, const u8 *key, u32 keylen,
-				u8 *digest, u32 digest_len)
+static int rk_ahash_fallback_digest(const char *alg_name, bool is_hmac,
+				    const u8 *key, u32 key_len,
+				    const u8 *msg, u32 msg_len,
+				    u8 *digest, u32 digest_len)
 {
 	struct crypto_ahash *ahash_tfm;
 	struct ahash_request *req;
@@ -276,7 +365,8 @@ static int rk_ahash_calc_digest(const char *alg_name, const u8 *key, u32 keylen,
 	struct scatterlist sg;
 	int ret;
 
-	CRYPTO_TRACE();
+	CRYPTO_TRACE("%s, is_hmac = %d, key_len = %u, msg_len = %u, digest_len = %u",
+		     alg_name, is_hmac, key_len, msg_len, digest_len);
 
 	ahash_tfm = crypto_alloc_ahash(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK);
 	if (IS_ERR(ahash_tfm))
@@ -290,10 +380,16 @@ static int rk_ahash_calc_digest(const char *alg_name, const u8 *key, u32 keylen,
 
 	init_completion(&wait.completion);
 
+	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+				   crypto_req_done, &wait);
+
 	crypto_ahash_clear_flags(ahash_tfm, ~0);
 
-	sg_init_one(&sg, key, keylen);
-	ahash_request_set_crypt(req, &sg, digest, keylen);
+	sg_init_one(&sg, msg, msg_len);
+	ahash_request_set_crypt(req, &sg, digest, msg_len);
+
+	if (is_hmac)
+		crypto_ahash_setkey(ahash_tfm, key, key_len);
 
 	ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
 	if (ret) {
@@ -309,78 +405,114 @@ exit:
 }
 
 static int rk_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key,
-				 unsigned int keylen)
+				unsigned int keylen)
 {
 	unsigned int blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
 	unsigned int digestsize = crypto_ahash_digestsize(tfm);
 	struct rk_crypto_algt *algt = rk_ahash_get_algt(tfm);
 	struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
 	const char *alg_name;
-	int ret;
+	int ret = 0;
 
 	CRYPTO_MSG();
 
-	crypto_ahash_clear_flags(ctx->fallback_tfm, ~0);
-	ret = crypto_ahash_setkey(ctx->fallback_tfm, key, keylen);
-	if (ret) {
-		CRYPTO_MSG("setkey failed, ret = %d\n", ret);
-		goto exit;
-	}
-
-	memset(ctx->authkey, 0, sizeof(ctx->authkey));
-
-	if (keylen <= blocksize) {
-		memcpy(ctx->authkey, key, keylen);
-		goto exit;
-	}
-
 	if (algt->algo >= ARRAY_SIZE(hash_algo2name)) {
 		CRYPTO_MSG("hash algo %d invalid\n", algt->algo);
 		return -EINVAL;
 	}
 
+	memset(ctx->authkey, 0, sizeof(ctx->authkey));
+
+	if (keylen <= blocksize) {
+		memcpy(ctx->authkey, key, keylen);
+		ctx->authkey_len = keylen;
+		goto exit;
+	}
+
 	alg_name = hash_algo2name[algt->algo];
 
 	CRYPTO_TRACE("calc key digest %s", alg_name);
 
-	ret = rk_ahash_calc_digest(alg_name, key, keylen, ctx->authkey, digestsize);
+	ret = rk_ahash_fallback_digest(alg_name, false, NULL, 0, key, keylen,
+				       ctx->authkey, digestsize);
 	if (ret) {
-		CRYPTO_MSG("rk_ahash_calc_digest error ret = %d\n", ret);
+		CRYPTO_MSG("rk_ahash_fallback_digest error ret = %d\n", ret);
 		goto exit;
 	}
 
+	ctx->authkey_len = digestsize;
 exit:
+	if (ret == 0)
+		write_key_reg(ctx->rk_dev, ctx->authkey, sizeof(ctx->authkey));
+
 	return ret;
 }
 
-static void rk_ahash_dma_start(struct rk_crypto_dev *rk_dev)
+static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, bool is_final)
 {
 	struct rk_hw_crypto_v2_info *hw_info =
 			(struct rk_hw_crypto_v2_info *)rk_dev->hw_info;
 	struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev);
+	u32 dma_ctl = hw_info->is_started ? CRYPTO_DMA_RESTART : CRYPTO_DMA_START;
 
-	CRYPTO_TRACE();
+	CRYPTO_TRACE("count %u Byte, is_started = %d, is_final = %d",
+		     alg_ctx->count, hw_info->is_started, is_final);
+
+	if (alg_ctx->count % RK_DMA_ALIGNMENT && !is_final) {
+		dev_err(rk_dev->dev, "count = %u is not aligned with [%u]\n",
+			alg_ctx->count, RK_DMA_ALIGNMENT);
+		return -EINVAL;
+	}
+
+	if (alg_ctx->count == 0) {
+		/* do nothing */
+		CRYPTO_TRACE("empty calc");
+		return 0;
+	}
 
 	memset(hw_info->desc, 0x00, sizeof(*hw_info->desc));
 
-	hw_info->desc->src_addr = alg_ctx->addr_in;
-	hw_info->desc->src_len  = alg_ctx->count;
-	hw_info->desc->next_addr = 0;
-	hw_info->desc->dma_ctrl = 0x00000401;
-	hw_info->desc->user_define = 0x7;
+	hw_info->desc->src_addr  = alg_ctx->addr_in;
+	hw_info->desc->src_len   = alg_ctx->count;
+	hw_info->desc->next_addr = hw_info->desc_dma;
+
+	hw_info->desc->dma_ctrl  = is_final ? LLI_DMA_CTRL_LAST : LLI_DMA_CTRL_PAUSE;
+	hw_info->desc->dma_ctrl |= LLI_DMA_CTRL_SRC_DONE;
+
+	if (!hw_info->is_started) {
+		hw_info->is_started = true;
+		hw_info->desc->user_define |= LLI_USER_CIPHER_START;
+		hw_info->desc->user_define |= LLI_USER_STRING_START;
+
+		CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma);
+		CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL,
+			     (CRYPTO_HASH_ENABLE << CRYPTO_WRITE_MASK_SHIFT) |
+			     CRYPTO_HASH_ENABLE);
+
+		hw_info->hash_calc_cnt = 0;
+	}
+
+	if (is_final && alg_ctx->left_bytes == 0) {
+		hw_info->desc->user_define |= LLI_USER_STRING_LAST;
+		hw_info->is_started = false;
+	}
+
+	CRYPTO_TRACE("dma_ctrl = %08x, user_define = %08x, len = %u",
+		     hw_info->desc->dma_ctrl, hw_info->desc->user_define, alg_ctx->count);
+
+	hw_info->hash_calc_cnt += alg_ctx->count;
 
 	dma_wmb();
 
-	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma);
-	CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL,
-		     (CRYPTO_HASH_ENABLE <<
-		      CRYPTO_WRITE_MASK_SHIFT) |
-		      CRYPTO_HASH_ENABLE);
+	/* enable crypto irq */
+	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x7f);
 
-	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, 0x00010001);/* start */
+	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, dma_ctl | dma_ctl << CRYPTO_WRITE_MASK_SHIFT);
+
+	return 0;
 }
 
-static int rk_ahash_set_data_start(struct rk_crypto_dev *rk_dev)
+static int rk_ahash_set_data_start(struct rk_crypto_dev *rk_dev, bool is_final)
 {
 	int err;
 	struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev);
@@ -389,58 +521,147 @@ static int rk_ahash_set_data_start(struct rk_crypto_dev *rk_dev)
 
 	err = rk_dev->load_data(rk_dev, alg_ctx->sg_src, alg_ctx->sg_dst);
 	if (!err)
-		rk_ahash_dma_start(rk_dev);
+		err = rk_ahash_dma_start(rk_dev, is_final);
+
 	return err;
 }
 
+static u32 rk_calc_lastc_new_len(u32 nbytes, u32 old_len)
+{
+	u32 total_len = nbytes + old_len;
+
+	if (total_len <= RK_DMA_ALIGNMENT)
+		return nbytes;
+
+	if (total_len % RK_DMA_ALIGNMENT)
+		return total_len % RK_DMA_ALIGNMENT;
+
+	return RK_DMA_ALIGNMENT;
+}
+
 static int rk_ahash_start(struct rk_crypto_dev *rk_dev)
 {
 	struct ahash_request *req = ahash_request_cast(rk_dev->async_req);
+	struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev);
+	struct rk_ahash_ctx *ctx = rk_ahash_ctx_cast(rk_dev);
+	struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
+	struct scatterlist *src_sg;
+	unsigned int nbytes;
+	int ret = 0;
+
+	CRYPTO_TRACE("origin: old_len = %u, new_len = %u, nbytes = %u, is_final = %d",
+		     ctx->hash_tmp_len, ctx->lastc_len, req->nbytes, rctx->is_final);
+
+	/* update 0Byte do nothing */
+	if (req->nbytes == 0 && !rctx->is_final)
+		goto no_calc;
+
+	if (ctx->lastc_len) {
+		/* move lastc saved last time to the head of this calculation */
+		memcpy(ctx->hash_tmp + ctx->hash_tmp_len, ctx->lastc, ctx->lastc_len);
+		ctx->hash_tmp_len = ctx->hash_tmp_len + ctx->lastc_len;
+		ctx->lastc_len = 0;
+	}
+
+	CRYPTO_TRACE("hash_tmp_len = %u", ctx->hash_tmp_len);
+
+	/* final request no need to save lastc_new */
+	if (!rctx->is_final) {
+		ctx->lastc_len = rk_calc_lastc_new_len(req->nbytes, ctx->hash_tmp_len);
+
+		CRYPTO_TRACE("nents = %u, ctx->lastc_len = %u, offset = %u",
+			sg_nents_for_len(req->src, req->nbytes), ctx->lastc_len,
+			req->nbytes - ctx->lastc_len);
+
+		if (!sg_pcopy_to_buffer(req->src, sg_nents_for_len(req->src, req->nbytes),
+			  ctx->lastc, ctx->lastc_len, req->nbytes - ctx->lastc_len)) {
+			ret = -EINVAL;
+			goto exit;
+		}
+
+		nbytes = ctx->hash_tmp_len + req->nbytes - ctx->lastc_len;
+
+		/* not enough data */
+		if (nbytes < RK_DMA_ALIGNMENT) {
+			CRYPTO_TRACE("nbytes = %u, not enough data", nbytes);
+			memcpy(ctx->hash_tmp + ctx->hash_tmp_len,
+			       ctx->lastc, ctx->lastc_len);
+			ctx->hash_tmp_len = ctx->hash_tmp_len + ctx->lastc_len;
+			ctx->lastc_len = 0;
+			goto no_calc;
+		}
+	} else {
+		/* final just calc lastc_old */
+		nbytes = ctx->hash_tmp_len;
+		CRYPTO_TRACE("nbytes = %u", nbytes);
+	}
+
+	if (ctx->hash_tmp_len) {
+		/* Concatenate old data to the header */
+		sg_init_table(ctx->hash_sg, ARRAY_SIZE(ctx->hash_sg));
+		sg_set_buf(ctx->hash_sg, ctx->hash_tmp, ctx->hash_tmp_len);
+		sg_chain(ctx->hash_sg, ARRAY_SIZE(ctx->hash_sg), req->src);
+
+		src_sg = &ctx->hash_sg[0];
+		ctx->hash_tmp_len = 0;
+	} else {
+		src_sg = req->src;
+	}
+
+	alg_ctx->total      = nbytes;
+	alg_ctx->left_bytes = nbytes;
+	alg_ctx->sg_src     = src_sg;
+	alg_ctx->req_src    = src_sg;
+	alg_ctx->src_nents  = sg_nents_for_len(src_sg, nbytes);
+
+	CRYPTO_TRACE("adjust: old_len = %u, new_len = %u, nbytes = %u",
+		     ctx->hash_tmp_len, ctx->lastc_len, nbytes);
+
+	if (nbytes)
+		ret = rk_ahash_set_data_start(rk_dev, rctx->is_final);
+
+exit:
+	return ret;
+no_calc:
+	CRYPTO_TRACE("no calc");
+	rk_alg_ctx_clear(alg_ctx);
+
+	return 0;
+}
+
+static int rk_ahash_get_zero_result(struct rk_crypto_dev *rk_dev,
+				    uint8_t *data, uint32_t data_len)
+{
+
+	struct ahash_request *req =
+		ahash_request_cast(rk_dev->async_req);
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 	struct rk_crypto_algt *algt = rk_ahash_get_algt(tfm);
 	struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
-	struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev);
-	u32 reg_ctrl = 0;
 
-	CRYPTO_TRACE();
+	return rk_ahash_fallback_digest(algt->name, algt->type == ALG_TYPE_HMAC,
+					ctx->authkey, ctx->authkey_len,
+					NULL, 0,
+					data, data_len);
 
-	alg_ctx->total      = req->nbytes;
-	alg_ctx->left_bytes = req->nbytes;
-	alg_ctx->sg_src     = req->src;
-	alg_ctx->req_src    = req->src;
-	alg_ctx->src_nents  = sg_nents_for_len(req->src, req->nbytes);
-
-	if (algt->algo >= ARRAY_SIZE(hash_algo2bc))
-		goto exit;
-
-	reg_ctrl |= hash_algo2bc[algt->algo];
-	clear_hash_out_reg(rk_dev);
-
-	reg_ctrl |= CRYPTO_HW_PAD_ENABLE;
-
-	if (IS_TYPE_HMAC(algt->type)) {
-		CRYPTO_TRACE("this is hmac");
-		reg_ctrl |= CRYPTO_HMAC_ENABLE;
-		write_key_reg(rk_dev, ctx->authkey, sizeof(ctx->authkey));
-	}
-
-	CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, reg_ctrl | CRYPTO_WRITE_MASK_ALL);
-
-	CRYPTO_WRITE(rk_dev, CRYPTO_FIFO_CTL, 0x00030003);
-	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x7f);
-
-	return rk_ahash_set_data_start(rk_dev);
-exit:
-	CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, CRYPTO_WRITE_MASK_ALL | 0);
-	return -EINVAL;
 }
-
 static int rk_ahash_get_result(struct rk_crypto_dev *rk_dev,
 			       uint8_t *data, uint32_t data_len)
 {
-	int ret;
+	int ret = 0;
 	u32 i, offset;
 	u32 reg_ctrl = 0;
+	struct rk_hw_crypto_v2_info *hw_info =
+			(struct rk_hw_crypto_v2_info *)rk_dev->hw_info;
+
+	hw_info->is_started = false;
+
+	/* use fallback hash */
+	if (hw_info->hash_calc_cnt == 0) {
+		CRYPTO_TRACE("use fallback hash");
+		ret = rk_ahash_get_zero_result(rk_dev, data, data_len);
+		goto exit;
+	}
 
 	ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_HASH_VALID,
 					reg_ctrl,
@@ -465,6 +686,7 @@ static int rk_ahash_get_result(struct rk_crypto_dev *rk_dev,
 	CRYPTO_WRITE(rk_dev, CRYPTO_HASH_VALID, CRYPTO_HASH_IS_VALID);
 
 exit:
+	hw_info->hash_calc_cnt = 0;
 	return ret;
 }
 
@@ -473,9 +695,9 @@ static int rk_ahash_crypto_rx(struct rk_crypto_dev *rk_dev)
 	int err = 0;
 	struct ahash_request *req = ahash_request_cast(rk_dev->async_req);
 	struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev);
-	struct crypto_ahash *tfm;
+	struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 
-	CRYPTO_TRACE("left bytes = %u", alg_ctx->left_bytes);
+	CRYPTO_TRACE("left bytes = %u, is_final = %d", alg_ctx->left_bytes, rctx->is_final);
 
 	err = rk_dev->unload_data(rk_dev);
 	if (err)
@@ -491,12 +713,21 @@ static int rk_ahash_crypto_rx(struct rk_crypto_dev *rk_dev)
 			}
 			alg_ctx->sg_src = sg_next(alg_ctx->sg_src);
 		}
-		err = rk_ahash_set_data_start(rk_dev);
+		err = rk_ahash_set_data_start(rk_dev, rctx->is_final);
 	} else {
 		/*
 		 * it will take some time to process date after last dma
 		 * transmission.
 		 */
+		struct crypto_ahash *tfm;
+
+		if (!rctx->is_final)
+			goto out_rx;
+
+		if (!req->result) {
+			err = -EINVAL;
+			goto out_rx;
+		}
 
 		tfm = crypto_ahash_reqtfm(req);
 
@@ -525,30 +756,25 @@ static int rk_cra_hash_init(struct crypto_tfm *tfm)
 	if (!rk_dev->request_crypto)
 		return -EFAULT;
 
-	rk_dev->request_crypto(rk_dev, alg_name);
-
-	alg_ctx->align_size     = 64;
+	alg_ctx->align_size     = RK_DMA_ALIGNMENT;
 
 	alg_ctx->ops.start      = rk_ahash_start;
 	alg_ctx->ops.update     = rk_ahash_crypto_rx;
 	alg_ctx->ops.complete   = rk_ahash_crypto_complete;
 	alg_ctx->ops.irq_handle = rk_crypto_irq_handle;
 
-	ctx->rk_dev = rk_dev;
-
-	/* for fallback */
-	ctx->fallback_tfm = crypto_alloc_ahash(alg_name, 0,
-					       CRYPTO_ALG_NEED_FALLBACK);
-	if (IS_ERR(ctx->fallback_tfm)) {
-		dev_err(rk_dev->dev, "Could not load fallback driver.\n");
-		return PTR_ERR(ctx->fallback_tfm);
+	ctx->rk_dev   = rk_dev;
+	ctx->hash_tmp = (u8 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32);
+	if (!ctx->hash_tmp) {
+		dev_err(rk_dev->dev, "Can't get zeroed page for hash tmp.\n");
+		return -ENOMEM;
 	}
 
-	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
-				 sizeof(struct rk_ahash_rctx) +
-				 crypto_ahash_reqsize(ctx->fallback_tfm));
+	rk_dev->request_crypto(rk_dev, alg_name);
 
-	algt->alg.hash.halg.statesize = crypto_ahash_statesize(ctx->fallback_tfm);
+	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct rk_ahash_rctx));
+
+	algt->alg.hash.halg.statesize = sizeof(struct rk_ahash_expt_ctx);
 
 	return 0;
 }
@@ -559,8 +785,10 @@ static void rk_cra_hash_exit(struct crypto_tfm *tfm)
 
 	CRYPTO_TRACE();
 
-	if (ctx->fallback_tfm)
-		crypto_free_ahash(ctx->fallback_tfm);
+	clean_hash_setting(ctx->rk_dev);
+
+	if (ctx->hash_tmp)
+		free_page((unsigned long)ctx->hash_tmp);
 
 	ctx->rk_dev->release_crypto(ctx->rk_dev, crypto_tfm_alg_name(tfm));
 }

drivers/crypto/rockchip/rk_crypto_v2_skcipher.c

@@ -9,6 +9,7 @@
  * Some ideas are from marvell-cesa.c and s5p-sss.c driver.
  */
 
+#include <linux/iopoll.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 
@@ -220,7 +221,9 @@ static bool is_calc_need_round_up(struct skcipher_request *req)
 
 static void rk_cipher_reset(struct rk_crypto_dev *rk_dev)
 {
+	int ret;
 	u32 tmp = 0, tmp_mask = 0;
+	unsigned int  pool_timeout_us = 1000;
 
 	CRYPTO_WRITE(rk_dev, CRYPTO_DMA_INT_EN, 0x00);
 
@@ -228,8 +231,13 @@ static void rk_cipher_reset(struct rk_crypto_dev *rk_dev)
 	tmp_mask = tmp << CRYPTO_WRITE_MASK_SHIFT;
 
 	CRYPTO_WRITE(rk_dev, CRYPTO_RST_CTL, tmp | tmp_mask);
-	while (CRYPTO_READ(rk_dev, CRYPTO_RST_CTL))
-		nop();
+
+	/* This is usually done in 20 clock cycles */
+	ret = readl_poll_timeout_atomic(rk_dev->reg + CRYPTO_RST_CTL,
+					tmp, !tmp, 0, pool_timeout_us);
+	if (ret)
+		dev_err(rk_dev->dev, "cipher reset pool timeout %ums.",
+			pool_timeout_us);
 
 	CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, 0xffff0000);
 }
@@ -255,7 +263,6 @@ static void rk_crypto_complete(struct crypto_async_request *base, int err)
 		pr_err("lli->dma_ctl = %08x\n", lli_desc->dma_ctrl);
 		pr_err("lli->usr_def = %08x\n", lli_desc->user_define);
 		pr_err("lli->next    = %08x\n\n\n", lli_desc->next_addr);
-
 	}
 
 	if (base->complete)
@@ -445,6 +452,8 @@ static void rk_ablk_hw_init(struct rk_crypto_dev *rk_dev)
 	struct rk_cipher_ctx *ctx = crypto_skcipher_ctx(cipher);
 	u32 ivsize;
 
+	rk_cipher_reset(rk_dev);
+
 	CRYPTO_WRITE(rk_dev, CRYPTO_BC_CTL, 0x00010000);
 
 	ivsize = crypto_skcipher_ivsize(cipher);