From 8a62dc1f10267a8f0382b8845a0467acedc810ac Mon Sep 17 00:00:00 2001 From: Matthew Shyu Date: Tue, 22 Nov 2016 13:29:00 +0800 Subject: [PATCH] crypto: hardware crypto engine support PD#138714: init add Hardware crypto engine support Amlogic hardware crypto driver for AES-128/192/256-CBC, AES-128/192-256-CTR, DES3-EDE-CBC, SHA1, SHA2, SHA256, HMAC Change-Id: I6d690f46222c84164021f3bd6244c708b476f6f0 Signed-off-by: Matthew Shyu --- .../devicetree/bindings/crypto/aml-crypto.txt | 116 ++ MAINTAINERS | 4 + arch/arm64/boot/dts/amlogic/mesongxl.dtsi | 27 + arch/arm64/boot/dts/amlogic/mesongxm.dtsi | 27 + arch/arm64/configs/meson64_defconfig | 1 + drivers/amlogic/Kconfig | 2 + drivers/amlogic/Makefile | 1 + drivers/amlogic/crypto/Kconfig | 10 + drivers/amlogic/crypto/Makefile | 4 + drivers/amlogic/crypto/aml-aes-dma.c | 962 ++++++++++++ drivers/amlogic/crypto/aml-crypto-dma.c | 109 ++ drivers/amlogic/crypto/aml-crypto-dma.h | 107 ++ drivers/amlogic/crypto/aml-sha-dma.c | 1305 +++++++++++++++++ drivers/amlogic/crypto/aml-tdes-dma.c | 968 ++++++++++++ 14 files changed, 3643 insertions(+) create mode 100644 Documentation/devicetree/bindings/crypto/aml-crypto.txt create mode 100644 drivers/amlogic/crypto/Kconfig create mode 100644 drivers/amlogic/crypto/Makefile create mode 100644 drivers/amlogic/crypto/aml-aes-dma.c create mode 100644 drivers/amlogic/crypto/aml-crypto-dma.c create mode 100644 drivers/amlogic/crypto/aml-crypto-dma.h create mode 100644 drivers/amlogic/crypto/aml-sha-dma.c create mode 100644 drivers/amlogic/crypto/aml-tdes-dma.c diff --git a/Documentation/devicetree/bindings/crypto/aml-crypto.txt b/Documentation/devicetree/bindings/crypto/aml-crypto.txt new file mode 100644 index 000000000000..aecf7cf07e2b --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/aml-crypto.txt @@ -0,0 +1,116 @@ +* Amlogic HW cryptographic accelerators + +These are the HW cryptographic accelerators found on Amlogic products. + +* Advanced Encryption Standard (AES) + +Required properties: +- compatible : Should be "amlogic,aes". +- dev_name : Should be "aml_aes" +- interrupts: Should contain the IRQ line for the AES. +- resets: Should contain the clock to enable the module +- reg: Should contain the base address of regs + +Example: +aml_aes{ + compatible = "amlogic,aes"; + dev_name = "aml_aes"; + interrupts = <0 36 1>; + resets = <&clock GCLK_IDX_BLK_MOV>; + reg = <0x0 0xc8832000 0x0 0x2c4 + 0x0 0xda832000 0x0 0xe4>; +}; + +* Triple Data Encryption Standard (Triple DES) + +Required properties: +- compatible : Should be "amlogic,des,tdes". +- dev_name : Should be "aml_aes" +- interrupts: Should contain the IRQ line for the TDES. +- resets: Should contain the clock to enable the module +- reg: Should contain the base address of regs + +Example: +aml_tdes{ + compatible = "amlogic,des,tdes"; + dev_name = "aml_tdes"; + interrupts = <0 36 1>; + resets = <&clock GCLK_IDX_BLK_MOV>; + reg = <0x0 0xc8832000 0x0 0x2c4 + 0x0 0xda832000 0x0 0xe4>; +}; + +* Secure Hash Algorithm (SHA1/SHA224/SHA256) + +Required properties: +- compatible : Should be "amlogic,sha". +- dev_name : Should be "aml_sha" +- interrupts: Should contain the IRQ line for the SHA. +- resets: Should contain the clock to enable the module +- reg: Should contain the base address of regs + +Example: +aml_sha{ + compatible = "amlogic,sha"; + dev_name = "aml_sha"; + interrupts = <0 36 1>; + resets = <&clock GCLK_IDX_BLK_MOV>; + reg = <0x0 0xc8832000 0x0 0x2c4 + 0x0 0xda832000 0x0 0xe4>; +}; + +* New DMA for GXL and beyond +* Advanced Encryption Standard (AES) + +Required properties: +- compatible : Should be "amlogic,aes". +- dev_name : Should be "aml_aes" +- interrupts: Should contain the IRQ line for the AES. +- resets: Should contain the clock to enable the module +- reg: Should contain the base address of regs + +Example: +aml_aes{ + compatible = "amlogic,aes_dma"; + dev_name = "aml_aes_dma"; + interrupts = <0 188 1 + 0 189 1>; + reg = <0x0 0xc883e000 0x0 0x28>; +}; + + +* Triple Data Encryption Standard (Triple DES) + +Required properties: +- compatible : Should be "amlogic,des,tdes". +- dev_name : Should be "aml_aes" +- interrupts: Should contain the IRQ line for the TDES. +- resets: Should contain the clock to enable the module +- reg: Should contain the base address of regs + +Example: +aml_tdes{ + compatible = "amlogic,des_dma,tdes_dma"; + dev_name = "aml_tdes_dma"; + interrupts = <0 188 1 + 0 189 1>; + reg = <0x0 0xc883e000 0x0 0x28>; +}; +* Secure Hash Algorithm (SHA1/SHA224/SHA256/HMAC) + +Required properties: +- compatible : Should be "amlogic,sha". +- dev_name : Should be "aml_sha" +- interrupts: Should contain the IRQ line for the SHA. +- resets: Should contain the clock to enable the module +- reg: Should contain the base address of regs + +Example: +aml_sha{ + compatible = "amlogic,sha_dma"; + dev_name = "aml_sha_dma"; + interrupts = <0 188 1 + 0 189 1>; + reg = <0x0 0xc883e000 0x0 0x28>; +}; + diff --git a/MAINTAINERS b/MAINTAINERS index 7ed8a5d50da7..8aaed3a74178 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -13442,3 +13442,7 @@ F: include/dt-bindings/reset/amlogic,gxl-reset.h AMLOGIC meson64_xxx_defconfigs M: Jianxin Pan F: scripts/amlogic/configs/ + +AMLOGIC CRYPTO DMA +M: Matthew Shyu +F: drivers/amlogic/crypto/* diff --git a/arch/arm64/boot/dts/amlogic/mesongxl.dtsi b/arch/arm64/boot/dts/amlogic/mesongxl.dtsi index aa38d4633e2e..b157c485e3a2 100644 --- a/arch/arm64/boot/dts/amlogic/mesongxl.dtsi +++ b/arch/arm64/boot/dts/amlogic/mesongxl.dtsi @@ -187,6 +187,7 @@ clock-output-names = "xtal"; #clock-cells = <0>; }; + soc { compatible = "simple-bus"; #address-cells = <2>; @@ -674,3 +675,29 @@ }; }; +/{ + aml_aes { + compatible = "amlogic,aes_dma"; + dev_name = "aml_aes_dma"; + status = "okay"; + interrupts = <0 188 1 0 189 1>; + reg = <0x0 0xc883e000 0x0 0x28>; + }; + + aml_tdes { + compatible = "amlogic,des_dma,tdes_dma"; + dev_name = "aml_tdes_dma"; + status = "okay"; + interrupts = <0 188 1 0 189 1>; + reg = <0x0 0xc883e000 0x0 0x28>; + }; + + aml_sha { + compatible = "amlogic,sha_dma"; + dev_name = "aml_sha_dma"; + status = "okay"; + interrupts = <0 188 1 0 189 1>; + reg = <0x0 0xc883e000 0x0 0x28>; + }; +}; + diff --git a/arch/arm64/boot/dts/amlogic/mesongxm.dtsi b/arch/arm64/boot/dts/amlogic/mesongxm.dtsi index 3129aaa42dc3..acc0ab7d140c 100644 --- a/arch/arm64/boot/dts/amlogic/mesongxm.dtsi +++ b/arch/arm64/boot/dts/amlogic/mesongxm.dtsi @@ -232,6 +232,7 @@ clock-output-names = "xtal"; #clock-cells = <0>; }; + soc { compatible = "simple-bus"; #address-cells = <2>; @@ -719,3 +720,29 @@ }; }; +/{ + aml_aes { + compatible = "amlogic,aes_dma"; + dev_name = "aml_aes_dma"; + status = "okay"; + interrupts = <0 188 1 0 189 1>; + reg = <0x0 0xc883e000 0x0 0x28>; + }; + + aml_tdes { + compatible = "amlogic,des_dma,tdes_dma"; + dev_name = "aml_tdes_dma"; + status = "okay"; + interrupts = <0 188 1 0 189 1>; + reg = <0x0 0xc883e000 0x0 0x28>; + }; + + aml_sha { + compatible = "amlogic,sha_dma"; + dev_name = "aml_sha_dma"; + status = "okay"; + interrupts = <0 188 1 0 189 1>; + reg = <0x0 0xc883e000 0x0 0x28>; + }; +}; + diff --git a/arch/arm64/configs/meson64_defconfig b/arch/arm64/configs/meson64_defconfig index a0106dfd82df..f41e73195a2e 100644 --- a/arch/arm64/configs/meson64_defconfig +++ b/arch/arm64/configs/meson64_defconfig @@ -167,6 +167,7 @@ CONFIG_AMLOGIC_REG_ACCESS=y CONFIG_AMLOGIC_TIMER=y CONFIG_AMLOGIC_BC_TIMER=y CONFIG_AMLOGIC_CLK=y +CONFIG_AMLOGIC_CRYPTO=y CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y diff --git a/drivers/amlogic/Kconfig b/drivers/amlogic/Kconfig index dbb84dd13bdf..0b277df52d90 100644 --- a/drivers/amlogic/Kconfig +++ b/drivers/amlogic/Kconfig @@ -30,5 +30,7 @@ source "drivers/amlogic/reg_access/Kconfig" source "drivers/amlogic/clocksource/Kconfig" source "drivers/amlogic/clk/Kconfig" + +source "drivers/amlogic/crypto/Kconfig" endmenu endif diff --git a/drivers/amlogic/Makefile b/drivers/amlogic/Makefile index b7171a08d501..b55af52c5478 100644 --- a/drivers/amlogic/Makefile +++ b/drivers/amlogic/Makefile @@ -28,3 +28,4 @@ obj-$(CONFIG_AMLOGIC_TIMER) += clocksource/ obj-$(CONFIG_AMLOGIC_CLK) += clk/ +obj-$(CONFIG_AMLOGIC_CRYPTO) += crypto/ diff --git a/drivers/amlogic/crypto/Kconfig b/drivers/amlogic/crypto/Kconfig new file mode 100644 index 000000000000..84e5ae184ad3 --- /dev/null +++ b/drivers/amlogic/crypto/Kconfig @@ -0,0 +1,10 @@ + +menu "Amlogic Crypto Support" + +config AMLOGIC_CRYPTO + bool "amlogic HW CRYPTO module" + select CRYPTO_ALGAPI + select CRYPTO_BLKCIPHER + select CRYPTO_HASH + default n +endmenu diff --git a/drivers/amlogic/crypto/Makefile b/drivers/amlogic/crypto/Makefile new file mode 100644 index 000000000000..5fafcc0c3de7 --- /dev/null +++ b/drivers/amlogic/crypto/Makefile @@ -0,0 +1,4 @@ +obj-$(CONFIG_AMLOGIC_CRYPTO) += aml-aes-dma.o +obj-$(CONFIG_AMLOGIC_CRYPTO) += aml-tdes-dma.o +obj-$(CONFIG_AMLOGIC_CRYPTO) += aml-crypto-dma.o +obj-$(CONFIG_AMLOGIC_CRYPTO) += aml-sha-dma.o diff --git a/drivers/amlogic/crypto/aml-aes-dma.c b/drivers/amlogic/crypto/aml-aes-dma.c new file mode 100644 index 000000000000..401f066d0486 --- /dev/null +++ b/drivers/amlogic/crypto/aml-aes-dma.c @@ -0,0 +1,962 @@ +/* + * drivers/amlogic/crypto/aml-aes-dma.c + * + * Copyright (C) 2017 Amlogic, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "aml-crypto-dma.h" + +/* AES flags */ +#define AES_FLAGS_MODE_MASK 0x07 +#define AES_FLAGS_ENCRYPT BIT(0) +#define AES_FLAGS_CBC BIT(1) +#define AES_FLAGS_CTR BIT(2) + +#define AES_FLAGS_INIT BIT(8) +#define AES_FLAGS_DMA BIT(9) +#define AES_FLAGS_FAST BIT(10) +#define AES_FLAGS_BUSY BIT(11) + +#define AML_AES_QUEUE_LENGTH 50 +#define AML_AES_DMA_THRESHOLD 16 + +#define DMA_THREAD_REG (DMA_T0 + AES_THREAD_INDEX) +#define DMA_STATUS_REG (DMA_STS0 + AES_THREAD_INDEX) + +u8 map_in_aes_dma; +struct aml_aes_dev; + +struct aml_aes_ctx { + struct aml_aes_dev *dd; + + int keylen; + u32 key[AES_KEYSIZE_256 / sizeof(u32)]; + + u16 block_size; +}; + +struct aml_aes_reqctx { + unsigned long mode; +}; + +struct aml_aes_dev { + struct list_head list; + + struct aml_aes_ctx *ctx; + struct device *dev; + int irq; + + unsigned long flags; + int err; + + spinlock_t lock; + struct crypto_queue queue; + + struct tasklet_struct done_task; + struct tasklet_struct queue_task; + + struct ablkcipher_request *req; + size_t total; + + struct scatterlist *in_sg; + size_t in_offset; + struct scatterlist *out_sg; + size_t out_offset; + + size_t buflen; + size_t dma_size; + + void *buf_in; + dma_addr_t dma_addr_in; + + void *buf_out; + dma_addr_t dma_addr_out; + + void *descriptor; + dma_addr_t dma_descript_tab; + + uint32_t fast_nents; +}; + +struct aml_aes_drv { + struct list_head dev_list; + spinlock_t lock; +}; + +static struct aml_aes_drv aml_aes = { + .dev_list = LIST_HEAD_INIT(aml_aes.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(aml_aes.lock), +}; + +static void set_aes_key_iv(struct aml_aes_dev *dd, u32 *key, + uint32_t keylen, u32 *iv, uint8_t swap) +{ + struct dma_dsc *dsc = dd->descriptor; + uint32_t key_iv[12]; + uint32_t *piv = key_iv + 8; + uint32_t len = keylen; + dma_addr_t dma_addr_key; + + memset(key_iv, 0, sizeof(key_iv)); + memcpy(key_iv, key, keylen); + if (iv) { + if (swap) { + *(piv + 3) = swap_ulong32(*iv); + *(piv + 2) = swap_ulong32(*(iv + 1)); + *(piv + 1) = swap_ulong32(*(iv + 2)); + *(piv + 0) = swap_ulong32(*(iv + 3)); + } else { + memcpy(piv, iv, 16); + } + len = 48; /* full key storage */ + } + + dma_addr_key = dma_map_single(dd->dev, key_iv, + sizeof(key_iv), DMA_TO_DEVICE); + + dsc->src_addr = (uint32_t)dma_addr_key; + dsc->tgt_addr = 0; + dsc->dsc_cfg.d32 = 0; + dsc->dsc_cfg.b.length = len; + dsc->dsc_cfg.b.mode = MODE_KEY; + dsc->dsc_cfg.b.eoc = 1; + dsc->dsc_cfg.b.owner = 1; + + dma_sync_single_for_device(dd->dev, dd->dma_descript_tab, + PAGE_SIZE, DMA_TO_DEVICE); + aml_write_crypto_reg(DMA_THREAD_REG, + (uintptr_t) dd->dma_descript_tab | 2); + aml_dma_debug(dsc, 1, __func__); + while (aml_read_crypto_reg(DMA_STATUS_REG) == 0) + ; + aml_write_crypto_reg(DMA_STATUS_REG, 0xf); + dma_unmap_single(dd->dev, dma_addr_key, + sizeof(key_iv), DMA_TO_DEVICE); +} + +static size_t aml_aes_sg_copy(struct scatterlist **sg, size_t *offset, + void *buf, size_t buflen, size_t total, int out) +{ + size_t count, off = 0; + + while (buflen && total) { + count = min((*sg)->length - *offset, total); + count = min(count, buflen); + + if (!count) + return off; + + scatterwalk_map_and_copy(buf + off, *sg, *offset, count, out); + + off += count; + buflen -= count; + *offset += count; + total -= count; + + if (*offset == (*sg)->length) { + *sg = sg_next(*sg); + if (*sg) + *offset = 0; + else + total = 0; + } + } + + return off; +} + +static size_t aml_aes_sg_dma(struct aml_aes_dev *dd, struct dma_dsc *dsc, + uint32_t *nents, size_t total) +{ + size_t count = 0; + uint32_t i = 0; + int err = 0; + struct scatterlist *in_sg = dd->in_sg; + struct scatterlist *out_sg = dd->out_sg; + dma_addr_t addr_in, addr_out; + + while (total && in_sg && out_sg && (in_sg->length == out_sg->length) + && *nents < MAX_NUM_TABLES) { + count += min_t(unsigned int, total, in_sg->length); + *nents += 1; + total -= count; + in_sg = sg_next(in_sg); + out_sg = sg_next(out_sg); + } + err = dma_map_sg(dd->dev, dd->in_sg, *nents, DMA_TO_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + return 0; + } + + err = dma_map_sg(dd->dev, dd->out_sg, *nents, + DMA_FROM_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + dma_unmap_sg(dd->dev, dd->in_sg, *nents, + DMA_TO_DEVICE); + return 0; + } + + in_sg = dd->in_sg; + out_sg = dd->out_sg; + for (i = 0; i < *nents; i++) { + addr_in = sg_dma_address(in_sg); + addr_out = sg_dma_address(out_sg); + dsc[i].src_addr = (uintptr_t)addr_in; + dsc[i].tgt_addr = (uintptr_t)addr_out; + dsc[i].dsc_cfg.d32 = 0; + dsc[i].dsc_cfg.b.length = sg_dma_len(in_sg); + in_sg = sg_next(in_sg); + out_sg = sg_next(out_sg); + } + return count; +} + +static struct aml_aes_dev *aml_aes_find_dev(struct aml_aes_ctx *ctx) +{ + struct aml_aes_dev *aes_dd = NULL; + struct aml_aes_dev *tmp; + + spin_lock_bh(&aml_aes.lock); + if (!ctx->dd) { + list_for_each_entry(tmp, &aml_aes.dev_list, list) { + aes_dd = tmp; + break; + } + ctx->dd = aes_dd; + } else { + aes_dd = ctx->dd; + } + + spin_unlock_bh(&aml_aes.lock); + + return aes_dd; +} + +static int aml_aes_hw_init(struct aml_aes_dev *dd) +{ + if (!(dd->flags & AES_FLAGS_INIT)) { + dd->flags |= AES_FLAGS_INIT; + dd->err = 0; + } + + return 0; +} + +static void aml_aes_finish_req(struct aml_aes_dev *dd, int32_t err) +{ + struct ablkcipher_request *req = dd->req; + + dd->flags &= ~AES_FLAGS_BUSY; + req->base.complete(&req->base, err); +} + + +static int aml_aes_crypt_dma(struct aml_aes_dev *dd, struct dma_dsc *dsc, + uint32_t nents) +{ + uint32_t op_mode = OP_MODE_ECB; + uint32_t i = 0; + + dd->flags |= AES_FLAGS_DMA; + + if (dd->flags & AES_FLAGS_CBC) + op_mode = OP_MODE_CBC; + else if (dd->flags & AES_FLAGS_CTR) + op_mode = OP_MODE_CTR; + + for (i = 0; i < nents; i++) { + dsc[i].dsc_cfg.b.enc_sha_only = dd->flags & AES_FLAGS_ENCRYPT; + dsc[i].dsc_cfg.b.mode = + ((dd->ctx->keylen == AES_KEYSIZE_128) ? MODE_AES128 : + ((dd->ctx->keylen == AES_KEYSIZE_192) ? + MODE_AES192 : MODE_AES256)); + dsc[i].dsc_cfg.b.op_mode = op_mode; + dsc[i].dsc_cfg.b.eoc = (i == (nents - 1)); + dsc[i].dsc_cfg.b.owner = 1; + } + + dma_sync_single_for_device(dd->dev, dd->dma_descript_tab, + PAGE_SIZE, DMA_TO_DEVICE); + + aml_dma_debug(dsc, nents, __func__); + aml_write_crypto_reg(DMA_THREAD_REG, dd->dma_descript_tab | 2); + return 0; +} + +static int aml_aes_crypt_dma_start(struct aml_aes_dev *dd) +{ + int err = 0, fast = 0; + int in, out; + size_t count; + dma_addr_t addr_in, addr_out; + struct dma_dsc *dsc = dd->descriptor; + uint32_t nents; + + /* fast dma */ + if ((!dd->in_offset) && (!dd->out_offset)) { + /* check for alignment */ + in = IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size); + out = IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size); + fast = in && out; + + if (dd->in_sg->length != dd->out_sg->length + || dd->total < dd->ctx->block_size) + fast = 0; + dd->fast_nents = 0; + } + + if (fast) { + count = aml_aes_sg_dma(dd, dsc, &dd->fast_nents, dd->total); + dd->flags |= AES_FLAGS_FAST; + nents = dd->fast_nents; + } else { + /* slow dma */ + /* use cache buffers */ + count = aml_aes_sg_copy(&dd->in_sg, &dd->in_offset, + dd->buf_in, dd->buflen, dd->total, 0); + addr_in = dd->dma_addr_in; + addr_out = dd->dma_addr_out; + dd->dma_size = count; + dma_sync_single_for_device(dd->dev, addr_in, dd->dma_size, + DMA_TO_DEVICE); + dsc->src_addr = (uint32_t)addr_in; + dsc->tgt_addr = (uint32_t)addr_out; + dsc->dsc_cfg.d32 = 0; + dsc->dsc_cfg.b.length = count; + nents = 1; + dd->flags &= ~AES_FLAGS_FAST; + } + dd->total -= count; + + err = aml_aes_crypt_dma(dd, dsc, nents); + + if (err && (dd->flags & AES_FLAGS_FAST)) { + dma_unmap_sg(dd->dev, dd->in_sg, + dd->fast_nents, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, + dd->fast_nents, DMA_TO_DEVICE); + } + + return err; +} + +static int aml_aes_write_ctrl(struct aml_aes_dev *dd) +{ + int err = 0; + + err = aml_aes_hw_init(dd); + + if (err) + return err; + + if (dd->flags & AES_FLAGS_CBC) + set_aes_key_iv(dd, dd->ctx->key, dd->ctx->keylen, + dd->req->info, 0); + else if (dd->flags & AES_FLAGS_CTR) + set_aes_key_iv(dd, dd->ctx->key, dd->ctx->keylen, + dd->req->info, 1); + else + set_aes_key_iv(dd, dd->ctx->key, dd->ctx->keylen, NULL, 0); + + return err; +} + +static int aml_aes_handle_queue(struct aml_aes_dev *dd, + struct ablkcipher_request *req) +{ + struct crypto_async_request *async_req, *backlog; + struct aml_aes_ctx *ctx; + struct aml_aes_reqctx *rctx; + unsigned long flags; + int32_t err, ret = 0; + + spin_lock_irqsave(&dd->lock, flags); + if (req) + ret = ablkcipher_enqueue_request(&dd->queue, req); + + if (dd->flags & AES_FLAGS_BUSY) { + spin_unlock_irqrestore(&dd->lock, flags); + return ret; + } + backlog = crypto_get_backlog(&dd->queue); + async_req = crypto_dequeue_request(&dd->queue); + if (async_req) + dd->flags |= AES_FLAGS_BUSY; + spin_unlock_irqrestore(&dd->lock, flags); + + if (!async_req) + return ret; + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + + req = ablkcipher_request_cast(async_req); + + /* assign new request to device */ + dd->req = req; + dd->total = req->nbytes; + dd->in_offset = 0; + dd->in_sg = req->src; + dd->out_offset = 0; + dd->out_sg = req->dst; + + rctx = ablkcipher_request_ctx(req); + ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); + rctx->mode &= AES_FLAGS_MODE_MASK; + dd->flags = (dd->flags & ~AES_FLAGS_MODE_MASK) | rctx->mode; + dd->ctx = ctx; + ctx->dd = dd; + + err = aml_aes_write_ctrl(dd); + if (!err) { + if (dd->total % AML_AES_DMA_THRESHOLD == 0) + err = aml_aes_crypt_dma_start(dd); + else { + pr_err("size %zd is not multiple of %d", + dd->total, AML_AES_DMA_THRESHOLD); + err = -EINVAL; + } + } + if (err) { + /* aes_task will not finish it, so do it here */ + aml_aes_finish_req(dd, err); + tasklet_schedule(&dd->queue_task); + } + + return ret; +} + +static int aml_aes_crypt_dma_stop(struct aml_aes_dev *dd) +{ + int err = -EINVAL; + size_t count; + + if (dd->flags & AES_FLAGS_DMA) { + err = 0; + dma_sync_single_for_cpu(dd->dev, dd->dma_descript_tab, + PAGE_SIZE, DMA_FROM_DEVICE); + if (dd->flags & AES_FLAGS_FAST) { + dma_unmap_sg(dd->dev, dd->out_sg, + dd->fast_nents, DMA_FROM_DEVICE); + dma_unmap_sg(dd->dev, dd->in_sg, + dd->fast_nents, DMA_TO_DEVICE); + } else { + dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, + dd->dma_size, DMA_FROM_DEVICE); + + /* copy data */ + count = aml_aes_sg_copy(&dd->out_sg, &dd->out_offset, + dd->buf_out, dd->buflen, + dd->dma_size, 1); + if (count != dd->dma_size) { + err = -EINVAL; + pr_err("not all data converted: %zu\n", count); + } + } + dd->flags &= ~AES_FLAGS_DMA; + } + + return err; +} + + +static int aml_aes_buff_init(struct aml_aes_dev *dd) +{ + int err = -ENOMEM; + + dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, 0); + dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, 0); + dd->descriptor = (void *)__get_free_pages(GFP_KERNEL, 0); + dd->buflen = PAGE_SIZE; + dd->buflen &= ~(AES_BLOCK_SIZE - 1); + + if (!dd->buf_in || !dd->buf_out || !dd->descriptor) { + dev_err(dd->dev, "unable to alloc pages.\n"); + goto err_alloc; + } + + /* MAP here */ + dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, + dd->buflen, DMA_TO_DEVICE); + if (dma_mapping_error(dd->dev, dd->dma_addr_in)) { + dev_err(dd->dev, "dma %zd bytes error\n", dd->buflen); + err = -EINVAL; + goto err_map_in; + } + + dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, + dd->buflen, DMA_FROM_DEVICE); + if (dma_mapping_error(dd->dev, dd->dma_addr_out)) { + dev_err(dd->dev, "dma %zd bytes error\n", dd->buflen); + err = -EINVAL; + goto err_map_out; + } + + dd->dma_descript_tab = dma_map_single(dd->dev, dd->descriptor, + PAGE_SIZE, DMA_TO_DEVICE); + + if (dma_mapping_error(dd->dev, dd->dma_descript_tab)) { + dev_err(dd->dev, "dma descriptor error\n"); + err = -EINVAL; + goto err_map_descriptor; + } + + return 0; + +err_map_descriptor: + dma_unmap_single(dd->dev, dd->dma_descript_tab, PAGE_SIZE, + DMA_TO_DEVICE); + +err_map_out: + dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, + DMA_TO_DEVICE); +err_map_in: + free_page((uintptr_t)dd->buf_out); + free_page((uintptr_t)dd->buf_in); + free_page((uintptr_t)dd->descriptor); +err_alloc: + if (err) + pr_err("error: %d\n", err); + return err; +} + +static void aml_aes_buff_cleanup(struct aml_aes_dev *dd) +{ + dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, + DMA_FROM_DEVICE); + dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, + DMA_TO_DEVICE); + dma_unmap_single(dd->dev, dd->dma_descript_tab, PAGE_SIZE, + DMA_TO_DEVICE); + free_page((uintptr_t)dd->buf_out); + free_page((uintptr_t)dd->buf_in); + free_page((uintptr_t)dd->descriptor); +} + +static int aml_aes_crypt(struct ablkcipher_request *req, unsigned long mode) +{ + struct aml_aes_ctx *ctx = crypto_ablkcipher_ctx( + crypto_ablkcipher_reqtfm(req)); + struct aml_aes_reqctx *rctx = ablkcipher_request_ctx(req); + struct aml_aes_dev *dd; + + if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) { + pr_err("request size is not exact amount of AES blocks\n"); + return -EINVAL; + } + ctx->block_size = AES_BLOCK_SIZE; + + dd = aml_aes_find_dev(ctx); + if (!dd) + return -ENODEV; + + rctx->mode = mode; + + return aml_aes_handle_queue(dd, req); +} + +static int aml_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct aml_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) { + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int aml_aes_ecb_encrypt(struct ablkcipher_request *req) +{ + return aml_aes_crypt(req, + AES_FLAGS_ENCRYPT); +} + +static int aml_aes_ecb_decrypt(struct ablkcipher_request *req) +{ + return aml_aes_crypt(req, + 0); +} + +static int aml_aes_cbc_encrypt(struct ablkcipher_request *req) +{ + return aml_aes_crypt(req, + AES_FLAGS_ENCRYPT | AES_FLAGS_CBC); +} + +static int aml_aes_cbc_decrypt(struct ablkcipher_request *req) +{ + return aml_aes_crypt(req, + AES_FLAGS_CBC); +} + +static int aml_aes_ctr_encrypt(struct ablkcipher_request *req) +{ + return aml_aes_crypt(req, + AES_FLAGS_ENCRYPT | AES_FLAGS_CTR); +} + +static int aml_aes_ctr_decrypt(struct ablkcipher_request *req) +{ + /* XXX: use encrypt to replace for decrypt */ + return aml_aes_crypt(req, + AES_FLAGS_ENCRYPT | AES_FLAGS_CTR); +} + +static int aml_aes_cra_init(struct crypto_tfm *tfm) +{ + tfm->crt_ablkcipher.reqsize = sizeof(struct aml_aes_reqctx); + + return 0; +} + +static void aml_aes_cra_exit(struct crypto_tfm *tfm) +{ +} + +static struct crypto_alg aes_algs[] = { + { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-aml", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aml_aes_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = aml_aes_cra_init, + .cra_exit = aml_aes_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aml_aes_setkey, + .encrypt = aml_aes_ecb_encrypt, + .decrypt = aml_aes_ecb_decrypt, + } + }, + { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-aml", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aml_aes_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = aml_aes_cra_init, + .cra_exit = aml_aes_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aml_aes_setkey, + .encrypt = aml_aes_cbc_encrypt, + .decrypt = aml_aes_cbc_decrypt, + } + }, + { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-aml", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aml_aes_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = aml_aes_cra_init, + .cra_exit = aml_aes_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aml_aes_setkey, + .encrypt = aml_aes_ctr_encrypt, + .decrypt = aml_aes_ctr_decrypt, + } + } +}; + +static void aml_aes_queue_task(unsigned long data) +{ + struct aml_aes_dev *dd = (struct aml_aes_dev *)data; + + aml_aes_handle_queue(dd, NULL); +} + +static void aml_aes_done_task(unsigned long data) +{ + struct aml_aes_dev *dd = (struct aml_aes_dev *) data; + int err; + + err = aml_aes_crypt_dma_stop(dd); + + aml_dma_debug(dd->descriptor, dd->fast_nents ? + dd->fast_nents : 1, __func__); + + err = dd->err ? : err; + + if (dd->total && !err) { + if (dd->flags & AES_FLAGS_FAST) { + uint32_t i = 0; + + for (i = 0; i < dd->fast_nents; i++) { + dd->in_sg = sg_next(dd->in_sg); + dd->out_sg = sg_next(dd->out_sg); + if (!dd->in_sg || !dd->out_sg) + err = -EINVAL; + } + } + + if (!err) + err = aml_aes_crypt_dma_start(dd); + if (!err) + return; /* DMA started. Not fininishing. */ + } + + aml_aes_finish_req(dd, err); + aml_aes_handle_queue(dd, NULL); +} + +static irqreturn_t aml_aes_irq(int irq, void *dev_id) +{ + struct aml_aes_dev *aes_dd = dev_id; + uint8_t status = aml_read_crypto_reg(DMA_STATUS_REG); + + if (status) { + if (status == 0x1) + pr_err("irq overwrite\n"); + if (AES_FLAGS_DMA & aes_dd->flags) { + aml_write_crypto_reg(DMA_STATUS_REG, 0xf); + tasklet_schedule(&aes_dd->done_task); + return IRQ_HANDLED; + } else { + return IRQ_NONE; + } + } + + return IRQ_NONE; +} + +static void aml_aes_unregister_algs(struct aml_aes_dev *dd) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(aes_algs); i++) + crypto_unregister_alg(&aes_algs[i]); +} + +static int aml_aes_register_algs(struct aml_aes_dev *dd) +{ + int err, i, j; + + for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { + err = crypto_register_alg(&aes_algs[i]); + if (err) + goto err_aes_algs; + } + + return 0; + +err_aes_algs: + for (j = 0; j < i; j++) + crypto_unregister_alg(&aes_algs[j]); + + return err; +} + +static int aml_aes_probe(struct platform_device *pdev) +{ + struct aml_aes_dev *aes_dd; + struct device *dev = &pdev->dev; + struct resource *res_irq = 0; + struct resource *res_base = 0; + int err = -EPERM; + + aes_dd = kzalloc(sizeof(struct aml_aes_dev), GFP_KERNEL); + if (aes_dd == NULL) { + err = -ENOMEM; + goto aes_dd_err; + } + + aes_dd->dev = dev; + platform_set_drvdata(pdev, aes_dd); + res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, AES_THREAD_INDEX); + res_base = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res_base) { + dev_err(dev, "error to get normal IORESOURCE_MEM.\n"); + goto aes_dd_err; + } else { + if (!cryptoreg_offset) { + cryptoreg_offset = ioremap(res_base->start, + resource_size(res_base)); + map_in_aes_dma = 1; + } + } + + INIT_LIST_HEAD(&aes_dd->list); + + tasklet_init(&aes_dd->done_task, aml_aes_done_task, + (unsigned long)aes_dd); + tasklet_init(&aes_dd->queue_task, aml_aes_queue_task, + (unsigned long)aes_dd); + + crypto_init_queue(&aes_dd->queue, AML_AES_QUEUE_LENGTH); + + aes_dd->irq = res_irq->start; + + err = request_irq(aes_dd->irq, aml_aes_irq, IRQF_SHARED, "aml-aes", + aes_dd); + if (err) { + dev_err(dev, "unable to request aes irq.\n"); + goto aes_irq_err; + } + + err = aml_aes_hw_init(aes_dd); + if (err) + goto err_aes_buff; + + err = aml_aes_buff_init(aes_dd); + if (err) + goto err_aes_buff; + + spin_lock(&aml_aes.lock); + list_add_tail(&aes_dd->list, &aml_aes.dev_list); + spin_unlock(&aml_aes.lock); + + err = aml_aes_register_algs(aes_dd); + if (err) + goto err_algs; + + dev_info(dev, "Aml AES_dma\n"); + + return 0; + +err_algs: + spin_lock(&aml_aes.lock); + list_del(&aes_dd->list); + spin_unlock(&aml_aes.lock); + aml_aes_buff_cleanup(aes_dd); +err_aes_buff: + free_irq(aes_dd->irq, aes_dd); +aes_irq_err: + + if (map_in_aes_dma) { + iounmap(cryptoreg_offset); + map_in_aes_dma = 0; + } + + tasklet_kill(&aes_dd->done_task); + tasklet_kill(&aes_dd->queue_task); + kfree(aes_dd); + aes_dd = NULL; +aes_dd_err: + dev_err(dev, "initialization failed.\n"); + + return err; +} + +static int aml_aes_remove(struct platform_device *pdev) +{ + static struct aml_aes_dev *aes_dd; + + aes_dd = platform_get_drvdata(pdev); + if (!aes_dd) + return -ENODEV; + spin_lock(&aml_aes.lock); + list_del(&aes_dd->list); + spin_unlock(&aml_aes.lock); + + aml_aes_unregister_algs(aes_dd); + + tasklet_kill(&aes_dd->done_task); + tasklet_kill(&aes_dd->queue_task); + + if (map_in_aes_dma) { + iounmap(cryptoreg_offset); + map_in_aes_dma = 0; + } + + + if (aes_dd->irq > 0) + free_irq(aes_dd->irq, aes_dd); + + kfree(aes_dd); + aes_dd = NULL; + + return 0; +} + +#ifdef CONFIG_OF +static const struct of_device_id aml_aes_dt_match[] = { + { .compatible = "amlogic,aes_dma", + }, + {}, +}; +#else +#define aml_aes_dt_match NULL +#endif + +static struct platform_driver aml_aes_driver = { + .probe = aml_aes_probe, + .remove = aml_aes_remove, + .driver = { + .name = "aml_aes_dma", + .owner = THIS_MODULE, + .of_match_table = aml_aes_dt_match, + }, +}; + +module_platform_driver(aml_aes_driver); + +MODULE_DESCRIPTION("Aml AES hw acceleration support."); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("matthew.shyu "); diff --git a/drivers/amlogic/crypto/aml-crypto-dma.c b/drivers/amlogic/crypto/aml-crypto-dma.c new file mode 100644 index 000000000000..4c7a0f554e82 --- /dev/null +++ b/drivers/amlogic/crypto/aml-crypto-dma.c @@ -0,0 +1,109 @@ +/* + * drivers/amlogic/crypto/aml-crypto-dma.c + * + * Copyright (C) 2017 Amlogic, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "aml-crypto-dma.h" +#if 1 +void __iomem *cryptoreg_offset; +u32 swap_ulong32(u32 val) +{ + u32 res = 0; + + res = ((val & 0xff) << 24) + (((val >> 8) & 0xff) << 16) + + (((val >> 16) & 0xff) << 8) + ((val >> 24) & 0xff); + return res; +} +void aml_write_crypto_reg(u32 addr, u32 data) +{ + writel(data, cryptoreg_offset + (addr << 2)); +} + +u32 aml_read_crypto_reg(u32 addr) +{ + return readl(cryptoreg_offset + (addr << 2)); +} +#endif +void aml_dma_debug(struct dma_dsc *dsc, u32 nents, const char *msg) +{ +#if AML_CRYPTO_DEBUG + uint32_t i = 0; + + pr_err("begin %s\n", msg); + for (i = 0; i < 2; i++) + pr_err("reg(%lu) = 0x%8x\n", (uintptr_t)(DMA_T0 + i), + aml_read_crypto_reg(DMA_T0 + i)); + for (i = 0; i < 2; i++) + pr_err("reg(%lu) = 0x%8x\n", (uintptr_t)(DMA_STS0 + i), + aml_read_crypto_reg(DMA_STS0 + i)); + pr_err("reg(%lu) = 0x%8x\n", (uintptr_t)(DMA_CFG), + aml_read_crypto_reg(DMA_CFG)); + for (i = 0; i < nents; i++) { + pr_err("desc (%4x) (len) = 0x%8x\n", i, + dsc[i].dsc_cfg.b.length); + pr_err("desc (%4x) (irq) = 0x%8x\n", i, + dsc[i].dsc_cfg.b.irq); + pr_err("desc (%4x) (eoc) = 0x%8x\n", i, + dsc[i].dsc_cfg.b.eoc); + pr_err("desc (%4x) (lop) = 0x%8x\n", i, + dsc[i].dsc_cfg.b.loop); + pr_err("desc (%4x) (mod) = 0x%8x\n", i, + dsc[i].dsc_cfg.b.mode); + pr_err("desc (%4x) (beg) = 0x%8x\n", i, + dsc[i].dsc_cfg.b.begin); + pr_err("desc (%4x) (end) = 0x%8x\n", i, + dsc[i].dsc_cfg.b.end); + pr_err("desc (%4x) (opm) = 0x%8x\n", i, + dsc[i].dsc_cfg.b.op_mode); + pr_err("desc (%4x) (enc) = 0x%8x\n", i, + dsc[i].dsc_cfg.b.enc_sha_only); + pr_err("desc (%4x) (blk) = 0x%8x\n", i, + dsc[i].dsc_cfg.b.block); + pr_err("desc (%4x) (err) = 0x%8x\n", i, + dsc[i].dsc_cfg.b.error); + pr_err("desc (%4x) (own) = 0x%8x\n", i, + dsc[i].dsc_cfg.b.owner); + pr_err("desc (%4x) (src) = 0x%8x\n", i, + dsc[i].src_addr); + pr_err("desc (%4x) (tgt) = 0x%8x\n", i, + dsc[i].tgt_addr); + } + pr_err("end %s\n", msg); +#endif +} diff --git a/drivers/amlogic/crypto/aml-crypto-dma.h b/drivers/amlogic/crypto/aml-crypto-dma.h new file mode 100644 index 000000000000..30c933c9a289 --- /dev/null +++ b/drivers/amlogic/crypto/aml-crypto-dma.h @@ -0,0 +1,107 @@ +/* + * drivers/amlogic/crypto/aml-crypto-dma.h + * + * Copyright (C) 2017 Amlogic, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ + +#ifndef _AML_CRYPTO_H_ +#define _AML_CRYPTO_H_ +#include + +#define AML_CRYPTO_DEBUG 0 + + /* Reserved 4096 bytes and table is 12 bytes each */ +#define MAX_NUM_TABLES 341 + +#define DMA_T0 0x00 +#define DMA_T1 0x01 +#define DMA_T2 0x02 +#define DMA_T3 0x03 +#define DMA_STS0 0x04 +#define DMA_STS1 0x05 +#define DMA_STS2 0x06 +#define DMA_STS3 0x07 +#define DMA_CFG 0x08 + +#define aml_write_reg(addr, data) \ + writel(data, (int *)addr) + +#define aml_read_reg(addr) \ + readl(addr) + +/* driver logic flags */ +#define TDES_KEY_LENGTH 32 +#define TDES_MIN_BLOCK_SIZE 8 + +#define OP_MODE_ECB 0 +#define OP_MODE_CBC 1 +#define OP_MODE_CTR 2 + +#define OP_MODE_SHA 0 +#define OP_MODE_HMAC_I 1 +#define OP_MODE_HMAC_O 2 + +#define MODE_DMA 0x0 +#define MODE_KEY 0x1 +#define MODE_MEMSET 0x2 +/* 0x3 is skipped */ +/* 0x4 is skipped */ +#define MODE_SHA1 0x5 +#define MODE_SHA256 0x6 +#define MODE_SHA224 0x7 +#define MODE_AES128 0x8 +#define MODE_AES192 0x9 +#define MODE_AES256 0xa +/* 0xb is skipped */ +#define MODE_DES 0xc +/* 0xd is skipped */ +#define MODE_TDES_2K 0xe +#define MODE_TDES_3K 0xf + +/* Thread 2, 3 are for secure threads */ +#define AES_THREAD_INDEX 0 +#define TDES_THREAD_INDEX 0 +#define SHA_THREAD_INDEX 0 +#define HMAC_THREAD_INDEX 0 + +struct dma_dsc { + union { + uint32_t d32; + struct { + unsigned length:17; + unsigned irq:1; + unsigned eoc:1; + unsigned loop:1; + unsigned mode:4; + unsigned begin:1; + unsigned end:1; + unsigned op_mode:2; + unsigned enc_sha_only:1; + unsigned block:1; + unsigned error:1; + unsigned owner:1; + } b; + } dsc_cfg; + uint32_t src_addr; + uint32_t tgt_addr; +}; + +extern void __iomem *cryptoreg_offset; +extern u32 secure_cryptoreg_offset; + +u32 swap_ulong32(u32 val); +void aml_write_crypto_reg(u32 addr, u32 data); +u32 aml_read_crypto_reg(u32 addr); +void aml_dma_debug(struct dma_dsc *dsc, u32 nents, const char *msg); +#endif diff --git a/drivers/amlogic/crypto/aml-sha-dma.c b/drivers/amlogic/crypto/aml-sha-dma.c new file mode 100644 index 000000000000..5ce69fa9ea26 --- /dev/null +++ b/drivers/amlogic/crypto/aml-sha-dma.c @@ -0,0 +1,1305 @@ +/* + * drivers/amlogic/crypto/aml-sha-dma.c + * + * Copyright (C) 2017 Amlogic, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "aml-crypto-dma.h" + +/* SHA flags */ +#define SHA_FLAGS_BUSY BIT(0) +#define SHA_FLAGS_FINAL BIT(1) +#define SHA_FLAGS_DMA_ACTIVE BIT(2) +#define SHA_FLAGS_OUTPUT_READY BIT(3) +#define SHA_FLAGS_INIT BIT(4) +#define SHA_FLAGS_DMA_READY BIT(5) +#define SHA_FLAGS_DMA_FAST BIT(6) + +#define SHA_FLAGS_FINUP BIT(16) +#define SHA_FLAGS_SHA1 BIT(17) +#define SHA_FLAGS_SHA224 BIT(18) +#define SHA_FLAGS_SHA256 BIT(19) +#define SHA_FLAGS_HMAC BIT(20) +#define SHA_FLAGS_ERROR BIT(21) + +#define SHA_OP_UPDATE 1 +#define SHA_OP_FINAL 2 + +#define SHA_BUFFER_LEN PAGE_SIZE + +#define DMA_THREAD_REG (DMA_T0 + SHA_THREAD_INDEX) +#define DMA_STATUS_REG (DMA_STS0 + SHA_THREAD_INDEX) +u8 map_in_sha_dma; +struct aml_sha_dev; + +struct aml_sha_reqctx { + struct aml_sha_dev *dd; + unsigned long flags; + unsigned long op; + + u64 digcnt[2]; + size_t bufcnt; + size_t buflen; + dma_addr_t dma_addr; + dma_addr_t hash_addr; + + /* walk state */ + struct scatterlist *sg; + unsigned int offset; /* offset in current sg */ + unsigned int total; /* total request */ + + size_t block_size; + uint32_t fast_nents; + + u8 *digest; + u8 buffer[0] __aligned(sizeof(u32)); +}; + +struct aml_sha_ctx { + struct aml_sha_dev *dd; + u8 key[SHA256_BLOCK_SIZE]; + u32 keylen; + unsigned long flags; +}; + +#define AML_SHA_QUEUE_LENGTH 50 + +struct aml_sha_dev { + struct list_head list; + struct device *dev; + int irq; + + spinlock_t lock; + int err; + struct tasklet_struct done_task; + + unsigned long flags; + struct crypto_queue queue; + struct ahash_request *req; + + void *descriptor; + dma_addr_t dma_descript_tab; +}; + +struct aml_sha_drv { + struct list_head dev_list; + spinlock_t lock; +}; + +static struct aml_sha_drv aml_sha = { + .dev_list = LIST_HEAD_INIT(aml_sha.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(aml_sha.lock), +}; + +static size_t aml_sha_append_sg(struct aml_sha_reqctx *ctx) +{ + size_t count = 0; + + while ((ctx->bufcnt < ctx->buflen) && ctx->total) { + count = min(ctx->sg->length - ctx->offset, ctx->total); + count = min(count, ctx->buflen - ctx->bufcnt); + + if (count <= 0) + break; + + scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg, + ctx->offset, count, 0); + + ctx->bufcnt += count; + ctx->offset += count; + ctx->total -= count; + + if (ctx->offset == ctx->sg->length) { + ctx->sg = sg_next(ctx->sg); + if (ctx->sg) + ctx->offset = 0; + else + ctx->total = 0; + } + } + + return 0; +} + +static int aml_sha_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct aml_sha_ctx *tctx = crypto_ahash_ctx(tfm); + struct aml_sha_reqctx *ctx = ahash_request_ctx(req); + struct aml_sha_dev *dd = NULL; + struct aml_sha_dev *tmp; + + spin_lock_bh(&aml_sha.lock); + if (!tctx->dd) { + list_for_each_entry(tmp, &aml_sha.dev_list, list) { + dd = tmp; + break; + } + tctx->dd = dd; + } else { + dd = tctx->dd; + } + + spin_unlock_bh(&aml_sha.lock); + + ctx->dd = dd; + + ctx->flags = 0; + +#if AML_CRYPTO_DEBUG + dev_info(dd->dev, "init: digest size: %d\n", + crypto_ahash_digestsize(tfm)); +#endif + + switch (crypto_ahash_digestsize(tfm)) { + case SHA1_DIGEST_SIZE: + ctx->flags |= SHA_FLAGS_SHA1; + ctx->block_size = SHA1_BLOCK_SIZE; + break; + case SHA224_DIGEST_SIZE: + ctx->flags |= SHA_FLAGS_SHA224; + ctx->block_size = SHA224_BLOCK_SIZE; + break; + case SHA256_DIGEST_SIZE: + ctx->flags |= SHA_FLAGS_SHA256; + ctx->block_size = SHA256_BLOCK_SIZE; + break; + default: + return -EINVAL; + } + + ctx->digest = 0; + ctx->bufcnt = 0; + ctx->digcnt[0] = 0; + ctx->digcnt[1] = 0; + ctx->fast_nents = 0; + ctx->buflen = SHA_BUFFER_LEN; + + return 0; +} + +static int aml_sha_xmit_dma(struct aml_sha_dev *dd, struct dma_dsc *dsc, + uint32_t nents, int final) +{ + int i = 0; + u32 mode; + struct aml_sha_ctx *tctx = crypto_tfm_ctx(dd->req->base.tfm); + struct aml_sha_reqctx *ctx = ahash_request_ctx(dd->req); + size_t length = 0; + +#if AML_CRYPTO_DEBUG + dev_info(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, nents: %u, final: %d\n", + ctx->digcnt[1], ctx->digcnt[0], nents, final); +#endif + + mode = MODE_SHA1; + + if (ctx->flags & SHA_FLAGS_SHA224) + mode = MODE_SHA224; + else if (ctx->flags & SHA_FLAGS_SHA256) + mode = MODE_SHA256; + + if (final) { + kfree(ctx->digest); + ctx->digest = kzalloc(SHA256_DIGEST_SIZE, GFP_KERNEL); + ctx->hash_addr = dma_map_single(dd->dev, ctx->digest, + SHA256_DIGEST_SIZE, DMA_FROM_DEVICE); + if (dma_mapping_error(dd->dev, ctx->hash_addr)) { + dev_err(dd->dev, "hash %d bytes error\n", + SHA256_DIGEST_SIZE); + return -EINVAL; + } + } else { + ctx->hash_addr = 0; + } + + for (i = 0; i < nents; i++) { + dsc[i].tgt_addr = (uintptr_t)ctx->hash_addr; + dsc[i].dsc_cfg.b.enc_sha_only = 1; + dsc[i].dsc_cfg.b.mode = mode; + dsc[i].dsc_cfg.b.begin = + (i == 0 && !(ctx->digcnt[0] || ctx->digcnt[1]) && + !(tctx->flags & SHA_FLAGS_HMAC)); + dsc[i].dsc_cfg.b.end = final; + dsc[i].dsc_cfg.b.op_mode = OP_MODE_SHA; + dsc[i].dsc_cfg.b.eoc = (i == (nents - 1)); + dsc[i].dsc_cfg.b.owner = 1; + } + dma_sync_single_for_device(dd->dev, dd->dma_descript_tab, + PAGE_SIZE, DMA_TO_DEVICE); + + aml_dma_debug(dsc, nents, __func__); + /* should be non-zero before next lines to disable clocks later */ + for (i = 0; i < nents; i++) { + length = dsc->dsc_cfg.b.length; + ctx->digcnt[0] += length; + if (ctx->digcnt[0] < length) + ctx->digcnt[1]++; + } + + if (final) + dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */ + + dd->flags |= SHA_FLAGS_DMA_ACTIVE; + +#if AML_CRYPTO_DEBUG + dev_info(dd->dev, + "xmit before : digcnt: 0x%llx 0x%llx, length: %zd, final: %d\n", + ctx->digcnt[1], ctx->digcnt[0], length, final); +#endif + /* Start DMA transfer */ + aml_write_crypto_reg(DMA_THREAD_REG, + (uintptr_t) dd->dma_descript_tab | 2); + + return -EINPROGRESS; +} + +static int aml_sha_xmit_start(struct aml_sha_dev *dd, struct dma_dsc *dsc, + uint32_t nents, int final) +{ + return aml_sha_xmit_dma(dd, dsc, nents, final); +} + +static int aml_sha_xmit_dma_map(struct aml_sha_dev *dd, + struct aml_sha_reqctx *ctx, + size_t length, int final) +{ + struct dma_dsc *dsc = dd->descriptor; + + ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, + ctx->buflen, DMA_TO_DEVICE); + if (dma_mapping_error(dd->dev, ctx->dma_addr)) { + dev_err(dd->dev, "dma %zd bytes error\n", ctx->buflen); + return -EINVAL; + } + + dma_sync_single_for_device(dd->dev, ctx->dma_addr, + SHA_BUFFER_LEN, DMA_TO_DEVICE); + + dsc->src_addr = (uintptr_t)ctx->dma_addr; + dsc->dsc_cfg.d32 = 0; + dsc->dsc_cfg.b.length = length; + + /* next call does not fail... so no unmap in the case of error */ + return aml_sha_xmit_start(dd, dsc, 1, final); +} + +static int aml_sha_update_dma_slow(struct aml_sha_dev *dd) +{ + struct aml_sha_reqctx *ctx = ahash_request_ctx(dd->req); + unsigned int final; + size_t count; + + ctx->flags &= ~SHA_FLAGS_DMA_FAST; + aml_sha_append_sg(ctx); + + final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; + +#if AML_CRYPTO_DEBUG + dev_info(dd->dev, + "slow: bufcnt: %zd, digcnt: 0x%llx 0x%llx, final: %d, total: %u\n", + ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final, ctx->total); +#endif + + if (IS_ALIGNED(ctx->bufcnt, ctx->block_size) || final) { + count = ctx->bufcnt; + ctx->bufcnt = 0; + return aml_sha_xmit_dma_map(dd, ctx, count, final); + } + return 0; +} + +static int aml_sha_update_dma_start(struct aml_sha_dev *dd) +{ + struct aml_sha_reqctx *ctx = ahash_request_ctx(dd->req); + unsigned int length = 0, final = 0, tail = 0; + struct scatterlist *sg; + struct dma_dsc *dsc = dd->descriptor; + +#if AML_CRYPTO_DEBUG + dev_info(dd->dev, "start: total: %u, fast_nents: %u offset: %u\n", + ctx->total, ctx->fast_nents, ctx->offset); +#endif + + if (!ctx->total) + return 0; + + /* walk across (nents - 1) sg */ + /* because the last sg maybe only partially used */ + while (ctx->fast_nents > 1) { + ctx->sg = sg_next(ctx->sg); + ctx->fast_nents--; + } + + if (ctx->offset == ctx->sg->length) { + ctx->sg = sg_next(ctx->sg); + if (ctx->sg) + ctx->offset = 0; + else + return 0; + } + + ctx->fast_nents = 0; + + if (ctx->bufcnt || ctx->offset || ctx->total < ctx->block_size) + return aml_sha_update_dma_slow(dd); + + sg = ctx->sg; + + while (ctx->total && ctx->fast_nents < MAX_NUM_TABLES && sg) { +#if AML_CRYPTO_DEBUG + dev_info(dd->dev, + "fast: dig: 0x%llx 0x%llx, bufcnt: %zd, total: %u, sglen: %u\n", + ctx->digcnt[1], ctx->digcnt[0], + ctx->bufcnt, ctx->total, ctx->sg->length); +#endif + + length = min(ctx->total, sg->length); + + if (!(ctx->flags & SHA_FLAGS_FINUP)) { + /* not last sg must be ctx->block_size aligned */ + tail = length & (ctx->block_size - 1); + length -= tail; + } + + ctx->total -= length; + ctx->offset = length; /* offset where to start slow */ + + final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total; + + dma_map_sg(dd->dev, sg, 1, DMA_TO_DEVICE); + + ctx->dma_addr = sg_dma_address(sg); + dsc[ctx->fast_nents].src_addr = (uintptr_t)ctx->dma_addr; + dsc[ctx->fast_nents].dsc_cfg.d32 = 0; + dsc[ctx->fast_nents].dsc_cfg.b.length = length; + + sg = sg_next(sg); + ctx->fast_nents++; + +#if AML_CRYPTO_DEBUG + dev_info(dd->dev, "fast: total: %u, offset: %u, tail: %u\n", + ctx->total, ctx->offset, tail); +#endif + + if (tail) + break; + } + if (ctx->fast_nents) { + ctx->flags |= SHA_FLAGS_DMA_FAST; + return aml_sha_xmit_start(dd, dsc, ctx->fast_nents, final); + } + + return 0; +} + +static int aml_sha_update_dma_stop(struct aml_sha_dev *dd) +{ + struct aml_sha_reqctx *ctx = ahash_request_ctx(dd->req); + + if (ctx->flags & SHA_FLAGS_DMA_FAST) + dma_unmap_sg(dd->dev, ctx->sg, ctx->fast_nents, DMA_TO_DEVICE); + else + dma_unmap_single(dd->dev, ctx->dma_addr, + ctx->buflen, DMA_TO_DEVICE); + if (ctx->hash_addr) + dma_unmap_single(dd->dev, ctx->hash_addr, + SHA256_DIGEST_SIZE, DMA_FROM_DEVICE); + return 0; +} + +static int aml_sha_update_req(struct aml_sha_dev *dd) +{ + int err; +#if AML_CRYPTO_DEBUG + struct ahash_request *req = dd->req; + struct aml_sha_reqctx *ctx = ahash_request_ctx(req); +#endif + +#if AML_CRYPTO_DEBUG + dev_info(dd->dev, "update_req: total: %u, digcnt: 0x%llx 0x%llx\n", + ctx->total, ctx->digcnt[1], ctx->digcnt[0]); +#endif + + err = aml_sha_update_dma_start(dd); + + return err; +} + +static int aml_sha_final_req(struct aml_sha_dev *dd) +{ + int err = 0; + + err = aml_sha_update_dma_slow(dd); + +#if AML_CRYPTO_DEBUG + dev_info(dd->dev, "final_req: err: %d\n", err); +#endif + + return err; +} + +static void aml_sha_copy_ready_hash(struct ahash_request *req) +{ + struct aml_sha_reqctx *ctx = ahash_request_ctx(req); + + if (!req->result) + return; + + if (!ctx->digest) + return; + + if (ctx->flags & SHA_FLAGS_SHA1) + memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE); + else if (ctx->flags & SHA_FLAGS_SHA224) + memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE); + else if (ctx->flags & SHA_FLAGS_SHA256) + memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE); + + kfree(ctx->digest); +} + +static int aml_sha_finish_hmac(struct ahash_request *req) +{ + struct aml_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); + struct aml_sha_reqctx *ctx = ahash_request_ctx(req); + struct aml_sha_dev *dd = ctx->dd; + struct dma_dsc *dsc = dd->descriptor; + u32 mode; + u32 ds = 0; + u8 *key; + dma_addr_t dma_key = 0; + + if (!ctx->digest) + return -1; + + ctx->hash_addr = dma_map_single(dd->dev, ctx->digest, + SHA256_DIGEST_SIZE, DMA_BIDIRECTIONAL); + + key = kmalloc(tctx->keylen, GFP_KERNEL); + memcpy(key, tctx->key, tctx->keylen); + dma_key = dma_map_single(dd->dev, key, + tctx->keylen, DMA_TO_DEVICE); + + mode = MODE_SHA1; + if (ctx->flags & SHA_FLAGS_SHA224) + mode = MODE_SHA224; + else if (ctx->flags & SHA_FLAGS_SHA256) + mode = MODE_SHA256; + + if (ctx->flags & SHA_FLAGS_SHA1) + ds = SHA1_DIGEST_SIZE; + else if (ctx->flags & SHA_FLAGS_SHA224) + ds = SHA224_DIGEST_SIZE; + else if (ctx->flags & SHA_FLAGS_SHA256) + ds = SHA256_DIGEST_SIZE; + /* opad */ + dsc[0].src_addr = (uintptr_t)dma_key; + dsc[0].tgt_addr = 0; + dsc[0].dsc_cfg.d32 = 0; + dsc[0].dsc_cfg.b.length = tctx->keylen; + dsc[0].dsc_cfg.b.mode = mode; + dsc[0].dsc_cfg.b.enc_sha_only = 1; + dsc[0].dsc_cfg.b.op_mode = OP_MODE_HMAC_O; + dsc[0].dsc_cfg.b.begin = 1; + dsc[0].dsc_cfg.b.end = 0; + dsc[0].dsc_cfg.b.eoc = 0; + dsc[0].dsc_cfg.b.owner = 1; + + /* 2nd stage hash */ + dsc[1].src_addr = (uintptr_t)ctx->hash_addr; + dsc[1].tgt_addr = (uintptr_t)ctx->hash_addr; + dsc[1].dsc_cfg.d32 = 0; + dsc[1].dsc_cfg.b.length = ds; + dsc[1].dsc_cfg.b.mode = mode; + dsc[1].dsc_cfg.b.enc_sha_only = 1; + dsc[1].dsc_cfg.b.begin = 0; + dsc[1].dsc_cfg.b.end = 1; + dsc[1].dsc_cfg.b.eoc = 1; + dsc[1].dsc_cfg.b.owner = 1; + + dma_sync_single_for_device(dd->dev, dd->dma_descript_tab, + PAGE_SIZE, DMA_TO_DEVICE); + aml_dma_debug(dsc, 2, __func__); + aml_write_crypto_reg(DMA_THREAD_REG, + (uintptr_t) dd->dma_descript_tab | 2); + while (aml_read_crypto_reg(DMA_STATUS_REG) == 0) + ; + aml_write_crypto_reg(DMA_STATUS_REG, 0xf); + dma_unmap_single(dd->dev, dma_key, + tctx->keylen, DMA_TO_DEVICE); + dma_unmap_single(dd->dev, ctx->hash_addr, + SHA256_DIGEST_SIZE, DMA_BIDIRECTIONAL); + + return 0; +} + +static int aml_sha_finish(struct ahash_request *req) +{ + struct aml_sha_reqctx *ctx = ahash_request_ctx(req); + struct aml_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); + int err = 0; +#if AML_CRYPTO_DEBUG + struct aml_sha_dev *dd = ctx->dd; +#endif + + if (ctx->digcnt[0] || ctx->digcnt[1]) { + if (tctx->flags & SHA_FLAGS_HMAC) + err = aml_sha_finish_hmac(req); + aml_sha_copy_ready_hash(req); + } + +#if AML_CRYPTO_DEBUG + dev_info(dd->dev, "finish digcnt: 0x%llx 0x%llx, bufcnt: %zd\n", + ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt); +#endif + + return err; +} + +static void aml_sha_finish_req(struct ahash_request *req, int err) +{ + struct aml_sha_reqctx *ctx = ahash_request_ctx(req); + struct aml_sha_dev *dd = ctx->dd; + + if (!err) { + if (SHA_FLAGS_FINAL & dd->flags) + err = aml_sha_finish(req); + } else { + ctx->flags |= SHA_FLAGS_ERROR; + } + + /* atomic operation is not needed here */ + dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | + SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY); + + if (req->base.complete) + req->base.complete(&req->base, err); + + /* handle new request */ + tasklet_schedule(&dd->done_task); +} + +static int aml_sha_hw_init(struct aml_sha_dev *dd) +{ + if (!(dd->flags & SHA_FLAGS_INIT)) { + dd->flags |= SHA_FLAGS_INIT; + dd->err = 0; + } + + return 0; +} + +static int aml_sha_buff_init(struct aml_sha_dev *dd) +{ + int err = -ENOMEM; + + dd->descriptor = (void *)__get_free_pages(GFP_KERNEL, 0); + if (!dd->descriptor) { + dev_err(dd->dev, "unable to alloc pages.\n"); + goto err_alloc; + } + + dd->dma_descript_tab = dma_map_single(dd->dev, dd->descriptor, + PAGE_SIZE, DMA_TO_DEVICE); + + if (dma_mapping_error(dd->dev, dd->dma_descript_tab)) { + dev_err(dd->dev, "dma descriptor error\n"); + err = -EINVAL; + goto err_map_descriptor; + } + + return 0; + +err_map_descriptor: + dma_unmap_single(dd->dev, dd->dma_descript_tab, PAGE_SIZE, + DMA_TO_DEVICE); +err_alloc: + if (err) + pr_err("error: %d\n", err); + return err; +} + +static int aml_sha_handle_queue(struct aml_sha_dev *dd, + struct ahash_request *req) +{ + struct crypto_async_request *async_req, *backlog; + struct aml_sha_reqctx *ctx; + unsigned long flags; + int err = 0, ret = 0; + + spin_lock_irqsave(&dd->lock, flags); + if (req) + ret = ahash_enqueue_request(&dd->queue, req); + + if (SHA_FLAGS_BUSY & dd->flags) { + spin_unlock_irqrestore(&dd->lock, flags); + return ret; + } + + backlog = crypto_get_backlog(&dd->queue); + async_req = crypto_dequeue_request(&dd->queue); + if (async_req) + dd->flags |= SHA_FLAGS_BUSY; + + spin_unlock_irqrestore(&dd->lock, flags); + + if (!async_req) + return ret; + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + + req = ahash_request_cast(async_req); + dd->req = req; + ctx = ahash_request_ctx(req); + +#if AML_CRYPTO_DEBUG + dev_info(dd->dev, "handling new req, op: %lu, nbytes: %d\n", + ctx->op, req->nbytes); +#endif + + err = aml_sha_hw_init(dd); + + if (err) + goto err1; + + + if (ctx->op == SHA_OP_UPDATE) { + err = aml_sha_update_req(dd); + /* no final() after finup() */ + if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP)) + err = aml_sha_final_req(dd); + } else if (ctx->op == SHA_OP_FINAL) { + err = aml_sha_final_req(dd); + } + +err1: + if (err != -EINPROGRESS) + /* done_task will not finish it, so do it here */ + aml_sha_finish_req(req, err); + + return ret; +} + +static int aml_sha_enqueue(struct ahash_request *req, unsigned int op) +{ + struct aml_sha_reqctx *ctx = ahash_request_ctx(req); + struct aml_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); + struct aml_sha_dev *dd = tctx->dd; + + ctx->op = op; + + return aml_sha_handle_queue(dd, req); +} + +static int aml_sha_update(struct ahash_request *req) +{ + struct aml_sha_reqctx *ctx = ahash_request_ctx(req); + + if (!req->nbytes) + return 0; + + ctx->total = req->nbytes; + ctx->sg = req->src; + ctx->offset = 0; + return aml_sha_enqueue(req, SHA_OP_UPDATE); +} + +static int aml_sha_final(struct ahash_request *req) +{ + struct aml_sha_reqctx *ctx = ahash_request_ctx(req); + + ctx->flags |= SHA_FLAGS_FINUP; + + if (ctx->flags & SHA_FLAGS_ERROR) + return 0; /* uncompleted hash is not needed */ + + if (ctx->bufcnt) + return aml_sha_enqueue(req, SHA_OP_FINAL); + else + return aml_sha_finish(req); /* copy ready hash */ +} + +static int aml_sha_finup(struct ahash_request *req) +{ + struct aml_sha_reqctx *ctx = ahash_request_ctx(req); + int err1, err2; + + ctx->flags |= SHA_FLAGS_FINUP; + + err1 = aml_sha_update(req); + if (err1 == -EINPROGRESS || err1 == -EBUSY) + return err1; + + /* + * final() has to be always called to cleanup resources + * even if update() failed, except EINPROGRESS + */ + err2 = aml_sha_final(req); + + return err1 ?: err2; +} + +static int aml_sha_digest(struct ahash_request *req) +{ + return aml_sha_init(req) ?: aml_sha_finup(req); +} + +static int aml_sha_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct aml_sha_ctx *tctx = crypto_ahash_ctx(tfm); + struct aml_sha_dev *dd = 0; + struct aml_sha_dev *tmp = 0; + struct dma_dsc *dsc = 0; + uint32_t bs = 0; + uint32_t ds = 0; + int err = 0; + dma_addr_t dma_key = 0; + uint8_t *key_raw = 0; + uint32_t mode = MODE_SHA1; + uint32_t map_len = 0; + uint32_t ipad = 0; + + spin_lock_bh(&aml_sha.lock); + if (!tctx->dd) { + list_for_each_entry(tmp, &aml_sha.dev_list, list) { + dd = tmp; + break; + } + tctx->dd = dd; + } else { + dd = tctx->dd; + } + dsc = dd->descriptor; + + spin_unlock_bh(&aml_sha.lock); + + switch (crypto_ahash_digestsize(tfm)) { + case SHA1_DIGEST_SIZE: + bs = SHA1_BLOCK_SIZE; + ds = SHA1_DIGEST_SIZE; + mode = MODE_SHA1; + break; + case SHA224_DIGEST_SIZE: + bs = SHA224_BLOCK_SIZE; + ds = SHA224_DIGEST_SIZE; + mode = MODE_SHA224; + break; + case SHA256_DIGEST_SIZE: + bs = SHA256_BLOCK_SIZE; + ds = SHA256_DIGEST_SIZE; + mode = MODE_SHA256; + break; + default: + return -EINVAL; + } + + if (keylen > bs) + key_raw = kzalloc(keylen, GFP_KERNEL); + else + key_raw = kzalloc(bs, GFP_KERNEL); + + memcpy(key_raw, key, keylen); + map_len = keylen > bs ? keylen : bs; + dma_key = dma_map_single(dd->dev, key_raw, + map_len, DMA_BIDIRECTIONAL); + if (keylen > bs) { + dsc[0].src_addr = (uintptr_t)dma_key; + dsc[0].tgt_addr = (uintptr_t)dma_key; + dsc[0].dsc_cfg.d32 = 0; + dsc[0].dsc_cfg.b.length = keylen; + dsc[0].dsc_cfg.b.mode = mode; + dsc[0].dsc_cfg.b.enc_sha_only = 1; + dsc[0].dsc_cfg.b.begin = 1; + dsc[0].dsc_cfg.b.end = 1; + dsc[0].dsc_cfg.b.op_mode = OP_MODE_SHA; + dsc[0].dsc_cfg.b.eoc = 0; + dsc[0].dsc_cfg.b.owner = 1; + keylen = ds; + ipad = 1; + } + + /* ipad */ + dsc[ipad].src_addr = (uintptr_t)dma_key; + dsc[ipad].tgt_addr = 0; + dsc[ipad].dsc_cfg.d32 = 0; + dsc[ipad].dsc_cfg.b.length = keylen; + dsc[ipad].dsc_cfg.b.mode = mode; + dsc[ipad].dsc_cfg.b.enc_sha_only = 1; + dsc[ipad].dsc_cfg.b.op_mode = OP_MODE_HMAC_I; + dsc[ipad].dsc_cfg.b.begin = 1; + dsc[ipad].dsc_cfg.b.end = 0; + dsc[ipad].dsc_cfg.b.eoc = 1; + dsc[ipad].dsc_cfg.b.owner = 1; + + dma_sync_single_for_device(dd->dev, dd->dma_descript_tab, + PAGE_SIZE, DMA_TO_DEVICE); + aml_dma_debug(dsc, ipad + 1, __func__); + aml_write_crypto_reg(DMA_THREAD_REG, + (uintptr_t) dd->dma_descript_tab | 2); + while (aml_read_crypto_reg(DMA_STATUS_REG) == 0) + ; + aml_write_crypto_reg(DMA_STATUS_REG, 0xf); + dma_unmap_single(dd->dev, dma_key, + map_len, DMA_BIDIRECTIONAL); + tctx->keylen = keylen; + memcpy(tctx->key, key_raw, keylen); + + kfree(key_raw); + return err; +} +static int aml_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct aml_sha_reqctx) + + SHA_BUFFER_LEN); + return 0; +} + +static int aml_sha_cra_init(struct crypto_tfm *tfm) +{ + return aml_sha_cra_init_alg(tfm, NULL); +} + +static void aml_sha_cra_exit(struct crypto_tfm *tfm) +{ +} + +static int aml_hmac_cra_init(struct crypto_tfm *tfm) +{ + struct aml_sha_ctx *tctx = crypto_tfm_ctx(tfm); + + tctx->flags |= SHA_FLAGS_HMAC; + return aml_sha_cra_init_alg(tfm, NULL); +} + +static void aml_hmac_cra_exit(struct crypto_tfm *tfm) +{ +} + +static struct ahash_alg sha_algs[] = { + { + .init = aml_sha_init, + .update = aml_sha_update, + .final = aml_sha_final, + .finup = aml_sha_finup, + .digest = aml_sha_digest, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + /* although we don't support import/export, */ + /* let's cheat it. */ + .statesize = sizeof(struct aml_sha_ctx), + .base = { + .cra_name = "sha1", + .cra_driver_name = "aml-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aml_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aml_sha_cra_init, + .cra_exit = aml_sha_cra_exit, + } + } + }, + { + .init = aml_sha_init, + .update = aml_sha_update, + .final = aml_sha_final, + .finup = aml_sha_finup, + .digest = aml_sha_digest, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + /* although we don't support import/export, */ + /* let's cheat it. */ + .statesize = sizeof(struct aml_sha_ctx), + .base = { + .cra_name = "sha256", + .cra_driver_name = "aml-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aml_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aml_sha_cra_init, + .cra_exit = aml_sha_cra_exit, + } + } + }, + { + .init = aml_sha_init, + .update = aml_sha_update, + .final = aml_sha_final, + .finup = aml_sha_finup, + .digest = aml_sha_digest, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + /* although we don't support import/export, */ + /* let's cheat it. */ + .statesize = sizeof(struct aml_sha_ctx), + .base = { + .cra_name = "sha224", + .cra_driver_name = "aml-sha224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aml_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aml_sha_cra_init, + .cra_exit = aml_sha_cra_exit, + } + } + }, + { + .init = aml_sha_init, + .update = aml_sha_update, + .final = aml_sha_final, + .finup = aml_sha_finup, + .digest = aml_sha_digest, + .setkey = aml_sha_setkey, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + /* although we don't support import/export, */ + /* let's cheat it. */ + .statesize = sizeof(struct aml_sha_ctx), + .base = { + .cra_name = "hmac(sha1)", + .cra_driver_name = "aml-hmac-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aml_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aml_hmac_cra_init, + .cra_exit = aml_hmac_cra_exit, + } + } + }, + { + .init = aml_sha_init, + .update = aml_sha_update, + .final = aml_sha_final, + .finup = aml_sha_finup, + .digest = aml_sha_digest, + .setkey = aml_sha_setkey, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + /* although we don't support import/export, */ + /* let's cheat it. */ + .statesize = sizeof(struct aml_sha_ctx), + .base = { + .cra_name = "hmac(sha224)", + .cra_driver_name = "aml-hmac-sha224", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aml_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aml_hmac_cra_init, + .cra_exit = aml_hmac_cra_exit, + } + } + }, + { + .init = aml_sha_init, + .update = aml_sha_update, + .final = aml_sha_final, + .finup = aml_sha_finup, + .digest = aml_sha_digest, + .setkey = aml_sha_setkey, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + /* although we don't support import/export, */ + /* let's cheat it. */ + .statesize = sizeof(struct aml_sha_ctx), + .base = { + .cra_name = "hmac(sha256)", + .cra_driver_name = "aml-hmac-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aml_sha_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_init = aml_hmac_cra_init, + .cra_exit = aml_hmac_cra_exit, + } + } + } +}; + +static void aml_sha_done_task(unsigned long data) +{ + struct aml_sha_dev *dd = (struct aml_sha_dev *)data; + struct aml_sha_reqctx *ctx = ahash_request_ctx(dd->req); + int err = 0; + + if (!(SHA_FLAGS_BUSY & dd->flags)) { + aml_sha_handle_queue(dd, NULL); + return; + } + + dma_sync_single_for_cpu(dd->dev, dd->dma_descript_tab, + PAGE_SIZE, DMA_FROM_DEVICE); + aml_dma_debug(dd->descriptor, ctx->fast_nents ? + ctx->fast_nents : 1, __func__); + + if (SHA_FLAGS_DMA_READY & dd->flags) { + if (SHA_FLAGS_DMA_ACTIVE & dd->flags) { + dd->flags &= ~SHA_FLAGS_DMA_ACTIVE; + aml_sha_update_dma_stop(dd); + if (dd->err) { + err = dd->err; + goto finish; + } + } + if (SHA_FLAGS_OUTPUT_READY & dd->flags) { + /* hash or semi-hash ready */ + dd->flags &= ~(SHA_FLAGS_DMA_READY | + SHA_FLAGS_OUTPUT_READY); + err = aml_sha_update_dma_start(dd); + if (err != -EINPROGRESS) + goto finish; + } + } + return; + +finish: + /* finish curent request */ + aml_sha_finish_req(dd->req, err); +} + +static irqreturn_t aml_sha_irq(int irq, void *dev_id) +{ + struct aml_sha_dev *sha_dd = dev_id; + uint8_t status = aml_read_crypto_reg(DMA_STATUS_REG); + + if (status) { + if (status == 0x1) + pr_err("irq overwrite\n"); + if (SHA_FLAGS_DMA_ACTIVE & sha_dd->flags) { + sha_dd->flags |= SHA_FLAGS_OUTPUT_READY; + sha_dd->flags |= SHA_FLAGS_DMA_READY; + aml_write_crypto_reg(DMA_STATUS_REG, 0xf); + tasklet_schedule(&sha_dd->done_task); + return IRQ_HANDLED; + } else { + return IRQ_NONE; + } + } + return IRQ_NONE; +} + +static void aml_sha_unregister_algs(struct aml_sha_dev *dd) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(sha_algs); i++) + crypto_unregister_ahash(&sha_algs[i]); +} + +static int aml_sha_register_algs(struct aml_sha_dev *dd) +{ + int err, i, j; + + for (i = 0; i < ARRAY_SIZE(sha_algs); i++) { + err = crypto_register_ahash(&sha_algs[i]); + if (err) + goto err_sha_algs; + } + + return 0; + +err_sha_algs: + for (j = 0; j < i; j++) + crypto_unregister_ahash(&sha_algs[j]); + return err; +} + +static int aml_sha_probe(struct platform_device *pdev) +{ + struct aml_sha_dev *sha_dd; + struct device *dev = &pdev->dev; + struct resource *res_irq = 0; + struct resource *res_base = 0; + int err = -EPERM; + + sha_dd = kzalloc(sizeof(struct aml_sha_dev), GFP_KERNEL); + if (sha_dd == NULL) { + err = -ENOMEM; + goto sha_dd_err; + } + + sha_dd->dev = dev; + + platform_set_drvdata(pdev, sha_dd); + + + res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, SHA_THREAD_INDEX); + + res_base = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res_base) { + dev_err(dev, "error to get normal IORESOURCE_MEM.\n"); + goto sha_dd_err; + } else { + if (cryptoreg_offset) { + cryptoreg_offset = ioremap(res_base->start, + resource_size(res_base)); + map_in_sha_dma = 1; + } + } + + INIT_LIST_HEAD(&sha_dd->list); + + tasklet_init(&sha_dd->done_task, aml_sha_done_task, + (unsigned long)sha_dd); + + crypto_init_queue(&sha_dd->queue, AML_SHA_QUEUE_LENGTH); + + sha_dd->irq = res_irq->start; + err = request_irq(sha_dd->irq, aml_sha_irq, IRQF_SHARED, "aml-sha", + sha_dd); + if (err) { + dev_err(dev, "unable to request sha irq.\n"); + goto res_err; + } + + aml_sha_hw_init(sha_dd); + + err = aml_sha_buff_init(sha_dd); + if (err) + goto res_err; + + spin_lock(&aml_sha.lock); + list_add_tail(&sha_dd->list, &aml_sha.dev_list); + spin_unlock(&aml_sha.lock); + + err = aml_sha_register_algs(sha_dd); + if (err) + goto err_algs; + + dev_info(dev, "Aml SHA1/SHA224/SHA256 dma\n"); + + return 0; + +err_algs: + spin_lock(&aml_sha.lock); + list_del(&sha_dd->list); + spin_unlock(&aml_sha.lock); + + if (map_in_sha_dma) { + + iounmap(cryptoreg_offset); + map_in_sha_dma = 0; + } + + free_irq(sha_dd->irq, sha_dd); +res_err: + tasklet_kill(&sha_dd->done_task); + kfree(sha_dd); + sha_dd = NULL; +sha_dd_err: + dev_err(dev, "initialization failed.\n"); + + return err; +} + +static int aml_sha_remove(struct platform_device *pdev) +{ + static struct aml_sha_dev *sha_dd; + + sha_dd = platform_get_drvdata(pdev); + if (!sha_dd) + return -ENODEV; + spin_lock(&aml_sha.lock); + list_del(&sha_dd->list); + spin_unlock(&aml_sha.lock); + + aml_sha_unregister_algs(sha_dd); + + tasklet_kill(&sha_dd->done_task); + + if (map_in_sha_dma) { + + iounmap(cryptoreg_offset); + map_in_sha_dma = 0; + } + + if (sha_dd->irq >= 0) + free_irq(sha_dd->irq, sha_dd); + + kfree(sha_dd); + sha_dd = NULL; + + return 0; +} + +#ifdef CONFIG_OF +static const struct of_device_id aml_sha_dt_match[] = { + { .compatible = "amlogic,sha_dma", + }, + {}, +}; +#else +#define aml_aes_dt_match NULL +#endif +static struct platform_driver aml_sha_driver = { + .probe = aml_sha_probe, + .remove = aml_sha_remove, + .driver = { + .name = "aml_sha_dma", + .owner = THIS_MODULE, + .of_match_table = aml_sha_dt_match, + }, +}; + +module_platform_driver(aml_sha_driver); + +MODULE_DESCRIPTION("Aml SHA (1/256/224) hw acceleration support."); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("matthew.shyu "); diff --git a/drivers/amlogic/crypto/aml-tdes-dma.c b/drivers/amlogic/crypto/aml-tdes-dma.c new file mode 100644 index 000000000000..5aaffa78a363 --- /dev/null +++ b/drivers/amlogic/crypto/aml-tdes-dma.c @@ -0,0 +1,968 @@ +/* + * drivers/amlogic/crypto/aml-tdes-dma.c + * + * Copyright (C) 2017 Amlogic, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "aml-crypto-dma.h" + +/* TDES flags */ +#define TDES_FLAGS_MODE_MASK 0x03 +#define TDES_FLAGS_ENCRYPT BIT(0) +#define TDES_FLAGS_CBC BIT(1) + +#define TDES_FLAGS_INIT BIT(8) +#define TDES_FLAGS_DMA BIT(9) +#define TDES_FLAGS_FAST BIT(10) +#define TDES_FLAGS_BUSY BIT(11) + +#define AML_TDES_QUEUE_LENGTH 50 + +#define DMA_THREAD_REG (DMA_T0 + TDES_THREAD_INDEX) +#define DMA_STATUS_REG (DMA_STS0 + TDES_THREAD_INDEX) +u8 map_in_tdes_dma; + +struct aml_tdes_dev; + +struct aml_tdes_ctx { + struct aml_tdes_dev *dd; + + u32 keylen; + u32 key[3*DES_KEY_SIZE / sizeof(u32)]; + + u16 block_size; +}; + +struct aml_tdes_reqctx { + unsigned long mode; +}; + +struct aml_tdes_dev { + struct list_head list; + + struct aml_tdes_ctx *ctx; + struct device *dev; + int irq; + + unsigned long flags; + int err; + + spinlock_t lock; + struct crypto_queue queue; + + struct tasklet_struct done_task; + struct tasklet_struct queue_task; + + struct ablkcipher_request *req; + size_t total; + + struct scatterlist *in_sg; + size_t in_offset; + struct scatterlist *out_sg; + size_t out_offset; + + size_t buflen; + size_t dma_size; + + void *buf_in; + dma_addr_t dma_addr_in; + + void *buf_out; + dma_addr_t dma_addr_out; + + void *descriptor; + dma_addr_t dma_descript_tab; + + uint32_t fast_nents; +}; + +struct aml_tdes_drv { + struct list_head dev_list; + spinlock_t lock; +}; + +static struct aml_tdes_drv aml_tdes = { + .dev_list = LIST_HEAD_INIT(aml_tdes.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(aml_tdes.lock), +}; + +static void set_tdes_key_iv(struct aml_tdes_dev *dd, + u32 *key, u32 keylen, u32 *iv) +{ + struct dma_dsc *dsc = dd->descriptor; + uint32_t key_iv[12]; + uint32_t *piv = key_iv + 8; + uint32_t len = keylen; + dma_addr_t dma_addr_key; + + memset(key_iv, 0, sizeof(key_iv)); + memcpy(key_iv, key, keylen); + if (iv) { + memcpy(piv, iv, 8); + len = 48; /* full key storage */ + } + + dma_addr_key = dma_map_single(dd->dev, key_iv, + sizeof(key_iv), DMA_TO_DEVICE); + + dsc->src_addr = (uintptr_t)dma_addr_key; + dsc->tgt_addr = 0; + dsc->dsc_cfg.d32 = 0; + dsc->dsc_cfg.b.length = len; + dsc->dsc_cfg.b.mode = MODE_KEY; + dsc->dsc_cfg.b.eoc = 1; + dsc->dsc_cfg.b.owner = 1; + + dma_sync_single_for_device(dd->dev, dd->dma_descript_tab, + PAGE_SIZE, DMA_TO_DEVICE); + aml_write_crypto_reg(DMA_THREAD_REG, + (uintptr_t) dd->dma_descript_tab | 2); + aml_dma_debug(dsc, 1, __func__); + while (aml_read_crypto_reg(DMA_STATUS_REG) == 0) + ; + aml_write_crypto_reg(DMA_STATUS_REG, 0xf); + dma_unmap_single(dd->dev, dma_addr_key, + sizeof(key_iv), DMA_TO_DEVICE); +} + + +static size_t aml_tdes_sg_copy(struct scatterlist **sg, size_t *offset, + void *buf, size_t buflen, size_t total, int out) +{ + size_t count, off = 0; + + while (buflen && total) { + count = min((*sg)->length - *offset, total); + count = min(count, buflen); + + if (!count) + return off; + + scatterwalk_map_and_copy(buf + off, *sg, *offset, count, out); + + off += count; + buflen -= count; + *offset += count; + total -= count; + + if (*offset == (*sg)->length) { + *sg = sg_next(*sg); + if (*sg) + *offset = 0; + else + total = 0; + } + } + + return off; +} + +static size_t aml_tdes_sg_dma(struct aml_tdes_dev *dd, struct dma_dsc *dsc, + uint32_t *nents, size_t total) +{ + size_t count = 0; + uint32_t i = 0; + int err = 0; + struct scatterlist *in_sg = dd->in_sg; + struct scatterlist *out_sg = dd->out_sg; + dma_addr_t addr_in, addr_out; + + while (total && in_sg && out_sg && (in_sg->length == out_sg->length) + && *nents < MAX_NUM_TABLES) { + count += min_t(unsigned int, total, in_sg->length); + *nents += 1; + total -= count; + in_sg = sg_next(in_sg); + out_sg = sg_next(out_sg); + } + err = dma_map_sg(dd->dev, dd->in_sg, *nents, DMA_TO_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + return 0; + } + + err = dma_map_sg(dd->dev, dd->out_sg, *nents, + DMA_FROM_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + dma_unmap_sg(dd->dev, dd->in_sg, *nents, + DMA_TO_DEVICE); + return 0; + } + + in_sg = dd->in_sg; + out_sg = dd->out_sg; + for (i = 0; i < *nents; i++) { + addr_in = sg_dma_address(in_sg); + addr_out = sg_dma_address(out_sg); + dsc[i].src_addr = (uintptr_t)addr_in; + dsc[i].tgt_addr = (uintptr_t)addr_out; + dsc[i].dsc_cfg.d32 = 0; + dsc[i].dsc_cfg.b.length = sg_dma_len(in_sg); + in_sg = sg_next(in_sg); + out_sg = sg_next(out_sg); + } + return count; +} + +static struct aml_tdes_dev *aml_tdes_find_dev(struct aml_tdes_ctx *ctx) +{ + struct aml_tdes_dev *tdes_dd = NULL; + struct aml_tdes_dev *tmp; + + spin_lock_bh(&aml_tdes.lock); + if (!ctx->dd) { + list_for_each_entry(tmp, &aml_tdes.dev_list, list) { + tdes_dd = tmp; + break; + } + ctx->dd = tdes_dd; + } else { + tdes_dd = ctx->dd; + } + + spin_unlock_bh(&aml_tdes.lock); + + return tdes_dd; +} + +static int aml_tdes_hw_init(struct aml_tdes_dev *dd) +{ + if (!(dd->flags & TDES_FLAGS_INIT)) { + dd->flags |= TDES_FLAGS_INIT; + dd->err = 0; + } + + return 0; +} + +static void aml_tdes_finish_req(struct aml_tdes_dev *dd, int err) +{ + struct ablkcipher_request *req = dd->req; + + dd->flags &= ~TDES_FLAGS_BUSY; + req->base.complete(&req->base, err); +} + +static int aml_tdes_crypt_dma(struct aml_tdes_dev *dd, struct dma_dsc *dsc, + uint32_t nents) +{ + uint32_t op_mode = OP_MODE_ECB; + uint32_t i = 0; + + dd->flags |= TDES_FLAGS_DMA; + + if (dd->flags & TDES_FLAGS_CBC) + op_mode = OP_MODE_CBC; + + for (i = 0; i < nents; i++) { + dsc[i].dsc_cfg.b.enc_sha_only = dd->flags & TDES_FLAGS_ENCRYPT; + dsc[i].dsc_cfg.b.mode = + ((dd->ctx->keylen == DES_KEY_SIZE) ? MODE_DES : + ((dd->ctx->keylen == 2 * DES_KEY_SIZE) ? + MODE_TDES_2K : MODE_TDES_3K)); + dsc[i].dsc_cfg.b.op_mode = op_mode; + dsc[i].dsc_cfg.b.eoc = (i == (nents - 1)); + dsc[i].dsc_cfg.b.owner = 1; + } + + dma_sync_single_for_device(dd->dev, dd->dma_descript_tab, + PAGE_SIZE, DMA_TO_DEVICE); + + aml_dma_debug(dsc, nents, __func__); + aml_write_crypto_reg(DMA_THREAD_REG, + (uintptr_t) dd->dma_descript_tab | 2); + return 0; +} + +static int aml_tdes_crypt_dma_start(struct aml_tdes_dev *dd) +{ + int err = 0, fast = 0; + int in, out; + size_t count; + dma_addr_t addr_in, addr_out; + struct dma_dsc *dsc = dd->descriptor; + uint32_t nents; + + /* fast dma */ + if ((!dd->in_offset) && (!dd->out_offset)) { + /* check for alignment */ + in = IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size); + out = IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size); + fast = in && out; + + if (dd->in_sg->length != dd->out_sg->length + || dd->total < dd->ctx->block_size) + fast = 0; + dd->fast_nents = 0; + } + + if (fast) { + count = aml_tdes_sg_dma(dd, dsc, &dd->fast_nents, dd->total); + dd->flags |= TDES_FLAGS_FAST; + nents = dd->fast_nents; + } else { + /* slow dma */ + /* use cache buffers */ + count = aml_tdes_sg_copy(&dd->in_sg, &dd->in_offset, + dd->buf_in, dd->buflen, dd->total, 0); + addr_in = dd->dma_addr_in; + addr_out = dd->dma_addr_out; + dd->dma_size = count; + dma_sync_single_for_device(dd->dev, addr_in, dd->dma_size, + DMA_TO_DEVICE); + dsc->src_addr = (uint32_t)addr_in; + dsc->tgt_addr = (uint32_t)addr_out; + dsc->dsc_cfg.d32 = 0; + dsc->dsc_cfg.b.length = count; + nents = 1; + dd->flags &= ~TDES_FLAGS_FAST; + } + + dd->total -= count; + err = aml_tdes_crypt_dma(dd, dsc, nents); + + return err; +} + +static int aml_tdes_write_ctrl(struct aml_tdes_dev *dd) +{ + int err = 0; + + err = aml_tdes_hw_init(dd); + + if (err) + return err; + + if (dd->flags & TDES_FLAGS_CBC) + set_tdes_key_iv(dd, dd->ctx->key, dd->ctx->keylen, + dd->req->info); + else + set_tdes_key_iv(dd, dd->ctx->key, dd->ctx->keylen, + NULL); + + return err; +} + +static int aml_tdes_handle_queue(struct aml_tdes_dev *dd, + struct ablkcipher_request *req) +{ + struct crypto_async_request *async_req, *backlog; + struct aml_tdes_ctx *ctx; + struct aml_tdes_reqctx *rctx; + unsigned long flags; + int err, ret = 0; + + spin_lock_irqsave(&dd->lock, flags); + if (req) + ret = ablkcipher_enqueue_request(&dd->queue, req); + + if (dd->flags & TDES_FLAGS_BUSY) { + spin_unlock_irqrestore(&dd->lock, flags); + return ret; + } + backlog = crypto_get_backlog(&dd->queue); + async_req = crypto_dequeue_request(&dd->queue); + if (async_req) + dd->flags |= TDES_FLAGS_BUSY; + spin_unlock_irqrestore(&dd->lock, flags); + + if (!async_req) + return ret; + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + + req = ablkcipher_request_cast(async_req); + + /* assign new request to device */ + dd->req = req; + dd->total = req->nbytes; + dd->in_offset = 0; + dd->in_sg = req->src; + dd->out_offset = 0; + dd->out_sg = req->dst; + + rctx = ablkcipher_request_ctx(req); + ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); + rctx->mode &= TDES_FLAGS_MODE_MASK; + dd->flags = (dd->flags & ~TDES_FLAGS_MODE_MASK) | rctx->mode; + dd->ctx = ctx; + ctx->dd = dd; + + err = aml_tdes_write_ctrl(dd); + if (!err) + err = aml_tdes_crypt_dma_start(dd); + + if (err) { + /* tdes_task will not finish it, so do it here */ + aml_tdes_finish_req(dd, err); + tasklet_schedule(&dd->queue_task); + } + + return ret; +} + +static int aml_tdes_crypt_dma_stop(struct aml_tdes_dev *dd) +{ + int err = -EINVAL; + size_t count; + + if (dd->flags & TDES_FLAGS_DMA) { + err = 0; + dma_sync_single_for_cpu(dd->dev, dd->dma_descript_tab, + PAGE_SIZE, DMA_FROM_DEVICE); + if (dd->flags & TDES_FLAGS_FAST) { + dma_unmap_sg(dd->dev, dd->out_sg, + dd->fast_nents, DMA_FROM_DEVICE); + dma_unmap_sg(dd->dev, dd->in_sg, + dd->fast_nents, DMA_TO_DEVICE); + } else { + dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, + dd->dma_size, DMA_FROM_DEVICE); + + /* copy data */ + count = aml_tdes_sg_copy(&dd->out_sg, &dd->out_offset, + dd->buf_out, dd->buflen, + dd->dma_size, 1); + if (count != dd->dma_size) { + err = -EINVAL; + pr_err("not all data converted: %zu\n", count); + } + } + dd->flags &= ~TDES_FLAGS_DMA; + } + + return err; +} + + +static int aml_tdes_buff_init(struct aml_tdes_dev *dd) +{ + int err = -ENOMEM; + + dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, 0); + dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, 0); + dd->descriptor = (void *)__get_free_pages(GFP_KERNEL, 0); + dd->buflen = PAGE_SIZE; + dd->buflen &= ~(DES_BLOCK_SIZE - 1); + + if (!dd->buf_in || !dd->buf_out || !dd->descriptor) { + dev_err(dd->dev, "unable to alloc pages.\n"); + goto err_alloc; + } + + /* MAP here */ + dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, + dd->buflen, DMA_TO_DEVICE); + if (dma_mapping_error(dd->dev, dd->dma_addr_in)) { + dev_err(dd->dev, "dma %zd bytes error\n", dd->buflen); + err = -EINVAL; + goto err_map_in; + } + + dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, + dd->buflen, DMA_FROM_DEVICE); + if (dma_mapping_error(dd->dev, dd->dma_addr_out)) { + dev_err(dd->dev, "dma %zd bytes error\n", dd->buflen); + err = -EINVAL; + goto err_map_out; + } + + dd->dma_descript_tab = dma_map_single(dd->dev, dd->descriptor, + PAGE_SIZE, DMA_TO_DEVICE); + + if (dma_mapping_error(dd->dev, dd->dma_descript_tab)) { + dev_err(dd->dev, "dma descriptor error\n"); + err = -EINVAL; + goto err_map_descriptor; + } + + + return 0; + +err_map_descriptor: + dma_unmap_single(dd->dev, dd->dma_descript_tab, PAGE_SIZE, + DMA_TO_DEVICE); + +err_map_out: + dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, + DMA_TO_DEVICE); +err_map_in: + free_page((uintptr_t)dd->buf_out); + free_page((uintptr_t)dd->buf_in); + free_page((uintptr_t)dd->descriptor); +err_alloc: + if (err) + pr_err("error: %d\n", err); + return err; +} + +static void aml_tdes_buff_cleanup(struct aml_tdes_dev *dd) +{ + dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, + DMA_FROM_DEVICE); + dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, + DMA_TO_DEVICE); + dma_unmap_single(dd->dev, dd->dma_descript_tab, PAGE_SIZE, + DMA_TO_DEVICE); + free_page((unsigned long)dd->buf_out); + free_page((unsigned long)dd->buf_in); + free_page((uintptr_t)dd->descriptor); +} + +static int aml_tdes_crypt(struct ablkcipher_request *req, unsigned long mode) +{ + struct aml_tdes_ctx *ctx = crypto_ablkcipher_ctx( + crypto_ablkcipher_reqtfm(req)); + struct aml_tdes_reqctx *rctx = ablkcipher_request_ctx(req); + struct aml_tdes_dev *dd; + + if (!IS_ALIGNED(req->nbytes, DES_BLOCK_SIZE)) { + pr_err("request size is not exact amount of TDES blocks\n"); + return -EINVAL; + } + ctx->block_size = DES_BLOCK_SIZE; + + dd = aml_tdes_find_dev(ctx); + if (!dd) + return -ENODEV; + + rctx->mode = mode; + + return aml_tdes_handle_queue(dd, req); +} + +static int aml_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + u32 keylen) +{ + u32 tmp[DES_EXPKEY_WORDS]; + int err; + struct crypto_tfm *ctfm = crypto_ablkcipher_tfm(tfm); + struct aml_tdes_ctx *ctx = crypto_ablkcipher_ctx(tfm); + + if (keylen != DES_KEY_SIZE) { + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + err = des_ekey(tmp, key); + if (err == 0 && (ctfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + ctfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; + return -EINVAL; + } + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int aml_tdes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + u32 keylen) +{ + struct aml_tdes_ctx *ctx = crypto_ablkcipher_ctx(tfm); + + if ((keylen != 2 * DES_KEY_SIZE) && (keylen != 3 * DES_KEY_SIZE)) { + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int aml_tdes_ecb_encrypt(struct ablkcipher_request *req) +{ + return aml_tdes_crypt(req, + TDES_FLAGS_ENCRYPT); +} + +static int aml_tdes_ecb_decrypt(struct ablkcipher_request *req) +{ + return aml_tdes_crypt(req, + 0); +} + +static int aml_tdes_cbc_encrypt(struct ablkcipher_request *req) +{ + return aml_tdes_crypt(req, + TDES_FLAGS_ENCRYPT | TDES_FLAGS_CBC); +} + +static int aml_tdes_cbc_decrypt(struct ablkcipher_request *req) +{ + return aml_tdes_crypt(req, + TDES_FLAGS_CBC); +} + +static int aml_tdes_cra_init(struct crypto_tfm *tfm) +{ + tfm->crt_ablkcipher.reqsize = sizeof(struct aml_tdes_reqctx); + + return 0; +} + +static void aml_tdes_cra_exit(struct crypto_tfm *tfm) +{ +} + +static struct crypto_alg tdes_algs[] = { + { + .cra_name = "ecb(des)", + .cra_driver_name = "ecb-des-aml", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aml_tdes_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = aml_tdes_cra_init, + .cra_exit = aml_tdes_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = aml_des_setkey, + .encrypt = aml_tdes_ecb_encrypt, + .decrypt = aml_tdes_ecb_decrypt, + } + }, + { + .cra_name = "cbc(des)", + .cra_driver_name = "cbc-des-aml", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aml_tdes_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = aml_tdes_cra_init, + .cra_exit = aml_tdes_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = aml_des_setkey, + .encrypt = aml_tdes_cbc_encrypt, + .decrypt = aml_tdes_cbc_decrypt, + } + }, + { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "ecb-tdes-aml", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aml_tdes_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = aml_tdes_cra_init, + .cra_exit = aml_tdes_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = 2 * DES_KEY_SIZE, + .max_keysize = 3 * DES_KEY_SIZE, + .setkey = aml_tdes_setkey, + .encrypt = aml_tdes_ecb_encrypt, + .decrypt = aml_tdes_ecb_decrypt, + } + }, + { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc-tdes-aml", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct aml_tdes_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = aml_tdes_cra_init, + .cra_exit = aml_tdes_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = 2 * DES_KEY_SIZE, + .max_keysize = 3 * DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = aml_tdes_setkey, + .encrypt = aml_tdes_cbc_encrypt, + .decrypt = aml_tdes_cbc_decrypt, + } + } +}; + +static void aml_tdes_queue_task(unsigned long data) +{ + struct aml_tdes_dev *dd = (struct aml_tdes_dev *)data; + + aml_tdes_handle_queue(dd, NULL); +} + +static void aml_tdes_done_task(unsigned long data) +{ + struct aml_tdes_dev *dd = (struct aml_tdes_dev *) data; + int err; + + err = aml_tdes_crypt_dma_stop(dd); + + aml_dma_debug(dd->descriptor, dd->fast_nents ? + dd->fast_nents : 1, __func__); + err = dd->err ? : err; + + if (dd->total && !err) { + if (dd->flags & TDES_FLAGS_FAST) { + uint32_t i = 0; + + for (i = 0; i < dd->fast_nents; i++) { + if (!dd->in_sg || !dd->out_sg) + err = -EINVAL; + dd->in_sg = sg_next(dd->in_sg); + dd->out_sg = sg_next(dd->out_sg); + } + } + + if (!err) + err = aml_tdes_crypt_dma_start(dd); + if (!err) + return; /* DMA started. Not fininishing. */ + } + + aml_tdes_finish_req(dd, err); + aml_tdes_handle_queue(dd, NULL); +} + +static irqreturn_t aml_tdes_irq(int irq, void *dev_id) +{ + struct aml_tdes_dev *tdes_dd = dev_id; + uint8_t status = aml_read_crypto_reg(DMA_STATUS_REG); + + if (status) { + if (status == 0x1) + pr_err("irq overwrite\n"); + if (TDES_FLAGS_DMA & tdes_dd->flags) { + aml_write_crypto_reg(DMA_STATUS_REG, 0xf); + tasklet_schedule(&tdes_dd->done_task); + return IRQ_HANDLED; + } else { + return IRQ_NONE; + } + } + + return IRQ_NONE; +} + +static void aml_tdes_unregister_algs(struct aml_tdes_dev *dd) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(tdes_algs); i++) + crypto_unregister_alg(&tdes_algs[i]); +} + +static int aml_tdes_register_algs(struct aml_tdes_dev *dd) +{ + int err, i, j; + + for (i = 0; i < ARRAY_SIZE(tdes_algs); i++) { + err = crypto_register_alg(&tdes_algs[i]); + if (err) + goto err_tdes_algs; + } + + return 0; + +err_tdes_algs: + for (j = 0; j < i; j++) + crypto_unregister_alg(&tdes_algs[j]); + + return err; +} + +static int aml_tdes_probe(struct platform_device *pdev) +{ + struct aml_tdes_dev *tdes_dd; + struct device *dev = &pdev->dev; + struct resource *res_irq = 0; + struct resource *res_base = 0; + int err = -EPERM; + + tdes_dd = kzalloc(sizeof(struct aml_tdes_dev), GFP_KERNEL); + if (tdes_dd == NULL) { + err = -ENOMEM; + goto tdes_dd_err; + } + + tdes_dd->dev = dev; + + platform_set_drvdata(pdev, tdes_dd); + res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, + TDES_THREAD_INDEX); + + res_base = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res_base) { + dev_err(dev, "error to get normal IORESOURCE_MEM.\n"); + goto tdes_dd_err; + } else { + if (!cryptoreg_offset) { + cryptoreg_offset = ioremap(res_base->start, + resource_size(res_base)); + map_in_tdes_dma = 1; + } + } + + INIT_LIST_HEAD(&tdes_dd->list); + + tasklet_init(&tdes_dd->done_task, aml_tdes_done_task, + (unsigned long)tdes_dd); + tasklet_init(&tdes_dd->queue_task, aml_tdes_queue_task, + (unsigned long)tdes_dd); + + crypto_init_queue(&tdes_dd->queue, AML_TDES_QUEUE_LENGTH); + + tdes_dd->irq = res_irq->start; + err = request_irq(tdes_dd->irq, aml_tdes_irq, IRQF_SHARED, "aml-tdes", + tdes_dd); + if (err) { + dev_err(dev, "unable to request tdes irq.\n"); + goto tdes_irq_err; + } + + err = aml_tdes_hw_init(tdes_dd); + if (err) + goto err_tdes_buff; + + err = aml_tdes_buff_init(tdes_dd); + if (err) + goto err_tdes_buff; + + spin_lock(&aml_tdes.lock); + list_add_tail(&tdes_dd->list, &aml_tdes.dev_list); + spin_unlock(&aml_tdes.lock); + + err = aml_tdes_register_algs(tdes_dd); + if (err) + goto err_algs; + + dev_info(dev, "Aml TDES_dma\n"); + + return 0; + +err_algs: + spin_lock(&aml_tdes.lock); + list_del(&tdes_dd->list); + spin_unlock(&aml_tdes.lock); + aml_tdes_buff_cleanup(tdes_dd); +err_tdes_buff: + free_irq(tdes_dd->irq, tdes_dd); +tdes_irq_err: + + if (map_in_tdes_dma) { + iounmap(cryptoreg_offset); + map_in_tdes_dma = 0; + } + + tasklet_kill(&tdes_dd->done_task); + tasklet_kill(&tdes_dd->queue_task); + kfree(tdes_dd); + tdes_dd = NULL; +tdes_dd_err: + dev_err(dev, "initialization failed.\n"); + + return err; +} + +static int aml_tdes_remove(struct platform_device *pdev) +{ + static struct aml_tdes_dev *tdes_dd; + + tdes_dd = platform_get_drvdata(pdev); + if (!tdes_dd) + return -ENODEV; + spin_lock(&aml_tdes.lock); + list_del(&tdes_dd->list); + spin_unlock(&aml_tdes.lock); + + aml_tdes_unregister_algs(tdes_dd); + + tasklet_kill(&tdes_dd->done_task); + tasklet_kill(&tdes_dd->queue_task); + + if (map_in_tdes_dma) { + iounmap(cryptoreg_offset); + map_in_tdes_dma = 0; + } + + if (tdes_dd->irq > 0) + free_irq(tdes_dd->irq, tdes_dd); + + kfree(tdes_dd); + tdes_dd = NULL; + + return 0; +} + +#ifdef CONFIG_OF +static const struct of_device_id aml_tdes_dt_match[] = { + { .compatible = "amlogic,des_dma,tdes_dma", + }, + {}, +}; +#else +#define aml_tdes_dt_match NULL +#endif + +static struct platform_driver aml_tdes_driver = { + .probe = aml_tdes_probe, + .remove = aml_tdes_remove, + .driver = { + .name = "aml_tdes_dma", + .owner = THIS_MODULE, + .of_match_table = aml_tdes_dt_match, + }, +}; + +module_platform_driver(aml_tdes_driver); + +MODULE_DESCRIPTION("Aml TDES hw acceleration support."); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("matthew.shyu ");