shinys000114/linux
1
0
Fork 0
mirror of https://github.com/hardkernel/linux.git synced 2026-06-06 19:08:57 +09:00
Code Issues Packages Projects Releases Wiki Activity

crypto: rockchip: add cryptodev_linux driver

Browse Source 
provide crypto api to user space, you can open
"/dev/crypto" to use it.

cryptodev-linux source repository:
  https://github.com/cryptodev-linux/cryptodev-linux.git

use commit 356a45e63bbce94b9cea73b8c1e20d0d8ec02f04
  Author: cristian-stoica <cristianmarian.stoica@nxp.com>
  Date:   Thu Nov 11 09:30:19 2021 +0200

Change-Id: I91ca3660060f4adcf531e3efb8e720308bbd9f0e
Signed-off-by: Lin Jinhan <troy.lin@rock-chips.com>
This commit is contained in:
Lin Jinhan
2018-06-20 11:08:21 +08:00
committed by Tao Huang
parent 32bf18bcd3
commit e398bee2a0
17 changed files with 3910 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/crypto/Kconfig
View File

@@ -771,6 +771,8 @@ config CRYPTO_DEV_ROCKCHIP
This driver interfaces with the hardware crypto accelerator.
Supporting cbc/ecb chainmode, and aes/des/des3_ede cipher mode.
source "drivers/crypto/rockchip/Kconfig"
config CRYPTO_DEV_ZYNQMP_AES
tristate "Support for Xilinx ZynqMP AES hw accelerator"
depends on ZYNQMP_FIRMWARE || COMPILE_TEST

9
drivers/crypto/rockchip/Kconfig Normal file
View File

@@ -0,0 +1,9 @@
# SPDX-License-Identifier: GPL-2.0-only
config CRYPTO_DEV_ROCKCHIP_DEV
tristate "Export rockchip crypto device for user space"
depends on CRYPTO_DEV_ROCKCHIP
default n
help
This is a /dev/crypto device driver.The main idea is to
access existing ciphers in kernel space from userspace,
thus enabling the re-use of a hardware implementation of a cipher.

2
drivers/crypto/rockchip/Makefile
View File

@@ -8,3 +8,5 @@ rk_crypto-objs := rk_crypto_core.o \
rk_crypto_v2_akcipher.o \
rk_crypto_v2_pka.o \
rk_crypto_bignum.o
obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP_DEV) += cryptodev_linux/

9
drivers/crypto/rockchip/cryptodev_linux/Makefile Normal file
View File

@@ -0,0 +1,9 @@
# SPDX-License-Identifier: GPL-2.0+
obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP_DEV) += cryptodev.o
cryptodev-objs := ioctl.o \
main.o \
cryptlib.o \
authenc.o \
zc.o \
util.o

850
drivers/crypto/rockchip/cryptodev_linux/authenc.c Normal file
View File

@@ -0,0 +1,850 @@
/*
* Driver for /dev/crypto device (aka CryptoDev)
*
* Copyright (c) 2011, 2012 OpenSSL Software Foundation, Inc.
*
* Author: Nikos Mavrogiannopoulos
*
* This file is part of linux cryptodev.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
/*
* This file handles the AEAD part of /dev/crypto.
*
*/
#include <crypto/hash.h>
#include <linux/crypto.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/ioctl.h>
#include <linux/random.h>
#include <linux/syscalls.h>
#include <linux/pagemap.h>
#include <linux/poll.h>
#include <linux/uaccess.h>
#include "crypto/cryptodev.h"
#include <crypto/scatterwalk.h>
#include <linux/scatterlist.h>
#include "cryptodev_int.h"
#include "zc.h"
#include "util.h"
#include "cryptlib.h"
#include "version.h"
/* make caop->dst available in scatterlist.
* (caop->src is assumed to be equal to caop->dst)
*/
static int get_userbuf_tls(struct csession *ses, struct kernel_crypt_auth_op *kcaop,
struct scatterlist **dst_sg)
{
int pagecount = 0;
struct crypt_auth_op *caop = &kcaop->caop;
int rc;
if (caop->dst == NULL)
return -EINVAL;
if (ses->alignmask) {
if (!IS_ALIGNED((unsigned long)caop->dst, ses->alignmask + 1))
dwarning(2, "careful - source address %p is not %d byte aligned",
caop->dst, ses->alignmask + 1);
}
if (kcaop->dst_len == 0) {
dwarning(1, "Destination length cannot be zero");
return -EINVAL;
}
pagecount = PAGECOUNT(caop->dst, kcaop->dst_len);
ses->used_pages = pagecount;
ses->readonly_pages = 0;
rc = adjust_sg_array(ses, pagecount);
if (rc)
return rc;
rc = __get_userbuf(caop->dst, kcaop->dst_len, 1, pagecount,
ses->pages, ses->sg, kcaop->task, kcaop->mm);
if (unlikely(rc)) {
derr(1, "failed to get user pages for data input");
return -EINVAL;
}
(*dst_sg) = ses->sg;
return 0;
}
#define MAX_SRTP_AUTH_DATA_DIFF 256
/* Makes caop->auth_src available as scatterlist.
* It also provides a pointer to caop->dst, which however,
* is assumed to be within the caop->auth_src buffer. If not
* (if their difference exceeds MAX_SRTP_AUTH_DATA_DIFF) it
* returns error.
*/
static int get_userbuf_srtp(struct csession *ses, struct kernel_crypt_auth_op *kcaop,
struct scatterlist **auth_sg, struct scatterlist **dst_sg)
{
int pagecount, diff;
int auth_pagecount = 0;
struct crypt_auth_op *caop = &kcaop->caop;
int rc;
if (caop->dst == NULL && caop->auth_src == NULL) {
derr(1, "dst and auth_src cannot be both null");
return -EINVAL;
}
if (ses->alignmask) {
if (!IS_ALIGNED((unsigned long)caop->dst, ses->alignmask + 1))
dwarning(2, "careful - source address %p is not %d byte aligned",
caop->dst, ses->alignmask + 1);
if (!IS_ALIGNED((unsigned long)caop->auth_src, ses->alignmask + 1))
dwarning(2, "careful - source address %p is not %d byte aligned",
caop->auth_src, ses->alignmask + 1);
}
if (unlikely(kcaop->dst_len == 0 || caop->auth_len == 0)) {
dwarning(1, "Destination length cannot be zero");
return -EINVAL;
}
/* Note that in SRTP auth data overlap with data to be encrypted (dst)
*/
auth_pagecount = PAGECOUNT(caop->auth_src, caop->auth_len);
diff = (int)(caop->src - caop->auth_src);
if (diff > MAX_SRTP_AUTH_DATA_DIFF || diff < 0) {
dwarning(1, "auth_src must overlap with src (diff: %d).", diff);
return -EINVAL;
}
pagecount = auth_pagecount;
rc = adjust_sg_array(ses, pagecount*2); /* double pages to have pages for dst(=auth_src) */
if (rc) {
derr(1, "cannot adjust sg array");
return rc;
}
rc = __get_userbuf(caop->auth_src, caop->auth_len, 1, auth_pagecount,
ses->pages, ses->sg, kcaop->task, kcaop->mm);
if (unlikely(rc)) {
derr(1, "failed to get user pages for data input");
return -EINVAL;
}
ses->used_pages = pagecount;
ses->readonly_pages = 0;
(*auth_sg) = ses->sg;
(*dst_sg) = ses->sg + auth_pagecount;
sg_init_table(*dst_sg, auth_pagecount);
sg_copy(ses->sg, (*dst_sg), caop->auth_len);
(*dst_sg) = sg_advance(*dst_sg, diff);
if (*dst_sg == NULL) {
release_user_pages(ses);
derr(1, "failed to get enough pages for auth data");
return -EINVAL;
}
return 0;
}
/*
* Return tag (digest) length for authenticated encryption
* If the cipher and digest are separate, hdata.init is set - just return
* digest length. Otherwise return digest length for aead ciphers
*/
static int cryptodev_get_tag_len(struct csession *ses_ptr)
{
if (ses_ptr->hdata.init)
return ses_ptr->hdata.digestsize;
else
return cryptodev_cipher_get_tag_size(&ses_ptr->cdata);
}
/*
* Calculate destination buffer length for authenticated encryption. The
* expectation is that user-space code allocates exactly the same space for
* destination buffer before calling cryptodev. The result is cipher-dependent.
*/
static int cryptodev_get_dst_len(struct crypt_auth_op *caop, struct csession *ses_ptr)
{
int dst_len = caop->len;
if (caop->op == COP_DECRYPT)
return dst_len;
dst_len += caop->tag_len;
/* for TLS always add some padding so the total length is rounded to
* cipher block size */
if (caop->flags & COP_FLAG_AEAD_TLS_TYPE) {
int bs = ses_ptr->cdata.blocksize;
dst_len += bs - (dst_len % bs);
}
return dst_len;
}
static int fill_kcaop_from_caop(struct kernel_crypt_auth_op *kcaop, struct fcrypt *fcr)
{
struct crypt_auth_op *caop = &kcaop->caop;
struct csession *ses_ptr;
int ret;
/* this also enters ses_ptr->sem */
ses_ptr = crypto_get_session_by_sid(fcr, caop->ses);
if (unlikely(!ses_ptr)) {
derr(1, "invalid session ID=0x%08X", caop->ses);
return -EINVAL;
}
if (caop->flags & COP_FLAG_AEAD_TLS_TYPE || caop->flags & COP_FLAG_AEAD_SRTP_TYPE) {
if (caop->src != caop->dst) {
derr(1, "Non-inplace encryption and decryption is not efficient and not implemented");
ret = -EINVAL;
goto out_unlock;
}
}
if (caop->tag_len == 0)
caop->tag_len = cryptodev_get_tag_len(ses_ptr);
kcaop->ivlen = caop->iv ? ses_ptr->cdata.ivsize : 0;
kcaop->dst_len = cryptodev_get_dst_len(caop, ses_ptr);
kcaop->task = current;
kcaop->mm = current->mm;
if (caop->iv) {
ret = copy_from_user(kcaop->iv, caop->iv, kcaop->ivlen);
if (unlikely(ret)) {
derr(1, "error copying IV (%d bytes), copy_from_user returned %d for address %p",
kcaop->ivlen, ret, caop->iv);
ret = -EFAULT;
goto out_unlock;
}
}
ret = 0;
out_unlock:
crypto_put_session(ses_ptr);
return ret;
}
static int fill_caop_from_kcaop(struct kernel_crypt_auth_op *kcaop, struct fcrypt *fcr)
{
int ret;
kcaop->caop.len = kcaop->dst_len;
if (kcaop->ivlen && kcaop->caop.flags & COP_FLAG_WRITE_IV) {
ret = copy_to_user(kcaop->caop.iv,
kcaop->iv, kcaop->ivlen);
if (unlikely(ret)) {
derr(1, "Error in copying to userspace");
return -EFAULT;
}
}
return 0;
}
int kcaop_from_user(struct kernel_crypt_auth_op *kcaop,
struct fcrypt *fcr, void __user *arg)
{
if (unlikely(copy_from_user(&kcaop->caop, arg, sizeof(kcaop->caop)))) {
derr(1, "Error in copying from userspace");
return -EFAULT;
}
return fill_kcaop_from_caop(kcaop, fcr);
}
int kcaop_to_user(struct kernel_crypt_auth_op *kcaop,
struct fcrypt *fcr, void __user *arg)
{
int ret;
ret = fill_caop_from_kcaop(kcaop, fcr);
if (unlikely(ret)) {
derr(1, "fill_caop_from_kcaop");
return ret;
}
if (unlikely(copy_to_user(arg, &kcaop->caop, sizeof(kcaop->caop)))) {
derr(1, "Error in copying to userspace");
return -EFAULT;
}
return 0;
}
static void copy_tls_hash(struct scatterlist *dst_sg, int len, void *hash, int hash_len)
{
scatterwalk_map_and_copy(hash, dst_sg, len, hash_len, 1);
}
static void read_tls_hash(struct scatterlist *dst_sg, int len, void *hash, int hash_len)
{
scatterwalk_map_and_copy(hash, dst_sg, len - hash_len, hash_len, 0);
}
#define TLS_MAX_PADDING_SIZE 256
static int pad_record(struct scatterlist *dst_sg, int len, int block_size)
{
uint8_t pad[TLS_MAX_PADDING_SIZE];
int pad_size = block_size - (len % block_size);
memset(pad, pad_size - 1, pad_size);
scatterwalk_map_and_copy(pad, dst_sg, len, pad_size, 1);
return pad_size;
}
static int verify_tls_record_pad(struct scatterlist *dst_sg, int len, int block_size)
{
uint8_t pad[TLS_MAX_PADDING_SIZE];
uint8_t pad_size;
int i;
scatterwalk_map_and_copy(&pad_size, dst_sg, len - 1, 1, 0);
if (pad_size + 1 > len) {
derr(1, "Pad size: %d", pad_size);
return -EBADMSG;
}
scatterwalk_map_and_copy(pad, dst_sg, len - pad_size - 1, pad_size + 1, 0);
for (i = 0; i < pad_size; i++)
if (pad[i] != pad_size) {
derr(1, "Pad size: %u, pad: %d", pad_size, pad[i]);
return -EBADMSG;
}
return pad_size + 1;
}
/* Authenticate and encrypt the TLS way (also perform padding).
* During decryption it verifies the pad and tag and returns -EBADMSG on error.
*/
static int
tls_auth_n_crypt(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop,
struct scatterlist *auth_sg, uint32_t auth_len,
struct scatterlist *dst_sg, uint32_t len)
{
int ret, fail = 0;
struct crypt_auth_op *caop = &kcaop->caop;
uint8_t vhash[AALG_MAX_RESULT_LEN];
uint8_t hash_output[AALG_MAX_RESULT_LEN];
/* TLS authenticates the plaintext except for the padding.
*/
if (caop->op == COP_ENCRYPT) {
if (ses_ptr->hdata.init != 0) {
if (auth_len > 0) {
ret = cryptodev_hash_update(&ses_ptr->hdata,
auth_sg, auth_len);
if (unlikely(ret)) {
derr(0, "cryptodev_hash_update: %d", ret);
return ret;
}
}
if (len > 0) {
ret = cryptodev_hash_update(&ses_ptr->hdata,
dst_sg, len);
if (unlikely(ret)) {
derr(0, "cryptodev_hash_update: %d", ret);
return ret;
}
}
ret = cryptodev_hash_final(&ses_ptr->hdata, hash_output);
if (unlikely(ret)) {
derr(0, "cryptodev_hash_final: %d", ret);
return ret;
}
copy_tls_hash(dst_sg, len, hash_output, caop->tag_len);
len += caop->tag_len;
}
if (ses_ptr->cdata.init != 0) {
if (ses_ptr->cdata.blocksize > 1) {
ret = pad_record(dst_sg, len, ses_ptr->cdata.blocksize);
len += ret;
}
ret = cryptodev_cipher_encrypt(&ses_ptr->cdata,
dst_sg, dst_sg, len);
if (unlikely(ret)) {
derr(0, "cryptodev_cipher_encrypt: %d", ret);
return ret;
}
}
} else {
if (ses_ptr->cdata.init != 0) {
ret = cryptodev_cipher_decrypt(&ses_ptr->cdata,
dst_sg, dst_sg, len);
if (unlikely(ret)) {
derr(0, "cryptodev_cipher_decrypt: %d", ret);
return ret;
}
if (ses_ptr->cdata.blocksize > 1) {
ret = verify_tls_record_pad(dst_sg, len, ses_ptr->cdata.blocksize);
if (unlikely(ret < 0)) {
derr(2, "verify_record_pad: %d", ret);
fail = 1;
} else {
len -= ret;
}
}
}
if (ses_ptr->hdata.init != 0) {
if (unlikely(caop->tag_len > sizeof(vhash) || caop->tag_len > len)) {
derr(1, "Illegal tag len size");
return -EINVAL;
}
read_tls_hash(dst_sg, len, vhash, caop->tag_len);
len -= caop->tag_len;
if (auth_len > 0) {
ret = cryptodev_hash_update(&ses_ptr->hdata,
auth_sg, auth_len);
if (unlikely(ret)) {
derr(0, "cryptodev_hash_update: %d", ret);
return ret;
}
}
if (len > 0) {
ret = cryptodev_hash_update(&ses_ptr->hdata,
dst_sg, len);
if (unlikely(ret)) {
derr(0, "cryptodev_hash_update: %d", ret);
return ret;
}
}
ret = cryptodev_hash_final(&ses_ptr->hdata, hash_output);
if (unlikely(ret)) {
derr(0, "cryptodev_hash_final: %d", ret);
return ret;
}
if (memcmp(vhash, hash_output, caop->tag_len) != 0 || fail != 0) {
derr(2, "MAC verification failed (tag_len: %d)", caop->tag_len);
return -EBADMSG;
}
}
}
kcaop->dst_len = len;
return 0;
}
/* Authenticate and encrypt the SRTP way. During decryption
* it verifies the tag and returns -EBADMSG on error.
*/
static int
srtp_auth_n_crypt(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop,
struct scatterlist *auth_sg, uint32_t auth_len,
struct scatterlist *dst_sg, uint32_t len)
{
int ret, fail = 0;
struct crypt_auth_op *caop = &kcaop->caop;
uint8_t vhash[AALG_MAX_RESULT_LEN];
uint8_t hash_output[AALG_MAX_RESULT_LEN];
/* SRTP authenticates the encrypted data.
*/
if (caop->op == COP_ENCRYPT) {
if (ses_ptr->cdata.init != 0) {
ret = cryptodev_cipher_encrypt(&ses_ptr->cdata,
dst_sg, dst_sg, len);
if (unlikely(ret)) {
derr(0, "cryptodev_cipher_encrypt: %d", ret);
return ret;
}
}
if (ses_ptr->hdata.init != 0) {
if (auth_len > 0) {
ret = cryptodev_hash_update(&ses_ptr->hdata,
auth_sg, auth_len);
if (unlikely(ret)) {
derr(0, "cryptodev_hash_update: %d", ret);
return ret;
}
}
ret = cryptodev_hash_final(&ses_ptr->hdata, hash_output);
if (unlikely(ret)) {
derr(0, "cryptodev_hash_final: %d", ret);
return ret;
}
if (unlikely(copy_to_user(caop->tag, hash_output, caop->tag_len)))
return -EFAULT;
}
} else {
if (ses_ptr->hdata.init != 0) {
if (unlikely(caop->tag_len > sizeof(vhash) || caop->tag_len > len)) {
derr(1, "Illegal tag len size");
return -EINVAL;
}
if (unlikely(copy_from_user(vhash, caop->tag, caop->tag_len)))
return -EFAULT;
ret = cryptodev_hash_update(&ses_ptr->hdata,
auth_sg, auth_len);
if (unlikely(ret)) {
derr(0, "cryptodev_hash_update: %d", ret);
return ret;
}
ret = cryptodev_hash_final(&ses_ptr->hdata, hash_output);
if (unlikely(ret)) {
derr(0, "cryptodev_hash_final: %d", ret);
return ret;
}
if (memcmp(vhash, hash_output, caop->tag_len) != 0 || fail != 0) {
derr(2, "MAC verification failed");
return -EBADMSG;
}
}
if (ses_ptr->cdata.init != 0) {
ret = cryptodev_cipher_decrypt(&ses_ptr->cdata,
dst_sg, dst_sg, len);
if (unlikely(ret)) {
derr(0, "cryptodev_cipher_decrypt: %d", ret);
return ret;
}
}
}
kcaop->dst_len = len;
return 0;
}
/* Typical AEAD (i.e. GCM) encryption/decryption.
* During decryption the tag is verified.
*/
static int
auth_n_crypt(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop,
struct scatterlist *auth_sg, uint32_t auth_len,
struct scatterlist *src_sg,
struct scatterlist *dst_sg, uint32_t len)
{
int ret;
struct crypt_auth_op *caop = &kcaop->caop;
int max_tag_len;
max_tag_len = cryptodev_cipher_get_tag_size(&ses_ptr->cdata);
if (unlikely(caop->tag_len > max_tag_len)) {
derr(0, "Illegal tag length: %d", caop->tag_len);
return -EINVAL;
}
if (caop->tag_len)
cryptodev_cipher_set_tag_size(&ses_ptr->cdata, caop->tag_len);
else
caop->tag_len = max_tag_len;
cryptodev_cipher_auth(&ses_ptr->cdata, auth_sg, auth_len);
if (caop->op == COP_ENCRYPT) {
ret = cryptodev_cipher_encrypt(&ses_ptr->cdata,
src_sg, dst_sg, len);
if (unlikely(ret)) {
derr(0, "cryptodev_cipher_encrypt: %d", ret);
return ret;
}
kcaop->dst_len = len + caop->tag_len;
caop->tag = caop->dst + len;
} else {
ret = cryptodev_cipher_decrypt(&ses_ptr->cdata,
src_sg, dst_sg, len);
if (unlikely(ret)) {
derr(0, "cryptodev_cipher_decrypt: %d", ret);
return ret;
}
kcaop->dst_len = len - caop->tag_len;
caop->tag = caop->dst + len - caop->tag_len;
}
return 0;
}
static int crypto_auth_zc_srtp(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop)
{
struct scatterlist *dst_sg, *auth_sg;
struct crypt_auth_op *caop = &kcaop->caop;
int ret;
if (unlikely(ses_ptr->cdata.init != 0 &&
(ses_ptr->cdata.stream == 0 || ses_ptr->cdata.aead != 0))) {
derr(0, "Only stream modes are allowed in SRTP mode (but not AEAD)");
return -EINVAL;
}
ret = get_userbuf_srtp(ses_ptr, kcaop, &auth_sg, &dst_sg);
if (unlikely(ret)) {
derr(1, "get_userbuf_srtp(): Error getting user pages.");
return ret;
}
ret = srtp_auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len,
dst_sg, caop->len);
release_user_pages(ses_ptr);
return ret;
}
static int crypto_auth_zc_tls(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop)
{
struct crypt_auth_op *caop = &kcaop->caop;
struct scatterlist *dst_sg, *auth_sg;
unsigned char *auth_buf = NULL;
struct scatterlist tmp;
int ret;
if (unlikely(caop->auth_len > PAGE_SIZE)) {
derr(1, "auth data len is excessive.");
return -EINVAL;
}
auth_buf = (char *)__get_free_page(GFP_KERNEL);
if (unlikely(!auth_buf)) {
derr(1, "unable to get a free page.");
return -ENOMEM;
}
if (caop->auth_src && caop->auth_len > 0) {
if (unlikely(copy_from_user(auth_buf, caop->auth_src, caop->auth_len))) {
derr(1, "unable to copy auth data from userspace.");
ret = -EFAULT;
goto free_auth_buf;
}
sg_init_one(&tmp, auth_buf, caop->auth_len);
auth_sg = &tmp;
} else {
auth_sg = NULL;
}
ret = get_userbuf_tls(ses_ptr, kcaop, &dst_sg);
if (unlikely(ret)) {
derr(1, "get_userbuf_tls(): Error getting user pages.");
goto free_auth_buf;
}
ret = tls_auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len,
dst_sg, caop->len);
release_user_pages(ses_ptr);
free_auth_buf:
free_page((unsigned long)auth_buf);
return ret;
}
static int crypto_auth_zc_aead(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop)
{
struct scatterlist *dst_sg;
struct scatterlist *src_sg;
struct crypt_auth_op *caop = &kcaop->caop;
unsigned char *auth_buf = NULL;
int ret;
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0))
struct scatterlist tmp;
struct scatterlist *auth_sg;
#else
struct scatterlist auth1[2];
struct scatterlist auth2[2];
#endif
if (unlikely(ses_ptr->cdata.init == 0 ||
(ses_ptr->cdata.stream == 0 && ses_ptr->cdata.aead == 0))) {
derr(0, "Only stream and AEAD ciphers are allowed for authenc");
return -EINVAL;
}
if (unlikely(caop->auth_len > PAGE_SIZE)) {
derr(1, "auth data len is excessive.");
return -EINVAL;
}
auth_buf = (char *)__get_free_page(GFP_KERNEL);
if (unlikely(!auth_buf)) {
derr(1, "unable to get a free page.");
return -ENOMEM;
}
ret = get_userbuf(ses_ptr, caop->src, caop->len, caop->dst, kcaop->dst_len,
kcaop->task, kcaop->mm, &src_sg, &dst_sg);
if (unlikely(ret)) {
derr(1, "get_userbuf(): Error getting user pages.");
goto free_auth_buf;
}
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 2, 0))
if (caop->auth_src && caop->auth_len > 0) {
if (unlikely(copy_from_user(auth_buf, caop->auth_src, caop->auth_len))) {
derr(1, "unable to copy auth data from userspace.");
ret = -EFAULT;
goto free_pages;
}
sg_init_one(&tmp, auth_buf, caop->auth_len);
auth_sg = &tmp;
} else {
auth_sg = NULL;
}
ret = auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len,
src_sg, dst_sg, caop->len);
#else
if (caop->auth_src && caop->auth_len > 0) {
if (unlikely(copy_from_user(auth_buf, caop->auth_src, caop->auth_len))) {
derr(1, "unable to copy auth data from userspace.");
ret = -EFAULT;
goto free_pages;
}
sg_init_table(auth1, 2);
sg_set_buf(auth1, auth_buf, caop->auth_len);
sg_chain(auth1, 2, src_sg);
if (src_sg == dst_sg) {
src_sg = auth1;
dst_sg = auth1;
} else {
sg_init_table(auth2, 2);
sg_set_buf(auth2, auth_buf, caop->auth_len);
sg_chain(auth2, 2, dst_sg);
src_sg = auth1;
dst_sg = auth2;
}
}
ret = auth_n_crypt(ses_ptr, kcaop, NULL, caop->auth_len,
src_sg, dst_sg, caop->len);
#endif
free_pages:
release_user_pages(ses_ptr);
free_auth_buf:
free_page((unsigned long)auth_buf);
return ret;
}
static int
__crypto_auth_run_zc(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop)
{
struct crypt_auth_op *caop = &kcaop->caop;
int ret;
if (caop->flags & COP_FLAG_AEAD_SRTP_TYPE) {
ret = crypto_auth_zc_srtp(ses_ptr, kcaop);
} else if (caop->flags & COP_FLAG_AEAD_TLS_TYPE &&
ses_ptr->cdata.aead == 0) {
ret = crypto_auth_zc_tls(ses_ptr, kcaop);
} else if (ses_ptr->cdata.aead) {
ret = crypto_auth_zc_aead(ses_ptr, kcaop);
} else {
ret = -EINVAL;
}
return ret;
}
int crypto_auth_run(struct fcrypt *fcr, struct kernel_crypt_auth_op *kcaop)
{
struct csession *ses_ptr;
struct crypt_auth_op *caop = &kcaop->caop;
int ret;
if (unlikely(caop->op != COP_ENCRYPT && caop->op != COP_DECRYPT)) {
ddebug(1, "invalid operation op=%u", caop->op);
return -EINVAL;
}
/* this also enters ses_ptr->sem */
ses_ptr = crypto_get_session_by_sid(fcr, caop->ses);
if (unlikely(!ses_ptr)) {
derr(1, "invalid session ID=0x%08X", caop->ses);
return -EINVAL;
}
if (unlikely(ses_ptr->cdata.init == 0)) {
derr(1, "cipher context not initialized");
ret = -EINVAL;
goto out_unlock;
}
/* If we have a hash/mac handle reset its state */
if (ses_ptr->hdata.init != 0) {
ret = cryptodev_hash_reset(&ses_ptr->hdata);
if (unlikely(ret)) {
derr(1, "error in cryptodev_hash_reset()");
goto out_unlock;
}
}
cryptodev_cipher_set_iv(&ses_ptr->cdata, kcaop->iv,
min(ses_ptr->cdata.ivsize, kcaop->ivlen));
ret = __crypto_auth_run_zc(ses_ptr, kcaop);
if (unlikely(ret)) {
derr(1, "error in __crypto_auth_run_zc()");
goto out_unlock;
}
ret = 0;
cryptodev_cipher_get_iv(&ses_ptr->cdata, kcaop->iv,
min(ses_ptr->cdata.ivsize, kcaop->ivlen));
out_unlock:
crypto_put_session(ses_ptr);
return ret;
}

58
drivers/crypto/rockchip/cryptodev_linux/cipherapi.h Normal file
View File

@@ -0,0 +1,58 @@
/* SPDX-License-Identifier: GPL-2.0+ */
#ifndef CIPHERAPI_H
# define CIPHERAPI_H
#include <linux/version.h>
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0))
# include <linux/crypto.h>
typedef struct crypto_ablkcipher cryptodev_crypto_blkcipher_t;
typedef struct ablkcipher_request cryptodev_blkcipher_request_t;
# define cryptodev_crypto_alloc_blkcipher crypto_alloc_ablkcipher
# define cryptodev_crypto_blkcipher_blocksize crypto_ablkcipher_blocksize
# define cryptodev_crypto_blkcipher_ivsize crypto_ablkcipher_ivsize
# define cryptodev_crypto_blkcipher_alignmask crypto_ablkcipher_alignmask
# define cryptodev_crypto_blkcipher_setkey crypto_ablkcipher_setkey
static inline void cryptodev_crypto_free_blkcipher(cryptodev_crypto_blkcipher_t *c) {
if (c)
crypto_free_ablkcipher(c);
}
# define cryptodev_blkcipher_request_alloc ablkcipher_request_alloc
# define cryptodev_blkcipher_request_set_callback ablkcipher_request_set_callback
static inline void cryptodev_blkcipher_request_free(cryptodev_blkcipher_request_t *r) {
if (r)
ablkcipher_request_free(r);
}
# define cryptodev_blkcipher_request_set_crypt ablkcipher_request_set_crypt
# define cryptodev_crypto_blkcipher_encrypt crypto_ablkcipher_encrypt
# define cryptodev_crypto_blkcipher_decrypt crypto_ablkcipher_decrypt
# define cryptodev_crypto_blkcipher_tfm crypto_ablkcipher_tfm
#else
#include <crypto/skcipher.h>
typedef struct crypto_skcipher cryptodev_crypto_blkcipher_t;
typedef struct skcipher_request cryptodev_blkcipher_request_t;
# define cryptodev_crypto_alloc_blkcipher crypto_alloc_skcipher
# define cryptodev_crypto_blkcipher_blocksize crypto_skcipher_blocksize
# define cryptodev_crypto_blkcipher_ivsize crypto_skcipher_ivsize
# define cryptodev_crypto_blkcipher_alignmask crypto_skcipher_alignmask
# define cryptodev_crypto_blkcipher_setkey crypto_skcipher_setkey
# define cryptodev_crypto_free_blkcipher crypto_free_skcipher
# define cryptodev_blkcipher_request_alloc skcipher_request_alloc
# define cryptodev_blkcipher_request_set_callback skcipher_request_set_callback
# define cryptodev_blkcipher_request_free skcipher_request_free
# define cryptodev_blkcipher_request_set_crypt skcipher_request_set_crypt
# define cryptodev_crypto_blkcipher_encrypt crypto_skcipher_encrypt
# define cryptodev_crypto_blkcipher_decrypt crypto_skcipher_decrypt
# define cryptodev_crypto_blkcipher_tfm crypto_skcipher_tfm
#endif
#endif

493
drivers/crypto/rockchip/cryptodev_linux/cryptlib.c Normal file
View File

@@ -0,0 +1,493 @@
/*
* Driver for /dev/crypto device (aka CryptoDev)
*
* Copyright (c) 2010,2011 Nikos Mavrogiannopoulos <nmav@gnutls.org>
* Portions Copyright (c) 2010 Michael Weiser
* Portions Copyright (c) 2010 Phil Sutter
*
* This file is part of linux cryptodev.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/ioctl.h>
#include <linux/random.h>
#include <linux/scatterlist.h>
#include <linux/uaccess.h>
#include <crypto/algapi.h>
#include <crypto/hash.h>
#include "crypto/cryptodev.h"
#include <crypto/aead.h>
#include <linux/rtnetlink.h>
#include <crypto/authenc.h>
#include "cryptodev_int.h"
#include "cipherapi.h"
#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0))
extern const struct crypto_type crypto_givcipher_type;
#endif
static void cryptodev_complete(struct crypto_async_request *req, int err)
{
struct cryptodev_result *res = req->data;
if (err == -EINPROGRESS)
return;
res->err = err;
complete(&res->completion);
}
int cryptodev_get_cipher_keylen(unsigned int *keylen, struct session_op *sop,
int aead)
{
/*
* For blockciphers (AES-CBC) or non-composite aead ciphers (like AES-GCM),
* the key length is simply the cipher keylen obtained from userspace. If
* the cipher is composite aead, the keylen is the sum of cipher keylen,
* hmac keylen and a key header length. This key format is the one used in
* Linux kernel for composite aead ciphers (crypto/authenc.c)
*/
unsigned int klen = sop->keylen;
if (unlikely(sop->keylen > CRYPTO_CIPHER_MAX_KEY_LEN))
return -EINVAL;
if (aead && sop->mackeylen) {
if (unlikely(sop->mackeylen > CRYPTO_HMAC_MAX_KEY_LEN))
return -EINVAL;
klen += sop->mackeylen;
klen += RTA_SPACE(sizeof(struct crypto_authenc_key_param));
}
*keylen = klen;
return 0;
}
int cryptodev_get_cipher_key(uint8_t *key, struct session_op *sop, int aead)
{
/*
* Get cipher key from user-space. For blockciphers just copy it from
* user-space. For composite aead ciphers combine it with the hmac key in
* the format used by Linux kernel in crypto/authenc.c:
*
* [[AUTHENC_KEY_HEADER + CIPHER_KEYLEN] [AUTHENTICATION KEY] [CIPHER KEY]]
*/
struct crypto_authenc_key_param *param;
struct rtattr *rta;
int ret = 0;
if (aead && sop->mackeylen) {
/*
* Composite aead ciphers. The first four bytes are the header type and
* header length for aead keys
*/
rta = (void *)key;
rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM;
rta->rta_len = RTA_LENGTH(sizeof(*param));
/*
* The next four bytes hold the length of the encryption key
*/
param = RTA_DATA(rta);
param->enckeylen = cpu_to_be32(sop->keylen);
/* Advance key pointer eight bytes and copy the hmac key */
key += RTA_SPACE(sizeof(*param));
if (unlikely(copy_from_user(key, sop->mackey, sop->mackeylen))) {
ret = -EFAULT;
goto error;
}
/* Advance key pointer past the hmac key */
key += sop->mackeylen;
}
/* now copy the blockcipher key */
if (unlikely(copy_from_user(key, sop->key, sop->keylen)))
ret = -EFAULT;
error:
return ret;
}
/* Was correct key length supplied? */
static int check_key_size(size_t keylen, const char *alg_name,
unsigned int min_keysize, unsigned int max_keysize)
{
if (max_keysize > 0 && unlikely((keylen < min_keysize) ||
(keylen > max_keysize))) {
ddebug(1, "Wrong keylen '%zu' for algorithm '%s'. Use %u to %u.",
keylen, alg_name, min_keysize, max_keysize);
return -EINVAL;
}
return 0;
}
int cryptodev_cipher_init(struct cipher_data *out, const char *alg_name,
uint8_t *keyp, size_t keylen, int stream, int aead)
{
int ret;
if (aead == 0) {
unsigned int min_keysize, max_keysize;
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
struct crypto_tfm *tfm;
#else
struct ablkcipher_alg *alg;
#endif
out->async.s = cryptodev_crypto_alloc_blkcipher(alg_name, 0, 0);
if (unlikely(IS_ERR(out->async.s))) {
ddebug(1, "Failed to load cipher %s", alg_name);
return -EINVAL;
}
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0))
tfm = crypto_skcipher_tfm(out->async.s);
#if (LINUX_VERSION_CODE <= KERNEL_VERSION(5, 4, 0))
if ((tfm->__crt_alg->cra_type == &crypto_ablkcipher_type)
#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0))
|| (tfm->__crt_alg->cra_type == &crypto_givcipher_type)
#endif
) {
struct ablkcipher_alg *alg;
alg = &tfm->__crt_alg->cra_ablkcipher;
min_keysize = alg->min_keysize;
max_keysize = alg->max_keysize;
} else
#endif
{
struct skcipher_alg *alg;
alg = crypto_skcipher_alg(out->async.s);
min_keysize = alg->min_keysize;
max_keysize = alg->max_keysize;
}
#else
alg = crypto_ablkcipher_alg(out->async.s);
min_keysize = alg->min_keysize;
max_keysize = alg->max_keysize;
#endif
ret = check_key_size(keylen, alg_name, min_keysize,
max_keysize);
if (ret)
goto error;
out->blocksize = cryptodev_crypto_blkcipher_blocksize(out->async.s);
out->ivsize = cryptodev_crypto_blkcipher_ivsize(out->async.s);
out->alignmask = cryptodev_crypto_blkcipher_alignmask(out->async.s);
ret = cryptodev_crypto_blkcipher_setkey(out->async.s, keyp, keylen);
} else {
out->async.as = crypto_alloc_aead(alg_name, 0, 0);
if (unlikely(IS_ERR(out->async.as))) {
ddebug(1, "Failed to load cipher %s", alg_name);
return -EINVAL;
}
out->blocksize = crypto_aead_blocksize(out->async.as);
out->ivsize = crypto_aead_ivsize(out->async.as);
out->alignmask = crypto_aead_alignmask(out->async.as);
ret = crypto_aead_setkey(out->async.as, keyp, keylen);
}
if (unlikely(ret)) {
ddebug(1, "Setting key failed for %s-%zu.", alg_name, keylen*8);
ret = -EINVAL;
goto error;
}
out->stream = stream;
out->aead = aead;
init_completion(&out->async.result.completion);
if (aead == 0) {
out->async.request = cryptodev_blkcipher_request_alloc(out->async.s, GFP_KERNEL);
if (unlikely(!out->async.request)) {
derr(1, "error allocating async crypto request");
ret = -ENOMEM;
goto error;
}
cryptodev_blkcipher_request_set_callback(out->async.request,
CRYPTO_TFM_REQ_MAY_BACKLOG,
cryptodev_complete, &out->async.result);
} else {
out->async.arequest = aead_request_alloc(out->async.as, GFP_KERNEL);
if (unlikely(!out->async.arequest)) {
derr(1, "error allocating async crypto request");
ret = -ENOMEM;
goto error;
}
aead_request_set_callback(out->async.arequest,
CRYPTO_TFM_REQ_MAY_BACKLOG,
cryptodev_complete, &out->async.result);
}
out->init = 1;
return 0;
error:
if (aead == 0) {
cryptodev_blkcipher_request_free(out->async.request);
cryptodev_crypto_free_blkcipher(out->async.s);
} else {
if (out->async.arequest)
aead_request_free(out->async.arequest);
if (out->async.as)
crypto_free_aead(out->async.as);
}
return ret;
}
void cryptodev_cipher_deinit(struct cipher_data *cdata)
{
if (cdata->init) {
if (cdata->aead == 0) {
cryptodev_blkcipher_request_free(cdata->async.request);
cryptodev_crypto_free_blkcipher(cdata->async.s);
} else {
if (cdata->async.arequest)
aead_request_free(cdata->async.arequest);
if (cdata->async.as)
crypto_free_aead(cdata->async.as);
}
cdata->init = 0;
}
}
static inline int waitfor(struct cryptodev_result *cr, ssize_t ret)
{
switch (ret) {
case 0:
break;
case -EINPROGRESS:
case -EBUSY:
wait_for_completion(&cr->completion);
/* At this point we known for sure the request has finished,
* because wait_for_completion above was not interruptible.
* This is important because otherwise hardware or driver
* might try to access memory which will be freed or reused for
* another request. */
if (unlikely(cr->err)) {
derr(0, "error from async request: %d", cr->err);
return cr->err;
}
break;
default:
return ret;
}
return 0;
}
ssize_t cryptodev_cipher_encrypt(struct cipher_data *cdata,
const struct scatterlist *src, struct scatterlist *dst,
size_t len)
{
int ret;
reinit_completion(&cdata->async.result.completion);
if (cdata->aead == 0) {
cryptodev_blkcipher_request_set_crypt(cdata->async.request,
(struct scatterlist *)src, dst,
len, cdata->async.iv);
ret = cryptodev_crypto_blkcipher_encrypt(cdata->async.request);
} else {
aead_request_set_crypt(cdata->async.arequest,
(struct scatterlist *)src, dst,
len, cdata->async.iv);
ret = crypto_aead_encrypt(cdata->async.arequest);
}
return waitfor(&cdata->async.result, ret);
}
ssize_t cryptodev_cipher_decrypt(struct cipher_data *cdata,
const struct scatterlist *src, struct scatterlist *dst,
size_t len)
{
int ret;
reinit_completion(&cdata->async.result.completion);
if (cdata->aead == 0) {
cryptodev_blkcipher_request_set_crypt(cdata->async.request,
(struct scatterlist *)src, dst,
len, cdata->async.iv);
ret = cryptodev_crypto_blkcipher_decrypt(cdata->async.request);
} else {
aead_request_set_crypt(cdata->async.arequest,
(struct scatterlist *)src, dst,
len, cdata->async.iv);
ret = crypto_aead_decrypt(cdata->async.arequest);
}
return waitfor(&cdata->async.result, ret);
}
/* Hash functions */
int cryptodev_hash_init(struct hash_data *hdata, const char *alg_name,
int hmac_mode, void *mackey, size_t mackeylen)
{
int ret;
hdata->async.s = crypto_alloc_ahash(alg_name, 0, 0);
if (unlikely(IS_ERR(hdata->async.s))) {
ddebug(1, "Failed to load transform for %s", alg_name);
return -EINVAL;
}
/* Copy the key from user and set to TFM. */
if (hmac_mode != 0) {
ret = crypto_ahash_setkey(hdata->async.s, mackey, mackeylen);
if (unlikely(ret)) {
ddebug(1, "Setting hmac key failed for %s-%zu.",
alg_name, mackeylen*8);
ret = -EINVAL;
goto error;
}
}
hdata->digestsize = crypto_ahash_digestsize(hdata->async.s);
hdata->alignmask = crypto_ahash_alignmask(hdata->async.s);
init_completion(&hdata->async.result.completion);
hdata->async.request = ahash_request_alloc(hdata->async.s, GFP_KERNEL);
if (unlikely(!hdata->async.request)) {
derr(0, "error allocating async crypto request");
ret = -ENOMEM;
goto error;
}
ahash_request_set_callback(hdata->async.request,
CRYPTO_TFM_REQ_MAY_BACKLOG,
cryptodev_complete, &hdata->async.result);
hdata->init = 1;
return 0;
error:
crypto_free_ahash(hdata->async.s);
return ret;
}
void cryptodev_hash_deinit(struct hash_data *hdata)
{
if (hdata->init) {
ahash_request_free(hdata->async.request);
crypto_free_ahash(hdata->async.s);
hdata->init = 0;
}
}
int cryptodev_hash_reset(struct hash_data *hdata)
{
int ret;
ret = crypto_ahash_init(hdata->async.request);
if (unlikely(ret)) {
derr(0, "error in crypto_hash_init()");
return ret;
}
return 0;
}
ssize_t cryptodev_hash_update(struct hash_data *hdata,
struct scatterlist *sg, size_t len)
{
int ret;
reinit_completion(&hdata->async.result.completion);
ahash_request_set_crypt(hdata->async.request, sg, NULL, len);
ret = crypto_ahash_update(hdata->async.request);
return waitfor(&hdata->async.result, ret);
}
int cryptodev_hash_final(struct hash_data *hdata, void *output)
{
int ret;
reinit_completion(&hdata->async.result.completion);
ahash_request_set_crypt(hdata->async.request, NULL, output, 0);
ret = crypto_ahash_final(hdata->async.request);
return waitfor(&hdata->async.result, ret);
}
#ifdef CIOCCPHASH
/* import the current hash state of src to dst */
int cryptodev_hash_copy(struct hash_data *dst, struct hash_data *src)
{
int ret, statesize;
void *statedata = NULL;
struct crypto_tfm *tfm;
if (unlikely(src == NULL || !src->init ||
dst == NULL || !dst->init)) {
return -EINVAL;
}
reinit_completion(&src->async.result.completion);
statesize = crypto_ahash_statesize(src->async.s);
if (unlikely(statesize <= 0)) {
return -EINVAL;
}
statedata = kzalloc(statesize, GFP_KERNEL);
if (unlikely(statedata == NULL)) {
return -ENOMEM;
}
ret = crypto_ahash_export(src->async.request, statedata);
if (unlikely(ret < 0)) {
if (unlikely(ret == -ENOSYS)) {
tfm = crypto_ahash_tfm(src->async.s);
derr(0, "cryptodev_hash_copy: crypto_ahash_export not implemented for "
"alg='%s', driver='%s'", crypto_tfm_alg_name(tfm),
crypto_tfm_alg_driver_name(tfm));
}
goto out;
}
ret = crypto_ahash_import(dst->async.request, statedata);
if (unlikely(ret == -ENOSYS)) {
tfm = crypto_ahash_tfm(dst->async.s);
derr(0, "cryptodev_hash_copy: crypto_ahash_import not implemented for "
"alg='%s', driver='%s'", crypto_tfm_alg_name(tfm),
crypto_tfm_alg_driver_name(tfm));
}
out:
kfree(statedata);
return ret;
}
#endif /* CIOCCPHASH */

111
drivers/crypto/rockchip/cryptodev_linux/cryptlib.h Normal file
View File

@@ -0,0 +1,111 @@
/* SPDX-License-Identifier: GPL-2.0+ */
#ifndef CRYPTLIB_H
# define CRYPTLIB_H
#include <linux/version.h>
struct cryptodev_result {
struct completion completion;
int err;
};
#include "cipherapi.h"
struct cipher_data {
int init; /* 0 uninitialized */
int blocksize;
int aead;
int stream;
int ivsize;
int alignmask;
struct {
/* block ciphers */
cryptodev_crypto_blkcipher_t *s;
cryptodev_blkcipher_request_t *request;
/* AEAD ciphers */
struct crypto_aead *as;
struct aead_request *arequest;
struct cryptodev_result result;
uint8_t iv[EALG_MAX_BLOCK_LEN];
} async;
};
int cryptodev_cipher_init(struct cipher_data *out, const char *alg_name,
uint8_t *key, size_t keylen, int stream, int aead);
void cryptodev_cipher_deinit(struct cipher_data *cdata);
int cryptodev_get_cipher_key(uint8_t *key, struct session_op *sop, int aead);
int cryptodev_get_cipher_keylen(unsigned int *keylen, struct session_op *sop,
int aead);
ssize_t cryptodev_cipher_decrypt(struct cipher_data *cdata,
const struct scatterlist *sg1,
struct scatterlist *sg2, size_t len);
ssize_t cryptodev_cipher_encrypt(struct cipher_data *cdata,
const struct scatterlist *sg1,
struct scatterlist *sg2, size_t len);
/* AEAD */
static inline void cryptodev_cipher_auth(struct cipher_data *cdata,
struct scatterlist *sg1, size_t len)
{
/* for some reason we _have_ to call that even for zero length sgs */
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0))
aead_request_set_assoc(cdata->async.arequest, len ? sg1 : NULL, len);
#else
aead_request_set_ad(cdata->async.arequest, len);
#endif
}
static inline void cryptodev_cipher_set_tag_size(struct cipher_data *cdata, int size)
{
if (likely(cdata->aead != 0))
crypto_aead_setauthsize(cdata->async.as, size);
}
static inline int cryptodev_cipher_get_tag_size(struct cipher_data *cdata)
{
if (likely(cdata->init && cdata->aead != 0))
return crypto_aead_authsize(cdata->async.as);
else
return 0;
}
static inline void cryptodev_cipher_set_iv(struct cipher_data *cdata,
void *iv, size_t iv_size)
{
memcpy(cdata->async.iv, iv, min(iv_size, sizeof(cdata->async.iv)));
}
static inline void cryptodev_cipher_get_iv(struct cipher_data *cdata,
void *iv, size_t iv_size)
{
memcpy(iv, cdata->async.iv, min(iv_size, sizeof(cdata->async.iv)));
}
/* Hash */
struct hash_data {
int init; /* 0 uninitialized */
int digestsize;
int alignmask;
struct {
struct crypto_ahash *s;
struct cryptodev_result result;
struct ahash_request *request;
} async;
};
int cryptodev_hash_final(struct hash_data *hdata, void *output);
ssize_t cryptodev_hash_update(struct hash_data *hdata,
struct scatterlist *sg, size_t len);
int cryptodev_hash_reset(struct hash_data *hdata);
void cryptodev_hash_deinit(struct hash_data *hdata);
int cryptodev_hash_init(struct hash_data *hdata, const char *alg_name,
int hmac_mode, void *mackey, size_t mackeylen);
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
int cryptodev_hash_copy(struct hash_data *dst, struct hash_data *src);
#endif
#endif

319
drivers/crypto/rockchip/cryptodev_linux/crypto/cryptodev.h Normal file
View File

@@ -0,0 +1,319 @@
/* SPDX-License-Identifier: GPL-2.0+ */
/* This is a source compatible implementation with the original API of
* cryptodev by Angelos D. Keromytis, found at openbsd cryptodev.h.
* Placed under public domain */
#ifndef L_CRYPTODEV_H
#define L_CRYPTODEV_H
#include <linux/types.h>
#include <linux/version.h>
#ifndef __KERNEL__
#define __user
#endif
/* API extensions for linux */
#define CRYPTO_HMAC_MAX_KEY_LEN 512
#define CRYPTO_CIPHER_MAX_KEY_LEN 64
/* All the supported algorithms
*/
enum cryptodev_crypto_op_t {
CRYPTO_DES_CBC = 1,
CRYPTO_3DES_CBC = 2,
CRYPTO_BLF_CBC = 3,
CRYPTO_CAST_CBC = 4,
CRYPTO_SKIPJACK_CBC = 5,
CRYPTO_MD5_HMAC = 6,
CRYPTO_SHA1_HMAC = 7,
CRYPTO_RIPEMD160_HMAC = 8,
CRYPTO_MD5_KPDK = 9,
CRYPTO_SHA1_KPDK = 10,
CRYPTO_RIJNDAEL128_CBC = 11,
CRYPTO_AES_CBC = CRYPTO_RIJNDAEL128_CBC,
CRYPTO_ARC4 = 12,
CRYPTO_MD5 = 13,
CRYPTO_SHA1 = 14,
CRYPTO_DEFLATE_COMP = 15,
CRYPTO_NULL = 16,
CRYPTO_LZS_COMP = 17,
CRYPTO_SHA2_256_HMAC = 18,
CRYPTO_SHA2_384_HMAC = 19,
CRYPTO_SHA2_512_HMAC = 20,
CRYPTO_AES_CTR = 21,
CRYPTO_AES_XTS = 22,
CRYPTO_AES_ECB = 23,
CRYPTO_AES_GCM = 50,
CRYPTO_CAMELLIA_CBC = 101,
CRYPTO_RIPEMD160,
CRYPTO_SHA2_224,
CRYPTO_SHA2_256,
CRYPTO_SHA2_384,
CRYPTO_SHA2_512,
CRYPTO_SHA2_224_HMAC,
CRYPTO_TLS11_AES_CBC_HMAC_SHA1,
CRYPTO_TLS12_AES_CBC_HMAC_SHA256,
CRYPTO_ALGORITHM_ALL, /* Keep updated - see below */
};
#define CRYPTO_ALGORITHM_MAX (CRYPTO_ALGORITHM_ALL - 1)
/* Values for ciphers */
#define DES_BLOCK_LEN 8
#define DES3_BLOCK_LEN 8
#define RIJNDAEL128_BLOCK_LEN 16
#define AES_BLOCK_LEN RIJNDAEL128_BLOCK_LEN
#define CAMELLIA_BLOCK_LEN 16
#define BLOWFISH_BLOCK_LEN 8
#define SKIPJACK_BLOCK_LEN 8
#define CAST128_BLOCK_LEN 8
/* the maximum of the above */
#define EALG_MAX_BLOCK_LEN 16
/* Values for hashes/MAC */
#define AALG_MAX_RESULT_LEN 64
/* maximum length of verbose alg names (depends on CRYPTO_MAX_ALG_NAME) */
#define CRYPTODEV_MAX_ALG_NAME 64
#define HASH_MAX_LEN 64
/* input of CIOCGSESSION */
struct session_op {
/* Specify either cipher or mac
*/
__u32 cipher; /* cryptodev_crypto_op_t */
__u32 mac; /* cryptodev_crypto_op_t */
__u32 keylen;
__u8 __user *key;
__u32 mackeylen;
__u8 __user *mackey;
__u32 ses; /* session identifier */
};
struct session_info_op {
__u32 ses; /* session identifier */
/* verbose names for the requested ciphers */
struct alg_info {
char cra_name[CRYPTODEV_MAX_ALG_NAME];
char cra_driver_name[CRYPTODEV_MAX_ALG_NAME];
} cipher_info, hash_info;
__u16 alignmask; /* alignment constraints */
__u32 flags; /* SIOP_FLAGS_* */
};
/* If this flag is set then this algorithm uses
* a driver only available in kernel (software drivers,
* or drivers based on instruction sets do not set this flag).
*
* If multiple algorithms are involved (as in AEAD case), then
* if one of them is kernel-driver-only this flag will be set.
*/
#define SIOP_FLAG_KERNEL_DRIVER_ONLY 1
#define COP_ENCRYPT 0
#define COP_DECRYPT 1
/* input of CIOCCRYPT */
struct crypt_op {
__u32 ses; /* session identifier */
__u16 op; /* COP_ENCRYPT or COP_DECRYPT */
__u16 flags; /* see COP_FLAG_* */
__u32 len; /* length of source data */
__u8 __user *src; /* source data */
__u8 __user *dst; /* pointer to output data */
/* pointer to output data for hash/MAC operations */
__u8 __user *mac;
/* initialization vector for encryption operations */
__u8 __user *iv;
};
/* input of CIOCAUTHCRYPT */
struct crypt_auth_op {
__u32 ses; /* session identifier */
__u16 op; /* COP_ENCRYPT or COP_DECRYPT */
__u16 flags; /* see COP_FLAG_AEAD_* */
__u32 len; /* length of source data */
__u32 auth_len; /* length of auth data */
__u8 __user *auth_src; /* authenticated-only data */
/* The current implementation is more efficient if data are
* encrypted in-place (src==dst). */
__u8 __user *src; /* data to be encrypted and authenticated */
__u8 __user *dst; /* pointer to output data. Must have
* space for tag. For TLS this should be at least
* len + tag_size + block_size for padding */
__u8 __user *tag; /* where the tag will be copied to. TLS mode
* doesn't use that as tag is copied to dst.
* SRTP mode copies tag there. */
__u32 tag_len; /* the length of the tag. Use zero for digest size or max tag. */
/* initialization vector for encryption operations */
__u8 __user *iv;
__u32 iv_len;
};
/* In plain AEAD mode the following are required:
* flags : 0
* iv : the initialization vector (12 bytes)
* auth_len: the length of the data to be authenticated
* auth_src: the data to be authenticated
* len : length of data to be encrypted
* src : the data to be encrypted
* dst : space to hold encrypted data. It must have
* at least a size of len + tag_size.
* tag_size: the size of the desired authentication tag or zero to use
* the maximum tag output.
*
* Note tag isn't being used because the Linux AEAD interface
* copies the tag just after data.
*/
/* In TLS mode (used for CBC ciphers that required padding)
* the following are required:
* flags : COP_FLAG_AEAD_TLS_TYPE
* iv : the initialization vector
* auth_len: the length of the data to be authenticated only
* len : length of data to be encrypted
* auth_src: the data to be authenticated
* src : the data to be encrypted
* dst : space to hold encrypted data (preferably in-place). It must have
* at least a size of len + tag_size + blocksize.
* tag_size: the size of the desired authentication tag or zero to use
* the default mac output.
*
* Note that the padding used is the minimum padding.
*/
/* In SRTP mode the following are required:
* flags : COP_FLAG_AEAD_SRTP_TYPE
* iv : the initialization vector
* auth_len: the length of the data to be authenticated. This must
* include the SRTP header + SRTP payload (data to be encrypted) + rest
*
* len : length of data to be encrypted
* auth_src: pointer the data to be authenticated. Should point at the same buffer as src.
* src : pointer to the data to be encrypted.
* dst : This is mandatory to be the same as src (in-place only).
* tag_size: the size of the desired authentication tag or zero to use
* the default mac output.
* tag : Pointer to an address where the authentication tag will be copied.
*/
/* struct crypt_op flags */
#define COP_FLAG_NONE (0 << 0) /* totally no flag */
#define COP_FLAG_UPDATE (1 << 0) /* multi-update hash mode */
#define COP_FLAG_FINAL (1 << 1) /* multi-update final hash mode */
#define COP_FLAG_WRITE_IV (1 << 2) /* update the IV during operation */
#define COP_FLAG_NO_ZC (1 << 3) /* do not zero-copy */
#define COP_FLAG_AEAD_TLS_TYPE (1 << 4) /* authenticate and encrypt using the
* TLS protocol rules */
#define COP_FLAG_AEAD_SRTP_TYPE (1 << 5) /* authenticate and encrypt using the
* SRTP protocol rules */
#define COP_FLAG_RESET (1 << 6) /* multi-update reset the state.
* should be used in combination
* with COP_FLAG_UPDATE */
/* Stuff for bignum arithmetic and public key
* cryptography - not supported yet by linux
* cryptodev.
*/
#define CRYPTO_ALG_FLAG_SUPPORTED 1
#define CRYPTO_ALG_FLAG_RNG_ENABLE 2
#define CRYPTO_ALG_FLAG_DSA_SHA 4
struct crparam {
__u8 *crp_p;
__u32 crp_nbits;
};
#define CRK_MAXPARAM 8
/* input of CIOCKEY */
struct crypt_kop {
__u32 crk_op; /* cryptodev_crk_op_t */
__u32 crk_status;
__u16 crk_iparams;
__u16 crk_oparams;
__u32 crk_pad1;
struct crparam crk_param[CRK_MAXPARAM];
};
enum cryptodev_crk_op_t {
CRK_MOD_EXP = 0,
CRK_MOD_EXP_CRT = 1,
CRK_DSA_SIGN = 2,
CRK_DSA_VERIFY = 3,
CRK_DH_COMPUTE_KEY = 4,
CRK_ALGORITHM_ALL
};
/* input of CIOCCPHASH
* dst_ses : destination session identifier
* src_ses : source session identifier
* dst_ses must have been created with CIOGSESSION first
*/
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
struct cphash_op {
__u32 dst_ses;
__u32 src_ses;
};
#endif
#define CRK_ALGORITHM_MAX (CRK_ALGORITHM_ALL-1)
/* features to be queried with CIOCASYMFEAT ioctl
*/
#define CRF_MOD_EXP (1 << CRK_MOD_EXP)
#define CRF_MOD_EXP_CRT (1 << CRK_MOD_EXP_CRT)
#define CRF_DSA_SIGN (1 << CRK_DSA_SIGN)
#define CRF_DSA_VERIFY (1 << CRK_DSA_VERIFY)
#define CRF_DH_COMPUTE_KEY (1 << CRK_DH_COMPUTE_KEY)
/* ioctl's. Compatible with old linux cryptodev.h
*/
#define CRIOGET _IOWR('c', 101, __u32)
#define CIOCGSESSION _IOWR('c', 102, struct session_op)
#define CIOCFSESSION _IOW('c', 103, __u32)
#define CIOCCRYPT _IOWR('c', 104, struct crypt_op)
#define CIOCKEY _IOWR('c', 105, struct crypt_kop)
#define CIOCASYMFEAT _IOR('c', 106, __u32)
#define CIOCGSESSINFO _IOWR('c', 107, struct session_info_op)
/* to indicate that CRIOGET is not required in linux
*/
#define CRIOGET_NOT_NEEDED 1
/* additional ioctls for AEAD */
#define CIOCAUTHCRYPT _IOWR('c', 109, struct crypt_auth_op)
/* additional ioctls for asynchronous operation.
* These are conditionally enabled since version 1.6.
*/
#define CIOCASYNCCRYPT _IOW('c', 110, struct crypt_op)
#define CIOCASYNCFETCH _IOR('c', 111, struct crypt_op)
/* additional ioctl for copying of hash/mac session state data
* between sessions.
* The cphash_op parameter should contain the session id of
* the source and destination sessions. Both sessions
* must have been created with CIOGSESSION.
*/
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29))
#define CIOCCPHASH _IOW('c', 112, struct cphash_op)
#endif
#endif /* L_CRYPTODEV_H */

151
drivers/crypto/rockchip/cryptodev_linux/cryptodev_int.h Normal file
View File

@@ -0,0 +1,151 @@
/* SPDX-License-Identifier: GPL-2.0+ */
/* cipher stuff */
#ifndef CRYPTODEV_INT_H
# define CRYPTODEV_INT_H
#include <linux/version.h>
#if (LINUX_VERSION_CODE < KERNEL_VERSION(3, 13, 0))
# define reinit_completion(x) INIT_COMPLETION(*(x))
#endif
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/scatterlist.h>
#include "crypto/cryptodev.h"
#include <crypto/aead.h>
#define PFX "cryptodev: "
#define dprintk(level, severity, format, a...) \
do { \
if (level <= cryptodev_verbosity) \
printk(severity PFX "%s[%u] (%s:%u): " format "\n", \
current->comm, current->pid, \
__func__, __LINE__, \
##a); \
} while (0)
#define derr(level, format, a...) dprintk(level, KERN_ERR, format, ##a)
#define dwarning(level, format, a...) dprintk(level, KERN_WARNING, format, ##a)
#define dinfo(level, format, a...) dprintk(level, KERN_INFO, format, ##a)
#define ddebug(level, format, a...) dprintk(level, KERN_DEBUG, format, ##a)
extern int cryptodev_verbosity;
struct fcrypt {
struct list_head list;
struct mutex sem;
};
/* compatibility stuff */
#ifdef CONFIG_COMPAT
#include <linux/compat.h>
/* input of CIOCGSESSION */
struct compat_session_op {
/* Specify either cipher or mac
*/
uint32_t cipher; /* cryptodev_crypto_op_t */
uint32_t mac; /* cryptodev_crypto_op_t */
uint32_t keylen;
compat_uptr_t key; /* pointer to key data */
uint32_t mackeylen;
compat_uptr_t mackey; /* pointer to mac key data */
uint32_t ses; /* session identifier */
};
/* input of CIOCCRYPT */
struct compat_crypt_op {
uint32_t ses; /* session identifier */
uint16_t op; /* COP_ENCRYPT or COP_DECRYPT */
uint16_t flags; /* see COP_FLAG_* */
uint32_t len; /* length of source data */
compat_uptr_t src; /* source data */
compat_uptr_t dst; /* pointer to output data */
compat_uptr_t mac;/* pointer to output data for hash/MAC operations */
compat_uptr_t iv;/* initialization vector for encryption operations */
};
/* compat ioctls, defined for the above structs */
#define COMPAT_CIOCGSESSION _IOWR('c', 102, struct compat_session_op)
#define COMPAT_CIOCCRYPT _IOWR('c', 104, struct compat_crypt_op)
#define COMPAT_CIOCASYNCCRYPT _IOW('c', 107, struct compat_crypt_op)
#define COMPAT_CIOCASYNCFETCH _IOR('c', 108, struct compat_crypt_op)
#endif /* CONFIG_COMPAT */
/* kernel-internal extension to struct crypt_op */
struct kernel_crypt_op {
struct crypt_op cop;
int ivlen;
__u8 iv[EALG_MAX_BLOCK_LEN];
int digestsize;
uint8_t hash_output[AALG_MAX_RESULT_LEN];
struct task_struct *task;
struct mm_struct *mm;
};
struct kernel_crypt_auth_op {
struct crypt_auth_op caop;
int dst_len; /* based on src_len + pad + tag */
int ivlen;
__u8 iv[EALG_MAX_BLOCK_LEN];
struct task_struct *task;
struct mm_struct *mm;
};
/* auth */
int kcaop_from_user(struct kernel_crypt_auth_op *kcop,
struct fcrypt *fcr, void __user *arg);
int kcaop_to_user(struct kernel_crypt_auth_op *kcaop,
struct fcrypt *fcr, void __user *arg);
int crypto_auth_run(struct fcrypt *fcr, struct kernel_crypt_auth_op *kcaop);
int crypto_run(struct fcrypt *fcr, struct kernel_crypt_op *kcop);
#include "cryptlib.h"
/* other internal structs */
struct csession {
struct list_head entry;
struct mutex sem;
struct cipher_data cdata;
struct hash_data hdata;
uint32_t sid;
uint32_t alignmask;
unsigned int array_size;
unsigned int used_pages; /* the number of pages that are used */
/* the number of pages marked as NOT-writable; they preceed writeables */
unsigned int readonly_pages;
struct page **pages;
struct scatterlist *sg;
};
struct csession *crypto_get_session_by_sid(struct fcrypt *fcr, uint32_t sid);
int
crypto_get_sessions_by_sid(struct fcrypt *fcr,
uint32_t sid_1, struct csession **ses_ptr_1,
uint32_t sid_2, struct csession **ses_ptr_2);
static inline void crypto_put_session(struct csession *ses_ptr)
{
mutex_unlock(&ses_ptr->sem);
}
int adjust_sg_array(struct csession *ses, int pagecount);
#endif /* CRYPTODEV_INT_H */

1280
drivers/crypto/rockchip/cryptodev_linux/ioctl.c Normal file
View File

File diff suppressed because it is too large Load Diff

267
drivers/crypto/rockchip/cryptodev_linux/main.c Normal file
View File

@@ -0,0 +1,267 @@
/*
* Driver for /dev/crypto device (aka CryptoDev)
*
* Copyright (c) 2004 Michal Ludvig <mludvig@logix.net.nz>, SuSE Labs
* Copyright (c) 2009-2013 Nikos Mavrogiannopoulos <nmav@gnutls.org>
*
* This file is part of linux cryptodev.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
/*
* Device /dev/crypto provides an interface for
* accessing kernel CryptoAPI algorithms (ciphers,
* hashes) from userspace programs.
*
* /dev/crypto interface was originally introduced in
* OpenBSD and this module attempts to keep the API.
*
*/
#include <crypto/hash.h>
#include <linux/crypto.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/ioctl.h>
#include <linux/random.h>
#include <linux/syscalls.h>
#include <linux/pagemap.h>
#include <linux/poll.h>
#include <linux/uaccess.h>
#include "crypto/cryptodev.h"
#include <crypto/scatterwalk.h>
#include <linux/scatterlist.h>
#include "cryptodev_int.h"
#include "zc.h"
#include "cryptlib.h"
#include "version.h"
/* This file contains the traditional operations of encryption
* and hashing of /dev/crypto.
*/
static int
hash_n_crypt(struct csession *ses_ptr, struct crypt_op *cop,
struct scatterlist *src_sg, struct scatterlist *dst_sg,
uint32_t len)
{
int ret;
/* Always hash before encryption and after decryption. Maybe
* we should introduce a flag to switch... TBD later on.
*/
if (cop->op == COP_ENCRYPT) {
if (ses_ptr->hdata.init != 0) {
ret = cryptodev_hash_update(&ses_ptr->hdata,
src_sg, len);
if (unlikely(ret))
goto out_err;
}
if (ses_ptr->cdata.init != 0) {
ret = cryptodev_cipher_encrypt(&ses_ptr->cdata,
src_sg, dst_sg, len);
if (unlikely(ret))
goto out_err;
}
} else {
if (ses_ptr->cdata.init != 0) {
ret = cryptodev_cipher_decrypt(&ses_ptr->cdata,
src_sg, dst_sg, len);
if (unlikely(ret))
goto out_err;
}
if (ses_ptr->hdata.init != 0) {
ret = cryptodev_hash_update(&ses_ptr->hdata,
dst_sg, len);
if (unlikely(ret))
goto out_err;
}
}
return 0;
out_err:
derr(0, "CryptoAPI failure: %d", ret);
return ret;
}
/* This is the main crypto function - feed it with plaintext
and get a ciphertext (or vice versa :-) */
static int
__crypto_run_std(struct csession *ses_ptr, struct crypt_op *cop)
{
char *data;
char __user *src, *dst;
struct scatterlist sg;
size_t nbytes, bufsize;
int ret = 0;
nbytes = cop->len;
data = (char *)__get_free_page(GFP_KERNEL);
if (unlikely(!data)) {
derr(1, "Error getting free page.");
return -ENOMEM;
}
bufsize = PAGE_SIZE < nbytes ? PAGE_SIZE : nbytes;
src = cop->src;
dst = cop->dst;
while (nbytes > 0) {
size_t current_len = nbytes > bufsize ? bufsize : nbytes;
if (unlikely(copy_from_user(data, src, current_len))) {
derr(1, "Error copying %zu bytes from user address %p.", current_len, src);
ret = -EFAULT;
break;
}
sg_init_one(&sg, data, current_len);
ret = hash_n_crypt(ses_ptr, cop, &sg, &sg, current_len);
if (unlikely(ret)) {
derr(1, "hash_n_crypt failed.");
break;
}
if (ses_ptr->cdata.init != 0) {
if (unlikely(copy_to_user(dst, data, current_len))) {
derr(1, "could not copy to user.");
ret = -EFAULT;
break;
}
}
dst += current_len;
nbytes -= current_len;
src += current_len;
}
free_page((unsigned long)data);
return ret;
}
/* This is the main crypto function - zero-copy edition */
static int
__crypto_run_zc(struct csession *ses_ptr, struct kernel_crypt_op *kcop)
{
struct scatterlist *src_sg, *dst_sg;
struct crypt_op *cop = &kcop->cop;
int ret = 0;
ret = get_userbuf(ses_ptr, cop->src, cop->len, cop->dst, cop->len,
kcop->task, kcop->mm, &src_sg, &dst_sg);
if (unlikely(ret)) {
derr(1, "Error getting user pages. Falling back to non zero copy.");
return __crypto_run_std(ses_ptr, cop);
}
ret = hash_n_crypt(ses_ptr, cop, src_sg, dst_sg, cop->len);
release_user_pages(ses_ptr);
return ret;
}
int crypto_run(struct fcrypt *fcr, struct kernel_crypt_op *kcop)
{
struct csession *ses_ptr;
struct crypt_op *cop = &kcop->cop;
int ret;
if (unlikely(cop->op != COP_ENCRYPT && cop->op != COP_DECRYPT)) {
ddebug(1, "invalid operation op=%u", cop->op);
return -EINVAL;
}
/* this also enters ses_ptr->sem */
ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
if (unlikely(!ses_ptr)) {
derr(1, "invalid session ID=0x%08X", cop->ses);
return -EINVAL;
}
if (ses_ptr->hdata.init != 0 && (cop->flags == 0 || cop->flags & COP_FLAG_RESET)) {
ret = cryptodev_hash_reset(&ses_ptr->hdata);
if (unlikely(ret)) {
derr(1, "error in cryptodev_hash_reset()");
goto out_unlock;
}
}
if (ses_ptr->cdata.init != 0) {
int blocksize = ses_ptr->cdata.blocksize;
if (unlikely(cop->len % blocksize)) {
derr(1, "data size (%u) isn't a multiple of block size (%u)",
cop->len, blocksize);
ret = -EINVAL;
goto out_unlock;
}
cryptodev_cipher_set_iv(&ses_ptr->cdata, kcop->iv,
min(ses_ptr->cdata.ivsize, kcop->ivlen));
}
if (likely(cop->len)) {
if (!(cop->flags & COP_FLAG_NO_ZC)) {
if (unlikely(ses_ptr->alignmask && !IS_ALIGNED((unsigned long)cop->src, ses_ptr->alignmask + 1))) {
dwarning(2, "source address %p is not %d byte aligned - disabling zero copy",
cop->src, ses_ptr->alignmask + 1);
cop->flags |= COP_FLAG_NO_ZC;
}
if (unlikely(ses_ptr->alignmask && !IS_ALIGNED((unsigned long)cop->dst, ses_ptr->alignmask + 1))) {
dwarning(2, "destination address %p is not %d byte aligned - disabling zero copy",
cop->dst, ses_ptr->alignmask + 1);
cop->flags |= COP_FLAG_NO_ZC;
}
}
if (cop->flags & COP_FLAG_NO_ZC)
ret = __crypto_run_std(ses_ptr, &kcop->cop);
else
ret = __crypto_run_zc(ses_ptr, kcop);
if (unlikely(ret))
goto out_unlock;
}
if (ses_ptr->cdata.init != 0) {
cryptodev_cipher_get_iv(&ses_ptr->cdata, kcop->iv,
min(ses_ptr->cdata.ivsize, kcop->ivlen));
}
if (ses_ptr->hdata.init != 0 &&
((cop->flags & COP_FLAG_FINAL) ||
(!(cop->flags & COP_FLAG_UPDATE) || cop->len == 0))) {
ret = cryptodev_hash_final(&ses_ptr->hdata, kcop->hash_output);
if (unlikely(ret)) {
derr(0, "CryptoAPI failure: %d", ret);
goto out_unlock;
}
kcop->digestsize = ses_ptr->hdata.digestsize;
}
out_unlock:
crypto_put_session(ses_ptr);
return ret;
}

80
drivers/crypto/rockchip/cryptodev_linux/util.c Normal file
View File

@@ -0,0 +1,80 @@
/*
* Copyright (c) 2011 Maxim Levitsky
*
* This file is part of linux cryptodev.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <crypto/scatterwalk.h>
#include <linux/scatterlist.h>
#include "util.h"
/* These were taken from Maxim Levitsky's patch to lkml.
*/
struct scatterlist *sg_advance(struct scatterlist *sg, int consumed)
{
while (consumed >= sg->length) {
consumed -= sg->length;
sg = sg_next(sg);
if (!sg)
break;
}
WARN_ON(!sg && consumed);
if (!sg)
return NULL;
sg->offset += consumed;
sg->length -= consumed;
if (sg->offset >= PAGE_SIZE) {
struct page *page =
nth_page(sg_page(sg), sg->offset / PAGE_SIZE);
sg_set_page(sg, page, sg->length, sg->offset % PAGE_SIZE);
}
return sg;
}
/**
* sg_copy - copies sg entries from sg_from to sg_to, such
* as sg_to covers first 'len' bytes from sg_from.
*/
int sg_copy(struct scatterlist *sg_from, struct scatterlist *sg_to, int len)
{
while (len > sg_from->length) {
len -= sg_from->length;
sg_set_page(sg_to, sg_page(sg_from),
sg_from->length, sg_from->offset);
sg_to = sg_next(sg_to);
sg_from = sg_next(sg_from);
if (len && (!sg_from || !sg_to))
return -ENOMEM;
}
if (len)
sg_set_page(sg_to, sg_page(sg_from),
len, sg_from->offset);
sg_mark_end(sg_to);
return 0;
}

8
drivers/crypto/rockchip/cryptodev_linux/util.h Normal file
View File

@@ -0,0 +1,8 @@
/* SPDX-License-Identifier: GPL-2.0+ */
#ifndef UTILS_H
#define UTILS_H
int sg_copy(struct scatterlist *sg_from, struct scatterlist *sg_to, int len);
struct scatterlist *sg_advance(struct scatterlist *sg, int consumed);
#endif

9
drivers/crypto/rockchip/cryptodev_linux/version.h Normal file
View File

@@ -0,0 +1,9 @@
/* SPDX-License-Identifier: GPL-2.0+ */
#ifndef VERSION_H
#define VERSION_H
#define VERSION "1.12"
#endif

235
drivers/crypto/rockchip/cryptodev_linux/zc.c Normal file
View File

@@ -0,0 +1,235 @@
/*
* Driver for /dev/crypto device (aka CryptoDev)
*
* Copyright (c) 2009-2013 Nikos Mavrogiannopoulos <nmav@gnutls.org>
* Copyright (c) 2010 Phil Sutter
* Copyright (c) 2011, 2012 OpenSSL Software Foundation, Inc.
*
* This file is part of linux cryptodev.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
#include <crypto/hash.h>
#include <linux/crypto.h>
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/ioctl.h>
#include <linux/random.h>
#include <linux/syscalls.h>
#include <linux/pagemap.h>
#include <linux/uaccess.h>
#include <crypto/scatterwalk.h>
#include <linux/scatterlist.h>
#include "cryptodev_int.h"
#include "zc.h"
#include "version.h"
/* Helper functions to assist zero copy.
* This needs to be redesigned and moved out of the session. --nmav
*/
/* offset of buf in it's first page */
#define PAGEOFFSET(buf) ((unsigned long)buf & ~PAGE_MASK)
/* fetch the pages addr resides in into pg and initialise sg with them */
int __get_userbuf(uint8_t __user *addr, uint32_t len, int write,
unsigned int pgcount, struct page **pg, struct scatterlist *sg,
struct task_struct *task, struct mm_struct *mm)
{
int ret, pglen, i = 0;
struct scatterlist *sgp;
if (unlikely(!pgcount || !len || !addr)) {
sg_mark_end(sg);
return 0;
}
#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
down_read(&mm->mmap_sem);
#else
mmap_read_lock(mm);
#endif
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 168))
ret = get_user_pages(task, mm,
(unsigned long)addr, pgcount, write, 0, pg, NULL);
#elif (LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0))
ret = get_user_pages(task, mm,
(unsigned long)addr, pgcount, write, pg, NULL);
#elif (LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0))
ret = get_user_pages_remote(task, mm,
(unsigned long)addr, pgcount, write, 0, pg, NULL);
#elif (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
ret = get_user_pages_remote(task, mm,
(unsigned long)addr, pgcount, write ? FOLL_WRITE : 0,
pg, NULL);
#elif (LINUX_VERSION_CODE < KERNEL_VERSION(5, 9, 0))
ret = get_user_pages_remote(task, mm,
(unsigned long)addr, pgcount, write ? FOLL_WRITE : 0,
pg, NULL, NULL);
#else
ret = get_user_pages_remote(mm,
(unsigned long)addr, pgcount, write ? FOLL_WRITE : 0,
pg, NULL, NULL);
#endif
#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 8, 0))
up_read(&mm->mmap_sem);
#else
mmap_read_unlock(mm);
#endif
if (ret != pgcount)
return -EINVAL;
sg_init_table(sg, pgcount);
pglen = min((ptrdiff_t)(PAGE_SIZE - PAGEOFFSET(addr)), (ptrdiff_t)len);
sg_set_page(sg, pg[i++], pglen, PAGEOFFSET(addr));
len -= pglen;
for (sgp = sg_next(sg); len; sgp = sg_next(sgp)) {
pglen = min((uint32_t)PAGE_SIZE, len);
sg_set_page(sgp, pg[i++], pglen, 0);
len -= pglen;
}
sg_mark_end(sg_last(sg, pgcount));
return 0;
}
int adjust_sg_array(struct csession *ses, int pagecount)
{
struct scatterlist *sg;
struct page **pages;
int array_size;
for (array_size = ses->array_size; array_size < pagecount;
array_size *= 2)
;
ddebug(0, "reallocating from %d to %d pages",
ses->array_size, array_size);
pages = krealloc(ses->pages, array_size * sizeof(struct page *),
GFP_KERNEL);
if (unlikely(!pages))
return -ENOMEM;
ses->pages = pages;
sg = krealloc(ses->sg, array_size * sizeof(struct scatterlist),
GFP_KERNEL);
if (unlikely(!sg))
return -ENOMEM;
ses->sg = sg;
ses->array_size = array_size;
return 0;
}
void release_user_pages(struct csession *ses)
{
unsigned int i;
for (i = 0; i < ses->used_pages; i++) {
if (!PageReserved(ses->pages[i]))
SetPageDirty(ses->pages[i]);
if (ses->readonly_pages == 0)
flush_dcache_page(ses->pages[i]);
else
ses->readonly_pages--;
put_page(ses->pages[i]);
}
ses->used_pages = 0;
}
/* make src and dst available in scatterlists.
* dst might be the same as src.
*/
int get_userbuf(struct csession *ses,
void *__user src, unsigned int src_len,
void *__user dst, unsigned int dst_len,
struct task_struct *task, struct mm_struct *mm,
struct scatterlist **src_sg,
struct scatterlist **dst_sg)
{
int src_pagecount, dst_pagecount;
int rc;
/* Empty input is a valid option to many algorithms & is tested by NIST/FIPS */
/* Make sure NULL input has 0 length */
if (!src && src_len)
src_len = 0;
/* I don't know that null output is ever useful, but we can handle it gracefully */
/* Make sure NULL output has 0 length */
if (!dst && dst_len)
dst_len = 0;
src_pagecount = PAGECOUNT(src, src_len);
dst_pagecount = PAGECOUNT(dst, dst_len);
ses->used_pages = (src == dst) ? max(src_pagecount, dst_pagecount)
: src_pagecount + dst_pagecount;
ses->readonly_pages = (src == dst) ? 0 : src_pagecount;
if (ses->used_pages > ses->array_size) {
rc = adjust_sg_array(ses, ses->used_pages);
if (rc)
return rc;
}
if (src == dst) { /* inplace operation */
/* When we encrypt for authenc modes we need to write
* more data than the ones we read. */
if (src_len < dst_len)
src_len = dst_len;
rc = __get_userbuf(src, src_len, 1, ses->used_pages,
ses->pages, ses->sg, task, mm);
if (unlikely(rc)) {
derr(1, "failed to get user pages for data IO");
return rc;
}
(*src_sg) = (*dst_sg) = ses->sg;
return 0;
}
*src_sg = NULL; /* default to no input */
*dst_sg = NULL; /* default to ignore output */
if (likely(src)) {
rc = __get_userbuf(src, src_len, 0, ses->readonly_pages,
ses->pages, ses->sg, task, mm);
if (unlikely(rc)) {
derr(1, "failed to get user pages for data input");
return rc;
}
*src_sg = ses->sg;
}
if (likely(dst)) {
const unsigned int writable_pages =
ses->used_pages - ses->readonly_pages;
struct page **dst_pages = ses->pages + ses->readonly_pages;
*dst_sg = ses->sg + ses->readonly_pages;
rc = __get_userbuf(dst, dst_len, 1, writable_pages,
dst_pages, *dst_sg, task, mm);
if (unlikely(rc)) {
derr(1, "failed to get user pages for data output");
release_user_pages(ses); /* FIXME: use __release_userbuf(src, ...) */
return rc;
}
}
return 0;
}

27
drivers/crypto/rockchip/cryptodev_linux/zc.h Normal file
View File

@@ -0,0 +1,27 @@
/* SPDX-License-Identifier: GPL-2.0+ */
#ifndef ZC_H
# define ZC_H
/* For zero copy */
int __get_userbuf(uint8_t __user *addr, uint32_t len, int write,
unsigned int pgcount, struct page **pg, struct scatterlist *sg,
struct task_struct *task, struct mm_struct *mm);
void release_user_pages(struct csession *ses);
int get_userbuf(struct csession *ses,
void *__user src, unsigned int src_len,
void *__user dst, unsigned int dst_len,
struct task_struct *task, struct mm_struct *mm,
struct scatterlist **src_sg,
struct scatterlist **dst_sg);
/* buflen ? (last page - first page + 1) : 0 */
#define PAGECOUNT(buf, buflen) ((buflen) \
? ((((unsigned long)(buf + buflen - 1)) >> PAGE_SHIFT) - \
(((unsigned long)(buf )) >> PAGE_SHIFT) + 1) \
: 0)
#define DEFAULT_PREALLOC_PAGES 32
#endif