From f09056c1decec889721c24c72f99730376918044 Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Mon, 19 Oct 2020 12:35:24 -0700 Subject: [PATCH 001/360] EDAC/amd64: Remove unneeded breaks A break is not needed if it is preceded by a return. Signed-off-by: Tom Rix Signed-off-by: Borislav Petkov Reviewed-by: Robert Richter Link: https://lkml.kernel.org/r/20201019193524.13391-1-trix@redhat.com --- drivers/edac/amd64_edac.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 1362274d840b..3bac76efd3f6 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -2461,14 +2461,11 @@ static int map_err_sym_to_channel(int err_sym, int sym_size) case 0x20: case 0x21: return 0; - break; case 0x22: case 0x23: return 1; - break; default: return err_sym >> 4; - break; } /* x8 symbols */ else @@ -2478,17 +2475,12 @@ static int map_err_sym_to_channel(int err_sym, int sym_size) WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n", err_sym); return -1; - break; - case 0x11: return 0; - break; case 0x12: return 1; - break; default: return err_sym >> 3; - break; } return -1; } From 880396c86a1f3663c22b74fef34353f05a1263ec Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Mon, 19 Oct 2020 13:06:29 -0700 Subject: [PATCH 002/360] x86/microcode/amd: Remove unneeded break A break is not needed if it is preceded by a return. Signed-off-by: Tom Rix Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201019200629.17247-1-trix@redhat.com --- arch/x86/kernel/cpu/microcode/amd.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index 3f6b137ef4e6..3d4a48336084 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -215,7 +215,6 @@ static unsigned int __verify_patch_size(u8 family, u32 sh_psize, size_t buf_size default: WARN(1, "%s: WTF family: 0x%x\n", __func__, family); return 0; - break; } if (sh_psize > min_t(u32, buf_size, max_size)) From 633cdaf29ec4aae29868320adb3a4f1c5b8c0eac Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Mon, 19 Oct 2020 13:08:03 -0700 Subject: [PATCH 003/360] x86/mce: Remove unneeded break A break is not needed if it is preceded by a return. Signed-off-by: Tom Rix Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201019200803.17619-1-trix@redhat.com --- arch/x86/kernel/cpu/mce/core.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 4102b866e7c0..51bf910b1e9d 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1811,11 +1811,9 @@ static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c) case X86_VENDOR_INTEL: intel_p5_mcheck_init(c); return 1; - break; case X86_VENDOR_CENTAUR: winchip_mcheck_init(c); return 1; - break; default: return 0; } From 632211cdd6ad0efeef32c53ac731901b4bed3b94 Mon Sep 17 00:00:00 2001 From: Steve Wahl Date: Mon, 19 Oct 2020 15:35:33 -0500 Subject: [PATCH 004/360] MAINTAINERS: Cleanup SGI-related entries UV platforms are the only ones which currently use the XP/XPC/XPNET driver, so it seems fair HPE should take some responsibility as maintainers of it; so add Mike Travis and Steve Wahl. Cliff Whickman's email address is no longer valid, so remove it. Robin Holt was contacted and wishes to remain as a maintainer. Update Dimitri Sivanich's email address for the SGI GRU driver. Add Mike Travis to HPE Superdome Flex (UV) platform. [ bp: Massage commit message. ] Acked-by: Robin Holt Signed-off-by: Steve Wahl Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201019203533.GA1203217@swahl-home.5wahls.com --- MAINTAINERS | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index e73636b75f29..e706e1473818 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -15833,13 +15833,14 @@ F: include/linux/sfp.h K: phylink\.h|struct\s+phylink|\.phylink|>phylink_|phylink_(autoneg|clear|connect|create|destroy|disconnect|ethtool|helper|mac|mii|of|set|start|stop|test|validate) SGI GRU DRIVER -M: Dimitri Sivanich +M: Dimitri Sivanich S: Maintained F: drivers/misc/sgi-gru/ SGI XP/XPC/XPNET DRIVER -M: Cliff Whickman M: Robin Holt +M: Steve Wahl +R: Mike Travis S: Maintained F: drivers/misc/sgi-xp/ @@ -19085,6 +19086,7 @@ F: arch/x86/platform X86 PLATFORM UV HPE SUPERDOME FLEX M: Steve Wahl +R: Mike Travis R: Dimitri Sivanich R: Russ Anderson S: Supported From f30795fb404edaa54213b0f4297db6987ec4eb35 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Mon, 26 Oct 2020 12:59:56 +0100 Subject: [PATCH 005/360] EDAC: Do not issue useless debug statements in the polling routine They have been spreading around the subsystem by example so remove them all. Reported-by: Raymond Bennett Suggested-by: Jason Baron Signed-off-by: Borislav Petkov --- drivers/edac/amd76x_edac.c | 1 - drivers/edac/e752x_edac.c | 1 - drivers/edac/e7xxx_edac.c | 1 - drivers/edac/i3000_edac.c | 1 - drivers/edac/i3200_edac.c | 1 - drivers/edac/i5000_edac.c | 2 +- drivers/edac/i5400_edac.c | 2 +- drivers/edac/i82443bxgx_edac.c | 1 - drivers/edac/i82860_edac.c | 1 - drivers/edac/i82875p_edac.c | 1 - drivers/edac/i82975x_edac.c | 1 - drivers/edac/ie31200_edac.c | 1 - drivers/edac/r82600_edac.c | 1 - drivers/edac/x38_edac.c | 1 - 14 files changed, 2 insertions(+), 14 deletions(-) diff --git a/drivers/edac/amd76x_edac.c b/drivers/edac/amd76x_edac.c index 9c6e326b4c14..2a49f68a7cf9 100644 --- a/drivers/edac/amd76x_edac.c +++ b/drivers/edac/amd76x_edac.c @@ -179,7 +179,6 @@ static int amd76x_process_error_info(struct mem_ctl_info *mci, static void amd76x_check(struct mem_ctl_info *mci) { struct amd76x_error_info info; - edac_dbg(3, "\n"); amd76x_get_error_info(mci, &info); amd76x_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/e752x_edac.c b/drivers/edac/e752x_edac.c index 313d08018166..ac7c9b42d4c7 100644 --- a/drivers/edac/e752x_edac.c +++ b/drivers/edac/e752x_edac.c @@ -980,7 +980,6 @@ static void e752x_check(struct mem_ctl_info *mci) { struct e752x_error_info info; - edac_dbg(3, "\n"); e752x_get_error_info(mci, &info); e752x_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/e7xxx_edac.c b/drivers/edac/e7xxx_edac.c index 75d7ce62b3be..497e710fca3d 100644 --- a/drivers/edac/e7xxx_edac.c +++ b/drivers/edac/e7xxx_edac.c @@ -333,7 +333,6 @@ static void e7xxx_check(struct mem_ctl_info *mci) { struct e7xxx_error_info info; - edac_dbg(3, "\n"); e7xxx_get_error_info(mci, &info); e7xxx_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/i3000_edac.c b/drivers/edac/i3000_edac.c index 5c1eea96230c..9065bc4386ff 100644 --- a/drivers/edac/i3000_edac.c +++ b/drivers/edac/i3000_edac.c @@ -273,7 +273,6 @@ static void i3000_check(struct mem_ctl_info *mci) { struct i3000_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); i3000_get_error_info(mci, &info); i3000_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/i3200_edac.c b/drivers/edac/i3200_edac.c index a8988db6d423..afccdebf5ac1 100644 --- a/drivers/edac/i3200_edac.c +++ b/drivers/edac/i3200_edac.c @@ -253,7 +253,6 @@ static void i3200_check(struct mem_ctl_info *mci) { struct i3200_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); i3200_get_and_clear_error_info(mci, &info); i3200_process_error_info(mci, &info); } diff --git a/drivers/edac/i5000_edac.c b/drivers/edac/i5000_edac.c index 1a6f69c859ab..ba46057d4220 100644 --- a/drivers/edac/i5000_edac.c +++ b/drivers/edac/i5000_edac.c @@ -765,7 +765,7 @@ static void i5000_clear_error(struct mem_ctl_info *mci) static void i5000_check_error(struct mem_ctl_info *mci) { struct i5000_error_info info; - edac_dbg(4, "MC%d\n", mci->mc_idx); + i5000_get_error_info(mci, &info); i5000_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/i5400_edac.c b/drivers/edac/i5400_edac.c index 92d63eb533ae..f76624ee82ef 100644 --- a/drivers/edac/i5400_edac.c +++ b/drivers/edac/i5400_edac.c @@ -686,7 +686,7 @@ static void i5400_clear_error(struct mem_ctl_info *mci) static void i5400_check_error(struct mem_ctl_info *mci) { struct i5400_error_info info; - edac_dbg(4, "MC%d\n", mci->mc_idx); + i5400_get_error_info(mci, &info); i5400_process_error_info(mci, &info); } diff --git a/drivers/edac/i82443bxgx_edac.c b/drivers/edac/i82443bxgx_edac.c index a2ca929e2168..933dcf3cfdff 100644 --- a/drivers/edac/i82443bxgx_edac.c +++ b/drivers/edac/i82443bxgx_edac.c @@ -176,7 +176,6 @@ static void i82443bxgx_edacmc_check(struct mem_ctl_info *mci) { struct i82443bxgx_edacmc_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); i82443bxgx_edacmc_get_error_info(mci, &info); i82443bxgx_edacmc_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/i82860_edac.c b/drivers/edac/i82860_edac.c index 3e3a80ffb322..fbec90d00f1e 100644 --- a/drivers/edac/i82860_edac.c +++ b/drivers/edac/i82860_edac.c @@ -135,7 +135,6 @@ static void i82860_check(struct mem_ctl_info *mci) { struct i82860_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); i82860_get_error_info(mci, &info); i82860_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/i82875p_edac.c b/drivers/edac/i82875p_edac.c index ceac925af38c..553880b9fc12 100644 --- a/drivers/edac/i82875p_edac.c +++ b/drivers/edac/i82875p_edac.c @@ -262,7 +262,6 @@ static void i82875p_check(struct mem_ctl_info *mci) { struct i82875p_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); i82875p_get_error_info(mci, &info); i82875p_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/i82975x_edac.c b/drivers/edac/i82975x_edac.c index 6be99e0d850d..d99f005832cf 100644 --- a/drivers/edac/i82975x_edac.c +++ b/drivers/edac/i82975x_edac.c @@ -330,7 +330,6 @@ static void i82975x_check(struct mem_ctl_info *mci) { struct i82975x_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); i82975x_get_error_info(mci, &info); i82975x_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c index c47963240b65..9a9ff5ad611a 100644 --- a/drivers/edac/ie31200_edac.c +++ b/drivers/edac/ie31200_edac.c @@ -333,7 +333,6 @@ static void ie31200_check(struct mem_ctl_info *mci) { struct ie31200_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); ie31200_get_and_clear_error_info(mci, &info); ie31200_process_error_info(mci, &info); } diff --git a/drivers/edac/r82600_edac.c b/drivers/edac/r82600_edac.c index 851e53e122aa..d0aef83dca2a 100644 --- a/drivers/edac/r82600_edac.c +++ b/drivers/edac/r82600_edac.c @@ -204,7 +204,6 @@ static void r82600_check(struct mem_ctl_info *mci) { struct r82600_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); r82600_get_error_info(mci, &info); r82600_process_error_info(mci, &info, 1); } diff --git a/drivers/edac/x38_edac.c b/drivers/edac/x38_edac.c index a65e2f78a402..49ab5721aab2 100644 --- a/drivers/edac/x38_edac.c +++ b/drivers/edac/x38_edac.c @@ -238,7 +238,6 @@ static void x38_check(struct mem_ctl_info *mci) { struct x38_error_info info; - edac_dbg(1, "MC%d\n", mci->mc_idx); x38_get_and_clear_error_info(mci, &info); x38_process_error_info(mci, &info); } From 375d4bfda57392f0865dae051e1c4bd2700e8d71 Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Sat, 3 Oct 2020 23:25:27 -0400 Subject: [PATCH 006/360] perf/x86: Avoid TIF_IA32 when checking 64bit mode In preparation to remove TIF_IA32, stop using it in perf events code. Tested by running perf on 32-bit, 64-bit and x32 applications. Suggested-by: Andy Lutomirski Signed-off-by: Gabriel Krisman Bertazi Signed-off-by: Thomas Gleixner Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/20201004032536.1229030-2-krisman@collabora.com --- arch/x86/events/core.c | 2 +- arch/x86/events/intel/ds.c | 2 +- arch/x86/events/intel/lbr.c | 2 +- arch/x86/kernel/perf_regs.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index a88c94d65693..77b963e5e70a 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -2602,7 +2602,7 @@ perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *ent struct stack_frame_ia32 frame; const struct stack_frame_ia32 __user *fp; - if (!test_thread_flag(TIF_IA32)) + if (user_64bit_mode(regs)) return 0; cs_base = get_segment_base(regs->cs); diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 404315df1e16..99a59f38a4e2 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1259,7 +1259,7 @@ static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs) old_to = to; #ifdef CONFIG_X86_64 - is_64bit = kernel_ip(to) || !test_thread_flag(TIF_IA32); + is_64bit = kernel_ip(to) || any_64bit_mode(regs); #endif insn_init(&insn, kaddr, size, is_64bit); insn_get_length(&insn); diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index 8961653c5dd2..1aadb253d296 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -1221,7 +1221,7 @@ static int branch_type(unsigned long from, unsigned long to, int abort) * on 64-bit systems running 32-bit apps */ #ifdef CONFIG_X86_64 - is64 = kernel_ip((unsigned long)addr) || !test_thread_flag(TIF_IA32); + is64 = kernel_ip((unsigned long)addr) || any_64bit_mode(current_pt_regs()); #endif insn_init(&insn, addr, bytes_read, is64); insn_get_opcode(&insn); diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c index bb7e1132290b..9332c49a64a8 100644 --- a/arch/x86/kernel/perf_regs.c +++ b/arch/x86/kernel/perf_regs.c @@ -123,7 +123,7 @@ int perf_reg_validate(u64 mask) u64 perf_reg_abi(struct task_struct *task) { - if (test_tsk_thread_flag(task, TIF_IA32)) + if (!user_64bit_mode(task_pt_regs(task))) return PERF_SAMPLE_REGS_ABI_32; else return PERF_SAMPLE_REGS_ABI_64; From 214f0e804358cdd13b5cbe4445189f23e30618b4 Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Sat, 3 Oct 2020 23:25:28 -0400 Subject: [PATCH 007/360] x86/compat: Simplify compat syscall userspace allocation When allocating user memory space for a compat system call, don't consider whether the originating code is IA32 or X32, just allocate from a safe region for both, beyond the redzone. This should be safe for IA32, and has the benefit of avoiding TIF_IA32, which is about to be removed. Suggested-by: Andy Lutomirski Signed-off-by: Gabriel Krisman Bertazi Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20201004032536.1229030-3-krisman@collabora.com --- arch/x86/include/asm/compat.h | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h index 0e327a01f50f..f145e3326c6d 100644 --- a/arch/x86/include/asm/compat.h +++ b/arch/x86/include/asm/compat.h @@ -177,14 +177,13 @@ typedef struct user_regs_struct compat_elf_gregset_t; static inline void __user *arch_compat_alloc_user_space(long len) { - compat_uptr_t sp; + compat_uptr_t sp = task_pt_regs(current)->sp; - if (test_thread_flag(TIF_IA32)) { - sp = task_pt_regs(current)->sp; - } else { - /* -128 for the x32 ABI redzone */ - sp = task_pt_regs(current)->sp - 128; - } + /* + * -128 for the x32 ABI redzone. For IA32, it is not strictly + * necessary, but not harmful. + */ + sp -= 128; return (void __user *)round_down(sp - len, 16); } From 47cd4dac1fb21759ffcfe0600827c87fa6acdfa7 Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Sat, 3 Oct 2020 23:25:29 -0400 Subject: [PATCH 008/360] x86/oprofile: Avoid TIF_IA32 when checking 64bit mode In preparation to remove TIF_IA32, stop using it in oprofile code. Use user_64bit_mode() instead. Signed-off-by: Gabriel Krisman Bertazi Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20201004032536.1229030-4-krisman@collabora.com --- arch/x86/oprofile/backtrace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/oprofile/backtrace.c b/arch/x86/oprofile/backtrace.c index a2488b6e27d6..1d8391fcca68 100644 --- a/arch/x86/oprofile/backtrace.c +++ b/arch/x86/oprofile/backtrace.c @@ -49,7 +49,7 @@ x86_backtrace_32(struct pt_regs * const regs, unsigned int depth) struct stack_frame_ia32 *head; /* User process is IA32 */ - if (!current || !test_thread_flag(TIF_IA32)) + if (!current || user_64bit_mode(regs)) return 0; head = (struct stack_frame_ia32 *) regs->bp; From 2656af0d5abfa26d7f1e40f92e9953fe155b950a Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Sat, 3 Oct 2020 23:25:30 -0400 Subject: [PATCH 009/360] x86/elf: Use e_machine to choose DLINFO in compat Since TIF_X32 is going away, avoid using it to find the ELF type on ARCH_DLINFO. According to SysV AMD64 ABI Draft, an AMD64 ELF object using ILP32 must have ELFCLASS32 with (E_MACHINE == EM_X86_64), so use that ELF field to differentiate a x32 object from a IA32 object when loading ARCH_DLINFO in compat mode. Signed-off-by: Gabriel Krisman Bertazi Signed-off-by: Thomas Gleixner Reviewed-by: Andy Lutomirski Link: https://lore.kernel.org/r/20201004032536.1229030-5-krisman@collabora.com --- arch/x86/include/asm/elf.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/include/asm/elf.h b/arch/x86/include/asm/elf.h index b9a5d488f1a5..9220efc65d78 100644 --- a/arch/x86/include/asm/elf.h +++ b/arch/x86/include/asm/elf.h @@ -361,7 +361,7 @@ do { \ #define AT_SYSINFO 32 #define COMPAT_ARCH_DLINFO \ -if (test_thread_flag(TIF_X32)) \ +if (exec->e_machine == EM_X86_64) \ ARCH_DLINFO_X32; \ else \ ARCH_DLINFO_IA32 From bc3d7bf61a9eaecccc84dc2ecc2a9a3fa4f5ec47 Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Sat, 3 Oct 2020 23:25:31 -0400 Subject: [PATCH 010/360] elf: Expose ELF header in compat_start_thread() Like it is done for SET_PERSONALITY with x86, which requires the ELF header to select correct personality parameters, x86 requires the headers on compat_start_thread() to choose starting CS for ELF32 binaries, instead of relying on the going-away TIF_IA32/X32 flags. Add an indirection macro to ELF invocations of START_THREAD, that x86 can reimplement to receive the extra parameter just for ELF files. This requires no changes to other architectures who don't need the header information, they can continue to use the original start_thread for ELF and non-ELF binaries, and it prevents affecting non-ELF code paths for x86. Signed-off-by: Gabriel Krisman Bertazi Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20201004032536.1229030-6-krisman@collabora.com --- fs/binfmt_elf.c | 2 +- fs/compat_binfmt_elf.c | 9 +++++++-- include/linux/elf.h | 5 +++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index b6b3d052ca86..b23f7553fe9b 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1307,7 +1307,7 @@ out_free_interp: #endif finalize_exec(bprm); - start_thread(regs, elf_entry, bprm->p); + START_THREAD(elf_ex, regs, elf_entry, bprm->p); retval = 0; out: return retval; diff --git a/fs/compat_binfmt_elf.c b/fs/compat_binfmt_elf.c index 2d24c765cbd7..12b991368f0a 100644 --- a/fs/compat_binfmt_elf.c +++ b/fs/compat_binfmt_elf.c @@ -106,8 +106,13 @@ #endif #ifdef compat_start_thread -#undef start_thread -#define start_thread compat_start_thread +#define COMPAT_START_THREAD(ex, regs, new_ip, new_sp) \ + compat_start_thread(regs, new_ip, new_sp) +#endif + +#ifdef COMPAT_START_THREAD +#undef START_THREAD +#define START_THREAD COMPAT_START_THREAD #endif #ifdef compat_arch_setup_additional_pages diff --git a/include/linux/elf.h b/include/linux/elf.h index 5d5b0321da0b..6dbcfe7a3fd7 100644 --- a/include/linux/elf.h +++ b/include/linux/elf.h @@ -22,6 +22,11 @@ SET_PERSONALITY(ex) #endif +#ifndef START_THREAD +#define START_THREAD(elf_ex, regs, elf_entry, start_stack) \ + start_thread(regs, elf_entry, start_stack) +#endif + #define ELF32_GNU_PROPERTY_ALIGN 4 #define ELF64_GNU_PROPERTY_ALIGN 8 From 2424b14605c71a7187c14edd525044eb36bdea47 Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Sat, 3 Oct 2020 23:25:32 -0400 Subject: [PATCH 011/360] x86/elf: Use e_machine to select start_thread for x32 Since TIF_X32 is going away, avoid using it to find the ELF type in compat_start_thread. According to SysV AMD64 ABI Draft, an AMD64 ELF object using ILP32 must have ELFCLASS32 with (E_MACHINE == EM_X86_64), so use that ELF field to differentiate a x32 object from a IA32 object when executing start_thread() in compat mode. Signed-off-by: Gabriel Krisman Bertazi Signed-off-by: Thomas Gleixner Reviewed-by: Andy Lutomirski Link: https://lore.kernel.org/r/20201004032536.1229030-7-krisman@collabora.com --- arch/x86/include/asm/elf.h | 5 +++-- arch/x86/kernel/process_64.c | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/include/asm/elf.h b/arch/x86/include/asm/elf.h index 9220efc65d78..109697a19eb1 100644 --- a/arch/x86/include/asm/elf.h +++ b/arch/x86/include/asm/elf.h @@ -186,8 +186,9 @@ static inline void elf_common_init(struct thread_struct *t, #define COMPAT_ELF_PLAT_INIT(regs, load_addr) \ elf_common_init(¤t->thread, regs, __USER_DS) -void compat_start_thread(struct pt_regs *regs, u32 new_ip, u32 new_sp); -#define compat_start_thread compat_start_thread +void compat_start_thread(struct pt_regs *regs, u32 new_ip, u32 new_sp, bool x32); +#define COMPAT_START_THREAD(ex, regs, new_ip, new_sp) \ + compat_start_thread(regs, new_ip, new_sp, ex->e_machine == EM_X86_64) void set_personality_ia32(bool); #define COMPAT_SET_PERSONALITY(ex) \ diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index df342bedea88..5fb4103101f5 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -511,11 +511,10 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp) EXPORT_SYMBOL_GPL(start_thread); #ifdef CONFIG_COMPAT -void compat_start_thread(struct pt_regs *regs, u32 new_ip, u32 new_sp) +void compat_start_thread(struct pt_regs *regs, u32 new_ip, u32 new_sp, bool x32) { start_thread_common(regs, new_ip, new_sp, - test_thread_flag(TIF_X32) - ? __USER_CS : __USER32_CS, + x32 ? __USER_CS : __USER32_CS, __USER_DS, __USER_DS); } #endif From 9a29a671902c2be05d636045a4dd365219ca716c Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Sat, 3 Oct 2020 23:25:33 -0400 Subject: [PATCH 012/360] elf: Expose ELF header on arch_setup_additional_pages() Like it is done for SET_PERSONALITY with ARM, which requires the ELF header to select correct personality parameters, x86 requires the headers when selecting which VDSO to load, instead of relying on the going-away TIF_IA32/X32 flags. Add an indirection macro to arch_setup_additional_pages(), that x86 can reimplement to receive the extra parameter just for ELF files. This requires no changes to other architectures, who can continue to use the original arch_setup_additional_pages for ELF and non-ELF binaries. Signed-off-by: Gabriel Krisman Bertazi Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20201004032536.1229030-8-krisman@collabora.com --- fs/binfmt_elf.c | 2 +- fs/compat_binfmt_elf.c | 11 ++++++++--- include/linux/elf.h | 5 +++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index b23f7553fe9b..aabc11f099cf 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1246,7 +1246,7 @@ out_free_interp: set_binfmt(&elf_format); #ifdef ARCH_HAS_SETUP_ADDITIONAL_PAGES - retval = arch_setup_additional_pages(bprm, !!interpreter); + retval = ARCH_SETUP_ADDITIONAL_PAGES(bprm, elf_ex, !!interpreter); if (retval < 0) goto out; #endif /* ARCH_HAS_SETUP_ADDITIONAL_PAGES */ diff --git a/fs/compat_binfmt_elf.c b/fs/compat_binfmt_elf.c index 12b991368f0a..2c557229696a 100644 --- a/fs/compat_binfmt_elf.c +++ b/fs/compat_binfmt_elf.c @@ -115,11 +115,16 @@ #define START_THREAD COMPAT_START_THREAD #endif -#ifdef compat_arch_setup_additional_pages +#ifdef compat_arch_setup_additional_pages +#define COMPAT_ARCH_SETUP_ADDITIONAL_PAGES(bprm, ex, interpreter) \ + compat_arch_setup_additional_pages(bprm, interpreter) +#endif + +#ifdef COMPAT_ARCH_SETUP_ADDITIONAL_PAGES #undef ARCH_HAS_SETUP_ADDITIONAL_PAGES #define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1 -#undef arch_setup_additional_pages -#define arch_setup_additional_pages compat_arch_setup_additional_pages +#undef ARCH_SETUP_ADDITIONAL_PAGES +#define ARCH_SETUP_ADDITIONAL_PAGES COMPAT_ARCH_SETUP_ADDITIONAL_PAGES #endif #ifdef compat_elf_read_implies_exec diff --git a/include/linux/elf.h b/include/linux/elf.h index 6dbcfe7a3fd7..c9a46c4e183b 100644 --- a/include/linux/elf.h +++ b/include/linux/elf.h @@ -27,6 +27,11 @@ start_thread(regs, elf_entry, start_stack) #endif +#if defined(ARCH_HAS_SETUP_ADDITIONAL_PAGES) && !defined(ARCH_SETUP_ADDITIONAL_PAGES) +#define ARCH_SETUP_ADDITIONAL_PAGES(bprm, ex, interpreter) \ + arch_setup_additional_pages(bprm, interpreter) +#endif + #define ELF32_GNU_PROPERTY_ALIGN 4 #define ELF64_GNU_PROPERTY_ALIGN 8 From 3316ec8ccd34e19690a12e65801d605d25155031 Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Sat, 3 Oct 2020 23:25:34 -0400 Subject: [PATCH 013/360] x86/elf: Use e_machine to check for x32/ia32 in setup_additional_pages() Since TIF_X32 is going away, avoid using it to find the ELF type when choosing which additional pages to set up. According to SysV AMD64 ABI Draft, an AMD64 ELF object using ILP32 must have ELFCLASS32 with (E_MACHINE == EM_X86_64), so use that ELF field to differentiate a x32 object from a IA32 object when executing setup_additional_pages() in compat mode. Signed-off-by: Gabriel Krisman Bertazi Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20201004032536.1229030-9-krisman@collabora.com --- arch/x86/entry/vdso/vma.c | 4 ++-- arch/x86/include/asm/elf.h | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c index 9185cb1d13b9..50e5d3a2e70a 100644 --- a/arch/x86/entry/vdso/vma.c +++ b/arch/x86/entry/vdso/vma.c @@ -413,10 +413,10 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) #ifdef CONFIG_COMPAT int compat_arch_setup_additional_pages(struct linux_binprm *bprm, - int uses_interp) + int uses_interp, bool x32) { #ifdef CONFIG_X86_X32_ABI - if (test_thread_flag(TIF_X32)) { + if (x32) { if (!vdso64_enabled) return 0; return map_vdso_randomized(&vdso_image_x32); diff --git a/arch/x86/include/asm/elf.h b/arch/x86/include/asm/elf.h index 109697a19eb1..44a9b9940535 100644 --- a/arch/x86/include/asm/elf.h +++ b/arch/x86/include/asm/elf.h @@ -383,8 +383,10 @@ struct linux_binprm; extern int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp); extern int compat_arch_setup_additional_pages(struct linux_binprm *bprm, - int uses_interp); -#define compat_arch_setup_additional_pages compat_arch_setup_additional_pages + int uses_interp, bool x32); +#define COMPAT_ARCH_SETUP_ADDITIONAL_PAGES(bprm, ex, interpreter) \ + compat_arch_setup_additional_pages(bprm, interpreter, \ + (ex->e_machine == EM_X86_64)) /* Do not change the values. See get_align_mask() */ enum align_flags { From ff170cd0595398a7b66cb40f249eb2f10c29b66d Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Sat, 3 Oct 2020 23:25:35 -0400 Subject: [PATCH 014/360] x86/mm: Convert mmu context ia32_compat into a proper flags field The ia32_compat attribute is a weird thing. It mirrors TIF_IA32 and TIF_X32 and is used only in two very unrelated places: (1) to decide if the vsyscall page is accessible (2) for uprobes to find whether the patched instruction is 32 or 64 bit. In preparation to remove the TIF flags, a new mechanism is required for ia32_compat, but given its odd semantics, adding a real flags field which configures these specific behaviours is the best option. So, set_personality_x64() can ask for the vsyscall page, which is not available in x32/ia32 and set_personality_ia32() can configure the uprobe code as needed. uprobe cannot rely on other methods like user_64bit_mode() to decide how to patch, so it needs some specific flag like this. Signed-off-by: Gabriel Krisman Bertazi Signed-off-by: Thomas Gleixner Acked-by: Andy Lutomirski Link: https://lore.kernel.org/r/20201004032536.1229030-10-krisman@collabora.com --- arch/x86/entry/vsyscall/vsyscall_64.c | 2 +- arch/x86/include/asm/mmu.h | 9 +++++++-- arch/x86/include/asm/mmu_context.h | 2 +- arch/x86/kernel/process_64.c | 17 +++++++++++------ 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/arch/x86/entry/vsyscall/vsyscall_64.c b/arch/x86/entry/vsyscall/vsyscall_64.c index 44c33103a955..1b40b9297083 100644 --- a/arch/x86/entry/vsyscall/vsyscall_64.c +++ b/arch/x86/entry/vsyscall/vsyscall_64.c @@ -316,7 +316,7 @@ static struct vm_area_struct gate_vma __ro_after_init = { struct vm_area_struct *get_gate_vma(struct mm_struct *mm) { #ifdef CONFIG_COMPAT - if (!mm || mm->context.ia32_compat) + if (!mm || !(mm->context.flags & MM_CONTEXT_HAS_VSYSCALL)) return NULL; #endif if (vsyscall_mode == NONE) diff --git a/arch/x86/include/asm/mmu.h b/arch/x86/include/asm/mmu.h index 9257667d13c5..5d7494631ea9 100644 --- a/arch/x86/include/asm/mmu.h +++ b/arch/x86/include/asm/mmu.h @@ -6,6 +6,12 @@ #include #include #include +#include + +/* Uprobes on this MM assume 32-bit code */ +#define MM_CONTEXT_UPROBE_IA32 BIT(0) +/* vsyscall page is accessible on this MM */ +#define MM_CONTEXT_HAS_VSYSCALL BIT(1) /* * x86 has arch-specific MMU state beyond what lives in mm_struct. @@ -33,8 +39,7 @@ typedef struct { #endif #ifdef CONFIG_X86_64 - /* True if mm supports a task running in 32 bit compatibility mode. */ - unsigned short ia32_compat; + unsigned short flags; #endif struct mutex lock; diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h index d98016b83755..054a79157323 100644 --- a/arch/x86/include/asm/mmu_context.h +++ b/arch/x86/include/asm/mmu_context.h @@ -177,7 +177,7 @@ static inline void arch_exit_mmap(struct mm_struct *mm) static inline bool is_64bit_mm(struct mm_struct *mm) { return !IS_ENABLED(CONFIG_IA32_EMULATION) || - !(mm->context.ia32_compat == TIF_IA32); + !(mm->context.flags & MM_CONTEXT_UPROBE_IA32); } #else static inline bool is_64bit_mm(struct mm_struct *mm) diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index 5fb4103101f5..d6efaf6623c9 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -646,10 +646,8 @@ void set_personality_64bit(void) /* Pretend that this comes from a 64bit execve */ task_pt_regs(current)->orig_ax = __NR_execve; current_thread_info()->status &= ~TS_COMPAT; - - /* Ensure the corresponding mm is not marked. */ if (current->mm) - current->mm->context.ia32_compat = 0; + current->mm->context.flags = MM_CONTEXT_HAS_VSYSCALL; /* TBD: overwrites user setup. Should have two bits. But 64bit processes have always behaved this way, @@ -664,7 +662,8 @@ static void __set_personality_x32(void) clear_thread_flag(TIF_IA32); set_thread_flag(TIF_X32); if (current->mm) - current->mm->context.ia32_compat = TIF_X32; + current->mm->context.flags = 0; + current->personality &= ~READ_IMPLIES_EXEC; /* * in_32bit_syscall() uses the presence of the x32 syscall bit @@ -684,8 +683,14 @@ static void __set_personality_ia32(void) #ifdef CONFIG_IA32_EMULATION set_thread_flag(TIF_IA32); clear_thread_flag(TIF_X32); - if (current->mm) - current->mm->context.ia32_compat = TIF_IA32; + if (current->mm) { + /* + * uprobes applied to this MM need to know this and + * cannot use user_64bit_mode() at that time. + */ + current->mm->context.flags = MM_CONTEXT_UPROBE_IA32; + } + current->personality |= force_personality32; /* Prepare the first "return" to user space */ task_pt_regs(current)->orig_ax = __NR_ia32_execve; From 8d71d2bf6efec3032208958c483a247f529ffb16 Mon Sep 17 00:00:00 2001 From: Gabriel Krisman Bertazi Date: Sat, 3 Oct 2020 23:25:36 -0400 Subject: [PATCH 015/360] x86: Reclaim TIF_IA32 and TIF_X32 Now that these flags are no longer used, reclaim those TIF bits. Signed-off-by: Gabriel Krisman Bertazi Signed-off-by: Thomas Gleixner Link: https://lore.kernel.org/r/20201004032536.1229030-11-krisman@collabora.com --- arch/x86/include/asm/thread_info.h | 4 ---- arch/x86/kernel/process_64.c | 6 ------ 2 files changed, 10 deletions(-) diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h index 44733a4bfc42..a12b9644193b 100644 --- a/arch/x86/include/asm/thread_info.h +++ b/arch/x86/include/asm/thread_info.h @@ -91,7 +91,6 @@ struct thread_info { #define TIF_NEED_FPU_LOAD 14 /* load FPU on return to userspace */ #define TIF_NOCPUID 15 /* CPUID is not accessible in userland */ #define TIF_NOTSC 16 /* TSC is not accessible in userland */ -#define TIF_IA32 17 /* IA32 compatibility process */ #define TIF_SLD 18 /* Restore split lock detection on context switch */ #define TIF_MEMDIE 20 /* is terminating due to OOM killer */ #define TIF_POLLING_NRFLAG 21 /* idle is polling for TIF_NEED_RESCHED */ @@ -101,7 +100,6 @@ struct thread_info { #define TIF_LAZY_MMU_UPDATES 27 /* task is updating the mmu lazily */ #define TIF_SYSCALL_TRACEPOINT 28 /* syscall tracepoint instrumentation */ #define TIF_ADDR32 29 /* 32-bit address space on 64 bits */ -#define TIF_X32 30 /* 32-bit native x86-64 binary */ #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME) @@ -120,7 +118,6 @@ struct thread_info { #define _TIF_NEED_FPU_LOAD (1 << TIF_NEED_FPU_LOAD) #define _TIF_NOCPUID (1 << TIF_NOCPUID) #define _TIF_NOTSC (1 << TIF_NOTSC) -#define _TIF_IA32 (1 << TIF_IA32) #define _TIF_SLD (1 << TIF_SLD) #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) #define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP) @@ -129,7 +126,6 @@ struct thread_info { #define _TIF_LAZY_MMU_UPDATES (1 << TIF_LAZY_MMU_UPDATES) #define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT) #define _TIF_ADDR32 (1 << TIF_ADDR32) -#define _TIF_X32 (1 << TIF_X32) /* flags to check in __switch_to() */ #define _TIF_WORK_CTXSW_BASE \ diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index d6efaf6623c9..ad582f9ac5a6 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c @@ -640,9 +640,7 @@ void set_personality_64bit(void) /* inherit personality from parent */ /* Make sure to be in 64bit mode */ - clear_thread_flag(TIF_IA32); clear_thread_flag(TIF_ADDR32); - clear_thread_flag(TIF_X32); /* Pretend that this comes from a 64bit execve */ task_pt_regs(current)->orig_ax = __NR_execve; current_thread_info()->status &= ~TS_COMPAT; @@ -659,8 +657,6 @@ void set_personality_64bit(void) static void __set_personality_x32(void) { #ifdef CONFIG_X86_X32 - clear_thread_flag(TIF_IA32); - set_thread_flag(TIF_X32); if (current->mm) current->mm->context.flags = 0; @@ -681,8 +677,6 @@ static void __set_personality_x32(void) static void __set_personality_ia32(void) { #ifdef CONFIG_IA32_EMULATION - set_thread_flag(TIF_IA32); - clear_thread_flag(TIF_X32); if (current->mm) { /* * uprobes applied to this MM need to know this and From 3adb776384f2042ef6bda876e91a7a7ac2872c5e Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Mon, 19 Oct 2020 21:08:03 -0700 Subject: [PATCH 016/360] x86, libnvdimm/test: Remove COPY_MC_TEST The COPY_MC_TEST facility has served its purpose for validating the early termination conditions of the copy_mc_fragile() implementation. Remove it and the EXPORT_SYMBOL_GPL of copy_mc_fragile(). Reported-by: Borislav Petkov Signed-off-by: Dan Williams Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/160316688322.3374697.8648308115165836243.stgit@dwillia2-desk3.amr.corp.intel.com --- arch/x86/Kconfig.debug | 3 - arch/x86/include/asm/copy_mc_test.h | 75 -------------------- arch/x86/lib/copy_mc.c | 4 -- arch/x86/lib/copy_mc_64.S | 10 --- tools/testing/nvdimm/test/nfit.c | 103 ---------------------------- 5 files changed, 195 deletions(-) delete mode 100644 arch/x86/include/asm/copy_mc_test.h diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug index 27b5e2bc6a01..80b57e7f4947 100644 --- a/arch/x86/Kconfig.debug +++ b/arch/x86/Kconfig.debug @@ -62,9 +62,6 @@ config EARLY_PRINTK_USB_XDBC You should normally say N here, unless you want to debug early crashes or need a very simple printk logging facility. -config COPY_MC_TEST - def_bool n - config EFI_PGT_DUMP bool "Dump the EFI pagetable" depends on EFI diff --git a/arch/x86/include/asm/copy_mc_test.h b/arch/x86/include/asm/copy_mc_test.h deleted file mode 100644 index e4991ba96726..000000000000 --- a/arch/x86/include/asm/copy_mc_test.h +++ /dev/null @@ -1,75 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _COPY_MC_TEST_H_ -#define _COPY_MC_TEST_H_ - -#ifndef __ASSEMBLY__ -#ifdef CONFIG_COPY_MC_TEST -extern unsigned long copy_mc_test_src; -extern unsigned long copy_mc_test_dst; - -static inline void copy_mc_inject_src(void *addr) -{ - if (addr) - copy_mc_test_src = (unsigned long) addr; - else - copy_mc_test_src = ~0UL; -} - -static inline void copy_mc_inject_dst(void *addr) -{ - if (addr) - copy_mc_test_dst = (unsigned long) addr; - else - copy_mc_test_dst = ~0UL; -} -#else /* CONFIG_COPY_MC_TEST */ -static inline void copy_mc_inject_src(void *addr) -{ -} - -static inline void copy_mc_inject_dst(void *addr) -{ -} -#endif /* CONFIG_COPY_MC_TEST */ - -#else /* __ASSEMBLY__ */ -#include - -#ifdef CONFIG_COPY_MC_TEST -.macro COPY_MC_TEST_CTL - .pushsection .data - .align 8 - .globl copy_mc_test_src - copy_mc_test_src: - .quad 0 - EXPORT_SYMBOL_GPL(copy_mc_test_src) - .globl copy_mc_test_dst - copy_mc_test_dst: - .quad 0 - EXPORT_SYMBOL_GPL(copy_mc_test_dst) - .popsection -.endm - -.macro COPY_MC_TEST_SRC reg count target - leaq \count(\reg), %r9 - cmp copy_mc_test_src, %r9 - ja \target -.endm - -.macro COPY_MC_TEST_DST reg count target - leaq \count(\reg), %r9 - cmp copy_mc_test_dst, %r9 - ja \target -.endm -#else -.macro COPY_MC_TEST_CTL -.endm - -.macro COPY_MC_TEST_SRC reg count target -.endm - -.macro COPY_MC_TEST_DST reg count target -.endm -#endif /* CONFIG_COPY_MC_TEST */ -#endif /* __ASSEMBLY__ */ -#endif /* _COPY_MC_TEST_H_ */ diff --git a/arch/x86/lib/copy_mc.c b/arch/x86/lib/copy_mc.c index c13e8c9ee926..80efd45a7761 100644 --- a/arch/x86/lib/copy_mc.c +++ b/arch/x86/lib/copy_mc.c @@ -10,10 +10,6 @@ #include #ifdef CONFIG_X86_MCE -/* - * See COPY_MC_TEST for self-test of the copy_mc_fragile() - * implementation. - */ static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key); void enable_copy_mc_fragile(void) diff --git a/arch/x86/lib/copy_mc_64.S b/arch/x86/lib/copy_mc_64.S index 892d8915f609..e5f77e293034 100644 --- a/arch/x86/lib/copy_mc_64.S +++ b/arch/x86/lib/copy_mc_64.S @@ -2,14 +2,11 @@ /* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */ #include -#include -#include #include #ifndef CONFIG_UML #ifdef CONFIG_X86_MCE -COPY_MC_TEST_CTL /* * copy_mc_fragile - copy memory with indication if an exception / fault happened @@ -38,8 +35,6 @@ SYM_FUNC_START(copy_mc_fragile) subl %ecx, %edx .L_read_leading_bytes: movb (%rsi), %al - COPY_MC_TEST_SRC %rsi 1 .E_leading_bytes - COPY_MC_TEST_DST %rdi 1 .E_leading_bytes .L_write_leading_bytes: movb %al, (%rdi) incq %rsi @@ -55,8 +50,6 @@ SYM_FUNC_START(copy_mc_fragile) .L_read_words: movq (%rsi), %r8 - COPY_MC_TEST_SRC %rsi 8 .E_read_words - COPY_MC_TEST_DST %rdi 8 .E_write_words .L_write_words: movq %r8, (%rdi) addq $8, %rsi @@ -73,8 +66,6 @@ SYM_FUNC_START(copy_mc_fragile) movl %edx, %ecx .L_read_trailing_bytes: movb (%rsi), %al - COPY_MC_TEST_SRC %rsi 1 .E_trailing_bytes - COPY_MC_TEST_DST %rdi 1 .E_trailing_bytes .L_write_trailing_bytes: movb %al, (%rdi) incq %rsi @@ -88,7 +79,6 @@ SYM_FUNC_START(copy_mc_fragile) .L_done: ret SYM_FUNC_END(copy_mc_fragile) -EXPORT_SYMBOL_GPL(copy_mc_fragile) .section .fixup, "ax" /* diff --git a/tools/testing/nvdimm/test/nfit.c b/tools/testing/nvdimm/test/nfit.c index 2ac0fff6dad8..9b185bf82da8 100644 --- a/tools/testing/nvdimm/test/nfit.c +++ b/tools/testing/nvdimm/test/nfit.c @@ -23,7 +23,6 @@ #include "nfit_test.h" #include "../watermark.h" -#include #include /* @@ -3284,107 +3283,6 @@ static struct platform_driver nfit_test_driver = { .id_table = nfit_test_id, }; -static char copy_mc_buf[PAGE_SIZE] __attribute__((__aligned__(PAGE_SIZE))); - -enum INJECT { - INJECT_NONE, - INJECT_SRC, - INJECT_DST, -}; - -static void copy_mc_test_init(char *dst, char *src, size_t size) -{ - size_t i; - - memset(dst, 0xff, size); - for (i = 0; i < size; i++) - src[i] = (char) i; -} - -static bool copy_mc_test_validate(unsigned char *dst, unsigned char *src, - size_t size, unsigned long rem) -{ - size_t i; - - for (i = 0; i < size - rem; i++) - if (dst[i] != (unsigned char) i) { - pr_info_once("%s:%d: offset: %zd got: %#x expect: %#x\n", - __func__, __LINE__, i, dst[i], - (unsigned char) i); - return false; - } - for (i = size - rem; i < size; i++) - if (dst[i] != 0xffU) { - pr_info_once("%s:%d: offset: %zd got: %#x expect: 0xff\n", - __func__, __LINE__, i, dst[i]); - return false; - } - return true; -} - -void copy_mc_test(void) -{ - char *inject_desc[] = { "none", "source", "destination" }; - enum INJECT inj; - - if (IS_ENABLED(CONFIG_COPY_MC_TEST)) { - pr_info("%s: run...\n", __func__); - } else { - pr_info("%s: disabled, skip.\n", __func__); - return; - } - - for (inj = INJECT_NONE; inj <= INJECT_DST; inj++) { - int i; - - pr_info("%s: inject: %s\n", __func__, inject_desc[inj]); - for (i = 0; i < 512; i++) { - unsigned long expect, rem; - void *src, *dst; - bool valid; - - switch (inj) { - case INJECT_NONE: - copy_mc_inject_src(NULL); - copy_mc_inject_dst(NULL); - dst = ©_mc_buf[2048]; - src = ©_mc_buf[1024 - i]; - expect = 0; - break; - case INJECT_SRC: - copy_mc_inject_src(©_mc_buf[1024]); - copy_mc_inject_dst(NULL); - dst = ©_mc_buf[2048]; - src = ©_mc_buf[1024 - i]; - expect = 512 - i; - break; - case INJECT_DST: - copy_mc_inject_src(NULL); - copy_mc_inject_dst(©_mc_buf[2048]); - dst = ©_mc_buf[2048 - i]; - src = ©_mc_buf[1024]; - expect = 512 - i; - break; - } - - copy_mc_test_init(dst, src, 512); - rem = copy_mc_fragile(dst, src, 512); - valid = copy_mc_test_validate(dst, src, 512, expect); - if (rem == expect && valid) - continue; - pr_info("%s: copy(%#lx, %#lx, %d) off: %d rem: %ld %s expect: %ld\n", - __func__, - ((unsigned long) dst) & ~PAGE_MASK, - ((unsigned long ) src) & ~PAGE_MASK, - 512, i, rem, valid ? "valid" : "bad", - expect); - } - } - - copy_mc_inject_src(NULL); - copy_mc_inject_dst(NULL); -} - static __init int nfit_test_init(void) { int rc, i; @@ -3393,7 +3291,6 @@ static __init int nfit_test_init(void) libnvdimm_test(); acpi_nfit_test(); device_dax_test(); - copy_mc_test(); dax_pmem_test(); dax_pmem_core_test(); #ifdef CONFIG_DEV_DAX_PMEM_COMPAT From d1b22e36e3188f47dbf8aaef54d9e79df8e91f4c Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Wed, 14 Oct 2020 00:49:26 +0000 Subject: [PATCH 017/360] Documentation/x86: Rename resctrl_ui.rst and add two errata to the file Intel Memory Bandwidth Monitoring (MBM) counters may report system memory bandwidth incorrectly on some Intel processors. This is reported in documented in erratum SKX99, erratum BDF102 and in the RDT reference manual, see Documentation/x86/index.rst. To work around the errata, MBM total and local readings are corrected using a correction factor table. Since the correction factor table is not publicly documented anywhere, document the table and the errata in Documentation/x86/resctrl.rst for future reference. [ bp: Move web links to the doc, massage. ] Suggested-by: Borislav Petkov Signed-off-by: Fenghua Yu Signed-off-by: Borislav Petkov Reviewed-by: Tony Luck Link: https://lkml.kernel.org/r/20201014004927.1839452-2-fenghua.yu@intel.com --- Documentation/x86/index.rst | 2 +- .../x86/{resctrl_ui.rst => resctrl.rst} | 93 +++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) rename Documentation/x86/{resctrl_ui.rst => resctrl.rst} (90%) diff --git a/Documentation/x86/index.rst b/Documentation/x86/index.rst index b224d12c880b..9c6ebf355f81 100644 --- a/Documentation/x86/index.rst +++ b/Documentation/x86/index.rst @@ -27,7 +27,7 @@ x86-specific Documentation pti mds microcode - resctrl_ui + resctrl tsx_async_abort usb-legacy-support i386/index diff --git a/Documentation/x86/resctrl_ui.rst b/Documentation/x86/resctrl.rst similarity index 90% rename from Documentation/x86/resctrl_ui.rst rename to Documentation/x86/resctrl.rst index e59b7b93a9b4..71a531061e4e 100644 --- a/Documentation/x86/resctrl_ui.rst +++ b/Documentation/x86/resctrl.rst @@ -1209,3 +1209,96 @@ View the llc occupancy snapshot:: # cat /sys/fs/resctrl/p1/mon_data/mon_L3_00/llc_occupancy 11234000 + +Intel RDT Errata +================ + +Intel MBM Counters May Report System Memory Bandwidth Incorrectly +----------------------------------------------------------------- + +Errata SKX99 for Skylake server and BDF102 for Broadwell server. + +Problem: Intel Memory Bandwidth Monitoring (MBM) counters track metrics +according to the assigned Resource Monitor ID (RMID) for that logical +core. The IA32_QM_CTR register (MSR 0xC8E), used to report these +metrics, may report incorrect system bandwidth for certain RMID values. + +Implication: Due to the errata, system memory bandwidth may not match +what is reported. + +Workaround: MBM total and local readings are corrected according to the +following correction factor table: + ++---------------+---------------+---------------+-----------------+ +|core count |rmid count |rmid threshold |correction factor| ++---------------+---------------+---------------+-----------------+ +|1 |8 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|2 |16 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|3 |24 |15 |0.969650 | ++---------------+---------------+---------------+-----------------+ +|4 |32 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|6 |48 |31 |0.969650 | ++---------------+---------------+---------------+-----------------+ +|7 |56 |47 |1.142857 | ++---------------+---------------+---------------+-----------------+ +|8 |64 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|9 |72 |63 |1.185115 | ++---------------+---------------+---------------+-----------------+ +|10 |80 |63 |1.066553 | ++---------------+---------------+---------------+-----------------+ +|11 |88 |79 |1.454545 | ++---------------+---------------+---------------+-----------------+ +|12 |96 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|13 |104 |95 |1.230769 | ++---------------+---------------+---------------+-----------------+ +|14 |112 |95 |1.142857 | ++---------------+---------------+---------------+-----------------+ +|15 |120 |95 |1.066667 | ++---------------+---------------+---------------+-----------------+ +|16 |128 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|17 |136 |127 |1.254863 | ++---------------+---------------+---------------+-----------------+ +|18 |144 |127 |1.185255 | ++---------------+---------------+---------------+-----------------+ +|19 |152 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|20 |160 |127 |1.066667 | ++---------------+---------------+---------------+-----------------+ +|21 |168 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|22 |176 |159 |1.454334 | ++---------------+---------------+---------------+-----------------+ +|23 |184 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|24 |192 |127 |0.969744 | ++---------------+---------------+---------------+-----------------+ +|25 |200 |191 |1.280246 | ++---------------+---------------+---------------+-----------------+ +|26 |208 |191 |1.230921 | ++---------------+---------------+---------------+-----------------+ +|27 |216 |0 |1.000000 | ++---------------+---------------+---------------+-----------------+ +|28 |224 |191 |1.143118 | ++---------------+---------------+---------------+-----------------+ + +If rmid > rmid threshold, MBM total and local values should be multiplied +by the correction factor. + +See: + +1. Erratum SKX99 in Intel Xeon Processor Scalable Family Specification Update: +http://web.archive.org/web/20200716124958/https://www.intel.com/content/www/us/en/processors/xeon/scalable/xeon-scalable-spec-update.html + +2. Erratum BDF102 in Intel Xeon E5-2600 v4 Processor Product Family Specification Update: +http://web.archive.org/web/20191125200531/https://www.intel.com/content/dam/www/public/us/en/documents/specification-updates/xeon-e5-v4-spec-update.pdf + +3. The errata in Intel Resource Director Technology (Intel RDT) on 2nd Generation Intel Xeon Scalable Processors Reference Manual: +https://software.intel.com/content/www/us/en/develop/articles/intel-resource-director-technology-rdt-reference-manual.html + +for further information. From 4868a61d498af3627c3b571aa48107a845001e51 Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Wed, 14 Oct 2020 00:49:27 +0000 Subject: [PATCH 018/360] x86/resctrl: Correct MBM total and local values Intel Memory Bandwidth Monitoring (MBM) counters may report system memory bandwidth incorrectly on some Intel processors. The errata SKX99 for Skylake server, BDF102 for Broadwell server, and the correction factor table are documented in Documentation/x86/resctrl.rst. Intel MBM counters track metrics according to the assigned Resource Monitor ID (RMID) for that logical core. The IA32_QM_CTR register (MSR 0xC8E) used to report these metrics, may report incorrect system bandwidth for certain RMID values. Due to the errata, system memory bandwidth may not match what is reported. To work around the errata, correct MBM total and local readings using a correction factor table. If rmid > rmid threshold, MBM total and local values should be multiplied by the correction factor. [ bp: Mark mbm_cf_table[] __initdata. ] Signed-off-by: Fenghua Yu Signed-off-by: Borislav Petkov Reviewed-by: Tony Luck Link: https://lkml.kernel.org/r/20201014004927.1839452-3-fenghua.yu@intel.com --- arch/x86/kernel/cpu/resctrl/core.c | 4 ++ arch/x86/kernel/cpu/resctrl/internal.h | 1 + arch/x86/kernel/cpu/resctrl/monitor.c | 82 +++++++++++++++++++++++++- 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index e5f4ee8f4c3b..1a681fde4c51 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -893,6 +893,10 @@ static __init void __check_quirks_intel(void) set_rdt_options("!cmt,!mbmtotal,!mbmlocal,!l3cat"); else set_rdt_options("!l3cat"); + fallthrough; + case INTEL_FAM6_BROADWELL_X: + intel_rdt_mbm_apply_quirk(); + break; } } diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h index 80fa997fae60..6cb068fcf501 100644 --- a/arch/x86/kernel/cpu/resctrl/internal.h +++ b/arch/x86/kernel/cpu/resctrl/internal.h @@ -616,6 +616,7 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, void mbm_setup_overflow_handler(struct rdt_domain *dom, unsigned long delay_ms); void mbm_handle_overflow(struct work_struct *work); +void __init intel_rdt_mbm_apply_quirk(void); bool is_mba_sc(struct rdt_resource *r); void setup_default_ctrlval(struct rdt_resource *r, u32 *dc, u32 *dm); u32 delay_bw_map(unsigned long bw, struct rdt_resource *r); diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 54dffe574e67..622073ffa715 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -64,6 +64,69 @@ unsigned int rdt_mon_features; */ unsigned int resctrl_cqm_threshold; +#define CF(cf) ((unsigned long)(1048576 * (cf) + 0.5)) + +/* + * The correction factor table is documented in Documentation/x86/resctrl.rst. + * If rmid > rmid threshold, MBM total and local values should be multiplied + * by the correction factor. + * + * The original table is modified for better code: + * + * 1. The threshold 0 is changed to rmid count - 1 so don't do correction + * for the case. + * 2. MBM total and local correction table indexed by core counter which is + * equal to (x86_cache_max_rmid + 1) / 8 - 1 and is from 0 up to 27. + * 3. The correction factor is normalized to 2^20 (1048576) so it's faster + * to calculate corrected value by shifting: + * corrected_value = (original_value * correction_factor) >> 20 + */ +static const struct mbm_correction_factor_table { + u32 rmidthreshold; + u64 cf; +} mbm_cf_table[] __initdata = { + {7, CF(1.000000)}, + {15, CF(1.000000)}, + {15, CF(0.969650)}, + {31, CF(1.000000)}, + {31, CF(1.066667)}, + {31, CF(0.969650)}, + {47, CF(1.142857)}, + {63, CF(1.000000)}, + {63, CF(1.185115)}, + {63, CF(1.066553)}, + {79, CF(1.454545)}, + {95, CF(1.000000)}, + {95, CF(1.230769)}, + {95, CF(1.142857)}, + {95, CF(1.066667)}, + {127, CF(1.000000)}, + {127, CF(1.254863)}, + {127, CF(1.185255)}, + {151, CF(1.000000)}, + {127, CF(1.066667)}, + {167, CF(1.000000)}, + {159, CF(1.454334)}, + {183, CF(1.000000)}, + {127, CF(0.969744)}, + {191, CF(1.280246)}, + {191, CF(1.230921)}, + {215, CF(1.000000)}, + {191, CF(1.143118)}, +}; + +static u32 mbm_cf_rmidthreshold __read_mostly = UINT_MAX; +static u64 mbm_cf __read_mostly; + +static inline u64 get_corrected_mbm_count(u32 rmid, unsigned long val) +{ + /* Correct MBM value. */ + if (rmid > mbm_cf_rmidthreshold) + val = (val * mbm_cf) >> 20; + + return val; +} + static inline struct rmid_entry *__rmid_entry(u32 rmid) { struct rmid_entry *entry; @@ -260,7 +323,8 @@ static int __mon_event_count(u32 rmid, struct rmid_read *rr) m->chunks += chunks; m->prev_msr = tval; - rr->val += m->chunks; + rr->val += get_corrected_mbm_count(rmid, m->chunks); + return 0; } @@ -280,7 +344,7 @@ static void mbm_bw_count(u32 rmid, struct rmid_read *rr) chunks = mbm_overflow_count(m->prev_bw_msr, tval, rr->r->mbm_width); m->chunks += chunks; - cur_bw = (chunks * r->mon_scale) >> 20; + cur_bw = (get_corrected_mbm_count(rmid, chunks) * r->mon_scale) >> 20; if (m->delta_comp) m->delta_bw = abs(cur_bw - m->prev_bw); @@ -644,3 +708,17 @@ int rdt_get_mon_l3_config(struct rdt_resource *r) return 0; } + +void __init intel_rdt_mbm_apply_quirk(void) +{ + int cf_index; + + cf_index = (boot_cpu_data.x86_cache_max_rmid + 1) / 8 - 1; + if (cf_index >= ARRAY_SIZE(mbm_cf_table)) { + pr_info("No MBM correction factor available\n"); + return; + } + + mbm_cf_rmidthreshold = mbm_cf_table[cf_index].rmidthreshold; + mbm_cf = mbm_cf_table[cf_index].cf; +} From 1fcd009102ee02e217f2e7635ab65517d785da8e Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Tue, 27 Oct 2020 19:06:48 -0400 Subject: [PATCH 019/360] x86/mm/ident_map: Check for errors from ident_pud_init() Commit ea3b5e60ce80 ("x86/mm/ident_map: Add 5-level paging support") added ident_p4d_init() to support 5-level paging, but this function doesn't check and return errors from ident_pud_init(). For example, the decompressor stub uses this code to create an identity mapping. If it runs out of pages while trying to allocate a PMD pagetable, the error will be currently ignored. Fix this to propagate errors. [ bp: Space out statements for better readability. ] Fixes: ea3b5e60ce80 ("x86/mm/ident_map: Add 5-level paging support") Signed-off-by: Arvind Sankar Signed-off-by: Borislav Petkov Reviewed-by: Joerg Roedel Acked-by: Kirill A. Shutemov Link: https://lkml.kernel.org/r/20201027230648.1885111-1-nivedita@alum.mit.edu --- arch/x86/mm/ident_map.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/arch/x86/mm/ident_map.c b/arch/x86/mm/ident_map.c index fe7a12599d8e..968d7005f4a7 100644 --- a/arch/x86/mm/ident_map.c +++ b/arch/x86/mm/ident_map.c @@ -62,6 +62,7 @@ static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page, unsigned long addr, unsigned long end) { unsigned long next; + int result; for (; addr < end; addr = next) { p4d_t *p4d = p4d_page + p4d_index(addr); @@ -73,13 +74,20 @@ static int ident_p4d_init(struct x86_mapping_info *info, p4d_t *p4d_page, if (p4d_present(*p4d)) { pud = pud_offset(p4d, 0); - ident_pud_init(info, pud, addr, next); + result = ident_pud_init(info, pud, addr, next); + if (result) + return result; + continue; } pud = (pud_t *)info->alloc_pgt_page(info->context); if (!pud) return -ENOMEM; - ident_pud_init(info, pud, addr, next); + + result = ident_pud_init(info, pud, addr, next); + if (result) + return result; + set_p4d(p4d, __p4d(__pa(pud) | info->kernpg_flag)); } From 0d847ce7c17613d63401ac82336ee1d5df749120 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Wed, 21 Oct 2020 18:39:47 +0200 Subject: [PATCH 020/360] x86/setup: Remove unused MCA variables Commit bb8187d35f82 ("MCA: delete all remaining traces of microchannel bus support.") removed the remaining traces of Micro Channel Architecture support but one trace remained - three variables in setup.c which have been unused since 2012 at least. Drop them finally. Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201021165614.23023-1-bp@alien8.de --- arch/x86/kernel/setup.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 84f581c91db4..a23130c86bdd 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -119,11 +119,6 @@ EXPORT_SYMBOL(boot_cpu_data); unsigned int def_to_bigsmp; -/* For MCA, but anyone else can use it if they want */ -unsigned int machine_id; -unsigned int machine_submodel_id; -unsigned int BIOS_revision; - struct apm_info apm_info; EXPORT_SYMBOL(apm_info); From ea3186b9572a1b0299448697cfc44920061872cf Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Thu, 29 Oct 2020 12:19:03 -0400 Subject: [PATCH 021/360] x86/build: Fix vmlinux size check on 64-bit Commit b4e0409a36f4 ("x86: check vmlinux limits, 64-bit") added a check that the size of the 64-bit kernel is less than KERNEL_IMAGE_SIZE. The check uses (_end - _text), but this is not enough. The initial PMD used in startup_64() (level2_kernel_pgt) can only map upto KERNEL_IMAGE_SIZE from __START_KERNEL_map, not from _text, and the modules area (MODULES_VADDR) starts at KERNEL_IMAGE_SIZE. The correct check is what is currently done for 32-bit, since LOAD_OFFSET is defined appropriately for the two architectures. Just check (_end - LOAD_OFFSET) against KERNEL_IMAGE_SIZE unconditionally. Note that on 32-bit, the limit is not strict: KERNEL_IMAGE_SIZE is not really used by the main kernel. The higher the kernel is located, the less the space available for the vmalloc area. However, it is used by KASLR in the compressed stub to limit the maximum address of the kernel to a safe value. Clean up various comments to clarify that despite the name, KERNEL_IMAGE_SIZE is not a limit on the size of the kernel image, but a limit on the maximum virtual address that the image can occupy. Signed-off-by: Arvind Sankar Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201029161903.2553528-1-nivedita@alum.mit.edu --- arch/x86/include/asm/page_32_types.h | 8 +++++++- arch/x86/include/asm/page_64_types.h | 6 ++++-- arch/x86/include/asm/pgtable_32.h | 18 ++++++------------ arch/x86/kernel/head_64.S | 20 +++++++++----------- arch/x86/kernel/vmlinux.lds.S | 12 +++--------- 5 files changed, 29 insertions(+), 35 deletions(-) diff --git a/arch/x86/include/asm/page_32_types.h b/arch/x86/include/asm/page_32_types.h index f462895a33e4..faf9cc1c14bb 100644 --- a/arch/x86/include/asm/page_32_types.h +++ b/arch/x86/include/asm/page_32_types.h @@ -53,7 +53,13 @@ #define STACK_TOP_MAX STACK_TOP /* - * Kernel image size is limited to 512 MB (see in arch/x86/kernel/head_32.S) + * In spite of the name, KERNEL_IMAGE_SIZE is a limit on the maximum virtual + * address for the kernel image, rather than the limit on the size itself. On + * 32-bit, this is not a strict limit, but this value is used to limit the + * link-time virtual address range of the kernel, and by KASLR to limit the + * randomized address from which the kernel is executed. A relocatable kernel + * can be loaded somewhat higher than KERNEL_IMAGE_SIZE as long as enough space + * remains for the vmalloc area. */ #define KERNEL_IMAGE_SIZE (512 * 1024 * 1024) diff --git a/arch/x86/include/asm/page_64_types.h b/arch/x86/include/asm/page_64_types.h index 3f49dac03617..645bd1d0ee07 100644 --- a/arch/x86/include/asm/page_64_types.h +++ b/arch/x86/include/asm/page_64_types.h @@ -98,8 +98,10 @@ #define STACK_TOP_MAX TASK_SIZE_MAX /* - * Maximum kernel image size is limited to 1 GiB, due to the fixmap living - * in the next 1 GiB (see level2_kernel_pgt in arch/x86/kernel/head_64.S). + * In spite of the name, KERNEL_IMAGE_SIZE is a limit on the maximum virtual + * address for the kernel image, rather than the limit on the size itself. + * This can be at most 1 GiB, due to the fixmap living in the next 1 GiB (see + * level2_kernel_pgt in arch/x86/kernel/head_64.S). * * On KASLR use 1 GiB by default, leaving 1 GiB for modules once the * page tables are fully set up. diff --git a/arch/x86/include/asm/pgtable_32.h b/arch/x86/include/asm/pgtable_32.h index d7acae4120d5..7c9c968a42ef 100644 --- a/arch/x86/include/asm/pgtable_32.h +++ b/arch/x86/include/asm/pgtable_32.h @@ -57,19 +57,13 @@ do { \ #endif /* - * This is how much memory in addition to the memory covered up to - * and including _end we need mapped initially. - * We need: - * (KERNEL_IMAGE_SIZE/4096) / 1024 pages (worst case, non PAE) - * (KERNEL_IMAGE_SIZE/4096) / 512 + 4 pages (worst case for PAE) + * This is used to calculate the .brk reservation for initial pagetables. + * Enough space is reserved to allocate pagetables sufficient to cover all + * of LOWMEM_PAGES, which is an upper bound on the size of the direct map of + * lowmem. * - * Modulo rounding, each megabyte assigned here requires a kilobyte of - * memory, which is currently unreclaimed. - * - * This should be a multiple of a page. - * - * KERNEL_IMAGE_SIZE should be greater than pa(_end) - * and small than max_low_pfn, otherwise will waste some page table entries + * With PAE paging (PTRS_PER_PMD > 1), we allocate PTRS_PER_PGD == 4 pages for + * the PMD's in addition to the pages required for the last level pagetables. */ #if PTRS_PER_PMD > 1 #define PAGE_TABLE_SIZE(pages) (((pages) / PTRS_PER_PMD) + PTRS_PER_PGD) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 7eb2a1c87969..d41fa5bb77fe 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -524,21 +524,19 @@ SYM_DATA_END(level3_kernel_pgt) SYM_DATA_START_PAGE_ALIGNED(level2_kernel_pgt) /* - * 512 MB kernel mapping. We spend a full page on this pagetable - * anyway. + * Kernel high mapping. * - * The kernel code+data+bss must not be bigger than that. + * The kernel code+data+bss must be located below KERNEL_IMAGE_SIZE in + * virtual address space, which is 1 GiB if RANDOMIZE_BASE is enabled, + * 512 MiB otherwise. * - * (NOTE: at +512MB starts the module area, see MODULES_VADDR. - * If you want to increase this then increase MODULES_VADDR - * too.) + * (NOTE: after that starts the module area, see MODULES_VADDR.) * - * This table is eventually used by the kernel during normal - * runtime. Care must be taken to clear out undesired bits - * later, like _PAGE_RW or _PAGE_GLOBAL in some cases. + * This table is eventually used by the kernel during normal runtime. + * Care must be taken to clear out undesired bits later, like _PAGE_RW + * or _PAGE_GLOBAL in some cases. */ - PMDS(0, __PAGE_KERNEL_LARGE_EXEC, - KERNEL_IMAGE_SIZE/PMD_SIZE) + PMDS(0, __PAGE_KERNEL_LARGE_EXEC, KERNEL_IMAGE_SIZE/PMD_SIZE) SYM_DATA_END(level2_kernel_pgt) SYM_DATA_START_PAGE_ALIGNED(level2_fixmap_pgt) diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index bf9e0adb5b7e..efd9e9ea17f2 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -454,13 +454,13 @@ SECTIONS ASSERT(SIZEOF(.rela.dyn) == 0, "Unexpected run-time relocations (.rela) detected!") } -#ifdef CONFIG_X86_32 /* * The ASSERT() sink to . is intentional, for binutils 2.14 compatibility: */ . = ASSERT((_end - LOAD_OFFSET <= KERNEL_IMAGE_SIZE), "kernel image bigger than KERNEL_IMAGE_SIZE"); -#else + +#ifdef CONFIG_X86_64 /* * Per-cpu symbols which need to be offset from __per_cpu_load * for the boot processor. @@ -470,18 +470,12 @@ INIT_PER_CPU(gdt_page); INIT_PER_CPU(fixed_percpu_data); INIT_PER_CPU(irq_stack_backing_store); -/* - * Build-time check on the image size: - */ -. = ASSERT((_end - _text <= KERNEL_IMAGE_SIZE), - "kernel image bigger than KERNEL_IMAGE_SIZE"); - #ifdef CONFIG_SMP . = ASSERT((fixed_percpu_data == 0), "fixed_percpu_data is not at start of per-cpu area"); #endif -#endif /* CONFIG_X86_32 */ +#endif /* CONFIG_X86_64 */ #ifdef CONFIG_KEXEC_CORE #include From daf88f3757ecb7aed4dd05a1ca8c8b4ac378d6c3 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 7 Oct 2020 13:43:45 +0100 Subject: [PATCH 022/360] crypto: qat - remove unused function Remove unused function qat_dh_get_params(). This is to fix the following warning when compiling the driver with CC=clang W=1 drivers/crypto/qat/qat_common/qat_asym_algs.c:207:34: warning: unused function 'qat_dh_get_params' [-Wunused-function] Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_asym_algs.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 846569ec9066..f112078be868 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -204,11 +204,6 @@ static unsigned long qat_dh_fn_id(unsigned int len, bool g2) }; } -static inline struct qat_dh_ctx *qat_dh_get_params(struct crypto_kpp *tfm) -{ - return kpp_tfm_ctx(tfm); -} - static int qat_dh_compute_value(struct kpp_request *req) { struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); From d72286943212fdb6d254d08d4cfccdc70603d0ca Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 8 Oct 2020 16:58:18 +1100 Subject: [PATCH 023/360] lib/mpi: Remove unused scalar_copied The scalar_copied variable is not as the scalar is never copied in that block. This patch removes it. Fixes: d58bb7e55a8a ("lib/mpi: Introduce ec implementation to...") Reported-by: Gustavo A. R. Silva Signed-off-by: Herbert Xu --- lib/mpi/ec.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/mpi/ec.c b/lib/mpi/ec.c index c21470122dfc..40f5908e57a4 100644 --- a/lib/mpi/ec.c +++ b/lib/mpi/ec.c @@ -1252,7 +1252,6 @@ void mpi_ec_mul_point(MPI_POINT result, MPI_POINT q1, q2, prd, sum; unsigned long sw; mpi_size_t rsize; - int scalar_copied = 0; /* Compute scalar point multiplication with Montgomery Ladder. * Note that we don't use Y-coordinate in the points at all. @@ -1314,8 +1313,6 @@ void mpi_ec_mul_point(MPI_POINT result, point_free(&p2); point_free(&p1_); point_free(&p2_); - if (scalar_copied) - mpi_free(scalar); return; } From 195404db27f9533c71fdcb78d32a77075c2cb4a2 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Thu, 8 Oct 2020 09:34:55 +0000 Subject: [PATCH 024/360] crypto: talitos - Endianess in current_desc_hdr() current_desc_hdr() compares the value of the current descriptor with the next_desc member of the talitos_desc struct. While the current descriptor is obtained from in_be32() which return CPU ordered bytes, next_desc member is in big endian order. Convert the current descriptor into big endian before comparing it with next_desc. This fixes a sparse warning. Fixes: 37b5e8897eb5 ("crypto: talitos - chain in buffered data for ahash on SEC1") Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 66773892f665..1de6b0138126 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -478,7 +478,7 @@ static u32 current_desc_hdr(struct device *dev, int ch) iter = tail; while (priv->chan[ch].fifo[iter].dma_desc != cur_desc && - priv->chan[ch].fifo[iter].desc->next_desc != cur_desc) { + priv->chan[ch].fifo[iter].desc->next_desc != cpu_to_be32(cur_desc)) { iter = (iter + 1) & (priv->fifo_len - 1); if (iter == tail) { dev_err(dev, "couldn't locate current descriptor\n"); @@ -486,7 +486,7 @@ static u32 current_desc_hdr(struct device *dev, int ch) } } - if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc) { + if (priv->chan[ch].fifo[iter].desc->next_desc == cpu_to_be32(cur_desc)) { struct talitos_edesc *edesc; edesc = container_of(priv->chan[ch].fifo[iter].desc, From 0237616173fd363a54bd272aa3bd376faa1d7caa Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Thu, 8 Oct 2020 09:34:56 +0000 Subject: [PATCH 025/360] crypto: talitos - Fix return type of current_desc_hdr() current_desc_hdr() returns a u32 but in fact this is a __be32, leading to a lot of sparse warnings. Change the return type to __be32 and ensure it is handled as sure by the caller. Fixes: 3e721aeb3df3 ("crypto: talitos - handle descriptor not found in error path") Signed-off-by: Christophe Leroy Signed-off-by: Herbert Xu --- drivers/crypto/talitos.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 1de6b0138126..a713a35dc502 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -460,7 +460,7 @@ DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE) /* * locate current (offending) descriptor */ -static u32 current_desc_hdr(struct device *dev, int ch) +static __be32 current_desc_hdr(struct device *dev, int ch) { struct talitos_private *priv = dev_get_drvdata(dev); int tail, iter; @@ -501,13 +501,13 @@ static u32 current_desc_hdr(struct device *dev, int ch) /* * user diagnostics; report root cause of error based on execution unit status */ -static void report_eu_error(struct device *dev, int ch, u32 desc_hdr) +static void report_eu_error(struct device *dev, int ch, __be32 desc_hdr) { struct talitos_private *priv = dev_get_drvdata(dev); int i; if (!desc_hdr) - desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF); + desc_hdr = cpu_to_be32(in_be32(priv->chan[ch].reg + TALITOS_DESCBUF)); switch (desc_hdr & DESC_HDR_SEL0_MASK) { case DESC_HDR_SEL0_AFEU: From b2161cc0c9c461feca04cc832a5defb0ac24a513 Mon Sep 17 00:00:00 2001 From: Shiju Jose Date: Fri, 9 Oct 2020 09:19:38 +0100 Subject: [PATCH 026/360] crypto: hisilicon - Fix doc warnings in sgl.c and qm.c Fix following warnings caused by mismatch between function parameters and function comments. drivers/crypto/hisilicon/sgl.c:256: warning: Excess function parameter 'hw_sgl_dma' description in 'hisi_acc_sg_buf_unmap' drivers/crypto/hisilicon/sgl.c:256: warning: Excess function parameter 'pool' description in 'hisi_acc_sg_buf_unmap' drivers/crypto/hisilicon/qm.c:1849: warning: Function parameter or member 'qp' not described in 'qm_drain_qp' drivers/crypto/hisilicon/qm.c:2420: warning: Function parameter or member 'qm' not described in 'hisi_qm_set_vft' drivers/crypto/hisilicon/qm.c:2420: warning: Function parameter or member 'fun_num' not described in 'hisi_qm_set_vft' drivers/crypto/hisilicon/qm.c:2420: warning: Function parameter or member 'base' not described in 'hisi_qm_set_vft' drivers/crypto/hisilicon/qm.c:2420: warning: Function parameter or member 'number' not described in 'hisi_qm_set_vft' drivers/crypto/hisilicon/qm.c:2620: warning: Function parameter or member 'qm' not described in 'qm_clear_queues' Signed-off-by: Shiju Jose Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 13 +++++++++++++ drivers/crypto/hisilicon/sgl.c | 2 -- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 530f23116d7c..050fe4e74523 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -1843,6 +1843,9 @@ int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg) EXPORT_SYMBOL_GPL(hisi_qm_start_qp); /** + * qm_drain_qp() - Drain a qp. + * @qp: The qp we want to drain. + * * Determine whether the queue is cleared by judging the tail pointers of * sq and cq. */ @@ -2486,6 +2489,12 @@ int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number) EXPORT_SYMBOL_GPL(hisi_qm_get_vft); /** + * hisi_qm_set_vft() - Set vft to a qm. + * @qm: The qm we want to set its vft. + * @fun_num: The function number. + * @base: The base number of queue in vft. + * @number: The number of queues in vft. + * * This function is alway called in PF driver, it is used to assign queues * among PF and VFs. * @@ -2690,7 +2699,11 @@ static int qm_stop_started_qp(struct hisi_qm *qm) return 0; } + /** + * qm_clear_queues() - Clear all queues memory in a qm. + * @qm: The qm in which the queues will be cleared. + * * This function clears all queues memory in a qm. Reset of accelerator can * use this to clear queues. */ diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 725a739800b0..3bff6394acaf 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -246,8 +246,6 @@ EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_map_to_hw_sgl); * @dev: The device which hw sgl belongs to. * @sgl: Related scatterlist. * @hw_sgl: Virtual address of hw sgl. - * @hw_sgl_dma: DMA address of hw sgl. - * @pool: Pool which hw sgl is allocated in. * * This function unmaps allocated hw sgl. */ From c98e233062cd9d0e2f10e445a671f0799daaef67 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Sat, 10 Oct 2020 17:47:36 +0100 Subject: [PATCH 027/360] crypto: inside-secure - Fix sizeof() mismatch An incorrect sizeof() is being used, sizeof(priv->ring[i].rdr_req) is not correct, it should be sizeof(*priv->ring[i].rdr_req). Note that since the size of ** is the same size as * this is not causing any issues. Addresses-Coverity: ("Sizeof not portable (SIZEOF_MISMATCH)") Fixes: 9744fec95f06 ("crypto: inside-secure - remove request list to improve performance") Signed-off-by: Colin Ian King Acked-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index eb2418450f12..2e1562108a85 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1639,7 +1639,7 @@ static int safexcel_probe_generic(void *pdev, priv->ring[i].rdr_req = devm_kcalloc(dev, EIP197_DEFAULT_RING_SIZE, - sizeof(priv->ring[i].rdr_req), + sizeof(*priv->ring[i].rdr_req), GFP_KERNEL); if (!priv->ring[i].rdr_req) return -ENOMEM; From e4e37acc3bb0ce6152077e24cf9faad71f3c10b6 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Mon, 12 Oct 2020 21:38:17 +0100 Subject: [PATCH 028/360] crypto: qat - update IV in software Do IV update calculations in software for AES-CBC and AES-CTR. This allows to embed the IV on the request descriptor and removes the allocation of the IV buffer in the data path. In addition, this change allows the support of QAT devices that are not capable of updating the IV buffer when performing an AES-CBC or AES-CTR operation. Signed-off-by: Marco Chiappero Co-developed-by: Mateusz Polrola Signed-off-by: Mateusz Polrola Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Tested-by: Indrasena Reddy Gali Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 136 ++++++++++++--------- drivers/crypto/qat/qat_common/qat_crypto.h | 11 +- 2 files changed, 89 insertions(+), 58 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index d552dbcfe0a0..a38afc61f6d2 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include "adf_accel_devices.h" @@ -90,6 +91,7 @@ struct qat_alg_skcipher_ctx { struct qat_crypto_instance *inst; struct crypto_skcipher *ftfm; bool fallback; + int mode; }; static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) @@ -214,24 +216,7 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, return 0; } -static void qat_alg_init_hdr_iv_updt(struct icp_qat_fw_comn_req_hdr *header) -{ - ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, - ICP_QAT_FW_CIPH_IV_64BIT_PTR); - ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_UPDATE_STATE); -} - -static void qat_alg_init_hdr_no_iv_updt(struct icp_qat_fw_comn_req_hdr *header) -{ - ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, - ICP_QAT_FW_CIPH_IV_16BYTE_DATA); - ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_UPDATE_STATE); -} - -static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, - int aead) +static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) { header->hdr_flags = ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); @@ -241,12 +226,12 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, QAT_COMN_PTR_TYPE_SGL); ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags, ICP_QAT_FW_LA_PARTIAL_NONE); - if (aead) - qat_alg_init_hdr_no_iv_updt(header); - else - qat_alg_init_hdr_iv_updt(header); + ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, + ICP_QAT_FW_CIPH_IV_16BYTE_DATA); ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_PROTO); + ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_UPDATE_STATE); } static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, @@ -281,7 +266,7 @@ static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, return -EFAULT; /* Request setup */ - qat_alg_init_common_hdr(header, 1); + qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH; ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER); @@ -368,7 +353,7 @@ static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, return -EFAULT; /* Request setup */ - qat_alg_init_common_hdr(header, 1); + qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER; ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER); @@ -432,7 +417,7 @@ static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx, struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl; memcpy(cd->aes.key, key, keylen); - qat_alg_init_common_hdr(header, 0); + qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; cd_pars->u.s.content_desc_params_sz = sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; @@ -787,6 +772,61 @@ static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp, areq->base.complete(&areq->base, res); } +static void qat_alg_update_iv_ctr_mode(struct qat_crypto_request *qat_req) +{ + struct skcipher_request *sreq = qat_req->skcipher_req; + u64 iv_lo_prev; + u64 iv_lo; + u64 iv_hi; + + memcpy(qat_req->iv, sreq->iv, AES_BLOCK_SIZE); + + iv_lo = be64_to_cpu(qat_req->iv_lo); + iv_hi = be64_to_cpu(qat_req->iv_hi); + + iv_lo_prev = iv_lo; + iv_lo += DIV_ROUND_UP(sreq->cryptlen, AES_BLOCK_SIZE); + if (iv_lo < iv_lo_prev) + iv_hi++; + + qat_req->iv_lo = cpu_to_be64(iv_lo); + qat_req->iv_hi = cpu_to_be64(iv_hi); +} + +static void qat_alg_update_iv_cbc_mode(struct qat_crypto_request *qat_req) +{ + struct skcipher_request *sreq = qat_req->skcipher_req; + int offset = sreq->cryptlen - AES_BLOCK_SIZE; + struct scatterlist *sgl; + + if (qat_req->encryption) + sgl = sreq->dst; + else + sgl = sreq->src; + + scatterwalk_map_and_copy(qat_req->iv, sgl, offset, AES_BLOCK_SIZE, 0); +} + +static void qat_alg_update_iv(struct qat_crypto_request *qat_req) +{ + struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx; + struct device *dev = &GET_DEV(ctx->inst->accel_dev); + + switch (ctx->mode) { + case ICP_QAT_HW_CIPHER_CTR_MODE: + qat_alg_update_iv_ctr_mode(qat_req); + break; + case ICP_QAT_HW_CIPHER_CBC_MODE: + qat_alg_update_iv_cbc_mode(qat_req); + break; + case ICP_QAT_HW_CIPHER_XTS_MODE: + break; + default: + dev_warn(dev, "Unsupported IV update for cipher mode %d\n", + ctx->mode); + } +} + static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp, struct qat_crypto_request *qat_req) { @@ -794,16 +834,16 @@ static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp, struct qat_crypto_instance *inst = ctx->inst; struct skcipher_request *sreq = qat_req->skcipher_req; u8 stat_filed = qat_resp->comn_resp.comn_status; - struct device *dev = &GET_DEV(ctx->inst->accel_dev); int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); qat_alg_free_bufl(inst, qat_req); if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) res = -EINVAL; + if (qat_req->encryption) + qat_alg_update_iv(qat_req); + memcpy(sreq->iv, qat_req->iv, AES_BLOCK_SIZE); - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); sreq->base.complete(&sreq->base, res); } @@ -981,6 +1021,8 @@ static int qat_alg_skcipher_setkey(struct crypto_skcipher *tfm, { struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); + ctx->mode = mode; + if (ctx->enc_cd) return qat_alg_skcipher_rekey(ctx, key, keylen, mode); else @@ -1035,23 +1077,14 @@ static int qat_alg_skcipher_encrypt(struct skcipher_request *req) struct qat_crypto_request *qat_req = skcipher_request_ctx(req); struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_bulk_req *msg; - struct device *dev = &GET_DEV(ctx->inst->accel_dev); int ret, ctr = 0; if (req->cryptlen == 0) return 0; - qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE, - &qat_req->iv_paddr, GFP_ATOMIC); - if (!qat_req->iv) - return -ENOMEM; - ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req); - if (unlikely(ret)) { - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); + if (unlikely(ret)) return ret; - } msg = &qat_req->req; *msg = ctx->enc_fw_req; @@ -1061,19 +1094,18 @@ static int qat_alg_skcipher_encrypt(struct skcipher_request *req) qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req; qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + qat_req->encryption = true; cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = req->cryptlen; cipher_param->cipher_offset = 0; - cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr; - memcpy(qat_req->iv, req->iv, AES_BLOCK_SIZE); + memcpy(cipher_param->u.cipher_IV_array, req->iv, AES_BLOCK_SIZE); + do { ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg); } while (ret == -EAGAIN && ctr++ < 10); if (ret == -EAGAIN) { qat_alg_free_bufl(ctx->inst, qat_req); - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); return -EBUSY; } return -EINPROGRESS; @@ -1113,23 +1145,14 @@ static int qat_alg_skcipher_decrypt(struct skcipher_request *req) struct qat_crypto_request *qat_req = skcipher_request_ctx(req); struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_bulk_req *msg; - struct device *dev = &GET_DEV(ctx->inst->accel_dev); int ret, ctr = 0; if (req->cryptlen == 0) return 0; - qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE, - &qat_req->iv_paddr, GFP_ATOMIC); - if (!qat_req->iv) - return -ENOMEM; - ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req); - if (unlikely(ret)) { - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); + if (unlikely(ret)) return ret; - } msg = &qat_req->req; *msg = ctx->dec_fw_req; @@ -1139,19 +1162,20 @@ static int qat_alg_skcipher_decrypt(struct skcipher_request *req) qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req; qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + qat_req->encryption = false; cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = req->cryptlen; cipher_param->cipher_offset = 0; - cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr; - memcpy(qat_req->iv, req->iv, AES_BLOCK_SIZE); + memcpy(cipher_param->u.cipher_IV_array, req->iv, AES_BLOCK_SIZE); + + qat_alg_update_iv(qat_req); + do { ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg); } while (ret == -EAGAIN && ctr++ < 10); if (ret == -EAGAIN) { qat_alg_free_bufl(ctx->inst, qat_req); - dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, - qat_req->iv_paddr); return -EBUSY; } return -EINPROGRESS; diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h index 12682d1e9f5f..8d11e94cbf08 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.h +++ b/drivers/crypto/qat/qat_common/qat_crypto.h @@ -3,6 +3,7 @@ #ifndef _QAT_CRYPTO_INSTANCE_H_ #define _QAT_CRYPTO_INSTANCE_H_ +#include #include #include #include "adf_accel_devices.h" @@ -44,8 +45,14 @@ struct qat_crypto_request { struct qat_crypto_request_buffs buf; void (*cb)(struct icp_qat_fw_la_resp *resp, struct qat_crypto_request *req); - void *iv; - dma_addr_t iv_paddr; + union { + struct { + __be64 iv_hi; + __be64 iv_lo; + }; + u8 iv[AES_BLOCK_SIZE]; + }; + bool encryption; }; #endif From 7b07ed5042c5d21467af5aa055f2b49b2e661a83 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:18 +0100 Subject: [PATCH 029/360] crypto: qat - mask device capabilities with soft straps Enable acceleration engines (AEs) and accelerators based on soft straps and fuses. When looping with a number of AEs or accelerators, ignore the ones that are disabled. This patch is based on earlier work done by Conor McLoughlin. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 34 +++++++++++++++---- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h | 1 + drivers/crypto/qat/qat_c3xxx/adf_drv.c | 6 ++-- .../qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 4 +-- drivers/crypto/qat/qat_c3xxxvf/adf_drv.c | 4 +-- .../crypto/qat/qat_c62x/adf_c62x_hw_data.c | 34 +++++++++++++++---- .../crypto/qat/qat_c62x/adf_c62x_hw_data.h | 1 + drivers/crypto/qat/qat_c62x/adf_drv.c | 6 ++-- .../qat/qat_c62xvf/adf_c62xvf_hw_data.c | 4 +-- drivers/crypto/qat/qat_c62xvf/adf_drv.c | 4 +-- .../crypto/qat/qat_common/adf_accel_devices.h | 5 +-- drivers/crypto/qat/qat_common/qat_hal.c | 27 ++++++++------- .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 20 +++++++---- drivers/crypto/qat/qat_dh895xcc/adf_drv.c | 4 +-- .../qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 4 +-- drivers/crypto/qat/qat_dh895xccvf/adf_drv.c | 4 +-- 16 files changed, 109 insertions(+), 53 deletions(-) diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index aee494d3da52..4b2f5aa83391 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -17,15 +17,33 @@ static struct adf_hw_device_class c3xxx_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { - return (~fuse) >> ADF_C3XXX_ACCELERATORS_REG_OFFSET & - ADF_C3XXX_ACCELERATORS_MASK; + u32 straps = self->straps; + u32 fuses = self->fuses; + u32 accel; + + accel = ~(fuses | straps) >> ADF_C3XXX_ACCELERATORS_REG_OFFSET; + accel &= ADF_C3XXX_ACCELERATORS_MASK; + + return accel; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { - return (~fuse) & ADF_C3XXX_ACCELENGINES_MASK; + u32 straps = self->straps; + u32 fuses = self->fuses; + unsigned long disabled; + u32 ae_disable; + int accel; + + /* If an accel is disabled, then disable the corresponding two AEs */ + disabled = ~get_accel_mask(self) & ADF_C3XXX_ACCELERATORS_MASK; + ae_disable = BIT(1) | BIT(0); + for_each_set_bit(accel, &disabled, ADF_C3XXX_MAX_ACCELERATORS) + straps |= ae_disable << (accel << 1); + + return ~(fuses | straps) & ADF_C3XXX_ACCELENGINES_MASK; } static u32 get_num_accels(struct adf_hw_device_data *self) @@ -109,11 +127,13 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_device = accel_dev->hw_device; struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_C3XXX_PMISC_BAR]; + unsigned long accel_mask = hw_device->accel_mask; + unsigned long ae_mask = hw_device->ae_mask; void __iomem *csr = misc_bar->virt_addr; unsigned int val, i; /* Enable Accel Engine error detection & correction */ - for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + for_each_set_bit(i, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { val = ADF_CSR_RD(csr, ADF_C3XXX_AE_CTX_ENABLES(i)); val |= ADF_C3XXX_ENABLE_AE_ECC_ERR; ADF_CSR_WR(csr, ADF_C3XXX_AE_CTX_ENABLES(i), val); @@ -123,7 +143,7 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) } /* Enable shared memory error detection & correction */ - for (i = 0; i < hw_device->get_num_accels(hw_device); i++) { + for_each_set_bit(i, &accel_mask, ADF_C3XXX_MAX_ACCELERATORS) { val = ADF_CSR_RD(csr, ADF_C3XXX_UERRSSMSH(i)); val |= ADF_C3XXX_ERRSSMSH_EN; ADF_CSR_WR(csr, ADF_C3XXX_UERRSSMSH(i), val); diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h index 8b5dd2c94ebf..94097816f68a 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h @@ -18,6 +18,7 @@ #define ADF_C3XXX_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) #define ADF_C3XXX_SMIA0_MASK 0xFFFF #define ADF_C3XXX_SMIA1_MASK 0x1 +#define ADF_C3XXX_SOFTSTRAP_CSR_OFFSET 0x2EC /* Error detection and correction */ #define ADF_C3XXX_AE_CTX_ENABLES(i) (i * 0x1000 + 0x20818) #define ADF_C3XXX_AE_MISC_CONTROL(i) (i * 0x1000 + 0x20960) diff --git a/drivers/crypto/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/qat/qat_c3xxx/adf_drv.c index ed0e8e33fe4b..da6e88026988 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_drv.c @@ -126,10 +126,12 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_C3XXX_SOFTSTRAP_CSR_OFFSET, + &hw_data->straps); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index d2fedbd7113c..cdf8c500ef2a 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -11,12 +11,12 @@ static struct adf_hw_device_class c3xxxiov_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { return ADF_C3XXXIOV_ACCELERATORS_MASK; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { return ADF_C3XXXIOV_ACCELENGINES_MASK; } diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c index 456979b136a2..1d1532e8fb6d 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c @@ -119,8 +119,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_c3xxxiov(accel_dev->hw_device); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index 844ad5ed33fc..c0b5751e9682 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -22,15 +22,33 @@ static struct adf_hw_device_class c62x_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { - return (~fuse) >> ADF_C62X_ACCELERATORS_REG_OFFSET & - ADF_C62X_ACCELERATORS_MASK; + u32 straps = self->straps; + u32 fuses = self->fuses; + u32 accel; + + accel = ~(fuses | straps) >> ADF_C62X_ACCELERATORS_REG_OFFSET; + accel &= ADF_C62X_ACCELERATORS_MASK; + + return accel; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { - return (~fuse) & ADF_C62X_ACCELENGINES_MASK; + u32 straps = self->straps; + u32 fuses = self->fuses; + unsigned long disabled; + u32 ae_disable; + int accel; + + /* If an accel is disabled, then disable the corresponding two AEs */ + disabled = ~get_accel_mask(self) & ADF_C62X_ACCELERATORS_MASK; + ae_disable = BIT(1) | BIT(0); + for_each_set_bit(accel, &disabled, ADF_C62X_MAX_ACCELERATORS) + straps |= ae_disable << (accel << 1); + + return ~(fuses | straps) & ADF_C62X_ACCELENGINES_MASK; } static u32 get_num_accels(struct adf_hw_device_data *self) @@ -119,11 +137,13 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_device = accel_dev->hw_device; struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_C62X_PMISC_BAR]; + unsigned long accel_mask = hw_device->accel_mask; + unsigned long ae_mask = hw_device->ae_mask; void __iomem *csr = misc_bar->virt_addr; unsigned int val, i; /* Enable Accel Engine error detection & correction */ - for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + for_each_set_bit(i, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { val = ADF_CSR_RD(csr, ADF_C62X_AE_CTX_ENABLES(i)); val |= ADF_C62X_ENABLE_AE_ECC_ERR; ADF_CSR_WR(csr, ADF_C62X_AE_CTX_ENABLES(i), val); @@ -133,7 +153,7 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) } /* Enable shared memory error detection & correction */ - for (i = 0; i < hw_device->get_num_accels(hw_device); i++) { + for_each_set_bit(i, &accel_mask, ADF_C62X_MAX_ACCELERATORS) { val = ADF_CSR_RD(csr, ADF_C62X_UERRSSMSH(i)); val |= ADF_C62X_ERRSSMSH_EN; ADF_CSR_WR(csr, ADF_C62X_UERRSSMSH(i), val); diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h index 88504d2bf30d..a2e2961a2102 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h @@ -19,6 +19,7 @@ #define ADF_C62X_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) #define ADF_C62X_SMIA0_MASK 0xFFFF #define ADF_C62X_SMIA1_MASK 0x1 +#define ADF_C62X_SOFTSTRAP_CSR_OFFSET 0x2EC /* Error detection and correction */ #define ADF_C62X_AE_CTX_ENABLES(i) (i * 0x1000 + 0x20818) #define ADF_C62X_AE_MISC_CONTROL(i) (i * 0x1000 + 0x20960) diff --git a/drivers/crypto/qat/qat_c62x/adf_drv.c b/drivers/crypto/qat/qat_c62x/adf_drv.c index d8e7c9c25590..3da697a566ad 100644 --- a/drivers/crypto/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/qat/qat_c62x/adf_drv.c @@ -126,10 +126,12 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_C62X_SOFTSTRAP_CSR_OFFSET, + &hw_data->straps); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 29fd3f1091ab..a2543f75e81f 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -11,12 +11,12 @@ static struct adf_hw_device_class c62xiov_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { return ADF_C62XIOV_ACCELERATORS_MASK; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { return ADF_C62XIOV_ACCELENGINES_MASK; } diff --git a/drivers/crypto/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/qat/qat_c62xvf/adf_drv.c index b9810f79eb84..04742a6d91ca 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_drv.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_drv.c @@ -119,8 +119,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_c62xiov(accel_dev->hw_device); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 06952ece53d9..411a505e1f59 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -104,8 +104,8 @@ struct adf_etr_ring_data; struct adf_hw_device_data { struct adf_hw_device_class *dev_class; - u32 (*get_accel_mask)(u32 fuse); - u32 (*get_ae_mask)(u32 fuse); + u32 (*get_accel_mask)(struct adf_hw_device_data *self); + u32 (*get_ae_mask)(struct adf_hw_device_data *self); u32 (*get_sram_bar_id)(struct adf_hw_device_data *self); u32 (*get_misc_bar_id)(struct adf_hw_device_data *self); u32 (*get_etr_bar_id)(struct adf_hw_device_data *self); @@ -131,6 +131,7 @@ struct adf_hw_device_data { const char *fw_name; const char *fw_mmp_name; u32 fuses; + u32 straps; u32 accel_capabilities_mask; u32 instance_id; u16 accel_mask; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 6b9d47682d04..bc07199459e7 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -346,11 +346,12 @@ static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle, static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned int base_cnt, cur_cnt; unsigned char ae; int times = MAX_RETRY_TIMES; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { base_cnt = qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT); base_cnt &= 0xffff; @@ -384,6 +385,7 @@ int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned int misc_ctl; unsigned char ae; @@ -393,7 +395,7 @@ static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl & (~MC_TIMESTAMP_ENABLE)); - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0); qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); } @@ -438,6 +440,7 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) #define SHRAM_INIT_CYCLES 2060 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned int ae_reset_csr; unsigned char ae; unsigned int clk_csr; @@ -464,7 +467,7 @@ int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) goto out_err; /* Set undefined power-up/reset states to reasonable default values */ - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, INIT_CTX_ENABLE_VALUE); qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX, @@ -570,10 +573,11 @@ static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle, static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned char ae; unsigned short reg; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) { qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS, reg, 0); @@ -585,6 +589,7 @@ static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle) static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned char ae; unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX; int times = MAX_RETRY_TIMES; @@ -592,7 +597,7 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) unsigned int savctx = 0; int ret = 0; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { csr_val = qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL); csr_val &= ~(1 << MMC_SHARE_CS_BITPOS); qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); @@ -613,7 +618,7 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0); qat_hal_enable_ctx(handle, ae, ctx_mask); } - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { /* wait for AE to finish */ do { ret = qat_hal_wait_cycles(handle, ae, 20, 1); @@ -654,6 +659,8 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct adf_bar *misc_bar = &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)]; + unsigned long ae_mask = hw_data->ae_mask; + unsigned int csr_val = 0; struct adf_bar *sram_bar; handle = kzalloc(sizeof(*handle), GFP_KERNEL); @@ -689,9 +696,7 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) /* create AE objects */ handle->hal_handle->upc_mask = 0x1ffff; handle->hal_handle->max_ustore = 0x4000; - for (ae = 0; ae < ICP_QAT_UCLO_MAX_AE; ae++) { - if (!(hw_data->ae_mask & (1 << ae))) - continue; + for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) { handle->hal_handle->aes[ae].free_addr = 0; handle->hal_handle->aes[ae].free_size = handle->hal_handle->max_ustore; @@ -714,9 +719,7 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) } /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - unsigned int csr_val = 0; - + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { csr_val = qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE); csr_val |= 0x1; qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index b975c263446d..6a0d01103136 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -24,15 +24,19 @@ static struct adf_hw_device_class dh895xcc_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { - return (~fuse) >> ADF_DH895XCC_ACCELERATORS_REG_OFFSET & - ADF_DH895XCC_ACCELERATORS_MASK; + u32 fuses = self->fuses; + + return ~fuses >> ADF_DH895XCC_ACCELERATORS_REG_OFFSET & + ADF_DH895XCC_ACCELERATORS_MASK; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { - return (~fuse) & ADF_DH895XCC_ACCELENGINES_MASK; + u32 fuses = self->fuses; + + return ~fuses & ADF_DH895XCC_ACCELENGINES_MASK; } static u32 get_num_accels(struct adf_hw_device_data *self) @@ -131,11 +135,13 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_device = accel_dev->hw_device; struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR]; + unsigned long accel_mask = hw_device->accel_mask; + unsigned long ae_mask = hw_device->ae_mask; void __iomem *csr = misc_bar->virt_addr; unsigned int val, i; /* Enable Accel Engine error detection & correction */ - for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + for_each_set_bit(i, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { val = ADF_CSR_RD(csr, ADF_DH895XCC_AE_CTX_ENABLES(i)); val |= ADF_DH895XCC_ENABLE_AE_ECC_ERR; ADF_CSR_WR(csr, ADF_DH895XCC_AE_CTX_ENABLES(i), val); @@ -145,7 +151,7 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) } /* Enable shared memory error detection & correction */ - for (i = 0; i < hw_device->get_num_accels(hw_device); i++) { + for_each_set_bit(i, &accel_mask, ADF_DH895XCC_MAX_ACCELERATORS) { val = ADF_CSR_RD(csr, ADF_DH895XCC_UERRSSMSH(i)); val |= ADF_DH895XCC_ERRSSMSH_EN; ADF_CSR_WR(csr, ADF_DH895XCC_UERRSSMSH(i), val); diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c index ecb4f6f20e22..d7941bc2bafd 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c @@ -128,8 +128,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) &hw_data->fuses); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index 5246f0524ca3..737f9132f71a 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -11,12 +11,12 @@ static struct adf_hw_device_class dh895xcciov_class = { .instances = 0 }; -static u32 get_accel_mask(u32 fuse) +static u32 get_accel_mask(struct adf_hw_device_data *self) { return ADF_DH895XCCIOV_ACCELERATORS_MASK; } -static u32 get_ae_mask(u32 fuse) +static u32 get_ae_mask(struct adf_hw_device_data *self) { return ADF_DH895XCCIOV_ACCELENGINES_MASK; } diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c index 404cf9df6922..c972554a755e 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c @@ -119,8 +119,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_dh895xcciov(accel_dev->hw_device); /* Get Accelerators and Accelerators Engines masks */ - hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); - hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); accel_pci_dev->sku = hw_data->get_sku(hw_data); /* Create dev top level debugfs entry */ From 70b9bd3929da903d54dbd7d1ca2465333077cfd9 Mon Sep 17 00:00:00 2001 From: Ahsan Atta Date: Mon, 12 Oct 2020 21:38:19 +0100 Subject: [PATCH 030/360] crypto: qat - num_rings_per_bank is device dependent This change is to allow support for QAT devices that may not have 16 rings per bank. The rings structure in bank is allocated dynamically based on the number of banks supported by a device. Note that in the error path in adf_init_bank(), ring->inflights is set to NULL after the free to silence a false positive double free reported by clang scan-build. Signed-off-by: Ahsan Atta Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 1 + .../qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 1 + .../crypto/qat/qat_c62x/adf_c62x_hw_data.c | 1 + .../qat/qat_c62xvf/adf_c62xvf_hw_data.c | 1 + .../crypto/qat/qat_common/adf_accel_devices.h | 3 ++ drivers/crypto/qat/qat_common/adf_transport.c | 42 +++++++++++++------ .../qat/qat_common/adf_transport_debug.c | 10 ++++- .../qat/qat_common/adf_transport_internal.h | 2 +- .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 1 + .../qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 1 + 10 files changed, 47 insertions(+), 16 deletions(-) diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 4b2f5aa83391..62b0b290ff85 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -176,6 +176,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->dev_class = &c3xxx_class; hw_data->instance_id = c3xxx_class.instances++; hw_data->num_banks = ADF_C3XXX_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_C3XXX_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_C3XXX_MAX_ACCELENGINES; diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index cdf8c500ef2a..80a355e85a72 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -69,6 +69,7 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &c3xxxiov_class; hw_data->num_banks = ADF_C3XXXIOV_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_C3XXXIOV_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_C3XXXIOV_MAX_ACCELENGINES; diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index c0b5751e9682..1334b43e46e4 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -186,6 +186,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->dev_class = &c62x_class; hw_data->instance_id = c62x_class.instances++; hw_data->num_banks = ADF_C62X_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_C62X_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_C62X_MAX_ACCELENGINES; diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c index a2543f75e81f..7725387e58f8 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -69,6 +69,7 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &c62xiov_class; hw_data->num_banks = ADF_C62XIOV_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_C62XIOV_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_C62XIOV_MAX_ACCELENGINES; diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 411a505e1f59..85b423d28f77 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -139,6 +139,7 @@ struct adf_hw_device_data { u16 tx_rings_mask; u8 tx_rx_gap; u8 num_banks; + u8 num_rings_per_bank; u8 num_accel; u8 num_logical_accel; u8 num_engines; @@ -156,6 +157,8 @@ struct adf_hw_device_data { #define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars) #define GET_HW_DATA(accel_dev) (accel_dev->hw_device) #define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks) +#define GET_NUM_RINGS_PER_BANK(accel_dev) \ + GET_HW_DATA(accel_dev)->num_rings_per_bank #define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines) #define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c index 2ad774017200..24ddaaaa55b1 100644 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -190,6 +190,7 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, struct adf_etr_ring_data **ring_ptr) { struct adf_etr_data *transport_data = accel_dev->transport; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev); struct adf_etr_bank_data *bank; struct adf_etr_ring_data *ring; char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; @@ -219,7 +220,7 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section, dev_err(&GET_DEV(accel_dev), "Can't get ring number\n"); return -EFAULT; } - if (ring_num >= ADF_ETR_MAX_RINGS_PER_BANK) { + if (ring_num >= num_rings_per_bank) { dev_err(&GET_DEV(accel_dev), "Invalid ring number\n"); return -EFAULT; } @@ -286,15 +287,15 @@ void adf_remove_ring(struct adf_etr_ring_data *ring) static void adf_ring_response_handler(struct adf_etr_bank_data *bank) { - u32 empty_rings, i; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(bank->accel_dev); + unsigned long empty_rings; + int i; empty_rings = READ_CSR_E_STAT(bank->csr_addr, bank->bank_number); empty_rings = ~empty_rings & bank->irq_mask; - for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; ++i) { - if (empty_rings & (1 << i)) - adf_handle_response(&bank->rings[i]); - } + for_each_set_bit(i, &empty_rings, num_rings_per_bank) + adf_handle_response(&bank->rings[i]); } void adf_response_handler(uintptr_t bank_addr) @@ -343,9 +344,12 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, u32 bank_num, void __iomem *csr_addr) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u8 num_rings_per_bank = hw_data->num_rings_per_bank; struct adf_etr_ring_data *ring; struct adf_etr_ring_data *tx_ring; u32 i, coalesc_enabled = 0; + unsigned long ring_mask; + int size; memset(bank, 0, sizeof(*bank)); bank->bank_number = bank_num; @@ -353,6 +357,13 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, bank->accel_dev = accel_dev; spin_lock_init(&bank->lock); + /* Allocate the rings in the bank */ + size = num_rings_per_bank * sizeof(struct adf_etr_ring_data); + bank->rings = kzalloc_node(size, GFP_KERNEL, + dev_to_node(&GET_DEV(accel_dev))); + if (!bank->rings) + return -ENOMEM; + /* Enable IRQ coalescing always. This will allow to use * the optimised flag and coalesc register. * If it is disabled in the config file just use min time value */ @@ -363,7 +374,7 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, else bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME; - for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { + for (i = 0; i < num_rings_per_bank; i++) { WRITE_CSR_RING_CONFIG(csr_addr, bank_num, i, 0); WRITE_CSR_RING_BASE(csr_addr, bank_num, i, 0); ring = &bank->rings[i]; @@ -394,11 +405,13 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, WRITE_CSR_INT_SRCSEL(csr_addr, bank_num); return 0; err: - for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { + ring_mask = hw_data->tx_rings_mask; + for_each_set_bit(i, &ring_mask, num_rings_per_bank) { ring = &bank->rings[i]; - if (hw_data->tx_rings_mask & (1 << i)) - kfree(ring->inflights); + kfree(ring->inflights); + ring->inflights = NULL; } + kfree(bank->rings); return -ENOMEM; } @@ -464,11 +477,12 @@ EXPORT_SYMBOL_GPL(adf_init_etr_data); static void cleanup_bank(struct adf_etr_bank_data *bank) { + struct adf_accel_dev *accel_dev = bank->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u8 num_rings_per_bank = hw_data->num_rings_per_bank; u32 i; - for (i = 0; i < ADF_ETR_MAX_RINGS_PER_BANK; i++) { - struct adf_accel_dev *accel_dev = bank->accel_dev; - struct adf_hw_device_data *hw_data = accel_dev->hw_device; + for (i = 0; i < num_rings_per_bank; i++) { struct adf_etr_ring_data *ring = &bank->rings[i]; if (bank->ring_mask & (1 << i)) @@ -477,6 +491,7 @@ static void cleanup_bank(struct adf_etr_bank_data *bank) if (hw_data->tx_rings_mask & (1 << i)) kfree(ring->inflights); } + kfree(bank->rings); adf_bank_debugfs_rm(bank); memset(bank, 0, sizeof(*bank)); } @@ -507,6 +522,7 @@ void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev) if (etr_data) { adf_cleanup_etr_handles(accel_dev); debugfs_remove(etr_data->debug); + kfree(etr_data->banks->rings); kfree(etr_data->banks); kfree(etr_data); accel_dev->transport = NULL; diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c index dac25ba47260..da79d734c035 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/qat/qat_common/adf_transport_debug.c @@ -117,11 +117,14 @@ void adf_ring_debugfs_rm(struct adf_etr_ring_data *ring) static void *adf_bank_start(struct seq_file *sfile, loff_t *pos) { + struct adf_etr_bank_data *bank = sfile->private; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(bank->accel_dev); + mutex_lock(&bank_read_lock); if (*pos == 0) return SEQ_START_TOKEN; - if (*pos >= ADF_ETR_MAX_RINGS_PER_BANK) + if (*pos >= num_rings_per_bank) return NULL; return pos; @@ -129,7 +132,10 @@ static void *adf_bank_start(struct seq_file *sfile, loff_t *pos) static void *adf_bank_next(struct seq_file *sfile, void *v, loff_t *pos) { - if (++(*pos) >= ADF_ETR_MAX_RINGS_PER_BANK) + struct adf_etr_bank_data *bank = sfile->private; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(bank->accel_dev); + + if (++(*pos) >= num_rings_per_bank) return NULL; return pos; diff --git a/drivers/crypto/qat/qat_common/adf_transport_internal.h b/drivers/crypto/qat/qat_common/adf_transport_internal.h index c7faf4e2d302..501bcf0f1809 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_internal.h +++ b/drivers/crypto/qat/qat_common/adf_transport_internal.h @@ -28,7 +28,7 @@ struct adf_etr_ring_data { }; struct adf_etr_bank_data { - struct adf_etr_ring_data rings[ADF_ETR_MAX_RINGS_PER_BANK]; + struct adf_etr_ring_data *rings; struct tasklet_struct resp_handler; void __iomem *csr_addr; u32 irq_coalesc_timer; diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 6a0d01103136..1f3ea3ba1cee 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -185,6 +185,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->dev_class = &dh895xcc_class; hw_data->instance_id = dh895xcc_class.instances++; hw_data->num_banks = ADF_DH895XCC_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_DH895XCC_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_DH895XCC_MAX_ACCELENGINES; diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index 737f9132f71a..eca144bc1d67 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -69,6 +69,7 @@ void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &dh895xcciov_class; hw_data->num_banks = ADF_DH895XCCIOV_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_ETR_MAX_RINGS_PER_BANK; hw_data->num_accel = ADF_DH895XCCIOV_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; hw_data->num_engines = ADF_DH895XCCIOV_MAX_ACCELENGINES; From c4e8428673425c5b0c7280387c5cb35d60959b97 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:20 +0100 Subject: [PATCH 031/360] crypto: qat - fix configuration of iov threads The number of AE2FUNC_MAP registers is different in every QAT device (c62x, c3xxx and dh895xcc) although the logic and the register offsets are the same across devices. This patch separates the logic that configures the iov threads in a common function that takes as input the number of AE2FUNC_MAP registers supported by a device. The function is then added to the adf_hw_device_data structure of each device, and called with the appropriate parameters. The configure iov thread logic is added to a new file, adf_gen2_hw_data.c, that is going to contain code that is shared across QAT GEN2 devices. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 9 +++ .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h | 4 ++ .../crypto/qat/qat_c62x/adf_c62x_hw_data.c | 9 +++ .../crypto/qat/qat_c62x/adf_c62x_hw_data.h | 4 ++ drivers/crypto/qat/qat_common/Makefile | 1 + .../crypto/qat/qat_common/adf_accel_devices.h | 2 + .../crypto/qat/qat_common/adf_gen2_hw_data.c | 38 +++++++++++ .../crypto/qat/qat_common/adf_gen2_hw_data.h | 30 +++++++++ drivers/crypto/qat/qat_common/adf_sriov.c | 63 ++----------------- .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 9 +++ .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.h | 5 ++ 11 files changed, 115 insertions(+), 59 deletions(-) create mode 100644 drivers/crypto/qat/qat_common/adf_gen2_hw_data.c create mode 100644 drivers/crypto/qat/qat_common/adf_gen2_hw_data.h diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 62b0b290ff85..f449b2a0e82d 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_c3xxx_hw_data.h" /* Worker thread to service arbiter mappings based on dev SKUs */ @@ -171,6 +172,13 @@ static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) return 0; } +static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) +{ + adf_gen2_cfg_iov_thds(accel_dev, enable, + ADF_C3XXX_AE2FUNC_MAP_GRP_A_NUM_REGS, + ADF_C3XXX_AE2FUNC_MAP_GRP_B_NUM_REGS); +} + void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &c3xxx_class; @@ -199,6 +207,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->fw_mmp_name = ADF_C3XXX_MMP; hw_data->init_admin_comms = adf_init_admin_comms; hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->configure_iov_threads = configure_iov_threads; hw_data->disable_iov = adf_disable_sriov; hw_data->send_admin_init = adf_send_admin_init; hw_data->init_arb = adf_init_arb; diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h index 94097816f68a..fece8e38025a 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h @@ -31,6 +31,10 @@ #define ADF_C3XXX_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) #define ADF_C3XXX_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) +/* AE to function mapping */ +#define ADF_C3XXX_AE2FUNC_MAP_GRP_A_NUM_REGS 48 +#define ADF_C3XXX_AE2FUNC_MAP_GRP_B_NUM_REGS 6 + /* Firmware Binary */ #define ADF_C3XXX_FW "qat_c3xxx.bin" #define ADF_C3XXX_MMP "qat_c3xxx_mmp.bin" diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index 1334b43e46e4..d7bed610ae86 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_c62x_hw_data.h" /* Worker thread to service arbiter mappings based on dev SKUs */ @@ -181,6 +182,13 @@ static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) return 0; } +static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) +{ + adf_gen2_cfg_iov_thds(accel_dev, enable, + ADF_C62X_AE2FUNC_MAP_GRP_A_NUM_REGS, + ADF_C62X_AE2FUNC_MAP_GRP_B_NUM_REGS); +} + void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &c62x_class; @@ -209,6 +217,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->fw_mmp_name = ADF_C62X_MMP; hw_data->init_admin_comms = adf_init_admin_comms; hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->configure_iov_threads = configure_iov_threads; hw_data->disable_iov = adf_disable_sriov; hw_data->send_admin_init = adf_send_admin_init; hw_data->init_arb = adf_init_arb; diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h index a2e2961a2102..53d3cb577f5b 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h @@ -32,6 +32,10 @@ #define ADF_C62X_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) #define ADF_C62X_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) +/* AE to function mapping */ +#define ADF_C62X_AE2FUNC_MAP_GRP_A_NUM_REGS 80 +#define ADF_C62X_AE2FUNC_MAP_GRP_B_NUM_REGS 10 + /* Firmware Binary */ #define ADF_C62X_FW "qat_c62x.bin" #define ADF_C62X_MMP "qat_c62x_mmp.bin" diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 47a8e3d8b81a..25d28516dcdd 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -10,6 +10,7 @@ intel_qat-objs := adf_cfg.o \ adf_transport.o \ adf_admin.o \ adf_hw_arbiter.o \ + adf_gen2_hw_data.o \ qat_crypto.o \ qat_algs.o \ qat_asym_algs.o \ diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 85b423d28f77..d7a27d15e137 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -125,6 +125,8 @@ struct adf_hw_device_data { void (*get_arb_mapping)(struct adf_accel_dev *accel_dev, const u32 **cfg); void (*disable_iov)(struct adf_accel_dev *accel_dev); + void (*configure_iov_threads)(struct adf_accel_dev *accel_dev, + bool enable); void (*enable_ints)(struct adf_accel_dev *accel_dev); int (*enable_vf2pf_comms)(struct adf_accel_dev *accel_dev); void (*reset_device)(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c new file mode 100644 index 000000000000..26e345e3d7c3 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ +#include "adf_gen2_hw_data.h" + +void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, + int num_a_regs, int num_b_regs) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + void __iomem *pmisc_addr; + struct adf_bar *pmisc; + int pmisc_id, i; + u32 reg; + + pmisc_id = hw_data->get_misc_bar_id(hw_data); + pmisc = &GET_BARS(accel_dev)[pmisc_id]; + pmisc_addr = pmisc->virt_addr; + + /* Set/Unset Valid bit in AE Thread to PCIe Function Mapping Group A */ + for (i = 0; i < num_a_regs; i++) { + reg = READ_CSR_AE2FUNCTION_MAP_A(pmisc_addr, i); + if (enable) + reg |= AE2FUNCTION_MAP_VALID; + else + reg &= ~AE2FUNCTION_MAP_VALID; + WRITE_CSR_AE2FUNCTION_MAP_A(pmisc_addr, i, reg); + } + + /* Set/Unset Valid bit in AE Thread to PCIe Function Mapping Group B */ + for (i = 0; i < num_b_regs; i++) { + reg = READ_CSR_AE2FUNCTION_MAP_B(pmisc_addr, i); + if (enable) + reg |= AE2FUNCTION_MAP_VALID; + else + reg &= ~AE2FUNCTION_MAP_VALID; + WRITE_CSR_AE2FUNCTION_MAP_B(pmisc_addr, i, reg); + } +} +EXPORT_SYMBOL_GPL(adf_gen2_cfg_iov_thds); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h new file mode 100644 index 000000000000..1d348425d5f4 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2020 Intel Corporation */ +#ifndef ADF_GEN2_HW_DATA_H_ +#define ADF_GEN2_HW_DATA_H_ + +#include "adf_accel_devices.h" + +/* AE to function map */ +#define AE2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190) +#define AE2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310) +#define AE2FUNCTION_MAP_REG_SIZE 4 +#define AE2FUNCTION_MAP_VALID BIT(7) + +#define READ_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index) \ + ADF_CSR_RD(pmisc_bar_addr, AE2FUNCTION_MAP_A_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index)) +#define WRITE_CSR_AE2FUNCTION_MAP_A(pmisc_bar_addr, index, value) \ + ADF_CSR_WR(pmisc_bar_addr, AE2FUNCTION_MAP_A_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index), value) +#define READ_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index) \ + ADF_CSR_RD(pmisc_bar_addr, AE2FUNCTION_MAP_B_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index)) +#define WRITE_CSR_AE2FUNCTION_MAP_B(pmisc_bar_addr, index, value) \ + ADF_CSR_WR(pmisc_bar_addr, AE2FUNCTION_MAP_B_OFFSET + \ + AE2FUNCTION_MAP_REG_SIZE * (index), value) + +void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, + int num_a_regs, int num_b_regs); + +#endif diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 963b2bea78f2..dde6c57ef15a 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -10,31 +10,6 @@ static struct workqueue_struct *pf2vf_resp_wq; -#define ME2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190) -#define ME2FUNCTION_MAP_A_NUM_REGS 96 - -#define ME2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310) -#define ME2FUNCTION_MAP_B_NUM_REGS 12 - -#define ME2FUNCTION_MAP_REG_SIZE 4 -#define ME2FUNCTION_MAP_VALID BIT(7) - -#define READ_CSR_ME2FUNCTION_MAP_A(pmisc_bar_addr, index) \ - ADF_CSR_RD(pmisc_bar_addr, ME2FUNCTION_MAP_A_OFFSET + \ - ME2FUNCTION_MAP_REG_SIZE * index) - -#define WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_bar_addr, index, value) \ - ADF_CSR_WR(pmisc_bar_addr, ME2FUNCTION_MAP_A_OFFSET + \ - ME2FUNCTION_MAP_REG_SIZE * index, value) - -#define READ_CSR_ME2FUNCTION_MAP_B(pmisc_bar_addr, index) \ - ADF_CSR_RD(pmisc_bar_addr, ME2FUNCTION_MAP_B_OFFSET + \ - ME2FUNCTION_MAP_REG_SIZE * index) - -#define WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_bar_addr, index, value) \ - ADF_CSR_WR(pmisc_bar_addr, ME2FUNCTION_MAP_B_OFFSET + \ - ME2FUNCTION_MAP_REG_SIZE * index, value) - struct adf_pf2vf_resp { struct work_struct pf2vf_resp_work; struct adf_accel_vf_info *vf_info; @@ -68,12 +43,8 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) struct pci_dev *pdev = accel_to_pci_dev(accel_dev); int totalvfs = pci_sriov_get_totalvfs(pdev); struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *pmisc = - &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; - void __iomem *pmisc_addr = pmisc->virt_addr; struct adf_accel_vf_info *vf_info; int i; - u32 reg; for (i = 0, vf_info = accel_dev->pf.vf_info; i < totalvfs; i++, vf_info++) { @@ -90,19 +61,8 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) DEFAULT_RATELIMIT_BURST); } - /* Set Valid bits in ME Thread to PCIe Function Mapping Group A */ - for (i = 0; i < ME2FUNCTION_MAP_A_NUM_REGS; i++) { - reg = READ_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i); - reg |= ME2FUNCTION_MAP_VALID; - WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i, reg); - } - - /* Set Valid bits in ME Thread to PCIe Function Mapping Group B */ - for (i = 0; i < ME2FUNCTION_MAP_B_NUM_REGS; i++) { - reg = READ_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i); - reg |= ME2FUNCTION_MAP_VALID; - WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i, reg); - } + /* Set Valid bits in AE Thread to PCIe Function Mapping */ + hw_data->configure_iov_threads(accel_dev, true); /* Enable VF to PF interrupts for all VFs */ adf_enable_vf2pf_interrupts(accel_dev, GENMASK_ULL(totalvfs - 1, 0)); @@ -127,12 +87,8 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) void adf_disable_sriov(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; - struct adf_bar *pmisc = - &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; - void __iomem *pmisc_addr = pmisc->virt_addr; int totalvfs = pci_sriov_get_totalvfs(accel_to_pci_dev(accel_dev)); struct adf_accel_vf_info *vf; - u32 reg; int i; if (!accel_dev->pf.vf_info) @@ -145,19 +101,8 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) /* Disable VF to PF interrupts */ adf_disable_vf2pf_interrupts(accel_dev, 0xFFFFFFFF); - /* Clear Valid bits in ME Thread to PCIe Function Mapping Group A */ - for (i = 0; i < ME2FUNCTION_MAP_A_NUM_REGS; i++) { - reg = READ_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i); - reg &= ~ME2FUNCTION_MAP_VALID; - WRITE_CSR_ME2FUNCTION_MAP_A(pmisc_addr, i, reg); - } - - /* Clear Valid bits in ME Thread to PCIe Function Mapping Group B */ - for (i = 0; i < ME2FUNCTION_MAP_B_NUM_REGS; i++) { - reg = READ_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i); - reg &= ~ME2FUNCTION_MAP_VALID; - WRITE_CSR_ME2FUNCTION_MAP_B(pmisc_addr, i, reg); - } + /* Clear Valid bits in AE Thread to PCIe Function Mapping */ + hw_data->configure_iov_threads(accel_dev, false); for (i = 0, vf = accel_dev->pf.vf_info; i < totalvfs; i++, vf++) { tasklet_disable(&vf->vf2pf_bh_tasklet); diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 1f3ea3ba1cee..7b2f13ff49fd 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_dh895xcc_hw_data.h" /* Worker thread to service arbiter mappings based on dev SKUs */ @@ -180,6 +181,13 @@ static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) return 0; } +static void configure_iov_threads(struct adf_accel_dev *accel_dev, bool enable) +{ + adf_gen2_cfg_iov_thds(accel_dev, enable, + ADF_DH895XCC_AE2FUNC_MAP_GRP_A_NUM_REGS, + ADF_DH895XCC_AE2FUNC_MAP_GRP_B_NUM_REGS); +} + void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &dh895xcc_class; @@ -208,6 +216,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->fw_mmp_name = ADF_DH895XCC_MMP; hw_data->init_admin_comms = adf_init_admin_comms; hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->configure_iov_threads = configure_iov_threads; hw_data->disable_iov = adf_disable_sriov; hw_data->send_admin_init = adf_send_admin_init; hw_data->init_arb = adf_init_arb; diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h index 082a04466dca..4d613923d155 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h @@ -36,6 +36,11 @@ #define ADF_DH895XCC_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) #define ADF_DH895XCC_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) + +/* AE to function mapping */ +#define ADF_DH895XCC_AE2FUNC_MAP_GRP_A_NUM_REGS 96 +#define ADF_DH895XCC_AE2FUNC_MAP_GRP_B_NUM_REGS 12 + /* FW names */ #define ADF_DH895XCC_FW "qat_895xcc.bin" #define ADF_DH895XCC_MMP "qat_895xcc_mmp.bin" From f9c697c4bfc6002d92e8015b65fe2a79acadb416 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:21 +0100 Subject: [PATCH 032/360] crypto: qat - split transport CSR access logic Abstract access to transport CSRs and move generation specific code into adf_gen2_hw_data.c in preparation for the introduction of the qat_4xxx driver. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 1 + .../qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 2 + .../crypto/qat/qat_c62x/adf_c62x_hw_data.c | 1 + .../qat/qat_c62xvf/adf_c62xvf_hw_data.c | 2 + .../crypto/qat/qat_common/adf_accel_devices.h | 27 ++++++ .../crypto/qat/qat_common/adf_gen2_hw_data.c | 85 ++++++++++++++++++ .../crypto/qat/qat_common/adf_gen2_hw_data.h | 1 + drivers/crypto/qat/qat_common/adf_isr.c | 4 +- drivers/crypto/qat/qat_common/adf_transport.c | 86 +++++++++++++------ .../qat/qat_common/adf_transport_debug.c | 22 ++--- drivers/crypto/qat/qat_common/adf_vf_isr.c | 5 +- .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 1 + .../qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 2 + 13 files changed, 198 insertions(+), 41 deletions(-) diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index f449b2a0e82d..7af38b947cfe 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -217,6 +217,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; hw_data->reset_device = adf_reset_flr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_c3xxx(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index 80a355e85a72..15f6b9bdfb22 100644 --- a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_c3xxxvf_hw_data.h" static struct adf_hw_device_class c3xxxiov_class = { @@ -98,6 +99,7 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index d7bed610ae86..c18fb77dd8ec 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -227,6 +227,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; hw_data->reset_device = adf_reset_flr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_c62x(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 7725387e58f8..d231583428c9 100644 --- a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_c62xvf_hw_data.h" static struct adf_hw_device_class c62xiov_class = { @@ -98,6 +99,7 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data) hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_c62xiov(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index d7a27d15e137..459e22076813 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -97,6 +97,31 @@ struct adf_hw_device_class { u32 instances; } __packed; +struct adf_hw_csr_ops { + u32 (*read_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, + u32 ring); + void (*write_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value); + u32 (*read_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, + u32 ring); + void (*write_csr_ring_tail)(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value); + u32 (*read_csr_e_stat)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_ring_config)(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value); + void (*write_csr_ring_base)(void __iomem *csr_base_addr, u32 bank, + u32 ring, dma_addr_t addr); + void (*write_csr_int_flag)(void __iomem *csr_base_addr, u32 bank, + u32 value); + void (*write_csr_int_srcsel)(void __iomem *csr_base_addr, u32 bank); + void (*write_csr_int_col_en)(void __iomem *csr_base_addr, u32 bank, + u32 value); + void (*write_csr_int_col_ctl)(void __iomem *csr_base_addr, u32 bank, + u32 value); + void (*write_csr_int_flag_and_col)(void __iomem *csr_base_addr, + u32 bank, u32 value); +}; + struct adf_cfg_device_data; struct adf_accel_dev; struct adf_etr_data; @@ -130,6 +155,7 @@ struct adf_hw_device_data { void (*enable_ints)(struct adf_accel_dev *accel_dev); int (*enable_vf2pf_comms)(struct adf_accel_dev *accel_dev); void (*reset_device)(struct adf_accel_dev *accel_dev); + struct adf_hw_csr_ops csr_ops; const char *fw_name; const char *fw_mmp_name; u32 fuses; @@ -162,6 +188,7 @@ struct adf_hw_device_data { #define GET_NUM_RINGS_PER_BANK(accel_dev) \ GET_HW_DATA(accel_dev)->num_rings_per_bank #define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines) +#define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops) #define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev struct adf_admin_comms; diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 26e345e3d7c3..07a9211bf7f9 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ #include "adf_gen2_hw_data.h" +#include "adf_transport_access_macros.h" void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs) @@ -36,3 +37,87 @@ void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, } } EXPORT_SYMBOL_GPL(adf_gen2_cfg_iov_thds); + +static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); +} + +static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); +} + +static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_E_STAT(csr_base_addr, bank); +} + +static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, + u32 ring, u32 value) +{ + WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); +} + +static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring, + dma_addr_t addr) +{ + WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); +} + +static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); +} + +static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) +{ + WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); +} + +static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); +} + +static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); +} + +static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); +} + +void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) +{ + csr_ops->read_csr_ring_head = read_csr_ring_head; + csr_ops->write_csr_ring_head = write_csr_ring_head; + csr_ops->read_csr_ring_tail = read_csr_ring_tail; + csr_ops->write_csr_ring_tail = write_csr_ring_tail; + csr_ops->read_csr_e_stat = read_csr_e_stat; + csr_ops->write_csr_ring_config = write_csr_ring_config; + csr_ops->write_csr_ring_base = write_csr_ring_base; + csr_ops->write_csr_int_flag = write_csr_int_flag; + csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; + csr_ops->write_csr_int_col_en = write_csr_int_col_en; + csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; + csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; +} +EXPORT_SYMBOL_GPL(adf_gen2_init_hw_csr_ops); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index 1d348425d5f4..e6d3919a56a1 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -26,5 +26,6 @@ void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs); +void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); #endif diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index 36136f7db509..5444f0ea0a1d 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -50,8 +50,10 @@ static void adf_disable_msix(struct adf_accel_pci *pci_dev_info) static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr) { struct adf_etr_bank_data *bank = bank_ptr; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); - WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, 0); + csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number, + 0); tasklet_hi_schedule(&bank->resp_handler); return IRQ_HANDLED; } diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c index 24ddaaaa55b1..03fb7812818b 100644 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -54,24 +54,32 @@ static void adf_unreserve_ring(struct adf_etr_bank_data *bank, u32 ring) static void adf_enable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + spin_lock_bh(&bank->lock); bank->irq_mask |= (1 << ring); spin_unlock_bh(&bank->lock); - WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask); - WRITE_CSR_INT_COL_CTL(bank->csr_addr, bank->bank_number, - bank->irq_coalesc_timer); + csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number, + bank->irq_mask); + csr_ops->write_csr_int_col_ctl(bank->csr_addr, bank->bank_number, + bank->irq_coalesc_timer); } static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); + spin_lock_bh(&bank->lock); bank->irq_mask &= ~(1 << ring); spin_unlock_bh(&bank->lock); - WRITE_CSR_INT_COL_EN(bank->csr_addr, bank->bank_number, bank->irq_mask); + csr_ops->write_csr_int_col_en(bank->csr_addr, bank->bank_number, + bank->irq_mask); } int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); + if (atomic_add_return(1, ring->inflights) > ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size)) { atomic_dec(ring->inflights); @@ -84,14 +92,17 @@ int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg) ring->tail = adf_modulo(ring->tail + ADF_MSG_SIZE_TO_BYTES(ring->msg_size), ADF_RING_SIZE_MODULO(ring->ring_size)); - WRITE_CSR_RING_TAIL(ring->bank->csr_addr, ring->bank->bank_number, - ring->ring_number, ring->tail); + csr_ops->write_csr_ring_tail(ring->bank->csr_addr, + ring->bank->bank_number, ring->ring_number, + ring->tail); spin_unlock_bh(&ring->lock); + return 0; } static int adf_handle_response(struct adf_etr_ring_data *ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); u32 msg_counter = 0; u32 *msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); @@ -105,30 +116,36 @@ static int adf_handle_response(struct adf_etr_ring_data *ring) msg_counter++; msg = (u32 *)((uintptr_t)ring->base_addr + ring->head); } - if (msg_counter > 0) - WRITE_CSR_RING_HEAD(ring->bank->csr_addr, - ring->bank->bank_number, - ring->ring_number, ring->head); + if (msg_counter > 0) { + csr_ops->write_csr_ring_head(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, ring->head); + } return 0; } static void adf_configure_tx_ring(struct adf_etr_ring_data *ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); u32 ring_config = BUILD_RING_CONFIG(ring->ring_size); - WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, ring->bank->bank_number, - ring->ring_number, ring_config); + csr_ops->write_csr_ring_config(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, ring_config); + } static void adf_configure_rx_ring(struct adf_etr_ring_data *ring) { + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev); u32 ring_config = BUILD_RESP_RING_CONFIG(ring->ring_size, ADF_RING_NEAR_WATERMARK_512, ADF_RING_NEAR_WATERMARK_0); - WRITE_CSR_RING_CONFIG(ring->bank->csr_addr, ring->bank->bank_number, - ring->ring_number, ring_config); + csr_ops->write_csr_ring_config(ring->bank->csr_addr, + ring->bank->bank_number, + ring->ring_number, ring_config); } static int adf_init_ring(struct adf_etr_ring_data *ring) @@ -136,6 +153,7 @@ static int adf_init_ring(struct adf_etr_ring_data *ring) struct adf_etr_bank_data *bank = ring->bank; struct adf_accel_dev *accel_dev = bank->accel_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); u64 ring_base; u32 ring_size_bytes = ADF_SIZE_TO_RING_SIZE_IN_BYTES(ring->ring_size); @@ -163,8 +181,9 @@ static int adf_init_ring(struct adf_etr_ring_data *ring) adf_configure_rx_ring(ring); ring_base = BUILD_RING_BASE_ADDR(ring->dma_addr, ring->ring_size); - WRITE_CSR_RING_BASE(ring->bank->csr_addr, ring->bank->bank_number, - ring->ring_number, ring_base); + csr_ops->write_csr_ring_base(ring->bank->csr_addr, + ring->bank->bank_number, ring->ring_number, + ring_base); spin_lock_init(&ring->lock); return 0; } @@ -269,15 +288,17 @@ err: void adf_remove_ring(struct adf_etr_ring_data *ring) { struct adf_etr_bank_data *bank = ring->bank; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); /* Disable interrupts for the given ring */ adf_disable_ring_irq(bank, ring->ring_number); /* Clear PCI config space */ - WRITE_CSR_RING_CONFIG(bank->csr_addr, bank->bank_number, - ring->ring_number, 0); - WRITE_CSR_RING_BASE(bank->csr_addr, bank->bank_number, - ring->ring_number, 0); + + csr_ops->write_csr_ring_config(bank->csr_addr, bank->bank_number, + ring->ring_number, 0); + csr_ops->write_csr_ring_base(bank->csr_addr, bank->bank_number, + ring->ring_number, 0); adf_ring_debugfs_rm(ring); adf_unreserve_ring(bank, ring->ring_number); /* Disable HW arbitration for the given ring */ @@ -287,11 +308,14 @@ void adf_remove_ring(struct adf_etr_ring_data *ring) static void adf_ring_response_handler(struct adf_etr_bank_data *bank) { - u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(bank->accel_dev); + struct adf_accel_dev *accel_dev = bank->accel_dev; + u8 num_rings_per_bank = GET_NUM_RINGS_PER_BANK(accel_dev); + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); unsigned long empty_rings; int i; - empty_rings = READ_CSR_E_STAT(bank->csr_addr, bank->bank_number); + empty_rings = csr_ops->read_csr_e_stat(bank->csr_addr, + bank->bank_number); empty_rings = ~empty_rings & bank->irq_mask; for_each_set_bit(i, &empty_rings, num_rings_per_bank) @@ -301,11 +325,13 @@ static void adf_ring_response_handler(struct adf_etr_bank_data *bank) void adf_response_handler(uintptr_t bank_addr) { struct adf_etr_bank_data *bank = (void *)bank_addr; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); /* Handle all the responses and reenable IRQs */ adf_ring_response_handler(bank); - WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, - bank->irq_mask); + + csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number, + bank->irq_mask); } static inline int adf_get_cfg_int(struct adf_accel_dev *accel_dev, @@ -345,6 +371,7 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, { struct adf_hw_device_data *hw_data = accel_dev->hw_device; u8 num_rings_per_bank = hw_data->num_rings_per_bank; + struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops; struct adf_etr_ring_data *ring; struct adf_etr_ring_data *tx_ring; u32 i, coalesc_enabled = 0; @@ -375,8 +402,9 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, bank->irq_coalesc_timer = ADF_COALESCING_MIN_TIME; for (i = 0; i < num_rings_per_bank; i++) { - WRITE_CSR_RING_CONFIG(csr_addr, bank_num, i, 0); - WRITE_CSR_RING_BASE(csr_addr, bank_num, i, 0); + csr_ops->write_csr_ring_config(csr_addr, bank_num, i, 0); + csr_ops->write_csr_ring_base(csr_addr, bank_num, i, 0); + ring = &bank->rings[i]; if (hw_data->tx_rings_mask & (1 << i)) { ring->inflights = @@ -401,8 +429,10 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, goto err; } - WRITE_CSR_INT_FLAG(csr_addr, bank_num, ADF_BANK_INT_FLAG_CLEAR_MASK); - WRITE_CSR_INT_SRCSEL(csr_addr, bank_num); + csr_ops->write_csr_int_flag(csr_addr, bank_num, + ADF_BANK_INT_FLAG_CLEAR_MASK); + csr_ops->write_csr_int_srcsel(csr_addr, bank_num); + return 0; err: ring_mask = hw_data->tx_rings_mask; diff --git a/drivers/crypto/qat/qat_common/adf_transport_debug.c b/drivers/crypto/qat/qat_common/adf_transport_debug.c index da79d734c035..1205186ad51e 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/qat/qat_common/adf_transport_debug.c @@ -42,16 +42,17 @@ static int adf_ring_show(struct seq_file *sfile, void *v) { struct adf_etr_ring_data *ring = sfile->private; struct adf_etr_bank_data *bank = ring->bank; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); void __iomem *csr = ring->bank->csr_addr; if (v == SEQ_START_TOKEN) { int head, tail, empty; - head = READ_CSR_RING_HEAD(csr, bank->bank_number, - ring->ring_number); - tail = READ_CSR_RING_TAIL(csr, bank->bank_number, - ring->ring_number); - empty = READ_CSR_E_STAT(csr, bank->bank_number); + head = csr_ops->read_csr_ring_head(csr, bank->bank_number, + ring->ring_number); + tail = csr_ops->read_csr_ring_tail(csr, bank->bank_number, + ring->ring_number); + empty = csr_ops->read_csr_e_stat(csr, bank->bank_number); seq_puts(sfile, "------- Ring configuration -------\n"); seq_printf(sfile, "ring name: %s\n", @@ -144,6 +145,7 @@ static void *adf_bank_next(struct seq_file *sfile, void *v, loff_t *pos) static int adf_bank_show(struct seq_file *sfile, void *v) { struct adf_etr_bank_data *bank = sfile->private; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev); if (v == SEQ_START_TOKEN) { seq_printf(sfile, "------- Bank %d configuration -------\n", @@ -157,11 +159,11 @@ static int adf_bank_show(struct seq_file *sfile, void *v) if (!(bank->ring_mask & 1 << ring_id)) return 0; - head = READ_CSR_RING_HEAD(csr, bank->bank_number, - ring->ring_number); - tail = READ_CSR_RING_TAIL(csr, bank->bank_number, - ring->ring_number); - empty = READ_CSR_E_STAT(csr, bank->bank_number); + head = csr_ops->read_csr_ring_head(csr, bank->bank_number, + ring->ring_number); + tail = csr_ops->read_csr_ring_tail(csr, bank->bank_number, + ring->ring_number); + empty = csr_ops->read_csr_e_stat(csr, bank->bank_number); seq_printf(sfile, "ring num %02d, head %04x, tail %04x, empty: %d\n", diff --git a/drivers/crypto/qat/qat_common/adf_vf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c index c4a44dc6af3e..38d316a42ba6 100644 --- a/drivers/crypto/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c @@ -156,6 +156,7 @@ static irqreturn_t adf_isr(int irq, void *privdata) { struct adf_accel_dev *accel_dev = privdata; struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops; struct adf_bar *pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; void __iomem *pmisc_bar_addr = pmisc->virt_addr; @@ -180,8 +181,8 @@ static irqreturn_t adf_isr(int irq, void *privdata) struct adf_etr_bank_data *bank = &etr_data->banks[0]; /* Disable Flag and Coalesce Ring Interrupts */ - WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, - 0); + csr_ops->write_csr_int_flag_and_col(bank->csr_addr, + bank->bank_number, 0); tasklet_hi_schedule(&bank->resp_handler); return IRQ_HANDLED; } diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 7b2f13ff49fd..39423316664b 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -226,6 +226,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; hw_data->reset_device = adf_reset_sbr; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index eca144bc1d67..f14fb82ed6df 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "adf_dh895xccvf_hw_data.h" static struct adf_hw_device_class dh895xcciov_class = { @@ -98,6 +99,7 @@ void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; hw_data->dev_class->instances++; adf_devmgr_update_class_index(hw_data); + adf_gen2_init_hw_csr_ops(&hw_data->csr_ops); } void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) From d1e86b4c2cdc2d4a15767705696beaabbdc3a57f Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:22 +0100 Subject: [PATCH 033/360] crypto: qat - relocate GEN2 CSR access code Move gen2 specific transport macros to adf_gen2_hw_data.c. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_gen2_hw_data.c | 1 - .../crypto/qat/qat_common/adf_gen2_hw_data.h | 68 +++++++++++++++++++ .../qat_common/adf_transport_access_macros.h | 64 ----------------- 3 files changed, 68 insertions(+), 65 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 07a9211bf7f9..9011c94156a9 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -1,7 +1,6 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ #include "adf_gen2_hw_data.h" -#include "adf_transport_access_macros.h" void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs) diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index e6d3919a56a1..592aee627762 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -5,6 +5,74 @@ #include "adf_accel_devices.h" +/* Transport access */ +#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL +#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL +#define ADF_RING_CSR_RING_CONFIG 0x000 +#define ADF_RING_CSR_RING_LBASE 0x040 +#define ADF_RING_CSR_RING_UBASE 0x080 +#define ADF_RING_CSR_RING_HEAD 0x0C0 +#define ADF_RING_CSR_RING_TAIL 0x100 +#define ADF_RING_CSR_E_STAT 0x14C +#define ADF_RING_CSR_INT_FLAG 0x170 +#define ADF_RING_CSR_INT_SRCSEL 0x174 +#define ADF_RING_CSR_INT_SRCSEL_2 0x178 +#define ADF_RING_CSR_INT_COL_EN 0x17C +#define ADF_RING_CSR_INT_COL_CTL 0x180 +#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 +#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 +#define ADF_RING_BUNDLE_SIZE 0x1000 + +#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2)) +#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2)) +#define READ_CSR_E_STAT(csr_base_addr, bank) \ + ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_E_STAT) +#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_CONFIG + ((ring) << 2), value) +#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ +do { \ + u32 l_base = 0, u_base = 0; \ + l_base = (u32)((value) & 0xFFFFFFFF); \ + u_base = (u32)(((value) & 0xFFFFFFFF00000000ULL) >> 32); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_LBASE + ((ring) << 2), l_base); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_UBASE + ((ring) << 2), u_base); \ +} while (0) + +#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) +#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) +#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_FLAG, value) +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ +do { \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK_0); \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_SRCSEL_2, ADF_BANK_INT_SRC_SEL_MASK_X); \ +} while (0) +#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_COL_EN, value) +#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_COL_CTL, \ + ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) +#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_FLAG_AND_COL, value) + /* AE to function map */ #define AE2FUNCTION_MAP_A_OFFSET (0x3A400 + 0x190) #define AE2FUNCTION_MAP_B_OFFSET (0x3A400 + 0x310) diff --git a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h index 950d1988556c..4642b0b5cfb0 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h +++ b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h @@ -4,23 +4,7 @@ #define ADF_TRANSPORT_ACCESS_MACROS_H #include "adf_accel_devices.h" -#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL -#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL #define ADF_BANK_INT_FLAG_CLEAR_MASK 0xFFFF -#define ADF_RING_CSR_RING_CONFIG 0x000 -#define ADF_RING_CSR_RING_LBASE 0x040 -#define ADF_RING_CSR_RING_UBASE 0x080 -#define ADF_RING_CSR_RING_HEAD 0x0C0 -#define ADF_RING_CSR_RING_TAIL 0x100 -#define ADF_RING_CSR_E_STAT 0x14C -#define ADF_RING_CSR_INT_FLAG 0x170 -#define ADF_RING_CSR_INT_SRCSEL 0x174 -#define ADF_RING_CSR_INT_SRCSEL_2 0x178 -#define ADF_RING_CSR_INT_COL_EN 0x17C -#define ADF_RING_CSR_INT_COL_CTL 0x180 -#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 -#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 -#define ADF_RING_BUNDLE_SIZE 0x1000 #define ADF_RING_CONFIG_NEAR_FULL_WM 0x0A #define ADF_RING_CONFIG_NEAR_EMPTY_WM 0x05 #define ADF_COALESCING_MIN_TIME 0x1FF @@ -74,52 +58,4 @@ | size) #define BUILD_RING_BASE_ADDR(addr, size) \ ((addr >> 6) & (0xFFFFFFFFFFFFFFFFULL << size)) -#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_HEAD + (ring << 2)) -#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_TAIL + (ring << 2)) -#define READ_CSR_E_STAT(csr_base_addr, bank) \ - ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_E_STAT) -#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_CONFIG + (ring << 2), value) -#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ -do { \ - u32 l_base = 0, u_base = 0; \ - l_base = (u32)(value & 0xFFFFFFFF); \ - u_base = (u32)((value & 0xFFFFFFFF00000000ULL) >> 32); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_LBASE + (ring << 2), l_base); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_UBASE + (ring << 2), u_base); \ -} while (0) -#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_HEAD + (ring << 2), value) -#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_RING_TAIL + (ring << 2), value) -#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_FLAG, value) -#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ -do { \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK_0); \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_SRCSEL_2, ADF_BANK_INT_SRC_SEL_MASK_X); \ -} while (0) -#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_COL_EN, value) -#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_COL_CTL, \ - ADF_RING_CSR_INT_COL_CTL_ENABLE | value) -#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ - ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ - ADF_RING_CSR_INT_FLAG_AND_COL, value) #endif From fc920eccd33b6b0ceb9ebe568ef783b5a816606f Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:23 +0100 Subject: [PATCH 034/360] crypto: qat - abstract admin interface Abstract access to admin interface and move generation specific code into adf_gen2_hw_data.c in preparation for the introduction of the qat_4xxx driver. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 1 + .../crypto/qat/qat_c62x/adf_c62x_hw_data.c | 1 + .../crypto/qat/qat_common/adf_accel_devices.h | 7 ++++++ drivers/crypto/qat/qat_common/adf_admin.c | 25 +++++++++++-------- .../crypto/qat/qat_common/adf_gen2_hw_data.c | 8 ++++++ .../crypto/qat/qat_common/adf_gen2_hw_data.h | 6 +++++ .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 1 + 7 files changed, 39 insertions(+), 10 deletions(-) diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 7af38b947cfe..f72ed415800e 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -202,6 +202,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_admin_info = adf_gen2_get_admin_info; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_C3XXX_FW; hw_data->fw_mmp_name = ADF_C3XXX_MMP; diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index c18fb77dd8ec..d4443523dc9d 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -212,6 +212,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_admin_info = adf_gen2_get_admin_info; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_C62X_FW; hw_data->fw_mmp_name = ADF_C62X_MMP; diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 459e22076813..5f57850c2e8d 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -97,6 +97,12 @@ struct adf_hw_device_class { u32 instances; } __packed; +struct admin_info { + u32 admin_msg_ur; + u32 admin_msg_lr; + u32 mailbox_offset; +}; + struct adf_hw_csr_ops { u32 (*read_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, u32 ring); @@ -138,6 +144,7 @@ struct adf_hw_device_data { u32 (*get_num_accels)(struct adf_hw_device_data *self); u32 (*get_pf2vf_offset)(u32 i); u32 (*get_vintmsk_offset)(u32 i); + void (*get_admin_info)(struct admin_info *admin_csrs_info); enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); int (*alloc_irq)(struct adf_accel_dev *accel_dev); void (*free_irq)(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index ec9b390276d6..3ae7c89ce82a 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -10,11 +10,7 @@ #include "adf_common_drv.h" #include "icp_qat_fw_init_admin.h" -/* Admin Messages Registers */ -#define ADF_DH895XCC_ADMINMSGUR_OFFSET (0x3A000 + 0x574) -#define ADF_DH895XCC_ADMINMSGLR_OFFSET (0x3A000 + 0x578) -#define ADF_DH895XCC_MAILBOX_BASE_OFFSET 0x20970 -#define ADF_DH895XCC_MAILBOX_STRIDE 0x1000 +#define ADF_ADMIN_MAILBOX_STRIDE 0x1000 #define ADF_ADMINMSG_LEN 32 #define ADF_CONST_TABLE_SIZE 1024 #define ADF_ADMIN_POLL_DELAY_US 20 @@ -118,7 +114,7 @@ static int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, u32 ae, struct adf_admin_comms *admin = accel_dev->admin; int offset = ae * ADF_ADMINMSG_LEN * 2; void __iomem *mailbox = admin->mailbox_addr; - int mb_offset = ae * ADF_DH895XCC_MAILBOX_STRIDE; + int mb_offset = ae * ADF_ADMIN_MAILBOX_STRIDE; struct icp_qat_fw_init_admin_req *request = in; mutex_lock(&admin->lock); @@ -225,8 +221,9 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev) struct adf_bar *pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; void __iomem *csr = pmisc->virt_addr; - void __iomem *mailbox = (void __iomem *)((uintptr_t)csr + - ADF_DH895XCC_MAILBOX_BASE_OFFSET); + struct admin_info admin_csrs_info; + u32 mailbox_offset, adminmsg_u, adminmsg_l; + void __iomem *mailbox; u64 reg_val; admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL, @@ -254,9 +251,17 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev) } memcpy(admin->virt_tbl_addr, const_tab, sizeof(const_tab)); + hw_data->get_admin_info(&admin_csrs_info); + + mailbox_offset = admin_csrs_info.mailbox_offset; + mailbox = (void __iomem *)((uintptr_t)csr + mailbox_offset); + adminmsg_u = admin_csrs_info.admin_msg_ur; + adminmsg_l = admin_csrs_info.admin_msg_lr; + reg_val = (u64)admin->phy_addr; - ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGUR_OFFSET, reg_val >> 32); - ADF_CSR_WR(csr, ADF_DH895XCC_ADMINMSGLR_OFFSET, reg_val); + ADF_CSR_WR(csr, adminmsg_u, upper_32_bits(reg_val)); + ADF_CSR_WR(csr, adminmsg_l, lower_32_bits(reg_val)); + mutex_init(&admin->lock); admin->mailbox_addr = mailbox; accel_dev->admin = admin; diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 9011c94156a9..15a0bc921d7e 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -37,6 +37,14 @@ void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, } EXPORT_SYMBOL_GPL(adf_gen2_cfg_iov_thds); +void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info) +{ + admin_csrs_info->mailbox_offset = ADF_MAILBOX_BASE_OFFSET; + admin_csrs_info->admin_msg_ur = ADF_ADMINMSGUR_OFFSET; + admin_csrs_info->admin_msg_lr = ADF_ADMINMSGLR_OFFSET; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_admin_info); + static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) { return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index 592aee627762..e9d2591b2be8 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -92,8 +92,14 @@ do { \ ADF_CSR_WR(pmisc_bar_addr, AE2FUNCTION_MAP_B_OFFSET + \ AE2FUNCTION_MAP_REG_SIZE * (index), value) +/* Admin Interface Offsets */ +#define ADF_ADMINMSGUR_OFFSET (0x3A000 + 0x574) +#define ADF_ADMINMSGLR_OFFSET (0x3A000 + 0x578) +#define ADF_MAILBOX_BASE_OFFSET 0x20970 + void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs); void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); +void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info); #endif diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 39423316664b..c568e9808cec 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -210,6 +210,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_admin_info = adf_gen2_get_admin_info; hw_data->get_sram_bar_id = get_sram_bar_id; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_DH895XCC_FW; From f9ff4d3820d5966bd7b86cc97f475bd8c3c04313 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:24 +0100 Subject: [PATCH 035/360] crypto: qat - add packed to init admin structures Add packed attribute to the structures icp_qat_fw_init_admin_req and icp_qat_fw_init_admin_resp as they are accessed by firmware. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h index d4d188cd7ed0..3868bcbed252 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h @@ -39,7 +39,7 @@ struct icp_qat_fw_init_admin_req { }; __u32 resrvd4; -}; +} __packed; struct icp_qat_fw_init_admin_resp { __u8 flags; @@ -92,7 +92,7 @@ struct icp_qat_fw_init_admin_resp { __u64 resrvd8; }; }; -}; +} __packed; #define ICP_QAT_FW_COMN_HEARTBEAT_OK 0 #define ICP_QAT_FW_COMN_HEARTBEAT_BLOCKED 1 From fa7e41e649e33b54d0780af83caca6cbe573b0bb Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:25 +0100 Subject: [PATCH 036/360] crypto: qat - rename ME in AE Rename occurrences of ME in the admin module with the acronym AE (Acceleration Engine) as the two are equivalent. This is to keep a single acronym for engined in the codebase and follow the documentation in https://01.org/intel-quickassist-technology. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_admin.c | 6 +++--- drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index 3ae7c89ce82a..13a5e8659682 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -163,7 +163,7 @@ static int adf_send_admin(struct adf_accel_dev *accel_dev, return 0; } -static int adf_init_me(struct adf_accel_dev *accel_dev) +static int adf_init_ae(struct adf_accel_dev *accel_dev) { struct icp_qat_fw_init_admin_req req; struct icp_qat_fw_init_admin_resp resp; @@ -172,7 +172,7 @@ static int adf_init_me(struct adf_accel_dev *accel_dev) memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); - req.cmd_id = ICP_QAT_FW_INIT_ME; + req.cmd_id = ICP_QAT_FW_INIT_AE; return adf_send_admin(accel_dev, &req, &resp, ae_mask); } @@ -206,7 +206,7 @@ int adf_send_admin_init(struct adf_accel_dev *accel_dev) { int ret; - ret = adf_init_me(accel_dev); + ret = adf_init_ae(accel_dev); if (ret) return ret; diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h index 3868bcbed252..f05ad17fbdd6 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_init_admin.h @@ -6,7 +6,7 @@ #include "icp_qat_fw.h" enum icp_qat_fw_init_admin_cmd_id { - ICP_QAT_FW_INIT_ME = 0, + ICP_QAT_FW_INIT_AE = 0, ICP_QAT_FW_TRNG_ENABLE = 1, ICP_QAT_FW_TRNG_DISABLE = 2, ICP_QAT_FW_CONSTANTS_CFG = 3, From 09b08885169998134ef40a6cad6f66e833582083 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:26 +0100 Subject: [PATCH 037/360] crypto: qat - change admin sequence Call adf_set_fw_constants() before adf_init_ae(). This is required by QAT GEN4 devices, which expect that the FW_CONSTANTS_CFG command is sent to the admin AEs before the FW_INIT_AE command. Swapping the order of the two commands (FW_INIT_AE and FW_CONSTANTS_CFG) is allowed in QAT GEN2 devices as the firmware can handle those in any order. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_admin.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index 13a5e8659682..6d94746d266f 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -206,11 +206,11 @@ int adf_send_admin_init(struct adf_accel_dev *accel_dev) { int ret; - ret = adf_init_ae(accel_dev); + ret = adf_set_fw_constants(accel_dev); if (ret) return ret; - return adf_set_fw_constants(accel_dev); + return adf_init_ae(accel_dev); } EXPORT_SYMBOL_GPL(adf_send_admin_init); From 2d499c049ecdde25106bc391de644722f48793b5 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:27 +0100 Subject: [PATCH 038/360] crypto: qat - use admin mask to send fw constants Introduce admin AE mask. If this mask set, the fw constant message is sent only to engines that belong to that set, otherwise it is sent to all engines. This is in preparation for the qat_4xxx driver where the constant message should be sent only to admin engines. In GEN2 devices (c62x, c3xxx and dh895xcc), the admin AE mask is 0 and the fw constants message is sent to all AEs. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_admin.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 5f57850c2e8d..779f62fde3bd 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -171,6 +171,7 @@ struct adf_hw_device_data { u32 instance_id; u16 accel_mask; u16 ae_mask; + u32 admin_ae_mask; u16 tx_rings_mask; u8 tx_rx_gap; u8 num_banks; diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index 6d94746d266f..dcd580d2afe2 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -182,7 +182,7 @@ static int adf_set_fw_constants(struct adf_accel_dev *accel_dev) struct icp_qat_fw_init_admin_req req; struct icp_qat_fw_init_admin_resp resp; struct adf_hw_device_data *hw_device = accel_dev->hw_device; - u32 ae_mask = hw_device->ae_mask; + u32 ae_mask = hw_device->admin_ae_mask ?: hw_device->ae_mask; memset(&req, 0, sizeof(req)); memset(&resp, 0, sizeof(resp)); From 59ab4d9aa5c9f5361d95cf8dc306797dc48e68b4 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:28 +0100 Subject: [PATCH 039/360] crypto: qat - update constants table Extend admin contansts table to support QAT GEN4 devices. This change does not affect QAT GEN2 devices (c62x, c3xxx and dh895xcc) as the table was extended in an unused area which is not referenced by any of those drivers and devices. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_admin.c | 42 +++++++++++------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index dcd580d2afe2..7c2ca54229aa 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -66,28 +66,28 @@ static const u8 const_tab[1024] __aligned(1024) = { 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52, 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, -0x7e, 0x21, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x7e, 0x21, 0x79, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x01, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x15, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x02, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x14, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x02, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x24, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, +0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x01, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x01, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x2B, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, +0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, From 8b5b80db8b85a78929fb0ad50acea32eeefa7ebd Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:29 +0100 Subject: [PATCH 040/360] crypto: qat - remove writes into WQCFG WQCFG registers contain the correct values after reset in all generations of QAT. No need to write into them. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_hw_arbiter.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index d4162783f970..cbb9f0b8ff74 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -11,7 +11,6 @@ #define ADF_ARB_REG_SLOT 0x1000 #define ADF_ARB_WTR_OFFSET 0x010 #define ADF_ARB_RO_EN_OFFSET 0x090 -#define ADF_ARB_WQCFG_OFFSET 0x100 #define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 #define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C @@ -28,10 +27,6 @@ ADF_ARB_WRK_2_SER_MAP_OFFSET) + \ (ADF_ARB_REG_SIZE * index), value) -#define WRITE_CSR_ARB_WQCFG(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ - ADF_ARB_WQCFG_OFFSET) + (ADF_ARB_REG_SIZE * index), value) - int adf_init_arb(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; @@ -45,10 +40,6 @@ int adf_init_arb(struct adf_accel_dev *accel_dev) for (arb = 0; arb < ADF_ARB_NUM; arb++) WRITE_CSR_ARB_SARCONFIG(csr, arb, arb_cfg); - /* Setup worker queue registers */ - for (i = 0; i < hw_data->num_engines; i++) - WRITE_CSR_ARB_WQCFG(csr, i, i); - /* Map worker threads to service arbiters */ hw_data->get_arb_mapping(accel_dev, &thd_2_arb_cfg); @@ -84,10 +75,6 @@ void adf_exit_arb(struct adf_accel_dev *accel_dev) for (i = 0; i < ADF_ARB_NUM; i++) WRITE_CSR_ARB_SARCONFIG(csr, i, 0); - /* Shutdown work queue */ - for (i = 0; i < hw_data->num_engines; i++) - WRITE_CSR_ARB_WQCFG(csr, i, 0); - /* Unmap worker threads to service arbiters */ for (i = 0; i < hw_data->num_engines; i++) WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, 0); From c685d7a7ceb2961dca61914bf473304e08138e11 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:30 +0100 Subject: [PATCH 041/360] crypto: qat - remove unused macros in arbiter module Remove the unused macros ADF_ARB_WTR_SIZE, ADF_ARB_WTR_OFFSET and ADF_ARB_RO_EN_OFFSET. These macros were left in commit 34074205bb9f ("crypto: qat - remove redundant arbiter configuration") that removed the logic that used those defines. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_hw_arbiter.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index cbb9f0b8ff74..be2fd264a223 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -6,11 +6,8 @@ #define ADF_ARB_NUM 4 #define ADF_ARB_REG_SIZE 0x4 -#define ADF_ARB_WTR_SIZE 0x20 #define ADF_ARB_OFFSET 0x30000 #define ADF_ARB_REG_SLOT 0x1000 -#define ADF_ARB_WTR_OFFSET 0x010 -#define ADF_ARB_RO_EN_OFFSET 0x090 #define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 #define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C From 369eb4aaae46b1e40142fbe0ef27b2646c21e1e9 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:31 +0100 Subject: [PATCH 042/360] crypto: qat - abstract arbiter access The arbiter configuration, the offset to the arbiter config CSR and the offset to the worker thread to service arbiter CSR are going to be different in QAT GEN4 devices although the logic that uses them is the same across all QAT generations. This patch reworks the gen-specific parts of the arbiter access code by introducing the arb_info structure, that contains the values that are generation specific, and a function in the structure adf_hw_device_data, get_arb_info(), that allows to get them. Since the arbiter values for QAT GEN2 devices (c62x, c3xxx and dh895xcc) are the same, a single function, adf_gen2_get_arb_info() is provided in adf_gen2_hw_data.c and referenced by each QAT GEN2 driver. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Reviewed-by: Wojciech Ziemba Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 1 + .../crypto/qat/qat_c62x/adf_c62x_hw_data.c | 1 + .../crypto/qat/qat_common/adf_accel_devices.h | 7 ++++ .../crypto/qat/qat_common/adf_gen2_hw_data.c | 8 ++++ .../crypto/qat/qat_common/adf_gen2_hw_data.h | 6 +++ .../crypto/qat/qat_common/adf_hw_arbiter.c | 41 ++++++++++++------- .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 1 + 7 files changed, 50 insertions(+), 15 deletions(-) diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index f72ed415800e..f3f33dc0c316 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -203,6 +203,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; hw_data->get_admin_info = adf_gen2_get_admin_info; + hw_data->get_arb_info = adf_gen2_get_arb_info; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_C3XXX_FW; hw_data->fw_mmp_name = ADF_C3XXX_MMP; diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index d4443523dc9d..53c03b2f763f 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -213,6 +213,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; hw_data->get_admin_info = adf_gen2_get_admin_info; + hw_data->get_arb_info = adf_gen2_get_arb_info; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_C62X_FW; hw_data->fw_mmp_name = ADF_C62X_MMP; diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 779f62fde3bd..951072feb176 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -97,6 +97,12 @@ struct adf_hw_device_class { u32 instances; } __packed; +struct arb_info { + u32 arb_cfg; + u32 arb_offset; + u32 wt2sam_offset; +}; + struct admin_info { u32 admin_msg_ur; u32 admin_msg_lr; @@ -144,6 +150,7 @@ struct adf_hw_device_data { u32 (*get_num_accels)(struct adf_hw_device_data *self); u32 (*get_pf2vf_offset)(u32 i); u32 (*get_vintmsk_offset)(u32 i); + void (*get_arb_info)(struct arb_info *arb_csrs_info); void (*get_admin_info)(struct admin_info *admin_csrs_info); enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self); int (*alloc_irq)(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 15a0bc921d7e..b2f770cc29d8 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -45,6 +45,14 @@ void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info) } EXPORT_SYMBOL_GPL(adf_gen2_get_admin_info); +void adf_gen2_get_arb_info(struct arb_info *arb_info) +{ + arb_info->arb_cfg = ADF_ARB_CONFIG; + arb_info->arb_offset = ADF_ARB_OFFSET; + arb_info->wt2sam_offset = ADF_ARB_WRK_2_SER_MAP_OFFSET; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_arb_info); + static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) { return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index e9d2591b2be8..fe4ea3220bca 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -97,9 +97,15 @@ do { \ #define ADF_ADMINMSGLR_OFFSET (0x3A000 + 0x578) #define ADF_MAILBOX_BASE_OFFSET 0x20970 +/* Arbiter configuration */ +#define ADF_ARB_OFFSET 0x30000 +#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 +#define ADF_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) + void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs); void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info); +void adf_gen2_get_arb_info(struct arb_info *arb_info); #endif diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index be2fd264a223..9dc9d58f6093 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -6,36 +6,39 @@ #define ADF_ARB_NUM 4 #define ADF_ARB_REG_SIZE 0x4 -#define ADF_ARB_OFFSET 0x30000 #define ADF_ARB_REG_SLOT 0x1000 -#define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 #define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \ ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ (ADF_ARB_REG_SLOT * index), value) -#define WRITE_CSR_ARB_SARCONFIG(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, ADF_ARB_OFFSET + \ - (ADF_ARB_REG_SIZE * index), value) +#define WRITE_CSR_ARB_SARCONFIG(csr_addr, arb_offset, index, value) \ + ADF_CSR_WR(csr_addr, (arb_offset) + \ + (ADF_ARB_REG_SIZE * (index)), value) -#define WRITE_CSR_ARB_WRK_2_SER_MAP(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ - ADF_ARB_WRK_2_SER_MAP_OFFSET) + \ - (ADF_ARB_REG_SIZE * index), value) +#define WRITE_CSR_ARB_WT2SAM(csr_addr, arb_offset, wt_offset, index, value) \ + ADF_CSR_WR(csr_addr, ((arb_offset) + (wt_offset)) + \ + (ADF_ARB_REG_SIZE * (index)), value) int adf_init_arb(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; void __iomem *csr = accel_dev->transport->banks[0].csr_addr; - u32 arb_cfg = 0x1 << 31 | 0x4 << 4 | 0x1; - u32 arb, i; + u32 arb_off, wt_off, arb_cfg; const u32 *thd_2_arb_cfg; + struct arb_info info; + int arb, i; + + hw_data->get_arb_info(&info); + arb_cfg = info.arb_cfg; + arb_off = info.arb_offset; + wt_off = info.wt2sam_offset; /* Service arb configured for 32 bytes responses and * ring flow control check enabled. */ for (arb = 0; arb < ADF_ARB_NUM; arb++) - WRITE_CSR_ARB_SARCONFIG(csr, arb, arb_cfg); + WRITE_CSR_ARB_SARCONFIG(csr, arb_off, arb, arb_cfg); /* Map worker threads to service arbiters */ hw_data->get_arb_mapping(accel_dev, &thd_2_arb_cfg); @@ -44,7 +47,7 @@ int adf_init_arb(struct adf_accel_dev *accel_dev) return -EFAULT; for (i = 0; i < hw_data->num_engines; i++) - WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, *(thd_2_arb_cfg + i)); + WRITE_CSR_ARB_WT2SAM(csr, arb_off, wt_off, i, thd_2_arb_cfg[i]); return 0; } @@ -60,21 +63,29 @@ void adf_update_ring_arb(struct adf_etr_ring_data *ring) void adf_exit_arb(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 arb_off, wt_off; + struct arb_info info; void __iomem *csr; unsigned int i; + hw_data->get_arb_info(&info); + arb_off = info.arb_offset; + wt_off = info.wt2sam_offset; + if (!accel_dev->transport) return; csr = accel_dev->transport->banks[0].csr_addr; + hw_data->get_arb_info(&info); + /* Reset arbiter configuration */ for (i = 0; i < ADF_ARB_NUM; i++) - WRITE_CSR_ARB_SARCONFIG(csr, i, 0); + WRITE_CSR_ARB_SARCONFIG(csr, arb_off, i, 0); /* Unmap worker threads to service arbiters */ for (i = 0; i < hw_data->num_engines; i++) - WRITE_CSR_ARB_WRK_2_SER_MAP(csr, i, 0); + WRITE_CSR_ARB_WT2SAM(csr, arb_off, wt_off, i, 0); /* Disable arbitration on all rings */ for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index c568e9808cec..2e7017a3ad46 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -211,6 +211,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->get_pf2vf_offset = get_pf2vf_offset; hw_data->get_vintmsk_offset = get_vintmsk_offset; hw_data->get_admin_info = adf_gen2_get_admin_info; + hw_data->get_arb_info = adf_gen2_get_arb_info; hw_data->get_sram_bar_id = get_sram_bar_id; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_DH895XCC_FW; From ad1332aa67eca6223b130cb593621ee16439b902 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Mon, 12 Oct 2020 21:38:32 +0100 Subject: [PATCH 043/360] crypto: qat - add support for capability detection Add logic to detect device capabilities for c62x, c3xxx and dh895xcc. Read fuses, straps and legfuses CSRs and build the device capabilities mask. This will be used to understand if a certain service is supported by a device. This patch is based on earlier work done by Conor McLoughlin. Signed-off-by: Marco Chiappero Co-developed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 2 ++ drivers/crypto/qat/qat_c3xxx/adf_drv.c | 5 ++-- .../crypto/qat/qat_c62x/adf_c62x_hw_data.c | 2 ++ drivers/crypto/qat/qat_c62x/adf_drv.c | 5 ++-- .../crypto/qat/qat_common/adf_accel_devices.h | 1 + .../crypto/qat/qat_common/adf_gen2_hw_data.c | 30 +++++++++++++++++++ .../crypto/qat/qat_common/adf_gen2_hw_data.h | 4 +++ drivers/crypto/qat/qat_common/icp_qat_hw.h | 23 ++++++++++++++ .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 25 ++++++++++++++++ drivers/crypto/qat/qat_dh895xcc/adf_drv.c | 5 ++-- 10 files changed, 93 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c index f3f33dc0c316..eb45f1b1ae3e 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -5,6 +5,7 @@ #include #include #include "adf_c3xxx_hw_data.h" +#include "icp_qat_hw.h" /* Worker thread to service arbiter mappings based on dev SKUs */ static const u32 thrd_to_arb_map_6_me_sku[] = { @@ -195,6 +196,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) hw_data->enable_error_correction = adf_enable_error_correction; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; + hw_data->get_accel_cap = adf_gen2_get_accel_cap; hw_data->get_num_accels = get_num_accels; hw_data->get_num_aes = get_num_aes; hw_data->get_sram_bar_id = get_sram_bar_id; diff --git a/drivers/crypto/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/qat/qat_c3xxx/adf_drv.c index da6e88026988..7fb3343ae8b0 100644 --- a/drivers/crypto/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/qat/qat_c3xxx/adf_drv.c @@ -177,9 +177,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_disable; } - /* Read accelerator capabilities mask */ - pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, - &hw_data->accel_capabilities_mask); + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); /* Find and map all the device's BARS */ i = 0; diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c index 53c03b2f763f..babdffbcb846 100644 --- a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -5,6 +5,7 @@ #include #include #include "adf_c62x_hw_data.h" +#include "icp_qat_hw.h" /* Worker thread to service arbiter mappings based on dev SKUs */ static const u32 thrd_to_arb_map_8_me_sku[] = { @@ -205,6 +206,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) hw_data->enable_error_correction = adf_enable_error_correction; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; + hw_data->get_accel_cap = adf_gen2_get_accel_cap; hw_data->get_num_accels = get_num_accels; hw_data->get_num_aes = get_num_aes; hw_data->get_sram_bar_id = get_sram_bar_id; diff --git a/drivers/crypto/qat/qat_c62x/adf_drv.c b/drivers/crypto/qat/qat_c62x/adf_drv.c index 3da697a566ad..1f5de442e1e6 100644 --- a/drivers/crypto/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/qat/qat_c62x/adf_drv.c @@ -177,9 +177,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_disable; } - /* Read accelerator capabilities mask */ - pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, - &hw_data->accel_capabilities_mask); + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); /* Find and map all the device's BARS */ i = (hw_data->fuses & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0; diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 951072feb176..692e39e5e878 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -143,6 +143,7 @@ struct adf_hw_device_data { struct adf_hw_device_class *dev_class; u32 (*get_accel_mask)(struct adf_hw_device_data *self); u32 (*get_ae_mask)(struct adf_hw_device_data *self); + u32 (*get_accel_cap)(struct adf_accel_dev *accel_dev); u32 (*get_sram_bar_id)(struct adf_hw_device_data *self); u32 (*get_misc_bar_id)(struct adf_hw_device_data *self); u32 (*get_etr_bar_id)(struct adf_hw_device_data *self); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index b2f770cc29d8..d5560e714167 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ #include "adf_gen2_hw_data.h" +#include "icp_qat_hw.h" +#include void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs) @@ -136,3 +138,31 @@ void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; } EXPORT_SYMBOL_GPL(adf_gen2_init_hw_csr_ops); + +u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 straps = hw_data->straps; + u32 fuses = hw_data->fuses; + u32 legfuses; + u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses); + + if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + if (legfuses & ICP_ACCEL_MASK_PKE_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + + if ((straps | fuses) & ADF_POWERGATE_PKE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + + return capabilities; +} +EXPORT_SYMBOL_GPL(adf_gen2_get_accel_cap); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index fe4ea3220bca..6c860aedb301 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -102,10 +102,14 @@ do { \ #define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 #define ADF_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +/* Power gating */ +#define ADF_POWERGATE_PKE BIT(24) + void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable, int num_a_regs, int num_b_regs); void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); void adf_gen2_get_admin_info(struct admin_info *admin_csrs_info); void adf_gen2_get_arb_info(struct arb_info *arb_info); +u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev); #endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/qat/qat_common/icp_qat_hw.h index c4b6ef1506ab..4aa5d724e11b 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hw.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hw.h @@ -65,6 +65,29 @@ struct icp_qat_hw_auth_config { __u32 reserved; }; +enum icp_qat_slice_mask { + ICP_ACCEL_MASK_CIPHER_SLICE = BIT(0), + ICP_ACCEL_MASK_AUTH_SLICE = BIT(1), + ICP_ACCEL_MASK_PKE_SLICE = BIT(2), + ICP_ACCEL_MASK_COMPRESS_SLICE = BIT(3), + ICP_ACCEL_MASK_LZS_SLICE = BIT(4), + ICP_ACCEL_MASK_EIA3_SLICE = BIT(5), + ICP_ACCEL_MASK_SHA3_SLICE = BIT(6), +}; + +enum icp_qat_capabilities_mask { + ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC = BIT(0), + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC = BIT(1), + ICP_ACCEL_CAPABILITIES_CIPHER = BIT(2), + ICP_ACCEL_CAPABILITIES_AUTHENTICATION = BIT(3), + ICP_ACCEL_CAPABILITIES_RESERVED_1 = BIT(4), + ICP_ACCEL_CAPABILITIES_COMPRESSION = BIT(5), + ICP_ACCEL_CAPABILITIES_LZS_COMPRESSION = BIT(6), + ICP_ACCEL_CAPABILITIES_RAND = BIT(7), + ICP_ACCEL_CAPABILITIES_ZUC = BIT(8), + ICP_ACCEL_CAPABILITIES_SHA3 = BIT(9), +}; + #define QAT_AUTH_MODE_BITPOS 4 #define QAT_AUTH_MODE_MASK 0xF #define QAT_AUTH_ALGO_BITPOS 0 diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 2e7017a3ad46..7970ebb67f28 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -5,6 +5,7 @@ #include #include #include "adf_dh895xcc_hw_data.h" +#include "icp_qat_hw.h" /* Worker thread to service arbiter mappings based on dev SKUs */ static const u32 thrd_to_arb_map_sku4[] = { @@ -83,6 +84,29 @@ static u32 get_sram_bar_id(struct adf_hw_device_data *self) return ADF_DH895XCC_SRAM_BAR; } +static u32 get_accel_cap(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 capabilities; + u32 legfuses; + + capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses); + + if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + if (legfuses & ICP_ACCEL_MASK_PKE_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + + return capabilities; +} + static enum dev_sku_info get_sku(struct adf_hw_device_data *self) { int sku = (self->fuses & ADF_DH895XCC_FUSECTL_SKU_MASK) @@ -204,6 +228,7 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->enable_error_correction = adf_enable_error_correction; hw_data->get_accel_mask = get_accel_mask; hw_data->get_ae_mask = get_ae_mask; + hw_data->get_accel_cap = get_accel_cap; hw_data->get_num_accels = get_num_accels; hw_data->get_num_aes = get_num_aes; hw_data->get_etr_bar_id = get_etr_bar_id; diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c index d7941bc2bafd..a9ec4357144c 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c @@ -177,9 +177,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_disable; } - /* Read accelerator capabilities mask */ - pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, - &hw_data->accel_capabilities_mask); + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); /* Find and map all the device's BARS */ i = 0; From dbf568755e7781a4ce8bd3c587c2c388600ae661 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:33 +0100 Subject: [PATCH 044/360] crypto: qat - register crypto instances based on capability Introduce the function adf_hw_dev_has_crypto() that returns true if a device supports symmetric crypto, asymmetric crypto and authentication services. If a device has crypto capabilities, add crypto instances to the configuration. This is done since the function that allows to retrieve crypto instances, qat_crypto_get_instance_node(), return instances that support all crypto services. Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_crypto.c | 7 ++++++- drivers/crypto/qat/qat_common/qat_crypto.h | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c index ab621b7dbd20..089d5d7b738e 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -117,10 +117,15 @@ int qat_crypto_dev_config(struct adf_accel_dev *accel_dev) { int cpus = num_online_cpus(); int banks = GET_MAX_BANKS(accel_dev); - int instances = min(cpus, banks); char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; int i; unsigned long val; + int instances; + + if (adf_hw_dev_has_crypto(accel_dev)) + instances = min(cpus, banks); + else + instances = 0; if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) goto err; diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h index 8d11e94cbf08..b6a4c95ae003 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.h +++ b/drivers/crypto/qat/qat_common/qat_crypto.h @@ -55,4 +55,19 @@ struct qat_crypto_request { bool encryption; }; +static inline bool adf_hw_dev_has_crypto(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + u32 mask = ~hw_device->accel_capabilities_mask; + + if (mask & ADF_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC) + return false; + if (mask & ADF_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC) + return false; + if (mask & ADF_ACCEL_CAPABILITIES_AUTHENTICATION) + return false; + + return true; +} + #endif From 070a34c9dfa3eeb0b18d63c6fec5dfe7861a43ac Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:34 +0100 Subject: [PATCH 045/360] crypto: qat - enable ring after pair is programmed Enable arbitration on the TX ring only after the RX ring is programmed. Before this change, arbitration was enabled on the TX ring before the RX ring was programmed allowing the HW to process a request before having the ring pair configured. With this change, the arbitration logic is programmed only if the TX half of the ring mask matches the RX half. This change does not affect QAT GEN2 devices (c62x, c3xxx and dh895xcc), but it is a must for QAT GEN4 devices since the CSRs of the ring pair are locked after arbitration is enabled on the TX ring. Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Maksim Lukoshkov Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_hw_arbiter.c | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index 9dc9d58f6093..bd03c8f54eb4 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -55,9 +55,27 @@ EXPORT_SYMBOL_GPL(adf_init_arb); void adf_update_ring_arb(struct adf_etr_ring_data *ring) { + struct adf_accel_dev *accel_dev = ring->bank->accel_dev; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + u32 tx_ring_mask = hw_data->tx_rings_mask; + u32 shift = hw_data->tx_rx_gap; + u32 arben, arben_tx, arben_rx; + u32 rx_ring_mask; + + /* + * Enable arbitration on a ring only if the TX half of the ring mask + * matches the RX part. This results in writes to CSR on both TX and + * RX update - only one is necessary, but both are done for + * simplicity. + */ + rx_ring_mask = tx_ring_mask << shift; + arben_tx = (ring->bank->ring_mask & tx_ring_mask) >> 0; + arben_rx = (ring->bank->ring_mask & rx_ring_mask) >> shift; + arben = arben_tx & arben_rx; + WRITE_CSR_ARB_RINGSRVARBEN(ring->bank->csr_addr, ring->bank->bank_number, - ring->bank->ring_mask & 0xFF); + arben); } void adf_exit_arb(struct adf_accel_dev *accel_dev) From 95a212bb7f1c9c75a5ae72277c734f8bf8fc04e4 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:35 +0100 Subject: [PATCH 046/360] crypto: qat - abstract build ring base Abstract the implementation of BUILD_RING_BASE_ADDR. This is in preparation for the introduction of the qat_4xxx driver since the value of the ring base differs between QAT GEN2 and QAT GEN4 devices. Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Maksim Lukoshkov Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_gen2_hw_data.c | 6 ++++++ drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 2 ++ drivers/crypto/qat/qat_common/adf_transport.c | 4 +++- drivers/crypto/qat/qat_common/adf_transport_access_macros.h | 2 -- 5 files changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 692e39e5e878..1fd32c56b119 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -110,6 +110,7 @@ struct admin_info { }; struct adf_hw_csr_ops { + u64 (*build_csr_ring_base_addr)(dma_addr_t addr, u32 size); u32 (*read_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, u32 ring); void (*write_csr_ring_head)(void __iomem *csr_base_addr, u32 bank, diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index d5560e714167..5de359165ab4 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -55,6 +55,11 @@ void adf_gen2_get_arb_info(struct arb_info *arb_info) } EXPORT_SYMBOL_GPL(adf_gen2_get_arb_info); +static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) +{ + return BUILD_RING_BASE_ADDR(addr, size); +} + static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) { return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); @@ -124,6 +129,7 @@ static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) { + csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; csr_ops->read_csr_ring_head = read_csr_ring_head; csr_ops->write_csr_ring_head = write_csr_ring_head; csr_ops->read_csr_ring_tail = read_csr_ring_tail; diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index 6c860aedb301..212ff395201f 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -23,6 +23,8 @@ #define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 #define ADF_RING_BUNDLE_SIZE 0x1000 +#define BUILD_RING_BASE_ADDR(addr, size) \ + (((addr) >> 6) & (0xFFFFFFFFFFFFFFFFULL << (size))) #define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ ADF_RING_CSR_RING_HEAD + ((ring) << 2)) diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c index 03fb7812818b..dd8f94fcb9a8 100644 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -180,7 +180,9 @@ static int adf_init_ring(struct adf_etr_ring_data *ring) else adf_configure_rx_ring(ring); - ring_base = BUILD_RING_BASE_ADDR(ring->dma_addr, ring->ring_size); + ring_base = csr_ops->build_csr_ring_base_addr(ring->dma_addr, + ring->ring_size); + csr_ops->write_csr_ring_base(ring->bank->csr_addr, ring->bank->bank_number, ring->ring_number, ring_base); diff --git a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h index 4642b0b5cfb0..12b1605a740e 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h +++ b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h @@ -56,6 +56,4 @@ ((watermark_nf << ADF_RING_CONFIG_NEAR_FULL_WM) \ | (watermark_ne << ADF_RING_CONFIG_NEAR_EMPTY_WM) \ | size) -#define BUILD_RING_BASE_ADDR(addr, size) \ - ((addr >> 6) & (0xFFFFFFFFFFFFFFFFULL << size)) #endif From fe779a46e85fa4433c650c8698065ea6f715cdba Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:36 +0100 Subject: [PATCH 047/360] crypto: qat - replace constant masks with GENMASK Replace constant 0xFFFFFFFFFFFFFFFFULL with GENMASK_ULL(63, 0) and 0xFFFFFFFF with GENMASK(31, 0) as they are masks. This makes code less error prone. Suggested-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 2 +- drivers/crypto/qat/qat_common/adf_sriov.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index 212ff395201f..04236a442f3c 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -24,7 +24,7 @@ #define ADF_RING_BUNDLE_SIZE 0x1000 #define BUILD_RING_BASE_ADDR(addr, size) \ - (((addr) >> 6) & (0xFFFFFFFFFFFFFFFFULL << (size))) + (((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) #define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ ADF_CSR_RD(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ ADF_RING_CSR_RING_HEAD + ((ring) << 2)) diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index dde6c57ef15a..0e8eab057d2d 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -99,7 +99,7 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) pci_disable_sriov(accel_to_pci_dev(accel_dev)); /* Disable VF to PF interrupts */ - adf_disable_vf2pf_interrupts(accel_dev, 0xFFFFFFFF); + adf_disable_vf2pf_interrupts(accel_dev, GENMASK(31, 0)); /* Clear Valid bits in AE Thread to PCIe Function Mapping */ hw_data->configure_iov_threads(accel_dev, false); From 097430ff7809c65697451a56b48fbe520b2ea27c Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:37 +0100 Subject: [PATCH 048/360] crypto: qat - use BIT_ULL() - 1 pattern for masks Replace occurrences of the pattern GENMASK_ULL(var - 1, 0)) with BIT_ULL(var) - 1 since it produces better code and it is easier to read. Suggested-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_sriov.c | 2 +- drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 0e8eab057d2d..9a0f6db83106 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -65,7 +65,7 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) hw_data->configure_iov_threads(accel_dev, true); /* Enable VF to PF interrupts for all VFs */ - adf_enable_vf2pf_interrupts(accel_dev, GENMASK_ULL(totalvfs - 1, 0)); + adf_enable_vf2pf_interrupts(accel_dev, BIT_ULL(totalvfs) - 1); /* * Due to the hardware design, when SR-IOV and the ring arbiter diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index 7970ebb67f28..1e83d9397b11 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -195,7 +195,7 @@ static void adf_enable_ints(struct adf_accel_dev *accel_dev) /* Enable bundle and misc interrupts */ ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET, accel_dev->pf.vf_info ? 0 : - GENMASK_ULL(GET_MAX_BANKS(accel_dev) - 1, 0)); + BIT_ULL(GET_MAX_BANKS(accel_dev)) - 1); ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET, ADF_DH895XCC_SMIA1_MASK); } From 02690ddcb8f17e33246e66826a55ad02750f8e76 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:38 +0100 Subject: [PATCH 049/360] crypto: qat - abstract writes to arbiter enable Abstract writes to the service arbiter enable register. This is in preparation for the introduction of the qat_4xxx driver since the arbitration enable register differes between QAT GEN2 and QAT GEN4 devices. Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Maksim Lukoshkov Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 2 ++ drivers/crypto/qat/qat_common/adf_gen2_hw_data.c | 7 +++++++ drivers/crypto/qat/qat_common/adf_gen2_hw_data.h | 6 ++++++ drivers/crypto/qat/qat_common/adf_hw_arbiter.c | 15 +++++---------- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 1fd32c56b119..a3b63dfe4d7b 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -133,6 +133,8 @@ struct adf_hw_csr_ops { u32 value); void (*write_csr_int_flag_and_col)(void __iomem *csr_base_addr, u32 bank, u32 value); + void (*write_csr_ring_srv_arb_en)(void __iomem *csr_base_addr, u32 bank, + u32 value); }; struct adf_cfg_device_data; diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c index 5de359165ab4..1aa17303838d 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.c @@ -127,6 +127,12 @@ static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); } +static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); +} + void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) { csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; @@ -142,6 +148,7 @@ void adf_gen2_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) csr_ops->write_csr_int_col_en = write_csr_int_col_en; csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; + csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; } EXPORT_SYMBOL_GPL(adf_gen2_init_hw_csr_ops); diff --git a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h index 04236a442f3c..3816e6500352 100644 --- a/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/qat/qat_common/adf_gen2_hw_data.h @@ -103,6 +103,12 @@ do { \ #define ADF_ARB_OFFSET 0x30000 #define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180 #define ADF_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +#define ADF_ARB_REG_SLOT 0x1000 +#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C + +#define WRITE_CSR_RING_SRV_ARB_EN(csr_addr, index, value) \ + ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ + (ADF_ARB_REG_SLOT * (index)), value) /* Power gating */ #define ADF_POWERGATE_PKE BIT(24) diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index bd03c8f54eb4..9f5240d9488b 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -6,12 +6,6 @@ #define ADF_ARB_NUM 4 #define ADF_ARB_REG_SIZE 0x4 -#define ADF_ARB_REG_SLOT 0x1000 -#define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C - -#define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ - (ADF_ARB_REG_SLOT * index), value) #define WRITE_CSR_ARB_SARCONFIG(csr_addr, arb_offset, index, value) \ ADF_CSR_WR(csr_addr, (arb_offset) + \ @@ -57,6 +51,7 @@ void adf_update_ring_arb(struct adf_etr_ring_data *ring) { struct adf_accel_dev *accel_dev = ring->bank->accel_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); u32 tx_ring_mask = hw_data->tx_rings_mask; u32 shift = hw_data->tx_rx_gap; u32 arben, arben_tx, arben_rx; @@ -73,14 +68,14 @@ void adf_update_ring_arb(struct adf_etr_ring_data *ring) arben_rx = (ring->bank->ring_mask & rx_ring_mask) >> shift; arben = arben_tx & arben_rx; - WRITE_CSR_ARB_RINGSRVARBEN(ring->bank->csr_addr, - ring->bank->bank_number, - arben); + csr_ops->write_csr_ring_srv_arb_en(ring->bank->csr_addr, + ring->bank->bank_number, arben); } void adf_exit_arb(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); u32 arb_off, wt_off; struct arb_info info; void __iomem *csr; @@ -107,6 +102,6 @@ void adf_exit_arb(struct adf_accel_dev *accel_dev) /* Disable arbitration on all rings */ for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) - WRITE_CSR_ARB_RINGSRVARBEN(csr, i, 0); + csr_ops->write_csr_ring_srv_arb_en(csr, i, 0); } EXPORT_SYMBOL_GPL(adf_exit_arb); From 264f590f2beb80d29dc33d5b60f93d2787f429b8 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:39 +0100 Subject: [PATCH 050/360] crypto: qat - remove hardcoded bank irq clear flag mask Replace hardcoded value of the bank interrupt clear flag mask with a value calculated on the fly which is based on the number of rings present in a bank. This is to support devices that have a number of rings per bank different than 16. Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_transport.c | 4 ++-- drivers/crypto/qat/qat_common/adf_transport_access_macros.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c index dd8f94fcb9a8..5a7030acdc33 100644 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -374,6 +374,7 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, struct adf_hw_device_data *hw_data = accel_dev->hw_device; u8 num_rings_per_bank = hw_data->num_rings_per_bank; struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops; + u32 irq_mask = BIT(num_rings_per_bank) - 1; struct adf_etr_ring_data *ring; struct adf_etr_ring_data *tx_ring; u32 i, coalesc_enabled = 0; @@ -431,8 +432,7 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, goto err; } - csr_ops->write_csr_int_flag(csr_addr, bank_num, - ADF_BANK_INT_FLAG_CLEAR_MASK); + csr_ops->write_csr_int_flag(csr_addr, bank_num, irq_mask); csr_ops->write_csr_int_srcsel(csr_addr, bank_num); return 0; diff --git a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h index 12b1605a740e..3b6b0267bbec 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h +++ b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h @@ -4,7 +4,6 @@ #define ADF_TRANSPORT_ACCESS_MACROS_H #include "adf_accel_devices.h" -#define ADF_BANK_INT_FLAG_CLEAR_MASK 0xFFFF #define ADF_RING_CONFIG_NEAR_FULL_WM 0x0A #define ADF_RING_CONFIG_NEAR_EMPTY_WM 0x05 #define ADF_COALESCING_MIN_TIME 0x1FF From 185aa9c3543e5b7303e3e9c97a9b8f06fec04afe Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:40 +0100 Subject: [PATCH 051/360] crypto: qat - call functions in adf_sriov if available Call the function configure_iov_threads(), adf_enable_vf2pf_interrupts() and adf_pf2vf_notify_restarting() only if present in the struct adf_hw_device_data of the device. This is to allow for QAT drivers that do not implement those functions. Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Maksim Lukoshkov Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_sriov.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 9a0f6db83106..d887640355d4 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -62,10 +62,12 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) } /* Set Valid bits in AE Thread to PCIe Function Mapping */ - hw_data->configure_iov_threads(accel_dev, true); + if (hw_data->configure_iov_threads) + hw_data->configure_iov_threads(accel_dev, true); /* Enable VF to PF interrupts for all VFs */ - adf_enable_vf2pf_interrupts(accel_dev, BIT_ULL(totalvfs) - 1); + if (hw_data->get_pf2vf_offset) + adf_enable_vf2pf_interrupts(accel_dev, BIT_ULL(totalvfs) - 1); /* * Due to the hardware design, when SR-IOV and the ring arbiter @@ -94,15 +96,18 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) if (!accel_dev->pf.vf_info) return; - adf_pf2vf_notify_restarting(accel_dev); + if (hw_data->get_pf2vf_offset) + adf_pf2vf_notify_restarting(accel_dev); pci_disable_sriov(accel_to_pci_dev(accel_dev)); /* Disable VF to PF interrupts */ - adf_disable_vf2pf_interrupts(accel_dev, GENMASK(31, 0)); + if (hw_data->get_pf2vf_offset) + adf_disable_vf2pf_interrupts(accel_dev, GENMASK(31, 0)); /* Clear Valid bits in AE Thread to PCIe Function Mapping */ - hw_data->configure_iov_threads(accel_dev, false); + if (hw_data->configure_iov_threads) + hw_data->configure_iov_threads(accel_dev, false); for (i = 0, vf = accel_dev->pf.vf_info; i < totalvfs; i++, vf++) { tasklet_disable(&vf->vf2pf_bh_tasklet); From 83ab8c4b2d893cf8718236e539287661c34df9ff Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:41 +0100 Subject: [PATCH 052/360] crypto: qat - remove unnecessary void* casts Remove superfluous casts to void* in function qat_crypto_dev_config(). Suggested-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_crypto.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c index 089d5d7b738e..ab1716f7044d 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -135,61 +135,61 @@ int qat_crypto_dev_config(struct adf_accel_dev *accel_dev) val = i; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); val = 128; if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; val = 512; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; val = 0; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; val = 2; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; val = 8; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; val = 10; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; val = ADF_COALESCING_DEF_TIME; snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", - key, (void *)&val, ADF_DEC)) + key, &val, ADF_DEC)) goto err; } val = i; if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - ADF_NUM_CY, (void *)&val, ADF_DEC)) + ADF_NUM_CY, &val, ADF_DEC)) goto err; set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); From 69d0fc22cf91c9488fcc6e6095b70e3f228ad580 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:42 +0100 Subject: [PATCH 053/360] crypto: qat - change return value in adf_cfg_add_key_value_param() If the parameter type provided to adf_cfg_add_key_value_param() is invalid, return -EINVAL instead of -1 that is treated as -EPERM and may confuse. Suggested-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_cfg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c index 22ae32838113..f2a29c70d61a 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -243,7 +243,7 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, } else { dev_err(&GET_DEV(accel_dev), "Unknown type given.\n"); kfree(key_val); - return -1; + return -EINVAL; } key_val->type = type; down_write(&cfg->lock); From 48710b1f31aef65918cb4396da530ad6ba00cd32 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:43 +0100 Subject: [PATCH 054/360] crypto: qat - change return value in adf_cfg_key_val_get() If a key is not found in the internal key value storage, return -ENODATA instead of -1 that is treated as -EPERM and may confuse. Suggested-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_cfg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c index f2a29c70d61a..575b6f002303 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -196,7 +196,7 @@ static int adf_cfg_key_val_get(struct adf_accel_dev *accel_dev, memcpy(val, keyval->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES); return 0; } - return -1; + return -ENODATA; } /** From 533179ce377f0dc4aa56857feeae012731dbef9a Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:44 +0100 Subject: [PATCH 055/360] crypto: qat - refactor qat_crypto_create_instances() Refactor function qat_crypto_create_instances() to propagate errors to the caller. Suggested-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_crypto.c | 68 +++++++++++++--------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c index ab1716f7044d..735463042987 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -202,83 +202,97 @@ EXPORT_SYMBOL_GPL(qat_crypto_dev_config); static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) { - int i; - unsigned long bank; - unsigned long num_inst, num_msg_sym, num_msg_asym; - int msg_size; - struct qat_crypto_instance *inst; + unsigned long bank, num_inst, num_msg_sym, num_msg_asym; char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + struct qat_crypto_instance *inst; + int msg_size; + int ret; + int i; INIT_LIST_HEAD(&accel_dev->crypto_list); - if (adf_cfg_get_param_value(accel_dev, SEC, ADF_NUM_CY, val)) - return -EFAULT; + ret = adf_cfg_get_param_value(accel_dev, SEC, ADF_NUM_CY, val); + if (ret) + return ret; - if (kstrtoul(val, 0, &num_inst)) - return -EFAULT; + ret = kstrtoul(val, 0, &num_inst); + if (ret) + return ret; for (i = 0; i < num_inst; i++) { inst = kzalloc_node(sizeof(*inst), GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev))); - if (!inst) + if (!inst) { + ret = -ENOMEM; goto err; + } list_add_tail(&inst->list, &accel_dev->crypto_list); inst->id = i; atomic_set(&inst->refctr, 0); inst->accel_dev = accel_dev; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); - if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) goto err; - if (kstrtoul(val, 10, &bank)) + ret = kstrtoul(val, 10, &bank); + if (ret) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); - if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) goto err; - if (kstrtoul(val, 10, &num_msg_sym)) + ret = kstrtoul(val, 10, &num_msg_sym); + if (ret) goto err; num_msg_sym = num_msg_sym >> 1; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); - if (adf_cfg_get_param_value(accel_dev, SEC, key, val)) + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) goto err; - if (kstrtoul(val, 10, &num_msg_asym)) + ret = kstrtoul(val, 10, &num_msg_asym); + if (ret) goto err; num_msg_asym = num_msg_asym >> 1; msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_sym, - msg_size, key, NULL, 0, &inst->sym_tx)) + ret = adf_create_ring(accel_dev, SEC, bank, num_msg_sym, + msg_size, key, NULL, 0, &inst->sym_tx); + if (ret) goto err; msg_size = msg_size >> 1; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, - msg_size, key, NULL, 0, &inst->pke_tx)) + ret = adf_create_ring(accel_dev, SEC, bank, num_msg_asym, + msg_size, key, NULL, 0, &inst->pke_tx); + if (ret) goto err; msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_sym, - msg_size, key, qat_alg_callback, 0, - &inst->sym_rx)) + ret = adf_create_ring(accel_dev, SEC, bank, num_msg_sym, + msg_size, key, qat_alg_callback, 0, + &inst->sym_rx); + if (ret) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); - if (adf_create_ring(accel_dev, SEC, bank, num_msg_asym, - msg_size, key, qat_alg_asym_callback, 0, - &inst->pke_rx)) + ret = adf_create_ring(accel_dev, SEC, bank, num_msg_asym, + msg_size, key, qat_alg_asym_callback, 0, + &inst->pke_rx); + if (ret) goto err; } return 0; err: qat_crypto_free_instances(accel_dev); - return -ENOMEM; + return ret; } static int qat_crypto_init(struct adf_accel_dev *accel_dev) From 54e3d7538fa53e35977f1bfadafba9c4bb2bd149 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:45 +0100 Subject: [PATCH 056/360] crypto: qat - refactor qat_crypto_dev_config() Refactor function qat_crypto_dev_config() to propagate errors to the caller. Suggested-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_crypto.c | 67 +++++++++++++--------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c index 735463042987..8d8ac5e48f47 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -115,88 +115,103 @@ struct qat_crypto_instance *qat_crypto_get_instance_node(int node) */ int qat_crypto_dev_config(struct adf_accel_dev *accel_dev) { - int cpus = num_online_cpus(); - int banks = GET_MAX_BANKS(accel_dev); char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - int i; + int banks = GET_MAX_BANKS(accel_dev); + int cpus = num_online_cpus(); unsigned long val; int instances; + int ret; + int i; if (adf_hw_dev_has_crypto(accel_dev)) instances = min(cpus, banks); else instances = 0; - if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) + ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); + if (ret) goto err; - if (adf_cfg_section_add(accel_dev, "Accelerator0")) + + ret = adf_cfg_section_add(accel_dev, "Accelerator0"); + if (ret) goto err; + for (i = 0; i < instances; i++) { val = i; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); val = 128; - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 512; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 0; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 2; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 8; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = 10; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) goto err; val = ADF_COALESCING_DEF_TIME; snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); - if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", - key, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, &val, ADF_DEC); + if (ret) goto err; } val = i; - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - ADF_NUM_CY, &val, ADF_DEC)) + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, + &val, ADF_DEC); + if (ret) goto err; set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); return 0; err: dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n"); - return -EINVAL; + return ret; } EXPORT_SYMBOL_GPL(qat_crypto_dev_config); From 657ad678e0c49a6e2f237f875b7a6c2d5065bbc8 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:46 +0100 Subject: [PATCH 057/360] crypto: qat - allow for instances in different banks Allow for crypto instances to be configured with symmetric crypto rings that belong to a bank that is different from the one where asymmetric crypto rings are located. This is to allow for devices with banks made of a single ring pair. In these, crypto instances will be composed of two separate banks. Changed string literals are not exposed to the user space. Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_cfg_strings.h | 3 +- drivers/crypto/qat/qat_common/qat_crypto.c | 34 ++++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_cfg_strings.h b/drivers/crypto/qat/qat_common/adf_cfg_strings.h index 314790f5b0af..09651e1f937a 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg_strings.h +++ b/drivers/crypto/qat/qat_common/adf_cfg_strings.h @@ -18,7 +18,8 @@ #define ADF_RING_DC_TX "RingTx" #define ADF_RING_DC_RX "RingRx" #define ADF_ETRMGR_BANK "Bank" -#define ADF_RING_BANK_NUM "BankNumber" +#define ADF_RING_SYM_BANK_NUM "BankSymNumber" +#define ADF_RING_ASYM_BANK_NUM "BankAsymNumber" #define ADF_CY "Cy" #define ADF_DC "Dc" #define ADF_ETRMGR_COALESCING_ENABLED "InterruptCoalescingEnabled" diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c index 8d8ac5e48f47..ece6776fbd53 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -138,7 +138,13 @@ int qat_crypto_dev_config(struct adf_accel_dev *accel_dev) for (i = 0; i < instances; i++) { val = i; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key, &val, ADF_DEC); if (ret) @@ -217,9 +223,10 @@ EXPORT_SYMBOL_GPL(qat_crypto_dev_config); static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) { - unsigned long bank, num_inst, num_msg_sym, num_msg_asym; + unsigned long num_inst, num_msg_sym, num_msg_asym; char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; + unsigned long sym_bank, asym_bank; struct qat_crypto_instance *inst; int msg_size; int ret; @@ -246,14 +253,25 @@ static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) inst->id = i; atomic_set(&inst->refctr, 0); inst->accel_dev = accel_dev; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); if (ret) goto err; - ret = kstrtoul(val, 10, &bank); + ret = kstrtoul(val, 10, &sym_bank); if (ret) goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); + ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); + if (ret) + goto err; + + ret = kstrtoul(val, 10, &asym_bank); + if (ret) + goto err; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); if (ret) @@ -277,28 +295,28 @@ static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); - ret = adf_create_ring(accel_dev, SEC, bank, num_msg_sym, + ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym, msg_size, key, NULL, 0, &inst->sym_tx); if (ret) goto err; msg_size = msg_size >> 1; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); - ret = adf_create_ring(accel_dev, SEC, bank, num_msg_asym, + ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym, msg_size, key, NULL, 0, &inst->pke_tx); if (ret) goto err; msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); - ret = adf_create_ring(accel_dev, SEC, bank, num_msg_sym, + ret = adf_create_ring(accel_dev, SEC, sym_bank, num_msg_sym, msg_size, key, qat_alg_callback, 0, &inst->sym_rx); if (ret) goto err; snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); - ret = adf_create_ring(accel_dev, SEC, bank, num_msg_asym, + ret = adf_create_ring(accel_dev, SEC, asym_bank, num_msg_asym, msg_size, key, qat_alg_asym_callback, 0, &inst->pke_rx); if (ret) From 3f9ae998355baa35284ed1a4c715b588c21dac08 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 12 Oct 2020 21:38:47 +0100 Subject: [PATCH 058/360] crypto: qat - extend ae_mask Change type of ae_mask in adf_hw_device_data to allow for devices with more than 16 Acceleration Engines (AEs). Signed-off-by: Giovanni Cabiddu Reviewed-by: Wojciech Ziemba Reviewed-by: Fiona Trahe Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index a3b63dfe4d7b..996d25565b11 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -181,7 +181,7 @@ struct adf_hw_device_data { u32 accel_capabilities_mask; u32 instance_id; u16 accel_mask; - u16 ae_mask; + u32 ae_mask; u32 admin_ae_mask; u16 tx_rings_mask; u8 tx_rx_gap; From d4b3984c9e627eacbbf5ee81f9a60eb3191bff8e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 13 Oct 2020 16:26:50 -0700 Subject: [PATCH 059/360] crypto: x86/aes - remove unused file aes_glue.c Commit 1d2c3279311e ("crypto: x86/aes - drop scalar assembler implementations") was meant to remove aes_glue.c, but it actually left it as an unused one-line file. Remove this unused file. Cc: Ard Biesheuvel Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/aes_glue.c | 1 - 1 file changed, 1 deletion(-) delete mode 100644 arch/x86/crypto/aes_glue.c diff --git a/arch/x86/crypto/aes_glue.c b/arch/x86/crypto/aes_glue.c deleted file mode 100644 index 7b7dc05fa1a4..000000000000 --- a/arch/x86/crypto/aes_glue.c +++ /dev/null @@ -1 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only From e68f9cf6601b53c7ee8ece552fbafcd9cfa47faa Mon Sep 17 00:00:00 2001 From: Longfang Liu Date: Thu, 15 Oct 2020 10:23:03 +0800 Subject: [PATCH 060/360] crypto: hisilicon - delete unused structure member variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Remove unused member‘pending_reqs' in‘sec_qp_ctx' structure. 2. Remove unused member‘status' in‘sec_dev' structure. Signed-off-by: Longfang Liu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 037762b531e2..08491912afd5 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -109,7 +109,6 @@ struct sec_qp_ctx { struct list_head backlog; struct hisi_acc_sgl_pool *c_in_pool; struct hisi_acc_sgl_pool *c_out_pool; - atomic_t pending_reqs; }; enum sec_alg_type { @@ -180,7 +179,6 @@ struct sec_dev { struct sec_debug debug; u32 ctx_q_num; bool iommu_used; - unsigned long status; }; void sec_destroy_qps(struct hisi_qp **qps, int qp_num); From 633e507fba02d62ed573a8dfb430fc3b881b23d6 Mon Sep 17 00:00:00 2001 From: Longfang Liu Date: Thu, 15 Oct 2020 10:23:04 +0800 Subject: [PATCH 061/360] crypto: hisilicon - fixes some coding style Clean up extra blank lines Signed-off-by: Longfang Liu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec_crypto.c | 17 +++++------- drivers/crypto/hisilicon/sec2/sec_main.c | 30 +++++++++------------- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index bb493423668c..87bc08afe567 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -101,6 +101,7 @@ static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) req->qp_ctx = qp_ctx; qp_ctx->req_list[req_id] = req; + return req_id; } @@ -317,6 +318,7 @@ static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) j * SEC_PBUF_PKG + pbuf_page_offset; } } + return 0; } @@ -345,12 +347,12 @@ static int sec_alg_resource_alloc(struct sec_ctx *ctx, } return 0; + alloc_pbuf_fail: if (ctx->alg_type == SEC_AEAD) sec_free_mac_resource(dev, qp_ctx->res); alloc_fail: sec_free_civ_resource(dev, res); - return ret; } @@ -419,7 +421,6 @@ err_free_c_in_pool: hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); err_destroy_idr: idr_destroy(&qp_ctx->req_idr); - return ret; } @@ -557,9 +558,9 @@ static int sec_skcipher_init(struct crypto_skcipher *tfm) goto err_cipher_init; return 0; + err_cipher_init: sec_ctx_base_uninit(ctx); - return ret; } @@ -740,7 +741,6 @@ static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, if (unlikely(pbuf_length != copy_size)) dev_err(dev, "copy pbuf data to dst error!\n"); - } static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, @@ -913,9 +913,9 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, } return 0; + bad_key: memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); - return -EINVAL; } @@ -966,7 +966,6 @@ static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) unmap_req_buf: ctx->req_op->buf_unmap(ctx, req); - return ret; } @@ -1107,7 +1106,6 @@ static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt); } - sk_req->base.complete(&sk_req->base, err); } @@ -1279,7 +1277,6 @@ err_send_req: sec_request_untransfer(ctx, req); err_uninit_req: sec_request_uninit(ctx, req); - return ret; } @@ -1349,7 +1346,6 @@ err_cipher_init: sec_auth_uninit(ctx); err_auth_init: sec_ctx_base_uninit(ctx); - return ret; } @@ -1437,8 +1433,8 @@ static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) } return 0; } - dev_err(dev, "skcipher algorithm error!\n"); + return -EINVAL; } @@ -1554,7 +1550,6 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) if (unlikely(c_alg != SEC_CALG_AES)) { dev_err(SEC_CTX_DEV(ctx), "aead crypto alg error!\n"); return -EINVAL; - } if (sreq->c_req.encrypt) sreq->c_req.c_len = req->cryptlen; diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 548896394c4b..2f52581b6d3c 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -660,12 +660,10 @@ static int sec_debugfs_init(struct hisi_qm *qm) if (ret) goto failed_to_create; - return 0; failed_to_create: debugfs_remove_recursive(sec_debugfs_root); - return ret; } @@ -683,13 +681,13 @@ static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts) while (errs->msg) { if (errs->int_msk & err_sts) { dev_err(dev, "%s [error status=0x%x] found\n", - errs->msg, errs->int_msk); + errs->msg, errs->int_msk); if (SEC_CORE_INT_STATUS_M_ECC & errs->int_msk) { err_val = readl(qm->io_base + SEC_CORE_SRAM_ECC_ERR_INFO); dev_err(dev, "multi ecc sram num=0x%x\n", - SEC_ECC_NUM(err_val)); + SEC_ECC_NUM(err_val)); } } errs++; @@ -724,13 +722,13 @@ static const struct hisi_qm_err_ini sec_err_ini = { .log_dev_hw_err = sec_log_hw_error, .open_axi_master_ooo = sec_open_axi_master_ooo, .err_info = { - .ce = QM_BASE_CE, - .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT | - QM_ACC_WB_NOT_READY_TIMEOUT, - .fe = 0, - .ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC, - .msi_wr_port = BIT(0), - .acpi_rst = "SRST", + .ce = QM_BASE_CE, + .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT | + QM_ACC_WB_NOT_READY_TIMEOUT, + .fe = 0, + .ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC, + .msi_wr_port = BIT(0), + .acpi_rst = "SRST", } }; @@ -899,17 +897,13 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) err_alg_unregister: hisi_qm_alg_unregister(qm, &sec_devices); - err_qm_stop: sec_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL); - err_probe_uninit: sec_probe_uninit(qm); - err_qm_uninit: sec_qm_uninit(qm); - return ret; } @@ -936,9 +930,9 @@ static void sec_remove(struct pci_dev *pdev) static const struct pci_error_handlers sec_err_handler = { .error_detected = hisi_qm_dev_err_detected, - .slot_reset = hisi_qm_dev_slot_reset, - .reset_prepare = hisi_qm_reset_prepare, - .reset_done = hisi_qm_reset_done, + .slot_reset = hisi_qm_dev_slot_reset, + .reset_prepare = hisi_qm_reset_prepare, + .reset_done = hisi_qm_reset_done, }; static struct pci_driver sec_pci_driver = { From ad0bb4e4d226e76950531cb4260402bbbb8997b5 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 15 Oct 2020 14:31:53 +1100 Subject: [PATCH 062/360] crypto: sa2ul - Reduce stack usage This patch reduces the stack usage in sa2ul: 1. Move the exported sha state into sa_prepare_iopads so that it can occupy the same space as the k_pad buffer. 2. Use one buffer for ipad/opad in sa_prepare_iopads. 3. Remove ipad/opad buffer from sa_set_sc_auth. 4. Use async skcipher fallback and remove on-stack request from sa_cipher_run. Reported-by: kernel test robot Fixes: d2c8ac187fc9 ("crypto: sa2ul - Add AEAD algorithm support") Signed-off-by: Herbert Xu --- drivers/crypto/sa2ul.c | 120 +++++++++++++++++++++-------------------- drivers/crypto/sa2ul.h | 6 +-- 2 files changed, 64 insertions(+), 62 deletions(-) diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index eda93fab95fe..c357010a159e 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -9,8 +9,10 @@ * Tero Kristo */ #include +#include #include #include +#include #include #include #include @@ -361,42 +363,45 @@ static void sa_swiz_128(u8 *in, u16 len) } /* Prepare the ipad and opad from key as per SHA algorithm step 1*/ -static void prepare_kiopad(u8 *k_ipad, u8 *k_opad, const u8 *key, u16 key_sz) +static void prepare_kipad(u8 *k_ipad, const u8 *key, u16 key_sz) { int i; - for (i = 0; i < key_sz; i++) { + for (i = 0; i < key_sz; i++) k_ipad[i] = key[i] ^ 0x36; - k_opad[i] = key[i] ^ 0x5c; - } /* Instead of XOR with 0 */ - for (; i < SHA1_BLOCK_SIZE; i++) { + for (; i < SHA1_BLOCK_SIZE; i++) k_ipad[i] = 0x36; - k_opad[i] = 0x5c; - } } -static void sa_export_shash(struct shash_desc *hash, int block_size, +static void prepare_kopad(u8 *k_opad, const u8 *key, u16 key_sz) +{ + int i; + + for (i = 0; i < key_sz; i++) + k_opad[i] = key[i] ^ 0x5c; + + /* Instead of XOR with 0 */ + for (; i < SHA1_BLOCK_SIZE; i++) + k_opad[i] = 0x5c; +} + +static void sa_export_shash(void *state, struct shash_desc *hash, int digest_size, __be32 *out) { - union { - struct sha1_state sha1; - struct sha256_state sha256; - struct sha512_state sha512; - } sha; - void *state; + struct sha1_state *sha1; + struct sha256_state *sha256; u32 *result; - int i; switch (digest_size) { case SHA1_DIGEST_SIZE: - state = &sha.sha1; - result = sha.sha1.state; + sha1 = state; + result = sha1->state; break; case SHA256_DIGEST_SIZE: - state = &sha.sha256; - result = sha.sha256.state; + sha256 = state; + result = sha256->state; break; default: dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__, @@ -406,8 +411,7 @@ static void sa_export_shash(struct shash_desc *hash, int block_size, crypto_shash_export(hash, state); - for (i = 0; i < digest_size >> 2; i++) - out[i] = cpu_to_be32(result[i]); + cpu_to_be32_array(out, result, digest_size / 4); } static void sa_prepare_iopads(struct algo_data *data, const u8 *key, @@ -416,24 +420,28 @@ static void sa_prepare_iopads(struct algo_data *data, const u8 *key, SHASH_DESC_ON_STACK(shash, data->ctx->shash); int block_size = crypto_shash_blocksize(data->ctx->shash); int digest_size = crypto_shash_digestsize(data->ctx->shash); - u8 k_ipad[SHA1_BLOCK_SIZE]; - u8 k_opad[SHA1_BLOCK_SIZE]; + union { + struct sha1_state sha1; + struct sha256_state sha256; + u8 k_pad[SHA1_BLOCK_SIZE]; + } sha; shash->tfm = data->ctx->shash; - prepare_kiopad(k_ipad, k_opad, key, key_sz); - - memzero_explicit(ipad, block_size); - memzero_explicit(opad, block_size); + prepare_kipad(sha.k_pad, key, key_sz); crypto_shash_init(shash); - crypto_shash_update(shash, k_ipad, block_size); - sa_export_shash(shash, block_size, digest_size, ipad); + crypto_shash_update(shash, sha.k_pad, block_size); + sa_export_shash(&sha, shash, digest_size, ipad); + + prepare_kopad(sha.k_pad, key, key_sz); crypto_shash_init(shash); - crypto_shash_update(shash, k_opad, block_size); + crypto_shash_update(shash, sha.k_pad, block_size); - sa_export_shash(shash, block_size, digest_size, opad); + sa_export_shash(&sha, shash, digest_size, opad); + + memzero_explicit(&sha, sizeof(sha)); } /* Derive the inverse key used in AES-CBC decryption operation */ @@ -506,7 +514,8 @@ static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz, static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz, u8 *sc_buf) { - __be32 ipad[64], opad[64]; + __be32 *ipad = (void *)(sc_buf + 32); + __be32 *opad = (void *)(sc_buf + 64); /* Set Authentication mode selector to hash processing */ sc_buf[0] = SA_HASH_PROCESSING; @@ -515,14 +524,9 @@ static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz, sc_buf[1] |= ad->auth_ctrl; /* Copy the keys or ipad/opad */ - if (ad->keyed_mac) { + if (ad->keyed_mac) ad->prep_iopad(ad, key, key_sz, ipad, opad); - - /* Copy ipad to AuthKey */ - memcpy(&sc_buf[32], ipad, ad->hash_size); - /* Copy opad to Aux-1 */ - memcpy(&sc_buf[64], opad, ad->hash_size); - } else { + else { /* basic hash */ sc_buf[1] |= SA_BASIC_HASH; } @@ -819,7 +823,7 @@ static void sa_cipher_cra_exit(struct crypto_skcipher *tfm) sa_free_ctx_info(&ctx->enc, data); sa_free_ctx_info(&ctx->dec, data); - crypto_free_sync_skcipher(ctx->fallback.skcipher); + crypto_free_skcipher(ctx->fallback.skcipher); } static int sa_cipher_cra_init(struct crypto_skcipher *tfm) @@ -827,6 +831,7 @@ static int sa_cipher_cra_init(struct crypto_skcipher *tfm) struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); const char *name = crypto_tfm_alg_name(&tfm->base); + struct crypto_skcipher *child; int ret; memzero_explicit(ctx, sizeof(*ctx)); @@ -841,14 +846,17 @@ static int sa_cipher_cra_init(struct crypto_skcipher *tfm) return ret; } - ctx->fallback.skcipher = - crypto_alloc_sync_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + child = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(ctx->fallback.skcipher)) { + if (IS_ERR(child)) { dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name); - return PTR_ERR(ctx->fallback.skcipher); + return PTR_ERR(child); } + ctx->fallback.skcipher = child; + crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) + + sizeof(struct skcipher_request)); + dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, ctx->dec.sc_id, &ctx->dec.sc_phys); @@ -859,6 +867,7 @@ static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen, struct algo_data *ad) { struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *child = ctx->fallback.skcipher; int cmdl_len; struct sa_cmdl_cfg cfg; int ret; @@ -874,12 +883,10 @@ static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key, cfg.enc_eng_id = ad->enc_eng.eng_id; cfg.iv_size = crypto_skcipher_ivsize(tfm); - crypto_sync_skcipher_clear_flags(ctx->fallback.skcipher, + crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(child, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); - crypto_sync_skcipher_set_flags(ctx->fallback.skcipher, - tfm->base.crt_flags & - CRYPTO_TFM_REQ_MASK); - ret = crypto_sync_skcipher_setkey(ctx->fallback.skcipher, key, keylen); + ret = crypto_skcipher_setkey(child, key, keylen); if (ret) return ret; @@ -1270,7 +1277,6 @@ static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc) crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); struct crypto_alg *alg = req->base.tfm->__crt_alg; struct sa_req sa_req = { 0 }; - int ret; if (!req->cryptlen) return 0; @@ -1282,20 +1288,18 @@ static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc) if (req->cryptlen > SA_MAX_DATA_SZ || (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN && req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) { - SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback.skcipher); + struct skcipher_request *subreq = skcipher_request_ctx(req); - skcipher_request_set_sync_tfm(subreq, ctx->fallback.skcipher); + skcipher_request_set_tfm(subreq, ctx->fallback.skcipher); skcipher_request_set_callback(subreq, req->base.flags, - NULL, NULL); + req->base.complete, + req->base.data); skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, req->iv); if (enc) - ret = crypto_skcipher_encrypt(subreq); + return crypto_skcipher_encrypt(subreq); else - ret = crypto_skcipher_decrypt(subreq); - - skcipher_request_zero(subreq); - return ret; + return crypto_skcipher_decrypt(subreq); } sa_req.size = req->cryptlen; diff --git a/drivers/crypto/sa2ul.h b/drivers/crypto/sa2ul.h index 7f7e3fe60d11..bb40df3876e5 100644 --- a/drivers/crypto/sa2ul.h +++ b/drivers/crypto/sa2ul.h @@ -12,10 +12,8 @@ #ifndef _K3_SA2UL_ #define _K3_SA2UL_ -#include -#include -#include #include +#include #define SA_ENGINE_ENABLE_CONTROL 0x1000 @@ -311,7 +309,7 @@ struct sa_tfm_ctx { struct crypto_shash *shash; /* for fallback */ union { - struct crypto_sync_skcipher *skcipher; + struct crypto_skcipher *skcipher; struct crypto_ahash *ahash; struct crypto_aead *aead; } fallback; From 1bc608b4655b8b1491fb100f4cf4f15ae64a8698 Mon Sep 17 00:00:00 2001 From: Tianjia Zhang Date: Thu, 15 Oct 2020 17:24:41 +0800 Subject: [PATCH 063/360] crypto: sm2 - remove unnecessary reset operations This is an algorithm optimization. The reset operation when setting the public key is repeated and redundant, so remove it. At the same time, `sm2_ecc_os2ec()` is optimized to make the function more simpler and more in line with the Linux code style. Signed-off-by: Tianjia Zhang Signed-off-by: Herbert Xu --- crypto/sm2.c | 75 ++++++++++++++++++++-------------------------------- 1 file changed, 29 insertions(+), 46 deletions(-) diff --git a/crypto/sm2.c b/crypto/sm2.c index 767e160333f6..b21addc3ac06 100644 --- a/crypto/sm2.c +++ b/crypto/sm2.c @@ -119,12 +119,6 @@ static void sm2_ec_ctx_deinit(struct mpi_ec_ctx *ec) memset(ec, 0, sizeof(*ec)); } -static int sm2_ec_ctx_reset(struct mpi_ec_ctx *ec) -{ - sm2_ec_ctx_deinit(ec); - return sm2_ec_ctx_init(ec); -} - /* RESULT must have been initialized and is set on success to the * point given by VALUE. */ @@ -132,55 +126,48 @@ static int sm2_ecc_os2ec(MPI_POINT result, MPI value) { int rc; size_t n; - const unsigned char *buf; - unsigned char *buf_memory; + unsigned char *buf; MPI x, y; - n = (mpi_get_nbits(value)+7)/8; - buf_memory = kmalloc(n, GFP_KERNEL); - rc = mpi_print(GCRYMPI_FMT_USG, buf_memory, n, &n, value); - if (rc) { - kfree(buf_memory); - return rc; - } - buf = buf_memory; + n = MPI_NBYTES(value); + buf = kmalloc(n, GFP_KERNEL); + if (!buf) + return -ENOMEM; - if (n < 1) { - kfree(buf_memory); - return -EINVAL; - } - if (*buf != 4) { - kfree(buf_memory); - return -EINVAL; /* No support for point compression. */ - } - if (((n-1)%2)) { - kfree(buf_memory); - return -EINVAL; - } - n = (n-1)/2; + rc = mpi_print(GCRYMPI_FMT_USG, buf, n, &n, value); + if (rc) + goto err_freebuf; + + rc = -EINVAL; + if (n < 1 || ((n - 1) % 2)) + goto err_freebuf; + /* No support for point compression */ + if (*buf != 0x4) + goto err_freebuf; + + rc = -ENOMEM; + n = (n - 1) / 2; x = mpi_read_raw_data(buf + 1, n); - if (!x) { - kfree(buf_memory); - return -ENOMEM; - } + if (!x) + goto err_freebuf; y = mpi_read_raw_data(buf + 1 + n, n); - kfree(buf_memory); - if (!y) { - mpi_free(x); - return -ENOMEM; - } + if (!y) + goto err_freex; mpi_normalize(x); mpi_normalize(y); - mpi_set(result->x, x); mpi_set(result->y, y); mpi_set_ui(result->z, 1); - mpi_free(x); - mpi_free(y); + rc = 0; - return 0; + mpi_free(y); +err_freex: + mpi_free(x); +err_freebuf: + kfree(buf); + return rc; } struct sm2_signature_ctx { @@ -399,10 +386,6 @@ static int sm2_set_pub_key(struct crypto_akcipher *tfm, MPI a; int rc; - rc = sm2_ec_ctx_reset(ec); - if (rc) - return rc; - ec->Q = mpi_point_new(0); if (!ec->Q) return -ENOMEM; From 383e8a823014532ffd81c787ef9009f1c2bd3b79 Mon Sep 17 00:00:00 2001 From: Zhang Qilong Date: Fri, 16 Oct 2020 17:05:36 +0800 Subject: [PATCH 064/360] crypto: omap-aes - fix the reference count leak of omap device pm_runtime_get_sync() will increment pm usage counter even when it returns an error code. We should call put operation in error handling paths of omap_aes_hw_init. Signed-off-by: Zhang Qilong Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 4fd14d90cc40..70ea5784a024 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -105,6 +105,7 @@ static int omap_aes_hw_init(struct omap_aes_dev *dd) err = pm_runtime_get_sync(dd->dev); if (err < 0) { + pm_runtime_put_noidle(dd->dev); dev_err(dd->dev, "failed to get sync: %d\n", err); return err; } From 1762818f25f3f99c5083caa13d69e5e5aa2e4b6f Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Sun, 25 Oct 2020 10:31:14 -0400 Subject: [PATCH 065/360] crypto: lib/sha256 - Use memzero_explicit() for clearing state Without the barrier_data() inside memzero_explicit(), the compiler may optimize away the state-clearing if it can tell that the state is not used afterwards. At least in lib/crypto/sha256.c:__sha256_final(), the function can get inlined into sha256(), in which case the memset is optimized away. Signed-off-by: Arvind Sankar Reviewed-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- lib/crypto/sha256.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 2321f6cb322f..d43bc39ab05e 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -265,7 +265,7 @@ static void __sha256_final(struct sha256_state *sctx, u8 *out, int digest_words) put_unaligned_be32(sctx->state[i], &dst[i]); /* Zeroize sensitive information. */ - memset(sctx, 0, sizeof(*sctx)); + memzero_explicit(sctx, sizeof(*sctx)); } void sha256_final(struct sha256_state *sctx, u8 *out) From 458c0480dcb338d7b72e89b2e88a622965adcea4 Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Sun, 25 Oct 2020 10:31:15 -0400 Subject: [PATCH 066/360] crypto: hash - Use memzero_explicit() for clearing state Without the barrier_data() inside memzero_explicit(), the compiler may optimize away the state-clearing if it can tell that the state is not used afterwards. Signed-off-by: Arvind Sankar Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/ghash-ce-glue.c | 2 +- arch/arm64/crypto/poly1305-glue.c | 2 +- arch/arm64/crypto/sha3-ce-glue.c | 2 +- arch/x86/crypto/poly1305_glue.c | 2 +- include/crypto/sha1_base.h | 3 ++- include/crypto/sha256_base.h | 3 ++- include/crypto/sha512_base.h | 3 ++- include/crypto/sm3_base.h | 3 ++- 8 files changed, 12 insertions(+), 8 deletions(-) diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 8536008e3e35..2427e2f3a9a1 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -168,7 +168,7 @@ static int ghash_final(struct shash_desc *desc, u8 *dst) put_unaligned_be64(ctx->digest[1], dst); put_unaligned_be64(ctx->digest[0], dst + 8); - *ctx = (struct ghash_desc_ctx){}; + memzero_explicit(ctx, sizeof(*ctx)); return 0; } diff --git a/arch/arm64/crypto/poly1305-glue.c b/arch/arm64/crypto/poly1305-glue.c index f33ada70c4ed..683de671741a 100644 --- a/arch/arm64/crypto/poly1305-glue.c +++ b/arch/arm64/crypto/poly1305-glue.c @@ -177,7 +177,7 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } poly1305_emit(&dctx->h, dst, dctx->s); - *dctx = (struct poly1305_desc_ctx){}; + memzero_explicit(dctx, sizeof(*dctx)); } EXPORT_SYMBOL(poly1305_final_arch); diff --git a/arch/arm64/crypto/sha3-ce-glue.c b/arch/arm64/crypto/sha3-ce-glue.c index 9a4bbfc45f40..e5a2936f0886 100644 --- a/arch/arm64/crypto/sha3-ce-glue.c +++ b/arch/arm64/crypto/sha3-ce-glue.c @@ -94,7 +94,7 @@ static int sha3_final(struct shash_desc *desc, u8 *out) if (digest_size & 4) put_unaligned_le32(sctx->st[i], (__le32 *)digest); - *sctx = (struct sha3_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index e508dbd91813..64d09520d279 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -209,7 +209,7 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } poly1305_simd_emit(&dctx->h, dst, dctx->s); - *dctx = (struct poly1305_desc_ctx){}; + memzero_explicit(dctx, sizeof(*dctx)); } EXPORT_SYMBOL(poly1305_final_arch); diff --git a/include/crypto/sha1_base.h b/include/crypto/sha1_base.h index 20fd1f7468af..a5d6033efef7 100644 --- a/include/crypto/sha1_base.h +++ b/include/crypto/sha1_base.h @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -101,7 +102,7 @@ static inline int sha1_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(__be32); i++) put_unaligned_be32(sctx->state[i], digest++); - *sctx = (struct sha1_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index 6ded110783ae..93f9fd21cc06 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -105,7 +106,7 @@ static inline int sha256_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be32)) put_unaligned_be32(sctx->state[i], digest++); - *sctx = (struct sha256_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h index fb19c77494dc..93ab73baa38e 100644 --- a/include/crypto/sha512_base.h +++ b/include/crypto/sha512_base.h @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -126,7 +127,7 @@ static inline int sha512_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be64)) put_unaligned_be64(sctx->state[i], digest++); - *sctx = (struct sha512_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } diff --git a/include/crypto/sm3_base.h b/include/crypto/sm3_base.h index 1cbf9aa1fe52..2f3a32ab97bb 100644 --- a/include/crypto/sm3_base.h +++ b/include/crypto/sm3_base.h @@ -13,6 +13,7 @@ #include #include #include +#include #include typedef void (sm3_block_fn)(struct sm3_state *sst, u8 const *src, int blocks); @@ -104,7 +105,7 @@ static inline int sm3_base_finish(struct shash_desc *desc, u8 *out) for (i = 0; i < SM3_DIGEST_SIZE / sizeof(__be32); i++) put_unaligned_be32(sctx->state[i], digest++); - *sctx = (struct sm3_state){}; + memzero_explicit(sctx, sizeof(*sctx)); return 0; } From 7a4295f6c9d54e082474667e552a227606b4a085 Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Sun, 25 Oct 2020 10:31:16 -0400 Subject: [PATCH 067/360] crypto: lib/sha256 - Don't clear temporary variables The assignments to clear a through h and t1/t2 are optimized out by the compiler because they are unused after the assignments. Clearing individual scalar variables is unlikely to be useful, as they may have been assigned to registers, and even if stack spilling was required, there may be compiler-generated temporaries that are impossible to clear in any case. So drop the clearing of a through h and t1/t2. Signed-off-by: Arvind Sankar Reviewed-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- lib/crypto/sha256.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index d43bc39ab05e..099cd11f83c1 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -202,7 +202,6 @@ static void sha256_transform(u32 *state, const u8 *input) state[4] += e; state[5] += f; state[6] += g; state[7] += h; /* clear any sensitive info... */ - a = b = c = d = e = f = g = h = t1 = t2 = 0; memzero_explicit(W, 64 * sizeof(u32)); } From b8399819b2dd6f29195ed7535217b66c01b1e57d Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Sun, 25 Oct 2020 10:31:17 -0400 Subject: [PATCH 068/360] crypto: lib/sha256 - Clear W[] in sha256_update() instead of sha256_transform() The temporary W[] array is currently zeroed out once every call to sha256_transform(), i.e. once every 64 bytes of input data. Moving it to sha256_update() instead so that it is cleared only once per update can save about 2-3% of the total time taken to compute the digest, with a reasonable memset() implementation, and considerably more (~20%) with a bad one (eg the x86 purgatory currently uses a memset() coded in C). Signed-off-by: Arvind Sankar Reviewed-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- lib/crypto/sha256.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 099cd11f83c1..c6bfeacc5b81 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -43,10 +43,9 @@ static inline void BLEND_OP(int I, u32 *W) W[I] = s1(W[I-2]) + W[I-7] + s0(W[I-15]) + W[I-16]; } -static void sha256_transform(u32 *state, const u8 *input) +static void sha256_transform(u32 *state, const u8 *input, u32 *W) { u32 a, b, c, d, e, f, g, h, t1, t2; - u32 W[64]; int i; /* load the input */ @@ -200,15 +199,13 @@ static void sha256_transform(u32 *state, const u8 *input) state[0] += a; state[1] += b; state[2] += c; state[3] += d; state[4] += e; state[5] += f; state[6] += g; state[7] += h; - - /* clear any sensitive info... */ - memzero_explicit(W, 64 * sizeof(u32)); } void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) { unsigned int partial, done; const u8 *src; + u32 W[64]; partial = sctx->count & 0x3f; sctx->count += len; @@ -223,11 +220,13 @@ void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) } do { - sha256_transform(sctx->state, src); + sha256_transform(sctx->state, src, W); done += 64; src = data + done; } while (done + 63 < len); + memzero_explicit(W, sizeof(W)); + partial = 0; } memcpy(sctx->buf + partial, src, len - done); From 63642d5c141f1bcbe97f7895467a724ad2e3f346 Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Sun, 25 Oct 2020 10:31:18 -0400 Subject: [PATCH 069/360] crypto: lib/sha256 - Unroll SHA256 loop 8 times intead of 64 This reduces code size substantially (on x86_64 with gcc-10 the size of sha256_update() goes from 7593 bytes to 1952 bytes including the new SHA256_K array), and on x86 is slightly faster than the full unroll (tested on Broadwell Xeon). Signed-off-by: Arvind Sankar Reviewed-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- lib/crypto/sha256.c | 174 ++++++++++---------------------------------- 1 file changed, 38 insertions(+), 136 deletions(-) diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index c6bfeacc5b81..e2e29d9b0ccd 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -18,6 +18,25 @@ #include #include +static const u32 SHA256_K[] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + static inline u32 Ch(u32 x, u32 y, u32 z) { return z ^ (x & (y ^ z)); @@ -43,9 +62,17 @@ static inline void BLEND_OP(int I, u32 *W) W[I] = s1(W[I-2]) + W[I-7] + s0(W[I-15]) + W[I-16]; } +#define SHA256_ROUND(i, a, b, c, d, e, f, g, h) do { \ + u32 t1, t2; \ + t1 = h + e1(e) + Ch(e, f, g) + SHA256_K[i] + W[i]; \ + t2 = e0(a) + Maj(a, b, c); \ + d += t1; \ + h = t1 + t2; \ +} while (0) + static void sha256_transform(u32 *state, const u8 *input, u32 *W) { - u32 a, b, c, d, e, f, g, h, t1, t2; + u32 a, b, c, d, e, f, g, h; int i; /* load the input */ @@ -61,141 +88,16 @@ static void sha256_transform(u32 *state, const u8 *input, u32 *W) e = state[4]; f = state[5]; g = state[6]; h = state[7]; /* now iterate */ - t1 = h + e1(e) + Ch(e, f, g) + 0x428a2f98 + W[0]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x71374491 + W[1]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0xb5c0fbcf + W[2]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0xe9b5dba5 + W[3]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x3956c25b + W[4]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x59f111f1 + W[5]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x923f82a4 + W[6]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0xab1c5ed5 + W[7]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0xd807aa98 + W[8]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x12835b01 + W[9]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x243185be + W[10]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x550c7dc3 + W[11]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x72be5d74 + W[12]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x80deb1fe + W[13]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x9bdc06a7 + W[14]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0xc19bf174 + W[15]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0xe49b69c1 + W[16]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0xefbe4786 + W[17]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x0fc19dc6 + W[18]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x240ca1cc + W[19]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x2de92c6f + W[20]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x4a7484aa + W[21]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x5cb0a9dc + W[22]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x76f988da + W[23]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0x983e5152 + W[24]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0xa831c66d + W[25]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0xb00327c8 + W[26]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0xbf597fc7 + W[27]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0xc6e00bf3 + W[28]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0xd5a79147 + W[29]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x06ca6351 + W[30]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x14292967 + W[31]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0x27b70a85 + W[32]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x2e1b2138 + W[33]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x4d2c6dfc + W[34]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x53380d13 + W[35]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x650a7354 + W[36]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x766a0abb + W[37]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x81c2c92e + W[38]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x92722c85 + W[39]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0xa2bfe8a1 + W[40]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0xa81a664b + W[41]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0xc24b8b70 + W[42]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0xc76c51a3 + W[43]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0xd192e819 + W[44]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0xd6990624 + W[45]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0xf40e3585 + W[46]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x106aa070 + W[47]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0x19a4c116 + W[48]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x1e376c08 + W[49]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x2748774c + W[50]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x34b0bcb5 + W[51]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x391c0cb3 + W[52]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0x4ed8aa4a + W[53]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0x5b9cca4f + W[54]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0x682e6ff3 + W[55]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; - - t1 = h + e1(e) + Ch(e, f, g) + 0x748f82ee + W[56]; - t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2; - t1 = g + e1(d) + Ch(d, e, f) + 0x78a5636f + W[57]; - t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2; - t1 = f + e1(c) + Ch(c, d, e) + 0x84c87814 + W[58]; - t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2; - t1 = e + e1(b) + Ch(b, c, d) + 0x8cc70208 + W[59]; - t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2; - t1 = d + e1(a) + Ch(a, b, c) + 0x90befffa + W[60]; - t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2; - t1 = c + e1(h) + Ch(h, a, b) + 0xa4506ceb + W[61]; - t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2; - t1 = b + e1(g) + Ch(g, h, a) + 0xbef9a3f7 + W[62]; - t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2; - t1 = a + e1(f) + Ch(f, g, h) + 0xc67178f2 + W[63]; - t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2; + for (i = 0; i < 64; i += 8) { + SHA256_ROUND(i + 0, a, b, c, d, e, f, g, h); + SHA256_ROUND(i + 1, h, a, b, c, d, e, f, g); + SHA256_ROUND(i + 2, g, h, a, b, c, d, e, f); + SHA256_ROUND(i + 3, f, g, h, a, b, c, d, e); + SHA256_ROUND(i + 4, e, f, g, h, a, b, c, d); + SHA256_ROUND(i + 5, d, e, f, g, h, a, b, c); + SHA256_ROUND(i + 6, c, d, e, f, g, h, a, b); + SHA256_ROUND(i + 7, b, c, d, e, f, g, h, a); + } state[0] += a; state[1] += b; state[2] += c; state[3] += d; state[4] += e; state[5] += f; state[6] += g; state[7] += h; From 18d05ca4486fe38991c3166b1f4df26b8a029665 Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Sun, 25 Oct 2020 10:31:19 -0400 Subject: [PATCH 070/360] crypto: lib/sha256 - Unroll LOAD and BLEND loops Unrolling the LOAD and BLEND loops improves performance by ~8% on x86_64 (tested on Broadwell Xeon) while not increasing code size too much. Signed-off-by: Arvind Sankar Reviewed-by: Eric Biggers Acked-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- lib/crypto/sha256.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index e2e29d9b0ccd..cdef37c05972 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -76,12 +76,28 @@ static void sha256_transform(u32 *state, const u8 *input, u32 *W) int i; /* load the input */ - for (i = 0; i < 16; i++) - LOAD_OP(i, W, input); + for (i = 0; i < 16; i += 8) { + LOAD_OP(i + 0, W, input); + LOAD_OP(i + 1, W, input); + LOAD_OP(i + 2, W, input); + LOAD_OP(i + 3, W, input); + LOAD_OP(i + 4, W, input); + LOAD_OP(i + 5, W, input); + LOAD_OP(i + 6, W, input); + LOAD_OP(i + 7, W, input); + } /* now blend */ - for (i = 16; i < 64; i++) - BLEND_OP(i, W); + for (i = 16; i < 64; i += 8) { + BLEND_OP(i + 0, W); + BLEND_OP(i + 1, W); + BLEND_OP(i + 2, W); + BLEND_OP(i + 3, W); + BLEND_OP(i + 4, W); + BLEND_OP(i + 5, W); + BLEND_OP(i + 6, W); + BLEND_OP(i + 7, W); + } /* load the state into our registers */ a = state[0]; b = state[1]; c = state[2]; d = state[3]; From 68299a42f84288537ee3420c431ac0115ccb90b1 Mon Sep 17 00:00:00 2001 From: Tony Luck Date: Fri, 30 Oct 2020 12:04:00 -0700 Subject: [PATCH 071/360] x86/mce: Enable additional error logging on certain Intel CPUs The Xeon versions of Sandy Bridge, Ivy Bridge and Haswell support an optional additional error logging mode which is enabled by an MSR. Previously, this mode was enabled from the mcelog(8) tool via /dev/cpu, but userspace should not be poking at MSRs. So move the enabling into the kernel. [ bp: Correct the explanation why this is done. ] Suggested-by: Boris Petkov Signed-off-by: Tony Luck Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201030190807.GA13884@agluck-desk2.amr.corp.intel.com --- arch/x86/include/asm/msr-index.h | 1 + arch/x86/kernel/cpu/mce/intel.c | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 972a34d93505..b2dd2648c0e2 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -139,6 +139,7 @@ #define MSR_IA32_MCG_CAP 0x00000179 #define MSR_IA32_MCG_STATUS 0x0000017a #define MSR_IA32_MCG_CTL 0x0000017b +#define MSR_ERROR_CONTROL 0x0000017f #define MSR_IA32_MCG_EXT_CTL 0x000004d0 #define MSR_OFFCORE_RSP_0 0x000001a6 diff --git a/arch/x86/kernel/cpu/mce/intel.c b/arch/x86/kernel/cpu/mce/intel.c index abe9fe0fb851..b47883e364b4 100644 --- a/arch/x86/kernel/cpu/mce/intel.c +++ b/arch/x86/kernel/cpu/mce/intel.c @@ -509,12 +509,32 @@ static void intel_ppin_init(struct cpuinfo_x86 *c) } } +/* + * Enable additional error logs from the integrated + * memory controller on processors that support this. + */ +static void intel_imc_init(struct cpuinfo_x86 *c) +{ + u64 error_control; + + switch (c->x86_model) { + case INTEL_FAM6_SANDYBRIDGE_X: + case INTEL_FAM6_IVYBRIDGE_X: + case INTEL_FAM6_HASWELL_X: + rdmsrl(MSR_ERROR_CONTROL, error_control); + error_control |= 2; + wrmsrl(MSR_ERROR_CONTROL, error_control); + break; + } +} + void mce_intel_feature_init(struct cpuinfo_x86 *c) { intel_init_thermal(c); intel_init_cmci(); intel_init_lmce(); intel_ppin_init(c); + intel_imc_init(c); } void mce_intel_feature_clear(struct cpuinfo_x86 *c) From 1e10cf448f841251053c7b379dfdf0ffa77790d7 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 8 Oct 2020 14:34:04 +0200 Subject: [PATCH 072/360] m68k: Avoid xchg() warning gcc warns about the value of xchg()/cmpxchg() being unused in some cases: net/core/filter.c: In function 'bpf_clear_redirect_map': arch/m68k/include/asm/cmpxchg.h:137:3: warning: value computed is not used [-Wunused-value] 106 | #define cmpxchg(ptr, o, n) cmpxchg_local((ptr), (o), (n)) net/core/filter.c:3595:4: note: in expansion of macro 'cmpxchg' 3595 | cmpxchg(&ri->map, map, NULL); Shut up that warning like we do on other architectures, by turning the macro into a statement expression. Signed-off-by: Arnd Bergmann Link: https://lore.kernel.org/r/20201008123429.1133896-1-arnd@arndb.de Signed-off-by: Geert Uytterhoeven --- arch/m68k/include/asm/cmpxchg.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/m68k/include/asm/cmpxchg.h b/arch/m68k/include/asm/cmpxchg.h index 3a3bdcfcd375..a4aa82021d3b 100644 --- a/arch/m68k/include/asm/cmpxchg.h +++ b/arch/m68k/include/asm/cmpxchg.h @@ -76,7 +76,7 @@ static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int siz } #endif -#define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) +#define xchg(ptr,x) ({(__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)));}) #include @@ -119,11 +119,11 @@ static inline unsigned long __cmpxchg(volatile void *p, unsigned long old, } #define cmpxchg(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ - (unsigned long)(n), sizeof(*(ptr)))) + ({(__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ + (unsigned long)(n), sizeof(*(ptr)));}) #define cmpxchg_local(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ - (unsigned long)(n), sizeof(*(ptr)))) + ({(__typeof__(*(ptr)))__cmpxchg((ptr), (unsigned long)(o), \ + (unsigned long)(n), sizeof(*(ptr)));}) #define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n)) From 1fe9bacab2ac5cca29b360e1032e6f7533fbdc78 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Fri, 9 Oct 2020 11:56:21 +0200 Subject: [PATCH 073/360] m68k: Remove unused mach_max_dma_address This information is unused since the discontinuous memory support has been introduced in 2007. Fixes: 12d810c1b8c2 ("m68k: discontinuous memory support") Signed-off-by: Laurent Vivier Link: https://lore.kernel.org/r/20201009095621.833192-1-laurent@vivier.eu Signed-off-by: Geert Uytterhoeven --- arch/m68k/amiga/config.c | 8 -------- arch/m68k/apollo/config.c | 1 - arch/m68k/atari/config.c | 1 - arch/m68k/bvme6000/config.c | 1 - arch/m68k/hp300/config.c | 1 - arch/m68k/include/asm/machdep.h | 1 - arch/m68k/kernel/setup_mm.c | 1 - arch/m68k/mac/config.c | 1 - arch/m68k/mvme147/config.c | 1 - arch/m68k/mvme16x/config.c | 1 - arch/m68k/q40/config.c | 5 ----- arch/m68k/sun3x/config.c | 2 -- 12 files changed, 24 deletions(-) diff --git a/arch/m68k/amiga/config.c b/arch/m68k/amiga/config.c index bee9f240f35d..3f6f7380d93c 100644 --- a/arch/m68k/amiga/config.c +++ b/arch/m68k/amiga/config.c @@ -383,14 +383,6 @@ void __init config_amiga(void) mach_init_IRQ = amiga_init_IRQ; mach_get_model = amiga_get_model; mach_get_hardware_list = amiga_get_hardware_list; - - /* - * default MAX_DMA=0xffffffff on all machines. If we don't do so, the SCSI - * code will not be able to allocate any mem for transfers, unless we are - * dealing with a Z2 mem only system. /Jes - */ - mach_max_dma_address = 0xffffffff; - mach_reset = amiga_reset; #if IS_ENABLED(CONFIG_INPUT_M68K_BEEP) mach_beep = amiga_mksound; diff --git a/arch/m68k/apollo/config.c b/arch/m68k/apollo/config.c index 762da5d7a415..2ebe5b6d229b 100644 --- a/arch/m68k/apollo/config.c +++ b/arch/m68k/apollo/config.c @@ -150,7 +150,6 @@ void __init config_apollo(void) mach_sched_init=dn_sched_init; /* */ mach_init_IRQ=dn_init_IRQ; - mach_max_dma_address = 0xffffffff; mach_hwclk = dn_dummy_hwclk; /* */ mach_reset = dn_dummy_reset; /* */ #ifdef CONFIG_HEARTBEAT diff --git a/arch/m68k/atari/config.c b/arch/m68k/atari/config.c index 7ec3161e8517..43b4e764a1ac 100644 --- a/arch/m68k/atari/config.c +++ b/arch/m68k/atari/config.c @@ -205,7 +205,6 @@ void __init config_atari(void) mach_get_model = atari_get_model; mach_get_hardware_list = atari_get_hardware_list; mach_reset = atari_reset; - mach_max_dma_address = 0xffffff; #if IS_ENABLED(CONFIG_INPUT_M68K_BEEP) mach_beep = atari_mksound; #endif diff --git a/arch/m68k/bvme6000/config.c b/arch/m68k/bvme6000/config.c index 50f4d01363df..8afaa73ec802 100644 --- a/arch/m68k/bvme6000/config.c +++ b/arch/m68k/bvme6000/config.c @@ -101,7 +101,6 @@ void __init config_bvme6000(void) bvme6000_set_vectors(); #endif - mach_max_dma_address = 0xffffffff; mach_sched_init = bvme6000_sched_init; mach_init_IRQ = bvme6000_init_IRQ; mach_hwclk = bvme6000_hwclk; diff --git a/arch/m68k/hp300/config.c b/arch/m68k/hp300/config.c index a161d44fd20b..ce1eb3d3d55d 100644 --- a/arch/m68k/hp300/config.c +++ b/arch/m68k/hp300/config.c @@ -260,7 +260,6 @@ void __init config_hp300(void) #ifdef CONFIG_HEARTBEAT mach_heartbeat = hp300_pulse; #endif - mach_max_dma_address = 0xffffffff; if (hp300_model >= HP_330 && hp300_model <= HP_433S && hp300_model != HP_350) { diff --git a/arch/m68k/include/asm/machdep.h b/arch/m68k/include/asm/machdep.h index 49bd3266b4b1..e564101790ab 100644 --- a/arch/m68k/include/asm/machdep.h +++ b/arch/m68k/include/asm/machdep.h @@ -27,7 +27,6 @@ extern void (*mach_halt)( void ); extern void (*mach_power_off)( void ); extern unsigned long (*mach_hd_init) (unsigned long, unsigned long); extern void (*mach_hd_setup)(char *, int *); -extern long mach_max_dma_address; extern void (*mach_heartbeat) (int); extern void (*mach_l2_flush) (int); extern void (*mach_beep) (unsigned int, unsigned int); diff --git a/arch/m68k/kernel/setup_mm.c b/arch/m68k/kernel/setup_mm.c index ab8aa7be260f..ade96b1ad946 100644 --- a/arch/m68k/kernel/setup_mm.c +++ b/arch/m68k/kernel/setup_mm.c @@ -99,7 +99,6 @@ EXPORT_SYMBOL(mach_set_rtc_pll); void (*mach_reset)( void ); void (*mach_halt)( void ); void (*mach_power_off)( void ); -long mach_max_dma_address = 0x00ffffff; /* default set to the lower 16MB */ #ifdef CONFIG_HEARTBEAT void (*mach_heartbeat) (int); EXPORT_SYMBOL(mach_heartbeat); diff --git a/arch/m68k/mac/config.c b/arch/m68k/mac/config.c index 0ac53d87493c..eff442e932cf 100644 --- a/arch/m68k/mac/config.c +++ b/arch/m68k/mac/config.c @@ -141,7 +141,6 @@ void __init config_mac(void) mach_reset = mac_reset; mach_halt = mac_poweroff; mach_power_off = mac_poweroff; - mach_max_dma_address = 0xffffffff; #if IS_ENABLED(CONFIG_INPUT_M68K_BEEP) mach_beep = mac_mksound; #endif diff --git a/arch/m68k/mvme147/config.c b/arch/m68k/mvme147/config.c index 490700aa2212..ead0e51ecd42 100644 --- a/arch/m68k/mvme147/config.c +++ b/arch/m68k/mvme147/config.c @@ -80,7 +80,6 @@ void __init mvme147_init_IRQ(void) void __init config_mvme147(void) { - mach_max_dma_address = 0x01000000; mach_sched_init = mvme147_sched_init; mach_init_IRQ = mvme147_init_IRQ; mach_hwclk = mvme147_hwclk; diff --git a/arch/m68k/mvme16x/config.c b/arch/m68k/mvme16x/config.c index 5b86d10e0f84..79896c331d55 100644 --- a/arch/m68k/mvme16x/config.c +++ b/arch/m68k/mvme16x/config.c @@ -268,7 +268,6 @@ void __init config_mvme16x(void) char id[40]; uint16_t brdno = be16_to_cpu(p->brdno); - mach_max_dma_address = 0xffffffff; mach_sched_init = mvme16x_sched_init; mach_init_IRQ = mvme16x_init_IRQ; mach_hwclk = mvme16x_hwclk; diff --git a/arch/m68k/q40/config.c b/arch/m68k/q40/config.c index 4627de3c0603..58566262127d 100644 --- a/arch/m68k/q40/config.c +++ b/arch/m68k/q40/config.c @@ -185,11 +185,6 @@ void __init config_q40(void) /* disable a few things that SMSQ might have left enabled */ q40_disable_irqs(); - - /* no DMA at all, but ide-scsi requires it.. make sure - * all physical RAM fits into the boundary - otherwise - * allocator may play costly and useless tricks */ - mach_max_dma_address = 1024*1024*1024; } diff --git a/arch/m68k/sun3x/config.c b/arch/m68k/sun3x/config.c index d806dee71a9c..37121a0f1253 100644 --- a/arch/m68k/sun3x/config.c +++ b/arch/m68k/sun3x/config.c @@ -44,8 +44,6 @@ void __init config_sun3x(void) sun3x_prom_init(); - mach_max_dma_address = 0xffffffff; /* we can DMA anywhere, whee */ - mach_sched_init = sun3x_sched_init; mach_init_IRQ = sun3_init_IRQ; From 4a2d2ed9bae16c14602e7aebba3f0c90f73fe786 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 23 Oct 2020 18:32:53 +0200 Subject: [PATCH 074/360] x86/mtrr: Fix a kernel-doc markup Kernel-doc markup should use this format: identifier - description Fix it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/2217cd4ae9e561da2825485eb97de77c65741489.1603469755.git.mchehab+huawei@kernel.org --- arch/x86/kernel/cpu/mtrr/mtrr.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/mtrr/mtrr.c b/arch/x86/kernel/cpu/mtrr/mtrr.c index 6a80f36b5d59..08a30c8e9431 100644 --- a/arch/x86/kernel/cpu/mtrr/mtrr.c +++ b/arch/x86/kernel/cpu/mtrr/mtrr.c @@ -813,7 +813,8 @@ void mtrr_ap_init(void) } /** - * Save current fixed-range MTRR state of the first cpu in cpu_online_mask. + * mtrr_save_state - Save current fixed-range MTRR state of the first + * cpu in cpu_online_mask. */ void mtrr_save_state(void) { From 24269999027e6b161c0078ad9c1557f9a1575128 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 23 Oct 2020 18:32:57 +0200 Subject: [PATCH 075/360] EDAC: Fix some kernel-doc markups Kernel-doc markup should use this format: identifier - description Correct that and also fix some enums' names in the kernel-doc markup. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/1d291393ba58c7b80908a3fedf02d2f53921ffe9.1603469755.git.mchehab+huawei@kernel.org --- drivers/edac/edac_device.h | 11 +++++------ include/linux/edac.h | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/drivers/edac/edac_device.h b/drivers/edac/edac_device.h index c4c0e0bdce14..fc2d2c218064 100644 --- a/drivers/edac/edac_device.h +++ b/drivers/edac/edac_device.h @@ -258,7 +258,7 @@ extern struct edac_device_ctl_info *edac_device_alloc_ctl_info( extern void edac_device_free_ctl_info(struct edac_device_ctl_info *ctl_info); /** - * edac_device_add_device: Insert the 'edac_dev' structure into the + * edac_device_add_device - Insert the 'edac_dev' structure into the * edac_device global list and create sysfs entries associated with * edac_device structure. * @@ -271,9 +271,8 @@ extern void edac_device_free_ctl_info(struct edac_device_ctl_info *ctl_info); extern int edac_device_add_device(struct edac_device_ctl_info *edac_dev); /** - * edac_device_del_device: - * Remove sysfs entries for specified edac_device structure and - * then remove edac_device structure from global list + * edac_device_del_device - Remove sysfs entries for specified edac_device + * structure and then remove edac_device structure from global list * * @dev: * Pointer to struct &device representing the edac device @@ -286,7 +285,7 @@ extern int edac_device_add_device(struct edac_device_ctl_info *edac_dev); extern struct edac_device_ctl_info *edac_device_del_device(struct device *dev); /** - * Log correctable errors. + * edac_device_handle_ce_count - Log correctable errors. * * @edac_dev: pointer to struct &edac_device_ctl_info * @inst_nr: number of the instance where the CE error happened @@ -299,7 +298,7 @@ void edac_device_handle_ce_count(struct edac_device_ctl_info *edac_dev, const char *msg); /** - * Log uncorrectable errors. + * edac_device_handle_ue_count - Log uncorrectable errors. * * @edac_dev: pointer to struct &edac_device_ctl_info * @inst_nr: number of the instance where the CE error happened diff --git a/include/linux/edac.h b/include/linux/edac.h index 15e8f3d8a895..52d7487f6bd4 100644 --- a/include/linux/edac.h +++ b/include/linux/edac.h @@ -229,7 +229,7 @@ enum mem_type { #define MEM_FLAG_NVDIMM BIT(MEM_NVDIMM) /** - * enum edac-type - Error Detection and Correction capabilities and mode + * enum edac_type - Error Detection and Correction capabilities and mode * @EDAC_UNKNOWN: Unknown if ECC is available * @EDAC_NONE: Doesn't support ECC * @EDAC_RESERVED: Reserved ECC type @@ -309,7 +309,7 @@ enum scrub_type { #define OP_OFFLINE 0x300 /** - * enum edac_mc_layer - memory controller hierarchy layer + * enum edac_mc_layer_type - memory controller hierarchy layer * * @EDAC_MC_LAYER_BRANCH: memory layer is named "branch" * @EDAC_MC_LAYER_CHANNEL: memory layer is named "channel" From 3b20369313a486246582c8ef6ff5d1d0b9c34613 Mon Sep 17 00:00:00 2001 From: Qiuxu Zhuo Date: Thu, 5 Nov 2020 15:48:51 +0800 Subject: [PATCH 076/360] EDAC: Add three new memory types There are {Low-Power DDR3/4, WIO2} types of memory. Add new entries to 'enum mem_type' and new strings to 'edac_mem_types[]' for the new types. Signed-off-by: Qiuxu Zhuo Signed-off-by: Tony Luck --- drivers/edac/edac_mc.c | 3 +++ include/linux/edac.h | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/drivers/edac/edac_mc.c b/drivers/edac/edac_mc.c index 01ff71f7b645..eef8724faae0 100644 --- a/drivers/edac/edac_mc.c +++ b/drivers/edac/edac_mc.c @@ -158,10 +158,13 @@ const char * const edac_mem_types[] = { [MEM_DDR3] = "Unbuffered-DDR3", [MEM_RDDR3] = "Registered-DDR3", [MEM_LRDDR3] = "Load-Reduced-DDR3-RAM", + [MEM_LPDDR3] = "Low-Power-DDR3-RAM", [MEM_DDR4] = "Unbuffered-DDR4", [MEM_RDDR4] = "Registered-DDR4", + [MEM_LPDDR4] = "Low-Power-DDR4-RAM", [MEM_LRDDR4] = "Load-Reduced-DDR4-RAM", [MEM_NVDIMM] = "Non-volatile-RAM", + [MEM_WIO2] = "Wide-IO-2", }; EXPORT_SYMBOL_GPL(edac_mem_types); diff --git a/include/linux/edac.h b/include/linux/edac.h index 15e8f3d8a895..8f63245f7f7c 100644 --- a/include/linux/edac.h +++ b/include/linux/edac.h @@ -175,11 +175,14 @@ static inline char *mc_event_error_type(const unsigned int err_type) * @MEM_RDDR3: Registered DDR3 RAM * This is a variant of the DDR3 memories. * @MEM_LRDDR3: Load-Reduced DDR3 memory. + * @MEM_LPDDR3: Low-Power DDR3 memory. * @MEM_DDR4: Unbuffered DDR4 RAM * @MEM_RDDR4: Registered DDR4 RAM * This is a variant of the DDR4 memories. * @MEM_LRDDR4: Load-Reduced DDR4 memory. + * @MEM_LPDDR4: Low-Power DDR4 memory. * @MEM_NVDIMM: Non-volatile RAM + * @MEM_WIO2: Wide I/O 2. */ enum mem_type { MEM_EMPTY = 0, @@ -200,10 +203,13 @@ enum mem_type { MEM_DDR3, MEM_RDDR3, MEM_LRDDR3, + MEM_LPDDR3, MEM_DDR4, MEM_RDDR4, MEM_LRDDR4, + MEM_LPDDR4, MEM_NVDIMM, + MEM_WIO2, }; #define MEM_FLAG_EMPTY BIT(MEM_EMPTY) @@ -223,10 +229,13 @@ enum mem_type { #define MEM_FLAG_XDR BIT(MEM_XDR) #define MEM_FLAG_DDR3 BIT(MEM_DDR3) #define MEM_FLAG_RDDR3 BIT(MEM_RDDR3) +#define MEM_FLAG_LPDDR3 BIT(MEM_LPDDR3) #define MEM_FLAG_DDR4 BIT(MEM_DDR4) #define MEM_FLAG_RDDR4 BIT(MEM_RDDR4) #define MEM_FLAG_LRDDR4 BIT(MEM_LRDDR4) +#define MEM_FLAG_LPDDR4 BIT(MEM_LPDDR4) #define MEM_FLAG_NVDIMM BIT(MEM_NVDIMM) +#define MEM_FLAG_WIO2 BIT(MEM_WIO2) /** * enum edac-type - Error Detection and Correction capabilities and mode From 77fdd94f091f88b045e4f45ea94cac463cbd3221 Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Mon, 19 Oct 2020 12:36:53 -0700 Subject: [PATCH 077/360] crypto: atmel-sha - remove unneeded break A break is not needed if it is preceded by a return Signed-off-by: Tom Rix Reviewed-by: Alexandre Belloni Signed-off-by: Herbert Xu --- drivers/crypto/atmel-sha.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 75ccf41a7cb9..0eb6f54e3b66 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -459,7 +459,6 @@ static int atmel_sha_init(struct ahash_request *req) break; default: return -EINVAL; - break; } ctx->bufcnt = 0; From da094e0f1b226124078dc406712264ba9533d49f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 26 Oct 2020 09:16:59 -0700 Subject: [PATCH 078/360] crypto: aead - add crypto_aead_driver_name() Add crypto_aead_driver_name(), which is analogous to crypto_skcipher_driver_name(). Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- include/crypto/aead.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/crypto/aead.h b/include/crypto/aead.h index c32a6f5664e9..fcc12c593ef8 100644 --- a/include/crypto/aead.h +++ b/include/crypto/aead.h @@ -191,6 +191,11 @@ static inline void crypto_free_aead(struct crypto_aead *tfm) crypto_destroy_tfm(tfm, crypto_aead_tfm(tfm)); } +static inline const char *crypto_aead_driver_name(struct crypto_aead *tfm) +{ + return crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)); +} + static inline struct aead_alg *crypto_aead_alg(struct crypto_aead *tfm) { return container_of(crypto_aead_tfm(tfm)->__crt_alg, From 79cafe9a8b16f7a1c2c800f14ff0f4d0b56f5721 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 26 Oct 2020 09:17:00 -0700 Subject: [PATCH 079/360] crypto: testmgr - always print the actual hash driver name When alg_test() is called from tcrypt.ko rather than from the algorithm registration code, "driver" is actually the algorithm name, not the driver name. So it shouldn't be used in places where a driver name is wanted, e.g. when reporting a test failure or when checking whether the driver is the generic driver or not. Fix this for the hash algorithm tests by getting the driver name from the crypto_ahash or crypto_shash that actually got allocated. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/testmgr.c | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index a64a639eddfa..ec64b70a5a83 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1171,8 +1171,7 @@ static inline const void *sg_data(struct scatterlist *sg) } /* Test one hash test vector in one configuration, using the shash API */ -static int test_shash_vec_cfg(const char *driver, - const struct hash_testvec *vec, +static int test_shash_vec_cfg(const struct hash_testvec *vec, const char *vec_name, const struct testvec_config *cfg, struct shash_desc *desc, @@ -1183,6 +1182,7 @@ static int test_shash_vec_cfg(const char *driver, const unsigned int alignmask = crypto_shash_alignmask(tfm); const unsigned int digestsize = crypto_shash_digestsize(tfm); const unsigned int statesize = crypto_shash_statesize(tfm); + const char *driver = crypto_shash_driver_name(tfm); const struct test_sg_division *divs[XBUFSIZE]; unsigned int i; u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN]; @@ -1355,8 +1355,7 @@ static int check_nonfinal_ahash_op(const char *op, int err, } /* Test one hash test vector in one configuration, using the ahash API */ -static int test_ahash_vec_cfg(const char *driver, - const struct hash_testvec *vec, +static int test_ahash_vec_cfg(const struct hash_testvec *vec, const char *vec_name, const struct testvec_config *cfg, struct ahash_request *req, @@ -1367,6 +1366,7 @@ static int test_ahash_vec_cfg(const char *driver, const unsigned int alignmask = crypto_ahash_alignmask(tfm); const unsigned int digestsize = crypto_ahash_digestsize(tfm); const unsigned int statesize = crypto_ahash_statesize(tfm); + const char *driver = crypto_ahash_driver_name(tfm); const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags; const struct test_sg_division *divs[XBUFSIZE]; DECLARE_CRYPTO_WAIT(wait); @@ -1521,8 +1521,7 @@ result_ready: driver, cfg); } -static int test_hash_vec_cfg(const char *driver, - const struct hash_testvec *vec, +static int test_hash_vec_cfg(const struct hash_testvec *vec, const char *vec_name, const struct testvec_config *cfg, struct ahash_request *req, @@ -1539,20 +1538,18 @@ static int test_hash_vec_cfg(const char *driver, */ if (desc) { - err = test_shash_vec_cfg(driver, vec, vec_name, cfg, desc, tsgl, + err = test_shash_vec_cfg(vec, vec_name, cfg, desc, tsgl, hashstate); if (err) return err; } - return test_ahash_vec_cfg(driver, vec, vec_name, cfg, req, tsgl, - hashstate); + return test_ahash_vec_cfg(vec, vec_name, cfg, req, tsgl, hashstate); } -static int test_hash_vec(const char *driver, const struct hash_testvec *vec, - unsigned int vec_num, struct ahash_request *req, - struct shash_desc *desc, struct test_sglist *tsgl, - u8 *hashstate) +static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num, + struct ahash_request *req, struct shash_desc *desc, + struct test_sglist *tsgl, u8 *hashstate) { char vec_name[16]; unsigned int i; @@ -1561,7 +1558,7 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, sprintf(vec_name, "%u", vec_num); for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) { - err = test_hash_vec_cfg(driver, vec, vec_name, + err = test_hash_vec_cfg(vec, vec_name, &default_hash_testvec_configs[i], req, desc, tsgl, hashstate); if (err) @@ -1576,7 +1573,7 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_hash_vec_cfg(driver, vec, vec_name, &cfg, + err = test_hash_vec_cfg(vec, vec_name, &cfg, req, desc, tsgl, hashstate); if (err) return err; @@ -1633,8 +1630,7 @@ done: * Test the hash algorithm represented by @req against the corresponding generic * implementation, if one is available. */ -static int test_hash_vs_generic_impl(const char *driver, - const char *generic_driver, +static int test_hash_vs_generic_impl(const char *generic_driver, unsigned int maxkeysize, struct ahash_request *req, struct shash_desc *desc, @@ -1646,6 +1642,7 @@ static int test_hash_vs_generic_impl(const char *driver, const unsigned int blocksize = crypto_ahash_blocksize(tfm); const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; const char *algname = crypto_hash_alg_common(tfm)->base.cra_name; + const char *driver = crypto_ahash_driver_name(tfm); char _generic_driver[CRYPTO_MAX_ALG_NAME]; struct crypto_shash *generic_tfm = NULL; struct shash_desc *generic_desc = NULL; @@ -1732,7 +1729,7 @@ static int test_hash_vs_generic_impl(const char *driver, vec_name, sizeof(vec_name)); generate_random_testvec_config(cfg, cfgname, sizeof(cfgname)); - err = test_hash_vec_cfg(driver, &vec, vec_name, cfg, + err = test_hash_vec_cfg(&vec, vec_name, cfg, req, desc, tsgl, hashstate); if (err) goto out; @@ -1749,8 +1746,7 @@ out: return err; } #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_hash_vs_generic_impl(const char *driver, - const char *generic_driver, +static int test_hash_vs_generic_impl(const char *generic_driver, unsigned int maxkeysize, struct ahash_request *req, struct shash_desc *desc, @@ -1820,6 +1816,7 @@ static int __alg_test_hash(const struct hash_testvec *vecs, driver, PTR_ERR(atfm)); return PTR_ERR(atfm); } + driver = crypto_ahash_driver_name(atfm); req = ahash_request_alloc(atfm, GFP_KERNEL); if (!req) { @@ -1859,13 +1856,12 @@ static int __alg_test_hash(const struct hash_testvec *vecs, } for (i = 0; i < num_vecs; i++) { - err = test_hash_vec(driver, &vecs[i], i, req, desc, tsgl, - hashstate); + err = test_hash_vec(&vecs[i], i, req, desc, tsgl, hashstate); if (err) goto out; cond_resched(); } - err = test_hash_vs_generic_impl(driver, generic_driver, maxkeysize, req, + err = test_hash_vs_generic_impl(generic_driver, maxkeysize, req, desc, tsgl, hashstate); out: kfree(hashstate); @@ -3602,6 +3598,7 @@ static int alg_test_crc32c(const struct alg_test_desc *desc, "%ld\n", driver, PTR_ERR(tfm)); return PTR_ERR(tfm); } + driver = crypto_shash_driver_name(tfm); do { SHASH_DESC_ON_STACK(shash, tfm); From 2257f4712dcfb3b15783e7b0845ef83db022018a Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 26 Oct 2020 09:17:01 -0700 Subject: [PATCH 080/360] crypto: testmgr - always print the actual AEAD driver name When alg_test() is called from tcrypt.ko rather than from the algorithm registration code, "driver" is actually the algorithm name, not the driver name. So it shouldn't be used in places where a driver name is wanted, e.g. when reporting a test failure or when checking whether the driver is the generic driver or not. Fix this for the AEAD algorithm tests by getting the driver name from the crypto_aead that actually got allocated. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/testmgr.c | 42 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index ec64b70a5a83..1b785b2f4987 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1919,8 +1919,7 @@ static int alg_test_hash(const struct alg_test_desc *desc, const char *driver, return err; } -static int test_aead_vec_cfg(const char *driver, int enc, - const struct aead_testvec *vec, +static int test_aead_vec_cfg(int enc, const struct aead_testvec *vec, const char *vec_name, const struct testvec_config *cfg, struct aead_request *req, @@ -1930,6 +1929,7 @@ static int test_aead_vec_cfg(const char *driver, int enc, const unsigned int alignmask = crypto_aead_alignmask(tfm); const unsigned int ivsize = crypto_aead_ivsize(tfm); const unsigned int authsize = vec->clen - vec->plen; + const char *driver = crypto_aead_driver_name(tfm); const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags; const char *op = enc ? "encryption" : "decryption"; DECLARE_CRYPTO_WAIT(wait); @@ -2102,9 +2102,8 @@ static int test_aead_vec_cfg(const char *driver, int enc, return 0; } -static int test_aead_vec(const char *driver, int enc, - const struct aead_testvec *vec, unsigned int vec_num, - struct aead_request *req, +static int test_aead_vec(int enc, const struct aead_testvec *vec, + unsigned int vec_num, struct aead_request *req, struct cipher_test_sglists *tsgls) { char vec_name[16]; @@ -2117,7 +2116,7 @@ static int test_aead_vec(const char *driver, int enc, sprintf(vec_name, "%u", vec_num); for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) { - err = test_aead_vec_cfg(driver, enc, vec, vec_name, + err = test_aead_vec_cfg(enc, vec, vec_name, &default_cipher_testvec_configs[i], req, tsgls); if (err) @@ -2132,7 +2131,7 @@ static int test_aead_vec(const char *driver, int enc, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_aead_vec_cfg(driver, enc, vec, vec_name, + err = test_aead_vec_cfg(enc, vec, vec_name, &cfg, req, tsgls); if (err) return err; @@ -2148,7 +2147,6 @@ static int test_aead_vec(const char *driver, int enc, struct aead_extra_tests_ctx { struct aead_request *req; struct crypto_aead *tfm; - const char *driver; const struct alg_test_desc *test_desc; struct cipher_test_sglists *tsgls; unsigned int maxdatasize; @@ -2354,7 +2352,7 @@ static int test_aead_inauthentic_inputs(struct aead_extra_tests_ctx *ctx) if (ctx->vec.novrfy) { generate_random_testvec_config(&ctx->cfg, ctx->cfgname, sizeof(ctx->cfgname)); - err = test_aead_vec_cfg(ctx->driver, DECRYPT, &ctx->vec, + err = test_aead_vec_cfg(DECRYPT, &ctx->vec, ctx->vec_name, &ctx->cfg, ctx->req, ctx->tsgls); if (err) @@ -2373,7 +2371,7 @@ static int test_aead_vs_generic_impl(struct aead_extra_tests_ctx *ctx) { struct crypto_aead *tfm = ctx->tfm; const char *algname = crypto_aead_alg(tfm)->base.cra_name; - const char *driver = ctx->driver; + const char *driver = crypto_aead_driver_name(tfm); const char *generic_driver = ctx->test_desc->generic_driver; char _generic_driver[CRYPTO_MAX_ALG_NAME]; struct crypto_aead *generic_tfm = NULL; @@ -2450,14 +2448,14 @@ static int test_aead_vs_generic_impl(struct aead_extra_tests_ctx *ctx) generate_random_testvec_config(&ctx->cfg, ctx->cfgname, sizeof(ctx->cfgname)); if (!ctx->vec.novrfy) { - err = test_aead_vec_cfg(driver, ENCRYPT, &ctx->vec, + err = test_aead_vec_cfg(ENCRYPT, &ctx->vec, ctx->vec_name, &ctx->cfg, ctx->req, ctx->tsgls); if (err) goto out; } if (ctx->vec.crypt_error == 0 || ctx->vec.novrfy) { - err = test_aead_vec_cfg(driver, DECRYPT, &ctx->vec, + err = test_aead_vec_cfg(DECRYPT, &ctx->vec, ctx->vec_name, &ctx->cfg, ctx->req, ctx->tsgls); if (err) @@ -2472,8 +2470,7 @@ out: return err; } -static int test_aead_extra(const char *driver, - const struct alg_test_desc *test_desc, +static int test_aead_extra(const struct alg_test_desc *test_desc, struct aead_request *req, struct cipher_test_sglists *tsgls) { @@ -2489,7 +2486,6 @@ static int test_aead_extra(const char *driver, return -ENOMEM; ctx->req = req; ctx->tfm = crypto_aead_reqtfm(req); - ctx->driver = driver; ctx->test_desc = test_desc; ctx->tsgls = tsgls; ctx->maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; @@ -2524,8 +2520,7 @@ out: return err; } #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_aead_extra(const char *driver, - const struct alg_test_desc *test_desc, +static int test_aead_extra(const struct alg_test_desc *test_desc, struct aead_request *req, struct cipher_test_sglists *tsgls) { @@ -2533,8 +2528,7 @@ static int test_aead_extra(const char *driver, } #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_aead(const char *driver, int enc, - const struct aead_test_suite *suite, +static int test_aead(int enc, const struct aead_test_suite *suite, struct aead_request *req, struct cipher_test_sglists *tsgls) { @@ -2542,8 +2536,7 @@ static int test_aead(const char *driver, int enc, int err; for (i = 0; i < suite->count; i++) { - err = test_aead_vec(driver, enc, &suite->vecs[i], i, req, - tsgls); + err = test_aead_vec(enc, &suite->vecs[i], i, req, tsgls); if (err) return err; cond_resched(); @@ -2571,6 +2564,7 @@ static int alg_test_aead(const struct alg_test_desc *desc, const char *driver, driver, PTR_ERR(tfm)); return PTR_ERR(tfm); } + driver = crypto_aead_driver_name(tfm); req = aead_request_alloc(tfm, GFP_KERNEL); if (!req) { @@ -2588,15 +2582,15 @@ static int alg_test_aead(const struct alg_test_desc *desc, const char *driver, goto out; } - err = test_aead(driver, ENCRYPT, suite, req, tsgls); + err = test_aead(ENCRYPT, suite, req, tsgls); if (err) goto out; - err = test_aead(driver, DECRYPT, suite, req, tsgls); + err = test_aead(DECRYPT, suite, req, tsgls); if (err) goto out; - err = test_aead_extra(driver, desc, req, tsgls); + err = test_aead_extra(desc, req, tsgls); out: free_cipher_test_sglists(tsgls); aead_request_free(req); From 6e5972fa4a0db6912714d465485bdae91a99ccc2 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 26 Oct 2020 09:17:02 -0700 Subject: [PATCH 081/360] crypto: testmgr - always print the actual skcipher driver name When alg_test() is called from tcrypt.ko rather than from the algorithm registration code, "driver" is actually the algorithm name, not the driver name. So it shouldn't be used in places where a driver name is wanted, e.g. when reporting a test failure or when checking whether the driver is the generic driver or not. Fix this for the skcipher algorithm tests by getting the driver name from the crypto_skcipher that actually got allocated. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/testmgr.c | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 1b785b2f4987..dcc1fa415e8e 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -2685,8 +2685,7 @@ out_nobuf: return ret; } -static int test_skcipher_vec_cfg(const char *driver, int enc, - const struct cipher_testvec *vec, +static int test_skcipher_vec_cfg(int enc, const struct cipher_testvec *vec, const char *vec_name, const struct testvec_config *cfg, struct skcipher_request *req, @@ -2695,6 +2694,7 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); const unsigned int alignmask = crypto_skcipher_alignmask(tfm); const unsigned int ivsize = crypto_skcipher_ivsize(tfm); + const char *driver = crypto_skcipher_driver_name(tfm); const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags; const char *op = enc ? "encryption" : "decryption"; DECLARE_CRYPTO_WAIT(wait); @@ -2849,8 +2849,7 @@ static int test_skcipher_vec_cfg(const char *driver, int enc, return 0; } -static int test_skcipher_vec(const char *driver, int enc, - const struct cipher_testvec *vec, +static int test_skcipher_vec(int enc, const struct cipher_testvec *vec, unsigned int vec_num, struct skcipher_request *req, struct cipher_test_sglists *tsgls) @@ -2865,7 +2864,7 @@ static int test_skcipher_vec(const char *driver, int enc, sprintf(vec_name, "%u", vec_num); for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) { - err = test_skcipher_vec_cfg(driver, enc, vec, vec_name, + err = test_skcipher_vec_cfg(enc, vec, vec_name, &default_cipher_testvec_configs[i], req, tsgls); if (err) @@ -2880,7 +2879,7 @@ static int test_skcipher_vec(const char *driver, int enc, for (i = 0; i < fuzz_iterations; i++) { generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); - err = test_skcipher_vec_cfg(driver, enc, vec, vec_name, + err = test_skcipher_vec_cfg(enc, vec, vec_name, &cfg, req, tsgls); if (err) return err; @@ -2951,8 +2950,7 @@ done: * Test the skcipher algorithm represented by @req against the corresponding * generic implementation, if one is available. */ -static int test_skcipher_vs_generic_impl(const char *driver, - const char *generic_driver, +static int test_skcipher_vs_generic_impl(const char *generic_driver, struct skcipher_request *req, struct cipher_test_sglists *tsgls) { @@ -2962,6 +2960,7 @@ static int test_skcipher_vs_generic_impl(const char *driver, const unsigned int blocksize = crypto_skcipher_blocksize(tfm); const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN; const char *algname = crypto_skcipher_alg(tfm)->base.cra_name; + const char *driver = crypto_skcipher_driver_name(tfm); char _generic_driver[CRYPTO_MAX_ALG_NAME]; struct crypto_skcipher *generic_tfm = NULL; struct skcipher_request *generic_req = NULL; @@ -3067,11 +3066,11 @@ static int test_skcipher_vs_generic_impl(const char *driver, vec_name, sizeof(vec_name)); generate_random_testvec_config(cfg, cfgname, sizeof(cfgname)); - err = test_skcipher_vec_cfg(driver, ENCRYPT, &vec, vec_name, + err = test_skcipher_vec_cfg(ENCRYPT, &vec, vec_name, cfg, req, tsgls); if (err) goto out; - err = test_skcipher_vec_cfg(driver, DECRYPT, &vec, vec_name, + err = test_skcipher_vec_cfg(DECRYPT, &vec, vec_name, cfg, req, tsgls); if (err) goto out; @@ -3089,8 +3088,7 @@ out: return err; } #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_skcipher_vs_generic_impl(const char *driver, - const char *generic_driver, +static int test_skcipher_vs_generic_impl(const char *generic_driver, struct skcipher_request *req, struct cipher_test_sglists *tsgls) { @@ -3098,8 +3096,7 @@ static int test_skcipher_vs_generic_impl(const char *driver, } #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_skcipher(const char *driver, int enc, - const struct cipher_test_suite *suite, +static int test_skcipher(int enc, const struct cipher_test_suite *suite, struct skcipher_request *req, struct cipher_test_sglists *tsgls) { @@ -3107,8 +3104,7 @@ static int test_skcipher(const char *driver, int enc, int err; for (i = 0; i < suite->count; i++) { - err = test_skcipher_vec(driver, enc, &suite->vecs[i], i, req, - tsgls); + err = test_skcipher_vec(enc, &suite->vecs[i], i, req, tsgls); if (err) return err; cond_resched(); @@ -3136,6 +3132,7 @@ static int alg_test_skcipher(const struct alg_test_desc *desc, driver, PTR_ERR(tfm)); return PTR_ERR(tfm); } + driver = crypto_skcipher_driver_name(tfm); req = skcipher_request_alloc(tfm, GFP_KERNEL); if (!req) { @@ -3153,16 +3150,15 @@ static int alg_test_skcipher(const struct alg_test_desc *desc, goto out; } - err = test_skcipher(driver, ENCRYPT, suite, req, tsgls); + err = test_skcipher(ENCRYPT, suite, req, tsgls); if (err) goto out; - err = test_skcipher(driver, DECRYPT, suite, req, tsgls); + err = test_skcipher(DECRYPT, suite, req, tsgls); if (err) goto out; - err = test_skcipher_vs_generic_impl(driver, desc->generic_driver, req, - tsgls); + err = test_skcipher_vs_generic_impl(desc->generic_driver, req, tsgls); out: free_cipher_test_sglists(tsgls); skcipher_request_free(req); From 09a5ef9644bc0e167984136b711eb08206733af8 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 26 Oct 2020 09:31:12 -0700 Subject: [PATCH 082/360] crypto: testmgr - WARN on test failure Currently, by default crypto self-test failures only result in a pr_warn() message and an "unknown" status in /proc/crypto. Both of these are easy to miss. There is also an option to panic the kernel when a test fails, but that can't be the default behavior. A crypto self-test failure always indicates a kernel bug, however, and there's already a standard way to report (recoverable) kernel bugs -- the WARN() family of macros. WARNs are noisier and harder to miss, and existing test systems already know to look for them in dmesg or via /proc/sys/kernel/tainted. Therefore, call WARN() when an algorithm fails its self-tests. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/testmgr.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index dcc1fa415e8e..321e38eef51b 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -5664,15 +5664,21 @@ int alg_test(const char *driver, const char *alg, u32 type, u32 mask) type, mask); test_done: - if (rc && (fips_enabled || panic_on_fail)) { - fips_fail_notify(); - panic("alg: self-tests for %s (%s) failed in %s mode!\n", - driver, alg, fips_enabled ? "fips" : "panic_on_fail"); + if (rc) { + if (fips_enabled || panic_on_fail) { + fips_fail_notify(); + panic("alg: self-tests for %s (%s) failed in %s mode!\n", + driver, alg, + fips_enabled ? "fips" : "panic_on_fail"); + } + WARN(1, "alg: self-tests for %s (%s) failed (rc=%d)", + driver, alg, rc); + } else { + if (fips_enabled) + pr_info("alg: self-tests for %s (%s) passed\n", + driver, alg); } - if (fips_enabled && !rc) - pr_info("alg: self-tests for %s (%s) passed\n", driver, alg); - return rc; notest: From 2d653936eb2cf613526290caa512b46e4c0d71ae Mon Sep 17 00:00:00 2001 From: Iuliana Prodan Date: Mon, 26 Oct 2020 21:06:26 +0200 Subject: [PATCH 083/360] crypto: caam - enable crypto-engine retry mechanism MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the new crypto_engine_alloc_init_and_set() function to initialize crypto-engine and enable retry mechanism. Set the maximum size for crypto-engine software queue based on Job Ring size (JOBR_DEPTH) and a threshold (reserved for the non-crypto-API requests that are not passed through crypto-engine). The callback for do_batch_requests is NULL, since CAAM doesn't support linked requests. Signed-off-by: Iuliana Prodan Reviewed-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/intern.h | 8 ++++++++ drivers/crypto/caam/jr.c | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 9112279a4de0..7d45b21bd55a 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -16,6 +16,14 @@ /* Currently comes from Kconfig param as a ^2 (driver-required) */ #define JOBR_DEPTH (1 << CONFIG_CRYPTO_DEV_FSL_CAAM_RINGSIZE) +/* + * Maximum size for crypto-engine software queue based on Job Ring + * size (JOBR_DEPTH) and a THRESHOLD (reserved for the non-crypto-API + * requests that are not passed through crypto-engine) + */ +#define THRESHOLD 15 +#define CRYPTO_ENGINE_MAX_QLEN (JOBR_DEPTH - THRESHOLD) + /* Kconfig params for interrupt coalescing if selected (else zero) */ #ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_INTC #define JOBR_INTC JRCFG_ICEN diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 6f669966ba2c..7f2b1101f567 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -550,7 +550,9 @@ static int caam_jr_probe(struct platform_device *pdev) } /* Initialize crypto engine */ - jrpriv->engine = crypto_engine_alloc_init(jrdev, false); + jrpriv->engine = crypto_engine_alloc_init_and_set(jrdev, true, NULL, + false, + CRYPTO_ENGINE_MAX_QLEN); if (!jrpriv->engine) { dev_err(jrdev, "Could not init crypto-engine\n"); return -ENOMEM; From 92eb6c3060ebe3adf381fd9899451c5b047bb14d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 26 Oct 2020 13:07:15 -0700 Subject: [PATCH 084/360] crypto: af_alg - avoid undefined behavior accessing salg_name Commit 3f69cc60768b ("crypto: af_alg - Allow arbitrarily long algorithm names") made the kernel start accepting arbitrarily long algorithm names in sockaddr_alg. However, the actual length of the salg_name field stayed at the original 64 bytes. This is broken because the kernel can access indices >= 64 in salg_name, which is undefined behavior -- even though the memory that is accessed is still located within the sockaddr structure. It would only be defined behavior if the array were properly marked as arbitrary-length (either by making it a flexible array, which is the recommended way these days, or by making it an array of length 0 or 1). We can't simply change salg_name into a flexible array, since that would break source compatibility with userspace programs that embed sockaddr_alg into another struct, or (more commonly) declare a sockaddr_alg like 'struct sockaddr_alg sa = { .salg_name = "foo" };'. One solution would be to change salg_name into a flexible array only when '#ifdef __KERNEL__'. However, that would keep userspace without an easy way to actually use the longer algorithm names. Instead, add a new structure 'sockaddr_alg_new' that has the flexible array field, and expose it to both userspace and the kernel. Make the kernel use it correctly in alg_bind(). This addresses the syzbot report "UBSAN: array-index-out-of-bounds in alg_bind" (https://syzkaller.appspot.com/bug?extid=92ead4eb8e26a26d465e). Reported-by: syzbot+92ead4eb8e26a26d465e@syzkaller.appspotmail.com Fixes: 3f69cc60768b ("crypto: af_alg - Allow arbitrarily long algorithm names") Cc: # v4.12+ Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/af_alg.c | 10 +++++++--- include/uapi/linux/if_alg.h | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/crypto/af_alg.c b/crypto/af_alg.c index d11db80d24cd..9acb9d2c4bcf 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -147,7 +147,7 @@ static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) const u32 allowed = CRYPTO_ALG_KERN_DRIVER_ONLY; struct sock *sk = sock->sk; struct alg_sock *ask = alg_sk(sk); - struct sockaddr_alg *sa = (void *)uaddr; + struct sockaddr_alg_new *sa = (void *)uaddr; const struct af_alg_type *type; void *private; int err; @@ -155,7 +155,11 @@ static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) if (sock->state == SS_CONNECTED) return -EINVAL; - if (addr_len < sizeof(*sa)) + BUILD_BUG_ON(offsetof(struct sockaddr_alg_new, salg_name) != + offsetof(struct sockaddr_alg, salg_name)); + BUILD_BUG_ON(offsetof(struct sockaddr_alg, salg_name) != sizeof(*sa)); + + if (addr_len < sizeof(*sa) + 1) return -EINVAL; /* If caller uses non-allowed flag, return error. */ @@ -163,7 +167,7 @@ static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) return -EINVAL; sa->salg_type[sizeof(sa->salg_type) - 1] = 0; - sa->salg_name[sizeof(sa->salg_name) + addr_len - sizeof(*sa) - 1] = 0; + sa->salg_name[addr_len - sizeof(*sa) - 1] = 0; type = alg_get_type(sa->salg_type); if (PTR_ERR(type) == -ENOENT) { diff --git a/include/uapi/linux/if_alg.h b/include/uapi/linux/if_alg.h index 60b7c2efd921..dc52a11ba6d1 100644 --- a/include/uapi/linux/if_alg.h +++ b/include/uapi/linux/if_alg.h @@ -24,6 +24,22 @@ struct sockaddr_alg { __u8 salg_name[64]; }; +/* + * Linux v4.12 and later removed the 64-byte limit on salg_name[]; it's now an + * arbitrary-length field. We had to keep the original struct above for source + * compatibility with existing userspace programs, though. Use the new struct + * below if support for very long algorithm names is needed. To do this, + * allocate 'sizeof(struct sockaddr_alg_new) + strlen(algname) + 1' bytes, and + * copy algname (including the null terminator) into salg_name. + */ +struct sockaddr_alg_new { + __u16 salg_family; + __u8 salg_type[14]; + __u32 salg_feat; + __u32 salg_mask; + __u8 salg_name[]; +}; + struct af_alg_iv { __u32 ivlen; __u8 iv[0]; From 519a0d7e495a6d3ce62594e485aea2a3a4a2ca0a Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 27 Oct 2020 00:00:27 +0100 Subject: [PATCH 085/360] crypto: arm64/poly1305-neon - reorder PAC authentication with SP update PAC pointer authentication signs the return address against the value of the stack pointer, to prevent stack overrun exploits from corrupting the control flow. However, this requires that the AUTIASP is issued with SP holding the same value as it held when the PAC value was generated. The Poly1305 NEON code got this wrong, resulting in crashes on PAC capable hardware. Fixes: f569ca164751 ("crypto: arm64/poly1305 - incorporate OpenSSL/CRYPTOGAMS ...") Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/poly1305-armv8.pl | 2 +- arch/arm64/crypto/poly1305-core.S_shipped | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/crypto/poly1305-armv8.pl b/arch/arm64/crypto/poly1305-armv8.pl index 6e5576d19af8..cbc980fb02e3 100644 --- a/arch/arm64/crypto/poly1305-armv8.pl +++ b/arch/arm64/crypto/poly1305-armv8.pl @@ -840,7 +840,6 @@ poly1305_blocks_neon: ldp d14,d15,[sp,#64] addp $ACC2,$ACC2,$ACC2 ldr x30,[sp,#8] - .inst 0xd50323bf // autiasp //////////////////////////////////////////////////////////////// // lazy reduction, but without narrowing @@ -882,6 +881,7 @@ poly1305_blocks_neon: str x4,[$ctx,#8] // set is_base2_26 ldr x29,[sp],#80 + .inst 0xd50323bf // autiasp ret .size poly1305_blocks_neon,.-poly1305_blocks_neon diff --git a/arch/arm64/crypto/poly1305-core.S_shipped b/arch/arm64/crypto/poly1305-core.S_shipped index 8d1c4e420ccd..fb2822abf63a 100644 --- a/arch/arm64/crypto/poly1305-core.S_shipped +++ b/arch/arm64/crypto/poly1305-core.S_shipped @@ -779,7 +779,6 @@ poly1305_blocks_neon: ldp d14,d15,[sp,#64] addp v21.2d,v21.2d,v21.2d ldr x30,[sp,#8] - .inst 0xd50323bf // autiasp //////////////////////////////////////////////////////////////// // lazy reduction, but without narrowing @@ -821,6 +820,7 @@ poly1305_blocks_neon: str x4,[x0,#8] // set is_base2_26 ldr x29,[sp],#80 + .inst 0xd50323bf // autiasp ret .size poly1305_blocks_neon,.-poly1305_blocks_neon From 76bb6a7b5b427fcbb1996f8d002888be81371320 Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Tue, 27 Oct 2020 12:01:25 -0700 Subject: [PATCH 086/360] crypto: cavium/nitrox - remove unneeded semicolon A semicolon is not needed after a switch statement. Signed-off-by: Tom Rix Signed-off-by: Herbert Xu --- drivers/crypto/cavium/nitrox/nitrox_mbx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.c b/drivers/crypto/cavium/nitrox/nitrox_mbx.c index b51b0449b478..73993f9e2311 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_mbx.c +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.c @@ -112,7 +112,7 @@ static void pf2vf_resp_handler(struct work_struct *work) case MBX_MSG_TYPE_ACK: case MBX_MSG_TYPE_NACK: break; - }; + } kfree(pf2vf_resp); } From cc9a99fb89a91b841556ff5aa97f7a9d0e01cbcc Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Tue, 27 Oct 2020 12:04:07 -0700 Subject: [PATCH 087/360] crypto: qat - remove unneeded semicolon A semicolon is not needed after a switch statement. Signed-off-by: Tom Rix Signed-off-by: Tom Rix Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 2 +- drivers/crypto/qat/qat_common/qat_asym_algs.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index a38afc61f6d2..0fab8bb8ca59 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -105,7 +105,7 @@ static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) return ICP_QAT_HW_SHA512_STATE1_SZ; default: return -EFAULT; - }; + } return -EFAULT; } diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index f112078be868..2c863d25327a 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -201,7 +201,7 @@ static unsigned long qat_dh_fn_id(unsigned int len, bool g2) return g2 ? PKE_DH_G2_4096 : PKE_DH_4096; default: return 0; - }; + } } static int qat_dh_compute_value(struct kpp_request *req) @@ -572,7 +572,7 @@ static unsigned long qat_rsa_enc_fn_id(unsigned int len) return PKE_RSA_EP_4096; default: return 0; - }; + } } #define PKE_RSA_DP1_512 0x1c161b3c @@ -601,7 +601,7 @@ static unsigned long qat_rsa_dec_fn_id(unsigned int len) return PKE_RSA_DP1_4096; default: return 0; - }; + } } #define PKE_RSA_DP2_512 0x1c131b57 @@ -630,7 +630,7 @@ static unsigned long qat_rsa_dec_fn_id_crt(unsigned int len) return PKE_RSA_DP2_4096; default: return 0; - }; + } } static int qat_rsa_enc(struct akcipher_request *req) From a2715fbdc6fc387e85211df917a4778761ec693d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Wed, 28 Oct 2020 11:03:20 +0200 Subject: [PATCH 088/360] crypto: arm/aes-neonbs - fix usage of cbc(aes) fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Loading the module deadlocks since: -local cbc(aes) implementation needs a fallback and -crypto API tries to find one but the request_module() resolves back to the same module Fix this by changing the module alias for cbc(aes) and using the NEED_FALLBACK flag when requesting for a fallback algorithm. Fixes: 00b99ad2bac2 ("crypto: arm/aes-neonbs - Use generic cbc encryption path") Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- arch/arm/crypto/aes-neonbs-glue.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index bda8bf17631e..f70af1d0514b 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -19,7 +19,7 @@ MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL v2"); MODULE_ALIAS_CRYPTO("ecb(aes)"); -MODULE_ALIAS_CRYPTO("cbc(aes)"); +MODULE_ALIAS_CRYPTO("cbc(aes)-all"); MODULE_ALIAS_CRYPTO("ctr(aes)"); MODULE_ALIAS_CRYPTO("xts(aes)"); @@ -191,7 +191,8 @@ static int cbc_init(struct crypto_skcipher *tfm) struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); unsigned int reqsize; - ctx->enc_tfm = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC); + ctx->enc_tfm = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(ctx->enc_tfm)) return PTR_ERR(ctx->enc_tfm); @@ -441,7 +442,8 @@ static struct skcipher_alg aes_algs[] = { { .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), .base.cra_module = THIS_MODULE, - .base.cra_flags = CRYPTO_ALG_INTERNAL, + .base.cra_flags = CRYPTO_ALG_INTERNAL | + CRYPTO_ALG_NEED_FALLBACK, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, From 5bd9938e9316a903f1586b8459fbaac396fc1188 Mon Sep 17 00:00:00 2001 From: Nigel Christian Date: Wed, 28 Oct 2020 20:52:17 -0400 Subject: [PATCH 089/360] hwrng: imx-rngc - irq already prints an error Clean up the check for irq. dev_err() is superfluous as platform_get_irq() already prints an error. Check for zero would indicate a bug. Remove curly braces to conform to styling requirements. Signed-off-by: Nigel Christian Signed-off-by: Herbert Xu --- drivers/char/hw_random/imx-rngc.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c index 61c844baf26e..b05d676ca814 100644 --- a/drivers/char/hw_random/imx-rngc.c +++ b/drivers/char/hw_random/imx-rngc.c @@ -252,10 +252,8 @@ static int imx_rngc_probe(struct platform_device *pdev) } irq = platform_get_irq(pdev, 0); - if (irq <= 0) { - dev_err(&pdev->dev, "Couldn't get irq %d\n", irq); + if (irq < 0) return irq; - } ret = clk_prepare_enable(rngc->clk); if (ret) From 1b5644f29983b2847162de4d3b5bce23faa136ab Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:01 +0800 Subject: [PATCH 090/360] crypto: hisilicon/qm - numbers are replaced by macros Some numbers are replaced by macros to avoid incomprehension. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 050fe4e74523..1de3aac1dd56 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -473,7 +473,7 @@ static int qm_wait_mb_ready(struct hisi_qm *qm) return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE, val, !((val >> QM_MB_BUSY_SHIFT) & - 0x1), 10, 1000); + 0x1), POLL_PERIOD, POLL_TIMEOUT); } /* 128 bit should be written to hardware at one time to trigger a mailbox */ @@ -583,7 +583,8 @@ static int qm_dev_mem_reset(struct hisi_qm *qm) writel(0x1, qm->io_base + QM_MEM_START_INIT); return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val, - val & BIT(0), 10, 1000); + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); } static u32 qm_get_irq_num_v1(struct hisi_qm *qm) @@ -804,7 +805,8 @@ static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, int ret; ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, - val & BIT(0), 10, 1000); + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); if (ret) return ret; @@ -818,7 +820,8 @@ static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, - val & BIT(0), 10, 1000); + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); } /* The config should be conducted after qm_dev_mem_reset() */ @@ -1785,10 +1788,11 @@ static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) INIT_QC_COMMON(cqc, qp->cqe_dma, pasid); if (ver == QM_HW_V1) { - cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, 4)); + cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, + QM_QC_CQE_SIZE)); cqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1); } else { - cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(4)); + cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE)); cqc->w8 = 0; } cqc->dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT); @@ -2011,7 +2015,8 @@ static void hisi_qm_cache_wb(struct hisi_qm *qm) writel(0x1, qm->io_base + QM_CACHE_WB_START); if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, - val, val & BIT(0), 10, 1000)) + val, val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT)) dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n"); } From e926d753a6128035a72a99490daa56a4f9a49f83 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:02 +0800 Subject: [PATCH 091/360] crypto: hisilicon/qm - modify the return type of function The returns of 'qm_get_hw_error_status' and 'qm_get_dev_err_status' are values from the hardware registers, which should not be defined as 'int', so update as 'u32'. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 1de3aac1dd56..d5d06ae106f2 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -3291,7 +3291,7 @@ pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, } EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected); -static int qm_get_hw_error_status(struct hisi_qm *qm) +static u32 qm_get_hw_error_status(struct hisi_qm *qm) { return readl(qm->io_base + QM_ABNORMAL_INT_STATUS); } @@ -3590,7 +3590,7 @@ restart_fail: return ret; } -static int qm_get_dev_err_status(struct hisi_qm *qm) +static u32 qm_get_dev_err_status(struct hisi_qm *qm) { return qm->err_ini->get_dev_hw_err_status(qm); } From a8ff38bd829420d4813c22d018f412d5f1bb65df Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:03 +0800 Subject: [PATCH 092/360] crypto: hisilicon/qm - modify the return type of debugfs interface Since 'qm_create_debugfs_file' always returns 0, change it as 'void'. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/hpre/hpre_main.c | 4 +--- drivers/crypto/hisilicon/qm.c | 19 ++++--------------- drivers/crypto/hisilicon/qm.h | 2 +- drivers/crypto/hisilicon/sec2/sec_main.c | 4 +--- drivers/crypto/hisilicon/zip/zip_main.c | 4 +--- 5 files changed, 8 insertions(+), 25 deletions(-) diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index a33394d91bbf..e5c991913f09 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -705,9 +705,7 @@ static int hpre_debugfs_init(struct hisi_qm *qm) qm->debug.sqe_mask_offset = HPRE_SQE_MASK_OFFSET; qm->debug.sqe_mask_len = HPRE_SQE_MASK_LEN; - ret = hisi_qm_debug_init(qm); - if (ret) - goto failed_to_create; + hisi_qm_debug_init(qm); if (qm->pdev->device == HPRE_PCI_DEVICE_ID) { ret = hpre_ctrl_debug_init(qm); diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index d5d06ae106f2..627479fb48e6 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -1520,7 +1520,7 @@ static const struct file_operations qm_cmd_fops = { .write = qm_cmd_write, }; -static int qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index) +static void qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index) { struct dentry *qm_d = qm->debug.qm_d; struct debugfs_file *file = qm->debug.files + index; @@ -1531,8 +1531,6 @@ static int qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index) file->index = index; mutex_init(&file->lock); file->debug = &qm->debug; - - return 0; } static void qm_hw_error_init_v1(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) @@ -2824,12 +2822,12 @@ DEFINE_DEBUGFS_ATTRIBUTE(qm_atomic64_ops, qm_debugfs_atomic64_get, * * Create qm related debugfs files. */ -int hisi_qm_debug_init(struct hisi_qm *qm) +void hisi_qm_debug_init(struct hisi_qm *qm) { struct qm_dfx *dfx = &qm->debug.dfx; struct dentry *qm_d; void *data; - int i, ret; + int i; qm_d = debugfs_create_dir("qm", qm->debug.debug_root); qm->debug.qm_d = qm_d; @@ -2837,10 +2835,7 @@ int hisi_qm_debug_init(struct hisi_qm *qm) /* only show this in PF */ if (qm->fun_type == QM_HW_PF) for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++) - if (qm_create_debugfs_file(qm, i)) { - ret = -ENOENT; - goto failed_to_create; - } + qm_create_debugfs_file(qm, i); debugfs_create_file("regs", 0444, qm->debug.qm_d, qm, &qm_regs_fops); @@ -2856,12 +2851,6 @@ int hisi_qm_debug_init(struct hisi_qm *qm) data, &qm_atomic64_ops); } - - return 0; - -failed_to_create: - debugfs_remove_recursive(qm_d); - return ret; } EXPORT_SYMBOL_GPL(hisi_qm_debug_init); diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index 0420f4ce7197..8624d1288afe 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -350,7 +350,7 @@ void hisi_qm_release_qp(struct hisi_qp *qp); int hisi_qp_send(struct hisi_qp *qp, const void *msg); int hisi_qm_get_free_qp_num(struct hisi_qm *qm); int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number); -int hisi_qm_debug_init(struct hisi_qm *qm); +void hisi_qm_debug_init(struct hisi_qm *qm); enum qm_hw_ver hisi_qm_get_hw_version(struct pci_dev *pdev); void hisi_qm_debug_regs_clear(struct hisi_qm *qm); int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs); diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 2f52581b6d3c..b35c1c2271a3 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -652,9 +652,7 @@ static int sec_debugfs_init(struct hisi_qm *qm) sec_debugfs_root); qm->debug.sqe_mask_offset = SEC_SQE_MASK_OFFSET; qm->debug.sqe_mask_len = SEC_SQE_MASK_LEN; - ret = hisi_qm_debug_init(qm); - if (ret) - goto failed_to_create; + hisi_qm_debug_init(qm); ret = sec_debug_init(qm); if (ret) diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 4bd2c811abba..3d1524b63ea8 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -590,9 +590,7 @@ static int hisi_zip_debugfs_init(struct hisi_qm *qm) qm->debug.sqe_mask_offset = HZIP_SQE_MASK_OFFSET; qm->debug.sqe_mask_len = HZIP_SQE_MASK_LEN; qm->debug.debug_root = dev_d; - ret = hisi_qm_debug_init(qm); - if (ret) - goto failed_to_create; + hisi_qm_debug_init(qm); if (qm->fun_type == QM_HW_PF) { ret = hisi_zip_ctrl_debug_init(qm); From cd1aff982af43d47557a44d1758d9d5c06247d59 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:04 +0800 Subject: [PATCH 093/360] crypto: hisilicon/qm - modify return type of 'qm_set_sqctype' Since 'qm_set_sqctype' always returns 0, change it as 'void'. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 627479fb48e6..17f84dbc8bbe 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -2118,7 +2118,7 @@ static void hisi_qm_uacce_stop_queue(struct uacce_queue *q) hisi_qm_stop_qp(q->priv); } -static int qm_set_sqctype(struct uacce_queue *q, u16 type) +static void qm_set_sqctype(struct uacce_queue *q, u16 type) { struct hisi_qm *qm = q->uacce->priv; struct hisi_qp *qp = q->priv; @@ -2126,8 +2126,6 @@ static int qm_set_sqctype(struct uacce_queue *q, u16 type) down_write(&qm->qps_lock); qp->alg_type = type; up_write(&qm->qps_lock); - - return 0; } static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, From 09493afbc62781bd9fba6224af89fd78fe33b8ba Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:05 +0800 Subject: [PATCH 094/360] crypto: hisilicon/qm - replace 'sprintf' with 'scnprintf' Replace 'sprintf' with 'scnprintf' to avoid overrun. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 17f84dbc8bbe..25c54146cca1 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -932,7 +932,8 @@ static ssize_t qm_debug_read(struct file *filp, char __user *buf, return -EINVAL; } mutex_unlock(&file->lock); - ret = sprintf(tbuf, "%u\n", val); + + ret = scnprintf(tbuf, QM_DBG_TMP_BUF_LEN, "%u\n", val); return simple_read_from_buffer(buf, count, pos, tbuf, ret); } From 3bf1ef9d513ea405672bbc245964acc1990f4e80 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:06 +0800 Subject: [PATCH 095/360] crypto: hisilicon/qm - split 'qm_qp_ctx_cfg' into smaller pieces 'qm_qp_ctx_cfg' initializes configuration of SQ and CQ, split it into two pieces to improve code readability. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 36 +++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 25c54146cca1..4c5cc60e9f7d 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -1735,19 +1735,15 @@ void hisi_qm_release_qp(struct hisi_qp *qp) } EXPORT_SYMBOL_GPL(hisi_qm_release_qp); -static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) +static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) { struct hisi_qm *qm = qp->qm; struct device *dev = &qm->pdev->dev; enum qm_hw_ver ver = qm->ver; struct qm_sqc *sqc; - struct qm_cqc *cqc; dma_addr_t sqc_dma; - dma_addr_t cqc_dma; int ret; - qm_init_qp_status(qp); - sqc = kzalloc(sizeof(struct qm_sqc), GFP_KERNEL); if (!sqc) return -ENOMEM; @@ -1772,12 +1768,23 @@ static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) ret = qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp_id, 0); dma_unmap_single(dev, sqc_dma, sizeof(struct qm_sqc), DMA_TO_DEVICE); kfree(sqc); - if (ret) - return ret; + + return ret; +} + +static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) +{ + struct hisi_qm *qm = qp->qm; + struct device *dev = &qm->pdev->dev; + enum qm_hw_ver ver = qm->ver; + struct qm_cqc *cqc; + dma_addr_t cqc_dma; + int ret; cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL); if (!cqc) return -ENOMEM; + cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc), DMA_TO_DEVICE); if (dma_mapping_error(dev, cqc_dma)) { @@ -1792,7 +1799,7 @@ static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) cqc->w8 = cpu_to_le16(QM_Q_DEPTH - 1); } else { cqc->dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE)); - cqc->w8 = 0; + cqc->w8 = 0; /* rand_qc */ } cqc->dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT); @@ -1803,6 +1810,19 @@ static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) return ret; } +static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) +{ + int ret; + + qm_init_qp_status(qp); + + ret = qm_sq_ctx_cfg(qp, qp_id, pasid); + if (ret) + return ret; + + return qm_cq_ctx_cfg(qp, qp_id, pasid); +} + static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg) { struct hisi_qm *qm = qp->qm; From 53737881da6e505a435f8a10d10a19dd0c888823 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:07 +0800 Subject: [PATCH 096/360] crypto: hisilicon/qm - split 'qm_eq_ctx_cfg' into smaller pieces 'qm_eq_ctx_cfg' initializes configuration of EQ and AEQ, split it into two pieces to improve code readability. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 44 +++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 4c5cc60e9f7d..6e8d20d73d54 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -1735,7 +1735,7 @@ void hisi_qm_release_qp(struct hisi_qp *qp) } EXPORT_SYMBOL_GPL(hisi_qm_release_qp); -static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) +static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) { struct hisi_qm *qm = qp->qm; struct device *dev = &qm->pdev->dev; @@ -1772,7 +1772,7 @@ static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) return ret; } -static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) +static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) { struct hisi_qm *qm = qp->qm; struct device *dev = &qm->pdev->dev; @@ -1784,7 +1784,6 @@ static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) cqc = kzalloc(sizeof(struct qm_cqc), GFP_KERNEL); if (!cqc) return -ENOMEM; - cqc_dma = dma_map_single(dev, cqc, sizeof(struct qm_cqc), DMA_TO_DEVICE); if (dma_mapping_error(dev, cqc_dma)) { @@ -1810,7 +1809,7 @@ static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) return ret; } -static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, int pasid) +static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid) { int ret; @@ -2550,14 +2549,10 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; struct qm_eqc *eqc; - struct qm_aeqc *aeqc; dma_addr_t eqc_dma; - dma_addr_t aeqc_dma; int ret; - qm_init_eq_aeq_status(qm); - - eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL); + eqc = kzalloc(sizeof(struct qm_eqc), GFP_KERNEL); //todo if (!eqc) return -ENOMEM; eqc_dma = dma_map_single(dev, eqc, sizeof(struct qm_eqc), @@ -2572,11 +2567,20 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) if (qm->ver == QM_HW_V1) eqc->dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE); eqc->dw6 = cpu_to_le32((QM_EQ_DEPTH - 1) | (1 << QM_EQC_PHASE_SHIFT)); + ret = qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0); dma_unmap_single(dev, eqc_dma, sizeof(struct qm_eqc), DMA_TO_DEVICE); kfree(eqc); - if (ret) - return ret; + + return ret; +} + +static int qm_aeq_ctx_cfg(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + struct qm_aeqc *aeqc; + dma_addr_t aeqc_dma; + int ret; aeqc = kzalloc(sizeof(struct qm_aeqc), GFP_KERNEL); if (!aeqc) @@ -2599,6 +2603,22 @@ static int qm_eq_ctx_cfg(struct hisi_qm *qm) return ret; } +static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + qm_init_eq_aeq_status(qm); + + ret = qm_eq_ctx_cfg(qm); + if (ret) { + dev_err(dev, "Set eqc failed!\n"); + return ret; + } + + return qm_aeq_ctx_cfg(qm); +} + static int __hisi_qm_start(struct hisi_qm *qm) { int ret; @@ -2615,7 +2635,7 @@ static int __hisi_qm_start(struct hisi_qm *qm) return ret; } - ret = qm_eq_ctx_cfg(qm); + ret = qm_eq_aeq_ctx_cfg(qm); if (ret) return ret; From fefc046f2b0e0bdbd4d5944f6b489039678e001e Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Sat, 31 Oct 2020 17:07:08 +0800 Subject: [PATCH 097/360] crypto: hisilicon/qm - split 'hisi_qm_init' into smaller pieces 'hisi_qm_init' initializes configuration of QM. To improve code readability, split it into two pieces. Signed-off-by: Weili Qian Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/qm.c | 86 +++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 33 deletions(-) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 6e8d20d73d54..f21ccae0e8ea 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -2442,6 +2442,16 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) qm->is_frozen = false; } +static void hisi_qm_pci_uninit(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + + pci_free_irq_vectors(pdev); + iounmap(qm->io_base); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + /** * hisi_qm_uninit() - Uninitialize qm. * @qm: The qm needed uninit. @@ -2460,9 +2470,6 @@ void hisi_qm_uninit(struct hisi_qm *qm) return; } - uacce_remove(qm->uacce); - qm->uacce = NULL; - hisi_qp_memory_uninit(qm, qm->qp_num); idr_destroy(&qm->qp_idr); @@ -2474,10 +2481,9 @@ void hisi_qm_uninit(struct hisi_qm *qm) } qm_irq_unregister(qm); - pci_free_irq_vectors(pdev); - iounmap(qm->io_base); - pci_release_mem_regions(pdev); - pci_disable_device(pdev); + hisi_qm_pci_uninit(qm); + uacce_remove(qm->uacce); + qm->uacce = NULL; up_write(&qm->qps_lock); } @@ -4038,34 +4044,22 @@ void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) } EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister); -/** - * hisi_qm_init() - Initialize configures about qm. - * @qm: The qm needing init. - * - * This function init qm, then we can call hisi_qm_start to put qm into work. - */ -int hisi_qm_init(struct hisi_qm *qm) +static int hisi_qm_pci_init(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; struct device *dev = &pdev->dev; unsigned int num_vec; int ret; - hisi_qm_pre_init(qm); - - ret = qm_alloc_uacce(qm); - if (ret < 0) - dev_warn(&pdev->dev, "fail to alloc uacce (%d)\n", ret); - ret = pci_enable_device_mem(pdev); if (ret < 0) { - dev_err(&pdev->dev, "Failed to enable device mem!\n"); - goto err_remove_uacce; + dev_err(dev, "Failed to enable device mem!\n"); + return ret; } ret = pci_request_mem_regions(pdev, qm->dev_name); if (ret < 0) { - dev_err(&pdev->dev, "Failed to request mem regions!\n"); + dev_err(dev, "Failed to request mem regions!\n"); goto err_disable_pcidev; } @@ -4093,9 +4087,42 @@ int hisi_qm_init(struct hisi_qm *qm) goto err_iounmap; } + return 0; + +err_iounmap: + iounmap(qm->io_base); +err_release_mem_regions: + pci_release_mem_regions(pdev); +err_disable_pcidev: + pci_disable_device(pdev); + return ret; +} + +/** + * hisi_qm_init() - Initialize configures about qm. + * @qm: The qm needing init. + * + * This function init qm, then we can call hisi_qm_start to put qm into work. + */ +int hisi_qm_init(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct device *dev = &pdev->dev; + int ret; + + hisi_qm_pre_init(qm); + + ret = qm_alloc_uacce(qm); + if (ret < 0) + dev_warn(dev, "fail to alloc uacce (%d)\n", ret); + + ret = hisi_qm_pci_init(qm); + if (ret) + goto err_remove_uacce; + ret = qm_irq_register(qm); if (ret) - goto err_free_irq_vectors; + goto err_pci_uninit; if (qm->fun_type == QM_HW_VF && qm->ver != QM_HW_V1) { /* v2 starts to support get vft by mailbox */ @@ -4118,14 +4145,8 @@ int hisi_qm_init(struct hisi_qm *qm) err_irq_unregister: qm_irq_unregister(qm); -err_free_irq_vectors: - pci_free_irq_vectors(pdev); -err_iounmap: - iounmap(qm->io_base); -err_release_mem_regions: - pci_release_mem_regions(pdev); -err_disable_pcidev: - pci_disable_device(pdev); +err_pci_uninit: + hisi_qm_pci_uninit(qm); err_remove_uacce: uacce_remove(qm->uacce); qm->uacce = NULL; @@ -4133,7 +4154,6 @@ err_remove_uacce: } EXPORT_SYMBOL_GPL(hisi_qm_init); - MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Zhou Wang "); MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver"); From ab95bd2aa904e4f53b7358efeea1d57693fb7889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Sun, 1 Nov 2020 22:05:53 +0200 Subject: [PATCH 098/360] crypto: caam - fix printing on xts fallback allocation error path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the time xts fallback tfm allocation fails the device struct hasn't been enabled yet in the caam xts tfm's private context. Fix this by using the device struct from xts algorithm's private context or, when not available, by replacing dev_err with pr_err. Fixes: 9d9b14dbe077 ("crypto: caam/jr - add fallback for XTS with more than 8B IV") Fixes: 83e8aa912138 ("crypto: caam/qi - add fallback for XTS with more than 8B IV") Fixes: 36e2d7cfdcf1 ("crypto: caam/qi2 - add fallback for XTS with more than 8B IV") Signed-off-by: Horia Geantă Reviewed-by: Iuliana Prodan Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg.c | 4 ++-- drivers/crypto/caam/caamalg_qi.c | 4 ++-- drivers/crypto/caam/caamalg_qi2.c | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index cf5bd7666dfc..8697ae53b063 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -3404,8 +3404,8 @@ static int caam_cra_init(struct crypto_skcipher *tfm) fallback = crypto_alloc_skcipher(tfm_name, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { - dev_err(ctx->jrdev, "Failed to allocate %s fallback: %ld\n", - tfm_name, PTR_ERR(fallback)); + pr_err("Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); return PTR_ERR(fallback); } diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index 66f60d78bdc8..a24ae966df4a 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -2502,8 +2502,8 @@ static int caam_cra_init(struct crypto_skcipher *tfm) fallback = crypto_alloc_skcipher(tfm_name, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { - dev_err(ctx->jrdev, "Failed to allocate %s fallback: %ld\n", - tfm_name, PTR_ERR(fallback)); + pr_err("Failed to allocate %s fallback: %ld\n", + tfm_name, PTR_ERR(fallback)); return PTR_ERR(fallback); } diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 98c1ff1744bb..a780e627838a 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -1611,7 +1611,8 @@ static int caam_cra_init_skcipher(struct crypto_skcipher *tfm) fallback = crypto_alloc_skcipher(tfm_name, 0, CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback)) { - dev_err(ctx->dev, "Failed to allocate %s fallback: %ld\n", + dev_err(caam_alg->caam.dev, + "Failed to allocate %s fallback: %ld\n", tfm_name, PTR_ERR(fallback)); return PTR_ERR(fallback); } From 77080929d56d87a57093869a15d2785b8b2d8cd5 Mon Sep 17 00:00:00 2001 From: Kaixu Xia Date: Fri, 6 Nov 2020 17:19:23 +0800 Subject: [PATCH 099/360] x86/mce: Assign boolean values to a bool variable Fix the following coccinelle warnings: ./arch/x86/kernel/cpu/mce/core.c:1765:3-20: WARNING: Assignment of 0/1 to bool variable ./arch/x86/kernel/cpu/mce/core.c:1584:2-9: WARNING: Assignment of 0/1 to bool variable [ bp: Massage commit message. ] Reported-by: Tosk Robot Signed-off-by: Kaixu Xia Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/1604654363-1463-1-git-send-email-kaixuxia@tencent.com --- arch/x86/kernel/cpu/mce/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 51bf910b1e9d..888248ae0b39 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1581,7 +1581,7 @@ static void __mcheck_cpu_mce_banks_init(void) * __mcheck_cpu_init_clear_banks() does the final bank setup. */ b->ctl = -1ULL; - b->init = 1; + b->init = true; } } @@ -1762,7 +1762,7 @@ static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c) */ if (c->x86 == 6 && c->x86_model < 0x1A && this_cpu_read(mce_num_banks) > 0) - mce_banks[0].init = 0; + mce_banks[0].init = false; /* * All newer Intel systems support MCE broadcasting. Enable From 15af36596ae305aefc8c502c2d3e8c58221709eb Mon Sep 17 00:00:00 2001 From: Zhen Lei Date: Fri, 6 Nov 2020 22:12:16 +0800 Subject: [PATCH 100/360] x86/mce: Correct the detection of invalid notifier priorities Commit c9c6d216ed28 ("x86/mce: Rename "first" function as "early"") changed the enumeration of MCE notifier priorities. Correct the check for notifier priorities to cover the new range. [ bp: Rewrite commit message, remove superfluous brackets in conditional. ] Fixes: c9c6d216ed28 ("x86/mce: Rename "first" function as "early"") Signed-off-by: Zhen Lei Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201106141216.2062-2-thunder.leizhen@huawei.com --- arch/x86/include/asm/mce.h | 3 ++- arch/x86/kernel/cpu/mce/core.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h index a0f147893a04..fc25c88c7ff2 100644 --- a/arch/x86/include/asm/mce.h +++ b/arch/x86/include/asm/mce.h @@ -177,7 +177,8 @@ enum mce_notifier_prios { MCE_PRIO_EXTLOG, MCE_PRIO_UC, MCE_PRIO_EARLY, - MCE_PRIO_CEC + MCE_PRIO_CEC, + MCE_PRIO_HIGHEST = MCE_PRIO_CEC }; struct notifier_block; diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 888248ae0b39..ccac4c2fad46 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -162,7 +162,8 @@ EXPORT_SYMBOL_GPL(mce_log); void mce_register_decode_chain(struct notifier_block *nb) { - if (WARN_ON(nb->priority > MCE_PRIO_MCELOG && nb->priority < MCE_PRIO_EDAC)) + if (WARN_ON(nb->priority < MCE_PRIO_LOWEST || + nb->priority > MCE_PRIO_HIGHEST)) return; blocking_notifier_chain_register(&x86_mce_decoder_chain, nb); From 13b5bd8af41ca56fd11cc0281f2a1201d8342233 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Tue, 13 Oct 2020 16:19:37 +0200 Subject: [PATCH 101/360] s390/head: set io/ext handlers to disabled wait Set io/ext handlers to disabled wait in the initial lowcore, so that they are effective right from the kernel start, when a boot method used does not rewrite this part of the lowcore for its own needs (i.e. kexec, z/vm ipl reader boot, qemu direct boot, load from removable media or server). When the kernel is loaded by zipl, scsi loader or qemu loader, some or all of the io/ext/pgm handlers addresses might be rewritten. Rewrite them to initial values again as early as possible. Reviewed-by: Heiko Carstens Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/boot/head.S | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/arch/s390/boot/head.S b/arch/s390/boot/head.S index 1a2c2b1ed964..bc34ad2c2e49 100644 --- a/arch/s390/boot/head.S +++ b/arch/s390/boot/head.S @@ -62,8 +62,12 @@ __HEAD .org __LC_RST_NEW_PSW # 0x1a0 .quad 0,iplstart + .org __LC_EXT_NEW_PSW # 0x1b0 + .quad 0x0002000180000000,0x1b0 # disabled wait .org __LC_PGM_NEW_PSW # 0x1d0 .quad 0x0000000180000000,startup_pgm_check_handler + .org __LC_IO_NEW_PSW # 0x1f0 + .quad 0x0002000180000000,0x1f0 # disabled wait .org 0x200 @@ -303,6 +307,9 @@ ENTRY(startup_kdump) sam64 # switch to 64 bit addressing mode basr %r13,0 # get base .LPG0: + mvc __LC_EXT_NEW_PSW(16),.Lext_new_psw-.LPG0(%r13) + mvc __LC_PGM_NEW_PSW(16),.Lpgm_new_psw-.LPG0(%r13) + mvc __LC_IO_NEW_PSW(16),.Lio_new_psw-.LPG0(%r13) xc 0x200(256),0x200 # partially clear lowcore xc 0x300(256),0x300 xc 0xe00(256),0xe00 @@ -320,7 +327,12 @@ ENTRY(startup_kdump) .long 0x8000 + (1<<(PAGE_SHIFT+BOOT_STACK_ORDER)) - STACK_FRAME_OVERHEAD .align 8 6: .long 0x7fffffff,0xffffffff - +.Lext_new_psw: + .quad 0x0002000180000000,0x1b0 # disabled wait +.Lpgm_new_psw: + .quad 0x0000000180000000,startup_pgm_check_handler +.Lio_new_psw: + .quad 0x0002000180000000,0x1f0 # disabled wait .Lctl: .quad 0x04040000 # cr0: AFP registers & secondary space .quad 0 # cr1: primary space segment table .quad .Lduct # cr2: dispatchable unit control table From 85cde0192a983b227341be11af2c3625d39bc374 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Tue, 13 Oct 2020 22:35:27 +0200 Subject: [PATCH 102/360] s390/udelay: make it work for the early code Currently udelay relies on working EXT interrupts handler, which is not the case during early startup. In such cases udelay_simple() has to be used instead. To avoid mistakes of calling udelay too early, which could happen from the common code as well - make udelay work for the early code by introducing static branch and redirecting all udelay calls to udelay_simple until EXT interrupts handler is fully initialized and async stack is allocated. Reviewed-by: Heiko Carstens Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/include/asm/delay.h | 1 + arch/s390/kernel/setup.c | 1 + arch/s390/lib/delay.c | 13 +++++++++++++ 3 files changed, 15 insertions(+) diff --git a/arch/s390/include/asm/delay.h b/arch/s390/include/asm/delay.h index 898323fd93d2..4a08379cd1eb 100644 --- a/arch/s390/include/asm/delay.h +++ b/arch/s390/include/asm/delay.h @@ -13,6 +13,7 @@ #ifndef _S390_DELAY_H #define _S390_DELAY_H +void udelay_enable(void); void __ndelay(unsigned long long nsecs); void __udelay(unsigned long long usecs); void udelay_simple(unsigned long long usecs); diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 4d843e64496f..2076415aee4b 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -336,6 +336,7 @@ int __init arch_early_irq_init(void) if (!stack) panic("Couldn't allocate async stack"); S390_lowcore.async_stack = stack + STACK_INIT_OFFSET; + udelay_enable(); return 0; } diff --git a/arch/s390/lib/delay.c b/arch/s390/lib/delay.c index daca7bad66de..1734a5c19834 100644 --- a/arch/s390/lib/delay.c +++ b/arch/s390/lib/delay.c @@ -13,11 +13,19 @@ #include #include #include +#include #include #include #include #include +static DEFINE_STATIC_KEY_FALSE(udelay_ready); + +void __init udelay_enable(void) +{ + static_branch_enable(&udelay_ready); +} + void __delay(unsigned long loops) { /* @@ -77,6 +85,11 @@ void __udelay(unsigned long long usecs) { unsigned long flags; + if (!static_branch_likely(&udelay_ready)) { + udelay_simple(usecs); + return; + } + preempt_disable(); local_irq_save(flags); if (in_irq()) { From f38b0a743904cc785f5ba0d65dd1f60e17e80387 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Thu, 24 Sep 2020 17:12:08 +0200 Subject: [PATCH 103/360] s390: remove unused s390_base_ext_handler s390_base_ext_handler_fn haven't been used since its introduction in commit ab14de6c37fa ("[S390] Convert memory detection into C code."). s390_base_ext_handler itself is currently falsely storing 16 registers at __LC_SAVE_AREA_ASYNC rewriting several following lowcore values: cpu_flags, return_psw, return_mcck_psw, sync_enter_timer and async_enter_timer. Besides that s390_base_ext_handler itself is only potentially hiding EXT interrupts which should not have happen in the first place. Any piece of code which requires EXT interrupts before fully functional ext_int_handler is enabled has to do it on its own, like this is done by sclp_early_cmd() which is doing EXT interrupts handling synchronously in sclp_early_wait_irq(). With s390_base_ext_handler removed unexpected EXT interrupt leads to disabled wait with the address 0x1b0 (__LC_EXT_NEW_PSW), which is currently setup in the decompressor. Reviewed-by: Heiko Carstens Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/include/asm/processor.h | 6 +----- arch/s390/kernel/base.S | 22 ---------------------- arch/s390/kernel/early.c | 4 +--- 3 files changed, 2 insertions(+), 30 deletions(-) diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 962da04234af..2043c562ec55 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -318,14 +318,10 @@ static __always_inline void __noreturn disabled_wait(void) } /* - * Basic Machine Check/Program Check Handler. + * Basic Program Check Handler. */ - extern void s390_base_pgm_handler(void); -extern void s390_base_ext_handler(void); - extern void (*s390_base_pgm_handler_fn)(void); -extern void (*s390_base_ext_handler_fn)(void); #define ARCH_LOW_ADDRESS_LIMIT 0x7fffffffUL diff --git a/arch/s390/kernel/base.S b/arch/s390/kernel/base.S index b79e0fd571f8..d255c69c1779 100644 --- a/arch/s390/kernel/base.S +++ b/arch/s390/kernel/base.S @@ -11,32 +11,10 @@ #include #include #include -#include GEN_BR_THUNK %r9 GEN_BR_THUNK %r14 -ENTRY(s390_base_ext_handler) - stmg %r0,%r15,__LC_SAVE_AREA_ASYNC - basr %r13,0 -0: aghi %r15,-STACK_FRAME_OVERHEAD - larl %r1,s390_base_ext_handler_fn - lg %r9,0(%r1) - ltgr %r9,%r9 - jz 1f - BASR_EX %r14,%r9 -1: lmg %r0,%r15,__LC_SAVE_AREA_ASYNC - ni __LC_EXT_OLD_PSW+1,0xfd # clear wait state bit - lpswe __LC_EXT_OLD_PSW -ENDPROC(s390_base_ext_handler) - - .section .bss - .align 8 - .globl s390_base_ext_handler_fn -s390_base_ext_handler_fn: - .quad 0 - .previous - ENTRY(s390_base_pgm_handler) stmg %r0,%r15,__LC_SAVE_AREA_SYNC basr %r13,0 diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 705844f73934..cc89763a4d3c 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -169,12 +169,10 @@ static noinline __init void setup_lowcore_early(void) { psw_t psw; + psw.addr = (unsigned long)s390_base_pgm_handler; psw.mask = PSW_MASK_BASE | PSW_DEFAULT_KEY | PSW_MASK_EA | PSW_MASK_BA; if (IS_ENABLED(CONFIG_KASAN)) psw.mask |= PSW_MASK_DAT; - psw.addr = (unsigned long) s390_base_ext_handler; - S390_lowcore.external_new_psw = psw; - psw.addr = (unsigned long) s390_base_pgm_handler; S390_lowcore.program_new_psw = psw; s390_base_pgm_handler_fn = early_pgm_check_handler; S390_lowcore.preempt_count = INIT_PREEMPT_COUNT; From a3453d923ece6760689894bad5b6d5e00c0ffe2d Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Thu, 15 Oct 2020 10:01:42 +0200 Subject: [PATCH 104/360] s390/kasan: remove 3-level paging support Compiling the kernel with Kasan disables automatic 3-level vs 4-level kernel space paging selection, because the shadow memory offset has to be known at compile time and there is no such offset which would be acceptable for both 3 and 4-level paging. Instead S390_4_LEVEL_PAGING option was introduced which allowed to pick how many paging levels to use under Kasan. With the introduction of protected virtualization, kernel memory layout may be affected due to ultravisor secure storage limit. This adds additional complexity into how memory layout would look like in combination with Kasan predefined shadow memory offsets. To simplify this make Kasan 4-level paging default and remove Kasan 3-level paging support. Suggested-by: Heiko Carstens Reviewed-by: Alexander Gordeev Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/Kconfig | 3 +- arch/s390/include/asm/kasan.h | 5 ---- arch/s390/mm/kasan_init.c | 52 +++++++---------------------------- lib/Kconfig.kasan | 9 ------ 4 files changed, 11 insertions(+), 58 deletions(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 4a2a12be04c9..28bdb11e9db2 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -53,8 +53,7 @@ config ARCH_SUPPORTS_UPROBES config KASAN_SHADOW_OFFSET hex depends on KASAN - default 0x18000000000000 if KASAN_S390_4_LEVEL_PAGING - default 0x30000000000 + default 0x18000000000000 config S390 def_bool y diff --git a/arch/s390/include/asm/kasan.h b/arch/s390/include/asm/kasan.h index e9bf486de136..528dfb9be77a 100644 --- a/arch/s390/include/asm/kasan.h +++ b/arch/s390/include/asm/kasan.h @@ -5,13 +5,8 @@ #ifdef CONFIG_KASAN #define KASAN_SHADOW_SCALE_SHIFT 3 -#ifdef CONFIG_KASAN_S390_4_LEVEL_PAGING #define KASAN_SHADOW_SIZE \ (_AC(1, UL) << (_REGION1_SHIFT - KASAN_SHADOW_SCALE_SHIFT)) -#else -#define KASAN_SHADOW_SIZE \ - (_AC(1, UL) << (_REGION2_SHIFT - KASAN_SHADOW_SCALE_SHIFT)) -#endif #define KASAN_SHADOW_OFFSET _AC(CONFIG_KASAN_SHADOW_OFFSET, UL) #define KASAN_SHADOW_START KASAN_SHADOW_OFFSET #define KASAN_SHADOW_END (KASAN_SHADOW_START + KASAN_SHADOW_SIZE) diff --git a/arch/s390/mm/kasan_init.c b/arch/s390/mm/kasan_init.c index 5646b39c728a..fcb2b533b235 100644 --- a/arch/s390/mm/kasan_init.c +++ b/arch/s390/mm/kasan_init.c @@ -123,8 +123,7 @@ static void __init kasan_early_vmemmap_populate(unsigned long address, pgd_populate(&init_mm, pg_dir, p4_dir); } - if (IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING) && - mode == POPULATE_SHALLOW) { + if (mode == POPULATE_SHALLOW) { address = (address + P4D_SIZE) & P4D_MASK; continue; } @@ -143,12 +142,6 @@ static void __init kasan_early_vmemmap_populate(unsigned long address, p4d_populate(&init_mm, p4_dir, pu_dir); } - if (!IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING) && - mode == POPULATE_SHALLOW) { - address = (address + PUD_SIZE) & PUD_MASK; - continue; - } - pu_dir = pud_offset(p4_dir, address); if (pud_none(*pu_dir)) { if (mode == POPULATE_ZERO_SHADOW && @@ -281,7 +274,6 @@ void __init kasan_early_init(void) unsigned long shadow_alloc_size; unsigned long vmax_unlimited; unsigned long initrd_end; - unsigned long asce_type; unsigned long memsize; unsigned long pgt_prot = pgprot_val(PAGE_KERNEL_RO); pte_t pte_z; @@ -304,25 +296,12 @@ void __init kasan_early_init(void) memsize = min(memsize, OLDMEM_SIZE); memsize = min(memsize, KASAN_SHADOW_START); - if (IS_ENABLED(CONFIG_KASAN_S390_4_LEVEL_PAGING)) { - /* 4 level paging */ - BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE)); - BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE)); - crst_table_init((unsigned long *)early_pg_dir, - _REGION2_ENTRY_EMPTY); - untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION1_SIZE; - if (has_uv_sec_stor_limit()) - kasan_vmax = min(vmax_unlimited, uv_info.max_sec_stor_addr); - asce_type = _ASCE_TYPE_REGION2; - } else { - /* 3 level paging */ - BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PUD_SIZE)); - BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PUD_SIZE)); - crst_table_init((unsigned long *)early_pg_dir, - _REGION3_ENTRY_EMPTY); - untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION2_SIZE; - asce_type = _ASCE_TYPE_REGION3; - } + BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE)); + BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE)); + crst_table_init((unsigned long *)early_pg_dir, _REGION2_ENTRY_EMPTY); + untracked_mem_end = kasan_vmax = vmax_unlimited = _REGION1_SIZE; + if (has_uv_sec_stor_limit()) + kasan_vmax = min(vmax_unlimited, uv_info.max_sec_stor_addr); /* init kasan zero shadow */ crst_table_init((unsigned long *)kasan_early_shadow_p4d, @@ -408,7 +387,7 @@ void __init kasan_early_init(void) pgalloc_freeable = pgalloc_pos; /* populate identity mapping */ kasan_early_vmemmap_populate(0, memsize, POPULATE_ONE2ONE); - kasan_set_pgd(early_pg_dir, asce_type); + kasan_set_pgd(early_pg_dir, _ASCE_TYPE_REGION2); kasan_enable_dat(); /* enable kasan */ init_task.kasan_depth = 0; @@ -428,24 +407,13 @@ void __init kasan_copy_shadow(pgd_t *pg_dir) pgd_t *pg_dir_dst; p4d_t *p4_dir_src; p4d_t *p4_dir_dst; - pud_t *pu_dir_src; - pud_t *pu_dir_dst; pg_dir_src = pgd_offset_raw(early_pg_dir, KASAN_SHADOW_START); pg_dir_dst = pgd_offset_raw(pg_dir, KASAN_SHADOW_START); p4_dir_src = p4d_offset(pg_dir_src, KASAN_SHADOW_START); p4_dir_dst = p4d_offset(pg_dir_dst, KASAN_SHADOW_START); - if (!p4d_folded(*p4_dir_src)) { - /* 4 level paging */ - memcpy(p4_dir_dst, p4_dir_src, - (KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t)); - return; - } - /* 3 level paging */ - pu_dir_src = pud_offset(p4_dir_src, KASAN_SHADOW_START); - pu_dir_dst = pud_offset(p4_dir_dst, KASAN_SHADOW_START); - memcpy(pu_dir_dst, pu_dir_src, - (KASAN_SHADOW_SIZE >> PUD_SHIFT) * sizeof(pud_t)); + memcpy(p4_dir_dst, p4_dir_src, + (KASAN_SHADOW_SIZE >> P4D_SHIFT) * sizeof(p4d_t)); } void __init kasan_free_early_identity(void) diff --git a/lib/Kconfig.kasan b/lib/Kconfig.kasan index 542a9c18398e..8fb097057fec 100644 --- a/lib/Kconfig.kasan +++ b/lib/Kconfig.kasan @@ -136,15 +136,6 @@ config KASAN_STACK default 1 if KASAN_STACK_ENABLE || CC_IS_GCC default 0 -config KASAN_S390_4_LEVEL_PAGING - bool "KASan: use 4-level paging" - depends on S390 - help - Compiling the kernel with KASan disables automatic 3-level vs - 4-level paging selection. 3-level paging is used by default (up - to 3TB of RAM with KASan enabled). This options allows to force - 4-level paging instead. - config KASAN_SW_TAGS_IDENTIFY bool "Enable memory corruption identification" depends on KASAN_SW_TAGS From 97b142b7400bdce93aa674df044a4bc58e88f08c Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Thu, 15 Oct 2020 10:20:08 +0200 Subject: [PATCH 105/360] s390: make sure vmemmap is top region table entry aligned Since commit 29d37e5b82f3 ("s390/protvirt: add ultravisor initialization") vmax is adjusted to the ultravisor secure storage limit. This limit is currently applied when 4-level paging is used. Later vmax is also used to align vmemmap address to the top region table entry border. When vmax is set to the ultravisor secure storage limit this is no longer the case. Instead of changing vmax, make only MODULES_END be affected by the secure storage limit, so that vmax stays intact for further vmemmap address alignment. Reviewed-by: Alexander Gordeev Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/kernel/setup.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 2076415aee4b..02bccee5ee85 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -568,13 +568,14 @@ static void __init setup_memory_end(void) vmax = _REGION2_SIZE; /* 3-level kernel page table */ else vmax = _REGION1_SIZE; /* 4-level kernel page table */ - if (is_prot_virt_host()) - adjust_to_uv_max(&vmax); -#ifdef CONFIG_KASAN - vmax = kasan_vmax; -#endif /* module area is at the end of the kernel address space. */ MODULES_END = vmax; + if (is_prot_virt_host()) + adjust_to_uv_max(&MODULES_END); +#ifdef CONFIG_KASAN + vmax = _REGION1_SIZE; + MODULES_END = kasan_vmax; +#endif MODULES_VADDR = MODULES_END - MODULES_LEN; VMALLOC_END = MODULES_VADDR; VMALLOC_START = VMALLOC_END - vmalloc_size; From fc67c880e32a85786ad32129faa880f57bb6de41 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 18 Sep 2020 12:25:36 +0200 Subject: [PATCH 106/360] s390/mm: extend default vmalloc area size to 512GB We've seen several occurences in the past where the default vmalloc size of 128GB is not sufficient. Therefore extend the default size. Reviewed-by: Claudio Imbrenda Reviewed-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/include/asm/pgtable.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index b5dbae78969b..a8edd96b2103 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -79,15 +79,15 @@ extern unsigned long zero_page_mask; /* * The vmalloc and module area will always be on the topmost area of the - * kernel mapping. We reserve 128GB (64bit) for vmalloc and modules. - * On 64 bit kernels we have a 2GB area at the top of the vmalloc area where - * modules will reside. That makes sure that inter module branches always - * happen without trampolines and in addition the placement within a 2GB frame - * is branch prediction unit friendly. + * kernel mapping. 512GB are reserved for vmalloc by default. + * At the top of the vmalloc area a 2GB area is reserved where modules + * will reside. That makes sure that inter module branches always + * happen without trampolines and in addition the placement within a + * 2GB frame is branch prediction unit friendly. */ extern unsigned long VMALLOC_START; extern unsigned long VMALLOC_END; -#define VMALLOC_DEFAULT_SIZE ((128UL << 30) - MODULES_LEN) +#define VMALLOC_DEFAULT_SIZE ((512UL << 30) - MODULES_LEN) extern struct page *vmemmap; extern unsigned long vmemmap_size; From 90178c1900798633dd0c83ab693254b8705177c5 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Fri, 18 Sep 2020 12:25:37 +0200 Subject: [PATCH 107/360] s390/mm: let vmalloc area size depend on physical memory size To make sure that the vmalloc area size is for almost all cases large enough let it depend on the (potential) physical memory size. There is still the possibility to override this with the vmalloc kernel command line parameter. Reviewed-by: Christian Borntraeger Reviewed-by: Claudio Imbrenda Reviewed-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/boot/boot.h | 1 + arch/s390/boot/ipl_parm.c | 5 ++++- arch/s390/boot/startup.c | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/arch/s390/boot/boot.h b/arch/s390/boot/boot.h index 2ea603f70c3b..4d4536299789 100644 --- a/arch/s390/boot/boot.h +++ b/arch/s390/boot/boot.h @@ -14,6 +14,7 @@ void print_pgm_check_info(void); unsigned long get_random_base(unsigned long safe_addr); extern int kaslr_enabled; +extern int vmalloc_size_set; extern const char kernel_version[]; unsigned long read_ipl_report(unsigned long safe_offset); diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c index f94b91d72620..33f43a7d03f3 100644 --- a/arch/s390/boot/ipl_parm.c +++ b/arch/s390/boot/ipl_parm.c @@ -21,6 +21,7 @@ unsigned long __bootdata(memory_end); int __bootdata(memory_end_set); int __bootdata(noexec_disabled); +int vmalloc_size_set; int kaslr_enabled; static inline int __diag308(unsigned long subcode, void *addr) @@ -242,8 +243,10 @@ void parse_boot_command_line(void) memory_end_set = 1; } - if (!strcmp(param, "vmalloc") && val) + if (!strcmp(param, "vmalloc") && val) { vmalloc_size = round_up(memparse(val, NULL), PAGE_SIZE); + vmalloc_size_set = 1; + } if (!strcmp(param, "dfltcc") && val) { if (!strcmp(val, "off")) diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index cc96b04cc0ba..95d13a252ed9 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -126,6 +126,21 @@ static void clear_bss_section(void) memset((void *)vmlinux.default_lma + vmlinux.image_size, 0, vmlinux.bss_size); } +/* + * Set vmalloc area size to an 8th of (potential) physical memory + * size, unless size has been set by kernel command line parameter. + */ +static void setup_vmalloc_size(void) +{ + unsigned long size; + + if (vmalloc_size_set) + return; + size = (memory_end ?: max_physmem_end) >> 3; + size = round_up(size, _SEGMENT_SIZE); + vmalloc_size = max(size, vmalloc_size); +} + void startup_kernel(void) { unsigned long random_lma; @@ -142,6 +157,7 @@ void startup_kernel(void) parse_boot_command_line(); setup_memory_end(); detect_memory(); + setup_vmalloc_size(); random_lma = __kaslr_offset = 0; if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && kaslr_enabled) { From 39f2899b9872d02ec178568a9acfe90804d7fd70 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Sat, 17 Oct 2020 15:55:10 +0200 Subject: [PATCH 108/360] s390/decompressor: fix build warning Fixes the following warning with CONFIG_KERNEL_UNCOMPRESSED=y arch/s390/boot/compressed/decompressor.h:6:46: warning: non-void function does not return a value [-Wreturn-type] static inline void *decompress_kernel(void) {} ^ Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/boot/compressed/decompressor.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/s390/boot/compressed/decompressor.h b/arch/s390/boot/compressed/decompressor.h index c15eb7114d83..41f0ad97a4db 100644 --- a/arch/s390/boot/compressed/decompressor.h +++ b/arch/s390/boot/compressed/decompressor.h @@ -2,8 +2,10 @@ #ifndef BOOT_COMPRESSED_DECOMPRESSOR_H #define BOOT_COMPRESSED_DECOMPRESSOR_H +#include + #ifdef CONFIG_KERNEL_UNCOMPRESSED -static inline void *decompress_kernel(void) {} +static inline void *decompress_kernel(void) { return NULL; } #else void *decompress_kernel(void); #endif From 92bca2fe61f5dbb23d9466d70b8fa3f2ad263ba8 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Mon, 5 Oct 2020 09:13:15 +0200 Subject: [PATCH 109/360] s390/kasan: avoid confusing naming Kasan has nothing to do with vmemmap, strip vmemmap from function names to avoid confusing people. Reviewed-by: Alexander Egorenkov Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/mm/kasan_init.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/arch/s390/mm/kasan_init.c b/arch/s390/mm/kasan_init.c index fcb2b533b235..a43381f030d4 100644 --- a/arch/s390/mm/kasan_init.c +++ b/arch/s390/mm/kasan_init.c @@ -87,7 +87,7 @@ enum populate_mode { POPULATE_ZERO_SHADOW, POPULATE_SHALLOW }; -static void __init kasan_early_vmemmap_populate(unsigned long address, +static void __init kasan_early_pgtable_populate(unsigned long address, unsigned long end, enum populate_mode mode) { @@ -367,26 +367,24 @@ void __init kasan_early_init(void) * +-----------------+ +- shadow end ---+ */ /* populate kasan shadow (for identity mapping and zero page mapping) */ - kasan_early_vmemmap_populate(__sha(0), __sha(memsize), POPULATE_MAP); + kasan_early_pgtable_populate(__sha(0), __sha(memsize), POPULATE_MAP); if (IS_ENABLED(CONFIG_MODULES)) untracked_mem_end = kasan_vmax - MODULES_LEN; if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) { untracked_mem_end = kasan_vmax - vmalloc_size - MODULES_LEN; /* shallowly populate kasan shadow for vmalloc and modules */ - kasan_early_vmemmap_populate(__sha(untracked_mem_end), - __sha(kasan_vmax), POPULATE_SHALLOW); + kasan_early_pgtable_populate(__sha(untracked_mem_end), __sha(kasan_vmax), + POPULATE_SHALLOW); } /* populate kasan shadow for untracked memory */ - kasan_early_vmemmap_populate(__sha(max_physmem_end), - __sha(untracked_mem_end), + kasan_early_pgtable_populate(__sha(max_physmem_end), __sha(untracked_mem_end), POPULATE_ZERO_SHADOW); - kasan_early_vmemmap_populate(__sha(kasan_vmax), - __sha(vmax_unlimited), + kasan_early_pgtable_populate(__sha(kasan_vmax), __sha(vmax_unlimited), POPULATE_ZERO_SHADOW); /* memory allocated for identity mapping structs will be freed later */ pgalloc_freeable = pgalloc_pos; /* populate identity mapping */ - kasan_early_vmemmap_populate(0, memsize, POPULATE_ONE2ONE); + kasan_early_pgtable_populate(0, memsize, POPULATE_ONE2ONE); kasan_set_pgd(early_pg_dir, _ASCE_TYPE_REGION2); kasan_enable_dat(); /* enable kasan */ From 54b52981bb39ec3f33dc3677583b2ea30772b292 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Mon, 5 Oct 2020 10:28:48 +0200 Subject: [PATCH 110/360] s390/kasan: remove obvious parameter with the only possible value Kasan early code is only working on init_mm, remove unneeded pgd parameter from kasan_copy_shadow and rename it to kasan_copy_shadow_mapping. Reviewed-by: Alexander Egorenkov Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/include/asm/kasan.h | 4 ++-- arch/s390/mm/init.c | 2 +- arch/s390/mm/kasan_init.c | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/s390/include/asm/kasan.h b/arch/s390/include/asm/kasan.h index 528dfb9be77a..0313f055f8c9 100644 --- a/arch/s390/include/asm/kasan.h +++ b/arch/s390/include/asm/kasan.h @@ -12,12 +12,12 @@ #define KASAN_SHADOW_END (KASAN_SHADOW_START + KASAN_SHADOW_SIZE) extern void kasan_early_init(void); -extern void kasan_copy_shadow(pgd_t *dst); +extern void kasan_copy_shadow_mapping(void); extern void kasan_free_early_identity(void); extern unsigned long kasan_vmax; #else static inline void kasan_early_init(void) { } -static inline void kasan_copy_shadow(pgd_t *dst) { } +static inline void kasan_copy_shadow_mapping(void) { } static inline void kasan_free_early_identity(void) { } #endif diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 77767850d0d0..69e6e2a5072e 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -105,7 +105,7 @@ void __init paging_init(void) S390_lowcore.user_asce = S390_lowcore.kernel_asce; crst_table_init((unsigned long *) init_mm.pgd, pgd_type); vmem_map_init(); - kasan_copy_shadow(init_mm.pgd); + kasan_copy_shadow_mapping(); /* enable virtual mapping in kernel mode */ __ctl_load(S390_lowcore.kernel_asce, 1, 1); diff --git a/arch/s390/mm/kasan_init.c b/arch/s390/mm/kasan_init.c index a43381f030d4..a9f561491110 100644 --- a/arch/s390/mm/kasan_init.c +++ b/arch/s390/mm/kasan_init.c @@ -393,7 +393,7 @@ void __init kasan_early_init(void) sclp_early_printk("KernelAddressSanitizer initialized\n"); } -void __init kasan_copy_shadow(pgd_t *pg_dir) +void __init kasan_copy_shadow_mapping(void) { /* * At this point we are still running on early pages setup early_pg_dir, @@ -407,7 +407,7 @@ void __init kasan_copy_shadow(pgd_t *pg_dir) p4d_t *p4_dir_dst; pg_dir_src = pgd_offset_raw(early_pg_dir, KASAN_SHADOW_START); - pg_dir_dst = pgd_offset_raw(pg_dir, KASAN_SHADOW_START); + pg_dir_dst = pgd_offset_raw(init_mm.pgd, KASAN_SHADOW_START); p4_dir_src = p4d_offset(pg_dir_src, KASAN_SHADOW_START); p4_dir_dst = p4d_offset(pg_dir_dst, KASAN_SHADOW_START); memcpy(p4_dir_dst, p4_dir_src, From e385b550faf356db47234083c53e9841e8ec05c5 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Mon, 5 Oct 2020 09:07:43 +0200 Subject: [PATCH 111/360] s390/kasan: make kasan header self-contained It is relying on _REGION1_SHIFT / _REGION2_SHIFT values which come from asm/pgtable.h, so include it. Reviewed-by: Alexander Egorenkov Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/include/asm/kasan.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/s390/include/asm/kasan.h b/arch/s390/include/asm/kasan.h index 0313f055f8c9..eea62586e719 100644 --- a/arch/s390/include/asm/kasan.h +++ b/arch/s390/include/asm/kasan.h @@ -2,6 +2,8 @@ #ifndef __ASM_KASAN_H #define __ASM_KASAN_H +#include + #ifdef CONFIG_KASAN #define KASAN_SHADOW_SCALE_SHIFT 3 From 0c4ec024a481774df98910c79a4b3a105d1bb996 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Mon, 5 Oct 2020 13:53:36 +0200 Subject: [PATCH 112/360] s390/kasan: move memory needs estimation into a function Also correct rounding downs in estimation calculations. Reviewed-by: Alexander Egorenkov Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/boot/kaslr.c | 30 ++++++++---------------------- arch/s390/include/asm/kasan.h | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/arch/s390/boot/kaslr.c b/arch/s390/boot/kaslr.c index d844a5ef9089..987a9eaf228a 100644 --- a/arch/s390/boot/kaslr.c +++ b/arch/s390/boot/kaslr.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "compressed/decompressor.h" #include "boot.h" @@ -179,34 +180,19 @@ unsigned long get_random_base(unsigned long safe_addr) if (memory_end_set) memory_limit = min(memory_limit, memory_end); + /* + * Avoid putting kernel in the end of physical memory + * which kasan will use for shadow memory and early pgtable + * mapping allocations. + */ + memory_limit -= kasan_estimate_memory_needs(memory_limit); + if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && INITRD_START && INITRD_SIZE) { if (safe_addr < INITRD_START + INITRD_SIZE) safe_addr = INITRD_START + INITRD_SIZE; } safe_addr = ALIGN(safe_addr, THREAD_SIZE); - if ((IS_ENABLED(CONFIG_KASAN))) { - /* - * Estimate kasan memory requirements, which it will reserve - * at the very end of available physical memory. To estimate - * that, we take into account that kasan would require - * 1/8 of available physical memory (for shadow memory) + - * creating page tables for the whole memory + shadow memory - * region (1 + 1/8). To keep page tables estimates simple take - * the double of combined ptes size. - */ - memory_limit = get_mem_detect_end(); - if (memory_end_set && memory_limit > memory_end) - memory_limit = memory_end; - - /* for shadow memory */ - kasan_needs = memory_limit / 8; - /* for paging structures */ - kasan_needs += (memory_limit + kasan_needs) / PAGE_SIZE / - _PAGE_ENTRIES * _PAGE_TABLE_SIZE * 2; - memory_limit -= kasan_needs; - } - kernel_size = vmlinux.image_size + vmlinux.bss_size; if (safe_addr + kernel_size > memory_limit) return 0; diff --git a/arch/s390/include/asm/kasan.h b/arch/s390/include/asm/kasan.h index eea62586e719..76f351bd6645 100644 --- a/arch/s390/include/asm/kasan.h +++ b/arch/s390/include/asm/kasan.h @@ -17,10 +17,36 @@ extern void kasan_early_init(void); extern void kasan_copy_shadow_mapping(void); extern void kasan_free_early_identity(void); extern unsigned long kasan_vmax; + +/* + * Estimate kasan memory requirements, which it will reserve + * at the very end of available physical memory. To estimate + * that, we take into account that kasan would require + * 1/8 of available physical memory (for shadow memory) + + * creating page tables for the whole memory + shadow memory + * region (1 + 1/8). To keep page tables estimates simple take + * the double of combined ptes size. + * + * physmem parameter has to be already adjusted if not entire physical memory + * would be used (e.g. due to effect of "mem=" option). + */ +static inline unsigned long kasan_estimate_memory_needs(unsigned long physmem) +{ + unsigned long kasan_needs; + unsigned long pages; + /* for shadow memory */ + kasan_needs = round_up(physmem / 8, PAGE_SIZE); + /* for paging structures */ + pages = DIV_ROUND_UP(physmem + kasan_needs, PAGE_SIZE); + kasan_needs += DIV_ROUND_UP(pages, _PAGE_ENTRIES) * _PAGE_TABLE_SIZE * 2; + + return kasan_needs; +} #else static inline void kasan_early_init(void) { } static inline void kasan_copy_shadow_mapping(void) { } static inline void kasan_free_early_identity(void) { } +static inline unsigned long kasan_estimate_memory_needs(unsigned long physmem) { return 0; } #endif #endif From d7e7fbba67a32a32c3c7fe1ee6fccef93a0a8cc5 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Fri, 9 Oct 2020 17:14:02 +0200 Subject: [PATCH 113/360] s390/early: rewrite program parameter setup in C And move it earlier in the decompressor. Reviewed-by: Heiko Carstens Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/boot/startup.c | 10 ++++++++++ arch/s390/kernel/head64.S | 7 +------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index 95d13a252ed9..eeda91fcf7cc 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -58,6 +59,14 @@ void error(char *x) disabled_wait(); } +static void setup_lpp(void) +{ + S390_lowcore.current_pid = 0; + S390_lowcore.lpp = LPP_MAGIC; + if (test_facility(40)) + lpp(&S390_lowcore.lpp); +} + #ifdef CONFIG_KERNEL_UNCOMPRESSED unsigned long mem_safe_offset(void) { @@ -147,6 +156,7 @@ void startup_kernel(void) unsigned long safe_addr; void *img; + setup_lpp(); store_ipl_parmblock(); safe_addr = mem_safe_offset(); safe_addr = read_ipl_report(safe_addr); diff --git a/arch/s390/kernel/head64.S b/arch/s390/kernel/head64.S index 8b88dbbda7df..0c253886da78 100644 --- a/arch/s390/kernel/head64.S +++ b/arch/s390/kernel/head64.S @@ -18,12 +18,7 @@ __HEAD ENTRY(startup_continue) - tm __LC_STFLE_FAC_LIST+5,0x80 # LPP available ? - jz 0f - xc __LC_LPP+1(7,0),__LC_LPP+1 # clear lpp and current_pid - mvi __LC_LPP,0x80 # and set LPP_MAGIC - .insn s,0xb2800000,__LC_LPP # load program parameter -0: larl %r1,tod_clock_base + larl %r1,tod_clock_base mvc 0(16,%r1),__LC_BOOT_CLOCK larl %r13,.LPG1 # get base # From a67a88b0b8de16b4cd6ad50bfe0e03605904dc75 Mon Sep 17 00:00:00 2001 From: Daniel Vetter Date: Wed, 21 Oct 2020 10:56:47 +0200 Subject: [PATCH 114/360] s390/pci: remove races against pte updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Way back it was a reasonable assumptions that iomem mappings never change the pfn range they point at. But this has changed: - gpu drivers dynamically manage their memory nowadays, invalidating ptes with unmap_mapping_range when buffers get moved - contiguous dma allocations have moved from dedicated carvetouts to cma regions. This means if we miss the unmap the pfn might contain pagecache or anon memory (well anything allocated with GFP_MOVEABLE) - even /dev/mem now invalidates mappings when the kernel requests that iomem region when CONFIG_IO_STRICT_DEVMEM is set, see commit 3234ac664a87 ("/dev/mem: Revoke mappings when a driver claims the region") Accessing pfns obtained from ptes without holding all the locks is therefore no longer a good idea. Fix this. Since zpci_memcpy_from|toio seems to not do anything nefarious with locks we just need to open code get_pfn and follow_pfn and make sure we drop the locks only after we're done. The write function also needs the copy_from_user move, since we can't take userspace faults while holding the mmap sem. Reviewed-by: Gerald Schaefer Signed-off-by: Daniel Vetter Cc: Jason Gunthorpe Cc: Dan Williams Cc: Kees Cook Cc: Andrew Morton Cc: John Hubbard Cc: Jérôme Glisse Cc: Jan Kara Cc: linux-mm@kvack.org Cc: linux-arm-kernel@lists.infradead.org Cc: linux-samsung-soc@vger.kernel.org Cc: linux-media@vger.kernel.org Cc: Gerald Schaefer Cc: linux-s390@vger.kernel.org Cc: Niklas Schnelle Signed-off-by: Daniel Vetter Signed-off-by: Niklas Schnelle Signed-off-by: Heiko Carstens --- arch/s390/pci/pci_mmio.c | 98 +++++++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 41 deletions(-) diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index 401cf670a243..1a6adbc68ee8 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -119,33 +119,15 @@ static inline int __memcpy_toio_inuser(void __iomem *dst, return rc; } -static long get_pfn(unsigned long user_addr, unsigned long access, - unsigned long *pfn) -{ - struct vm_area_struct *vma; - long ret; - - mmap_read_lock(current->mm); - ret = -EINVAL; - vma = find_vma(current->mm, user_addr); - if (!vma) - goto out; - ret = -EACCES; - if (!(vma->vm_flags & access)) - goto out; - ret = follow_pfn(vma, user_addr, pfn); -out: - mmap_read_unlock(current->mm); - return ret; -} - SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, const void __user *, user_buffer, size_t, length) { u8 local_buf[64]; void __iomem *io_addr; void *buf; - unsigned long pfn; + struct vm_area_struct *vma; + pte_t *ptep; + spinlock_t *ptl; long ret; if (!zpci_is_enabled()) @@ -158,7 +140,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, * We only support write access to MIO capable devices if we are on * a MIO enabled system. Otherwise we would have to check for every * address if it is a special ZPCI_ADDR and would have to do - * a get_pfn() which we don't need for MIO capable devices. Currently + * a pfn lookup which we don't need for MIO capable devices. Currently * ISM devices are the only devices without MIO support and there is no * known need for accessing these from userspace. */ @@ -176,21 +158,37 @@ SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, } else buf = local_buf; - ret = get_pfn(mmio_addr, VM_WRITE, &pfn); + ret = -EFAULT; + if (copy_from_user(buf, user_buffer, length)) + goto out_free; + + mmap_read_lock(current->mm); + ret = -EINVAL; + vma = find_vma(current->mm, mmio_addr); + if (!vma) + goto out_unlock_mmap; + if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) + goto out_unlock_mmap; + ret = -EACCES; + if (!(vma->vm_flags & VM_WRITE)) + goto out_unlock_mmap; + + ret = follow_pte_pmd(vma->vm_mm, mmio_addr, NULL, &ptep, NULL, &ptl); if (ret) - goto out; - io_addr = (void __iomem *)((pfn << PAGE_SHIFT) | + goto out_unlock_mmap; + + io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) | (mmio_addr & ~PAGE_MASK)); - ret = -EFAULT; if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) - goto out; - - if (copy_from_user(buf, user_buffer, length)) - goto out; + goto out_unlock_pt; ret = zpci_memcpy_toio(io_addr, buf, length); -out: +out_unlock_pt: + pte_unmap_unlock(ptep, ptl); +out_unlock_mmap: + mmap_read_unlock(current->mm); +out_free: if (buf != local_buf) kfree(buf); return ret; @@ -274,7 +272,9 @@ SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, u8 local_buf[64]; void __iomem *io_addr; void *buf; - unsigned long pfn; + struct vm_area_struct *vma; + pte_t *ptep; + spinlock_t *ptl; long ret; if (!zpci_is_enabled()) @@ -287,7 +287,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, * We only support read access to MIO capable devices if we are on * a MIO enabled system. Otherwise we would have to check for every * address if it is a special ZPCI_ADDR and would have to do - * a get_pfn() which we don't need for MIO capable devices. Currently + * a pfn lookup which we don't need for MIO capable devices. Currently * ISM devices are the only devices without MIO support and there is no * known need for accessing these from userspace. */ @@ -306,22 +306,38 @@ SYSCALL_DEFINE3(s390_pci_mmio_read, unsigned long, mmio_addr, buf = local_buf; } - ret = get_pfn(mmio_addr, VM_READ, &pfn); + mmap_read_lock(current->mm); + ret = -EINVAL; + vma = find_vma(current->mm, mmio_addr); + if (!vma) + goto out_unlock_mmap; + if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) + goto out_unlock_mmap; + ret = -EACCES; + if (!(vma->vm_flags & VM_WRITE)) + goto out_unlock_mmap; + + ret = follow_pte_pmd(vma->vm_mm, mmio_addr, NULL, &ptep, NULL, &ptl); if (ret) - goto out; - io_addr = (void __iomem *)((pfn << PAGE_SHIFT) | (mmio_addr & ~PAGE_MASK)); + goto out_unlock_mmap; + + io_addr = (void __iomem *)((pte_pfn(*ptep) << PAGE_SHIFT) | + (mmio_addr & ~PAGE_MASK)); if ((unsigned long) io_addr < ZPCI_IOMAP_ADDR_BASE) { ret = -EFAULT; - goto out; + goto out_unlock_pt; } ret = zpci_memcpy_fromio(buf, io_addr, length); - if (ret) - goto out; - if (copy_to_user(user_buffer, buf, length)) + +out_unlock_pt: + pte_unmap_unlock(ptep, ptl); +out_unlock_mmap: + mmap_read_unlock(current->mm); + + if (!ret && copy_to_user(user_buffer, buf, length)) ret = -EFAULT; -out: if (buf != local_buf) kfree(buf); return ret; From d041315ef75cf52df19613f56a2da2c5911c163c Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Tue, 29 Sep 2020 10:03:51 +0200 Subject: [PATCH 115/360] s390/trng: set quality to 1024 The s390-trng does provide 100% entropy. The quality value is supported to be between 1 and 1024 and not 1..1000. Use 1024 to make this driver the preferred one. If we ever have a better driver that has the same quality but is faster we can change this again when merging the new driver. No need to be conservative. This makes sure that the hw variant is preferred over things like virtio-rng, where the hypervisor has a potential to be misconfigured and thus should have a slightly lower confidence. Cc: Harald Freudenberger Signed-off-by: Christian Borntraeger Signed-off-by: Heiko Carstens --- drivers/char/hw_random/s390-trng.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/char/hw_random/s390-trng.c b/drivers/char/hw_random/s390-trng.c index 413cacbb08e2..7c673afd7241 100644 --- a/drivers/char/hw_random/s390-trng.c +++ b/drivers/char/hw_random/s390-trng.c @@ -192,14 +192,15 @@ static int trng_hwrng_read(struct hwrng *rng, void *data, size_t max, bool wait) /* * hwrng register struct - * The trng is suppost to have 100% entropy, and thus - * we register with a very high quality value. + * The trng is supposed to have 100% entropy, and thus we register with a very + * high quality value. If we ever have a better driver in the future, we should + * change this value again when we merge this driver. */ static struct hwrng trng_hwrng_dev = { .name = "s390-trng", .data_read = trng_hwrng_data_read, .read = trng_hwrng_read, - .quality = 999, + .quality = 1024, }; From 837cd1059a985ee610e4b72dcdddb287e7b97de0 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Mon, 31 Aug 2020 10:16:26 +0200 Subject: [PATCH 116/360] s390/ap: ap bus userspace notifications for some bus conditions This patch adds notifications to userspace for two important conditions of the ap bus: I) Initial ap bus scan done. This indicates that the initial scan of all the ap devices (cards, queues) is complete and ap devices have been build up for all the hardware found. This condition is signaled with 1) An ap bus change uevent send to userspace with an environment key/value pair "INITSCAN=done": # udevadm monitor -k -p ... KERNEL[97.830919] change /devices/ap (ap) ACTION=change DEVPATH=/devices/ap SUBSYSTEM=ap INITSCAN=done SEQNUM=10421 2) A sysfs attribute /sys/bus/ap/scans which shows the number of completed ap bus scans done since bus init. So a value of 1 or greater signals that the initial ap bus scan is complete. Note: The initial ap bus scan complete condition is fulfilled and will be signaled even if there was no ap resource found. II) APQN driver bindings complete. This indicates that all APQNs have been bound to an zcrypt or alternate device driver. Only with the help of an device driver an APQN can be used for crypto load. So the binding complete condition is the starting point for user space to be sure all crypto resources on the ap bus are available for use. This condition is signaled with 1) An ap bus change uevent send to userspace with an environment key/value pair "BINDINGS=complete": # udevadm monitor -k -p ... KERNEL[97.830975] change /devices/ap (ap) ACTION=change DEVPATH=/devices/ap SUBSYSTEM=ap BINDINGS=complete SEQNUM=10422 2) A sysfs attribute /sys/bus/ap/bindings showing "/ (complete)" when all available apqns have been bound to device drivers, or "/" when there are some apqns not bound to an device driver. Note: The binding complete condition is also fulfilled, when there are no apqns available to bind any device driver. In this case the binding complete will be signaled AFTER init scan is done. Note: This condition may arise multiple times when after initial scan modifications on the bindings take place. For example a manual unbind of an APQN switches the binding complete condition off. When at a later time the unbound APQNs are bound with an device driver the binding is (again) complete resulting in another uevent and marking the bindings sysfs attribute with '(complete)'. There is also a new function to be used within the kernel: int ap_wait_init_apqn_bindings_complete(unsigned long timeout) Interface to wait for the AP bus to have done one initial ap bus scan and all detected APQNs have been bound to device drivers. If these both conditions are not fulfilled, this function blocks on a condition with wait_for_completion_interruptible_timeout(). If these both conditions are fulfilled (before the timeout hits) the return value is 0. If the timeout (in jiffies) hits instead -ETIME is returned. On failures negative return values are returned to the caller. Please note that further unbind/bind actions after initial binding complete is through do not cause this function to block again. Reviewed-by: Ingo Franzki Signed-off-by: Harald Freudenberger Signed-off-by: Heiko Carstens --- drivers/s390/crypto/ap_bus.c | 164 ++++++++++++++++++++++++++++++++--- drivers/s390/crypto/ap_bus.h | 12 +++ 2 files changed, 166 insertions(+), 10 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index ef738b42a092..13bd6b27f00e 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -1,11 +1,12 @@ // SPDX-License-Identifier: GPL-2.0+ /* - * Copyright IBM Corp. 2006, 2012 + * Copyright IBM Corp. 2006, 2020 * Author(s): Cornelia Huck * Martin Schwidefsky * Ralph Wuerthner * Felix Beck * Holger Dengler + * Harald Freudenberger * * Adjunct processor bus. */ @@ -73,6 +74,12 @@ EXPORT_SYMBOL(ap_perms); DEFINE_MUTEX(ap_perms_mutex); EXPORT_SYMBOL(ap_perms_mutex); +/* # of bus scans since init */ +static atomic64_t ap_scan_bus_count; + +/* completion for initial APQN bindings complete */ +static DECLARE_COMPLETION(ap_init_apqn_bindings_complete); + static struct ap_config_info *ap_qci_info; /* @@ -577,23 +584,125 @@ static int ap_bus_match(struct device *dev, struct device_driver *drv) */ static int ap_uevent(struct device *dev, struct kobj_uevent_env *env) { + int rc; struct ap_device *ap_dev = to_ap_dev(dev); - int retval = 0; - if (!ap_dev) - return -ENODEV; + /* Uevents from ap bus core don't need extensions to the env */ + if (dev == ap_root_device) + return 0; /* Set up DEV_TYPE environment variable. */ - retval = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type); - if (retval) - return retval; + rc = add_uevent_var(env, "DEV_TYPE=%04X", ap_dev->device_type); + if (rc) + return rc; /* Add MODALIAS= */ - retval = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type); + rc = add_uevent_var(env, "MODALIAS=ap:t%02X", ap_dev->device_type); + if (rc) + return rc; - return retval; + return 0; } +static void ap_send_init_scan_done_uevent(void) +{ + char *envp[] = { "INITSCAN=done", NULL }; + + kobject_uevent_env(&ap_root_device->kobj, KOBJ_CHANGE, envp); +} + +static void ap_send_bindings_complete_uevent(void) +{ + char *envp[] = { "BINDINGS=complete", NULL }; + + kobject_uevent_env(&ap_root_device->kobj, KOBJ_CHANGE, envp); +} + +/* + * calc # of bound APQNs + */ + +struct __ap_calc_ctrs { + unsigned int apqns; + unsigned int bound; +}; + +static int __ap_calc_helper(struct device *dev, void *arg) +{ + struct __ap_calc_ctrs *pctrs = (struct __ap_calc_ctrs *) arg; + + if (is_queue_dev(dev)) { + pctrs->apqns++; + if ((to_ap_dev(dev))->drv) + pctrs->bound++; + } + + return 0; +} + +static void ap_calc_bound_apqns(unsigned int *apqns, unsigned int *bound) +{ + struct __ap_calc_ctrs ctrs; + + memset(&ctrs, 0, sizeof(ctrs)); + bus_for_each_dev(&ap_bus_type, NULL, (void *) &ctrs, __ap_calc_helper); + + *apqns = ctrs.apqns; + *bound = ctrs.bound; +} + +/* + * After initial ap bus scan do check if all existing APQNs are + * bound to device drivers. + */ +static void ap_check_bindings_complete(void) +{ + unsigned int apqns, bound; + + if (atomic64_read(&ap_scan_bus_count) >= 1) { + ap_calc_bound_apqns(&apqns, &bound); + if (bound == apqns) { + if (!completion_done(&ap_init_apqn_bindings_complete)) { + complete_all(&ap_init_apqn_bindings_complete); + AP_DBF(DBF_INFO, "%s complete\n", __func__); + } + ap_send_bindings_complete_uevent(); + } + } +} + +/* + * Interface to wait for the AP bus to have done one initial ap bus + * scan and all detected APQNs have been bound to device drivers. + * If these both conditions are not fulfilled, this function blocks + * on a condition with wait_for_completion_interruptible_timeout(). + * If these both conditions are fulfilled (before the timeout hits) + * the return value is 0. If the timeout (in jiffies) hits instead + * -ETIME is returned. On failures negative return values are + * returned to the caller. + */ +int ap_wait_init_apqn_bindings_complete(unsigned long timeout) +{ + long l; + + if (completion_done(&ap_init_apqn_bindings_complete)) + return 0; + + if (timeout) + l = wait_for_completion_interruptible_timeout( + &ap_init_apqn_bindings_complete, timeout); + else + l = wait_for_completion_interruptible( + &ap_init_apqn_bindings_complete); + if (l < 0) + return l == -ERESTARTSYS ? -EINTR : l; + else if (l == 0 && timeout) + return -ETIME; + + return 0; +} +EXPORT_SYMBOL(ap_wait_init_apqn_bindings_complete); + static int __ap_queue_devices_with_id_unregister(struct device *dev, void *data) { if (is_queue_dev(dev) && @@ -719,7 +828,8 @@ static int ap_device_probe(struct device *dev) hash_del(&to_ap_queue(dev)->hnode); spin_unlock_bh(&ap_queues_lock); ap_dev->drv = NULL; - } + } else + ap_check_bindings_complete(); out: if (rc) @@ -749,6 +859,7 @@ static int ap_device_remove(struct device *dev) if (is_queue_dev(dev)) hash_del(&to_ap_queue(dev)->hnode); spin_unlock_bh(&ap_queues_lock); + ap_dev->drv = NULL; put_device(dev); @@ -1166,6 +1277,30 @@ static ssize_t aqmask_store(struct bus_type *bus, const char *buf, static BUS_ATTR_RW(aqmask); +static ssize_t scans_show(struct bus_type *bus, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%llu\n", + atomic64_read(&ap_scan_bus_count)); +} + +static BUS_ATTR_RO(scans); + +static ssize_t bindings_show(struct bus_type *bus, char *buf) +{ + int rc; + unsigned int apqns, n; + + ap_calc_bound_apqns(&apqns, &n); + if (atomic64_read(&ap_scan_bus_count) >= 1 && n == apqns) + rc = scnprintf(buf, PAGE_SIZE, "%u/%u (complete)\n", n, apqns); + else + rc = scnprintf(buf, PAGE_SIZE, "%u/%u\n", n, apqns); + + return rc; +} + +static BUS_ATTR_RO(bindings); + static struct bus_attribute *const ap_bus_attrs[] = { &bus_attr_ap_domain, &bus_attr_ap_control_domain_mask, @@ -1179,6 +1314,8 @@ static struct bus_attribute *const ap_bus_attrs[] = { &bus_attr_ap_max_adapter_id, &bus_attr_apmask, &bus_attr_aqmask, + &bus_attr_scans, + &bus_attr_bindings, NULL, }; @@ -1608,6 +1745,12 @@ static void ap_scan_bus(struct work_struct *unused) ap_domain_index); } + if (atomic64_inc_return(&ap_scan_bus_count) == 1) { + AP_DBF(DBF_DEBUG, "%s init scan complete\n", __func__); + ap_send_init_scan_done_uevent(); + ap_check_bindings_complete(); + } + mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ); } @@ -1705,6 +1848,7 @@ static int __init ap_module_init(void) rc = PTR_ERR_OR_ZERO(ap_root_device); if (rc) goto out_bus; + ap_root_device->bus = &ap_bus_type; /* Setup the AP bus rescan timer. */ timer_setup(&ap_config_timer, ap_config_timeout, 0); diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index 5029b80132aa..472efd3a755c 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -350,4 +350,16 @@ int ap_parse_mask_str(const char *str, unsigned long *bitmap, int bits, struct mutex *lock); +/* + * Interface to wait for the AP bus to have done one initial ap bus + * scan and all detected APQNs have been bound to device drivers. + * If these both conditions are not fulfilled, this function blocks + * on a condition with wait_for_completion_killable_timeout(). + * If these both conditions are fulfilled (before the timeout hits) + * the return value is 0. If the timeout (in jiffies) hits instead + * -ETIME is returned. On failures negative return values are + * returned to the caller. + */ +int ap_wait_init_apqn_bindings_complete(unsigned long timeout); + #endif /* _AP_BUS_H_ */ From 43cb5a7c61186cde6eba344c43489b9cf95c68f8 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 1 Sep 2020 12:48:14 +0200 Subject: [PATCH 117/360] s390/zcrypt/pkey: introduce zcrypt_wait_api_operational() function The zcrypt api provides a new function to wait until the zcrypt api is operational: int zcrypt_wait_api_operational(void); The AP bus scan and the binding of ap devices to device drivers is an asynchronous job. This function waits until these initial jobs are done and so the zcrypt api should be ready to serve crypto requests - if there are resources available. The function uses an internal timeout of 60s. The very first caller will either wait for ap bus bindings complete or the timeout happens. This state will be remembered for further callers which will only be blocked until a decision is made (timeout or bindings complete). Reviewed-by: Ingo Franzki Signed-off-by: Harald Freudenberger Signed-off-by: Heiko Carstens --- drivers/s390/crypto/pkey_api.c | 15 ++++++++ drivers/s390/crypto/zcrypt_api.c | 66 ++++++++++++++++++++++++++++++++ drivers/s390/crypto/zcrypt_api.h | 2 + 3 files changed, 83 insertions(+) diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index dd84995049b9..cf23ce1b1146 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -150,6 +150,8 @@ static int pkey_skey2pkey(const u8 *key, struct pkey_protkey *pkey) u16 cardnr, domain; struct keytoken_header *hdr = (struct keytoken_header *)key; + zcrypt_wait_api_operational(); + /* * The cca_xxx2protkey call may fail when a card has been * addressed where the master key was changed after last fetch @@ -197,6 +199,8 @@ static int pkey_clr2ep11key(const u8 *clrkey, size_t clrkeylen, u16 card, dom; u32 nr_apqns, *apqns = NULL; + zcrypt_wait_api_operational(); + /* build a list of apqns suitable for ep11 keys with cpacf support */ rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, ZCRYPT_CEX7, EP11_API_V, NULL); @@ -230,6 +234,8 @@ static int pkey_ep11key2pkey(const u8 *key, struct pkey_protkey *pkey) u32 nr_apqns, *apqns = NULL; struct ep11keyblob *kb = (struct ep11keyblob *) key; + zcrypt_wait_api_operational(); + /* build a list of apqns suitable for this key */ rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF, ZCRYPT_CEX7, EP11_API_V, kb->wkvp); @@ -436,6 +442,7 @@ static int pkey_nonccatok2pkey(const u8 *key, u32 keylen, if (rc == 0) break; /* PCKMO failed, so try the CCA secure key way */ + zcrypt_wait_api_operational(); rc = cca_clr2seckey(0xFFFF, 0xFFFF, t->keytype, ckey.clrkey, tmpbuf); if (rc == 0) @@ -625,6 +632,8 @@ static int pkey_clr2seckey2(const struct pkey_apqn *apqns, size_t nr_apqns, return -EINVAL; } + zcrypt_wait_api_operational(); + /* simple try all apqns from the list */ for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { card = apqns[i].card; @@ -801,6 +810,8 @@ static int pkey_keyblob2pkey2(const struct pkey_apqn *apqns, size_t nr_apqns, return -EINVAL; } + zcrypt_wait_api_operational(); + /* simple try all apqns from the list */ for (i = 0, rc = -ENODEV; i < nr_apqns; i++) { card = apqns[i].card; @@ -838,6 +849,8 @@ static int pkey_apqns4key(const u8 *key, size_t keylen, u32 flags, if (keylen < sizeof(struct keytoken_header) || flags == 0) return -EINVAL; + zcrypt_wait_api_operational(); + if (hdr->type == TOKTYPE_NON_CCA && (hdr->version == TOKVER_EP11_AES_WITH_HEADER || hdr->version == TOKVER_EP11_ECC_WITH_HEADER) @@ -941,6 +954,8 @@ static int pkey_apqns4keytype(enum pkey_key_type ktype, int rc; u32 _nr_apqns, *_apqns = NULL; + zcrypt_wait_api_operational(); + if (ktype == PKEY_TYPE_CCA_DATA || ktype == PKEY_TYPE_CCA_CIPHER) { u64 cur_mkvp = 0, old_mkvp = 0; int minhwtype = ZCRYPT_CEX3C; diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index f60f9fb25214..10206e4498d0 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -1992,6 +1992,72 @@ void zcrypt_rng_device_remove(void) mutex_unlock(&zcrypt_rng_mutex); } +/* + * Wait until the zcrypt api is operational. + * The AP bus scan and the binding of ap devices to device drivers is + * an asynchronous job. This function waits until these initial jobs + * are done and so the zcrypt api should be ready to serve crypto + * requests - if there are resources available. The function uses an + * internal timeout of 60s. The very first caller will either wait for + * ap bus bindings complete or the timeout happens. This state will be + * remembered for further callers which will only be blocked until a + * decision is made (timeout or bindings complete). + * On timeout -ETIME is returned, on success the return value is 0. + */ +int zcrypt_wait_api_operational(void) +{ + static DEFINE_MUTEX(zcrypt_wait_api_lock); + static int zcrypt_wait_api_state; + int rc; + + rc = mutex_lock_interruptible(&zcrypt_wait_api_lock); + if (rc) + return rc; + + switch (zcrypt_wait_api_state) { + case 0: + /* initial state, invoke wait for the ap bus complete */ + rc = ap_wait_init_apqn_bindings_complete( + msecs_to_jiffies(60 * 1000)); + switch (rc) { + case 0: + /* ap bus bindings are complete */ + zcrypt_wait_api_state = 1; + break; + case -EINTR: + /* interrupted, go back to caller */ + break; + case -ETIME: + /* timeout */ + ZCRYPT_DBF(DBF_WARN, + "%s ap_wait_init_apqn_bindings_complete() returned with ETIME\n", + __func__); + zcrypt_wait_api_state = -ETIME; + break; + default: + /* other failure */ + ZCRYPT_DBF(DBF_DEBUG, + "%s ap_wait_init_apqn_bindings_complete() failure rc=%d\n", + __func__, rc); + break; + } + break; + case 1: + /* a previous caller already found ap bus bindings complete */ + rc = 0; + break; + default: + /* a previous caller had timeout or other failure */ + rc = zcrypt_wait_api_state; + break; + } + + mutex_unlock(&zcrypt_wait_api_lock); + + return rc; +} +EXPORT_SYMBOL(zcrypt_wait_api_operational); + int __init zcrypt_debug_init(void) { zcrypt_dbf_info = debug_register("zcrypt", 1, 1, diff --git a/drivers/s390/crypto/zcrypt_api.h b/drivers/s390/crypto/zcrypt_api.h index 51c0b8bdef50..16219efb2f61 100644 --- a/drivers/s390/crypto/zcrypt_api.h +++ b/drivers/s390/crypto/zcrypt_api.h @@ -162,6 +162,8 @@ void zcrypt_device_status_mask_ext(struct zcrypt_device_status_ext *devstatus); int zcrypt_device_status_ext(int card, int queue, struct zcrypt_device_status_ext *devstatus); +int zcrypt_wait_api_operational(void); + static inline unsigned long z_copy_from_user(bool userspace, void *to, const void __user *from, From 211f737ac76ac317b67fa903742e92236d5776df Mon Sep 17 00:00:00 2001 From: Tony Luck Date: Thu, 5 Nov 2020 10:44:06 -0800 Subject: [PATCH 118/360] MAINTAINERS: Clean up the F: entries for some EDAC drivers The edac_altera entry stopped at the "." and needed "[ch]" to match both the driver and the header file. The edac_skx entry only matched on ".c" files so didn't include skx_common.h Signed-off-by: Tony Luck --- MAINTAINERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index e73636b75f29..aee8921cb470 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2486,7 +2486,7 @@ F: drivers/clk/socfpga/ ARM/SOCFPGA EDAC SUPPORT M: Dinh Nguyen S: Maintained -F: drivers/edac/altera_edac. +F: drivers/edac/altera_edac.[ch] ARM/SPREADTRUM SoC SUPPORT M: Orson Zhai @@ -6402,7 +6402,7 @@ EDAC-SKYLAKE M: Tony Luck L: linux-edac@vger.kernel.org S: Maintained -F: drivers/edac/skx_*.c +F: drivers/edac/skx_*.[ch] EDAC-TI M: Tero Kristo From ab177c5d00cda2655fd814356ea034aaf179cf05 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 10 Nov 2020 14:51:32 +0100 Subject: [PATCH 119/360] s390/mm: remove unused clear_user_asce() Signed-off-by: Heiko Carstens --- arch/s390/include/asm/mmu_context.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index c9f3d8a52756..cec19ae164eb 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -78,13 +78,6 @@ static inline void set_user_asce(struct mm_struct *mm) clear_cpu_flag(CIF_ASCE_PRIMARY); } -static inline void clear_user_asce(void) -{ - S390_lowcore.user_asce = S390_lowcore.kernel_asce; - __ctl_load(S390_lowcore.kernel_asce, 1, 1); - set_cpu_flag(CIF_ASCE_PRIMARY); -} - mm_segment_t enable_sacf_uaccess(void); void disable_sacf_uaccess(mm_segment_t old_fs); From 6569e3097f1c4a490bdf2b23d326855e04942dfd Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Mon, 2 Nov 2020 14:48:15 +0100 Subject: [PATCH 120/360] crypto: Kconfig - CRYPTO_MANAGER_EXTRA_TESTS requires the manager The extra tests in the manager actually require the manager to be selected too. Otherwise the linker gives errors like: ld: arch/x86/crypto/chacha_glue.o: in function `chacha_simd_stream_xor': chacha_glue.c:(.text+0x422): undefined reference to `crypto_simd_disabled_for_test' Fixes: 2343d1529aff ("crypto: Kconfig - allow tests to be disabled when manager is disabled") Signed-off-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- crypto/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 094ef56ab7b4..37de7d006858 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -145,7 +145,7 @@ config CRYPTO_MANAGER_DISABLE_TESTS config CRYPTO_MANAGER_EXTRA_TESTS bool "Enable extra run-time crypto self tests" - depends on DEBUG_KERNEL && !CRYPTO_MANAGER_DISABLE_TESTS + depends on DEBUG_KERNEL && !CRYPTO_MANAGER_DISABLE_TESTS && CRYPTO_MANAGER help Enable extra run-time self tests of registered crypto algorithms, including randomized fuzz tests. From ec3c5b32fccc537e7184dcf39387b2e57d12f517 Mon Sep 17 00:00:00 2001 From: Adam Guerin Date: Mon, 2 Nov 2020 17:04:54 +0000 Subject: [PATCH 121/360] crypto: qat - remove cast for mailbox CSR Remove cast for mailbox CSR in adf_admin.c as it is not needed. Suggested-by: Andy Shevchenko Signed-off-by: Adam Guerin Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_admin.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index 7c2ca54229aa..43680e178242 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -254,7 +254,7 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev) hw_data->get_admin_info(&admin_csrs_info); mailbox_offset = admin_csrs_info.mailbox_offset; - mailbox = (void __iomem *)((uintptr_t)csr + mailbox_offset); + mailbox = csr + mailbox_offset; adminmsg_u = admin_csrs_info.admin_msg_ur; adminmsg_l = admin_csrs_info.admin_msg_lr; From 86cd97ec4b943af35562a74688bc4e909b32c3d1 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 3 Nov 2020 17:28:09 +0100 Subject: [PATCH 122/360] crypto: arm/chacha-neon - optimize for non-block size multiples The current NEON based ChaCha implementation for ARM is optimized for multiples of 4x the ChaCha block size (64 bytes). This makes sense for block encryption, but given that ChaCha is also often used in the context of networking, it makes sense to consider arbitrary length inputs as well. For example, WireGuard typically uses 1420 byte packets, and performing ChaCha encryption involves 5 invocations of chacha_4block_xor_neon() and 3 invocations of chacha_block_xor_neon(), where the last one also involves a memcpy() using a buffer on the stack to process the final chunk of 1420 % 64 == 12 bytes. Let's optimize for this case as well, by letting chacha_4block_xor_neon() deal with any input size between 64 and 256 bytes, using NEON permutation instructions and overlapping loads and stores. This way, the 140 byte tail of a 1420 byte input buffer can simply be processed in one go. This results in the following performance improvements for 1420 byte blocks, without significant impact on power-of-2 input sizes. (Note that Raspberry Pi is widely used in combination with a 32-bit kernel, even though the core is 64-bit capable) Cortex-A8 (BeagleBone) : 7% Cortex-A15 (Calxeda Midway) : 21% Cortex-A53 (Raspberry Pi 3) : 3% Cortex-A72 (Raspberry Pi 4) : 19% Cc: Eric Biggers Cc: "Jason A . Donenfeld" Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/chacha-glue.c | 34 +++++------ arch/arm/crypto/chacha-neon-core.S | 97 +++++++++++++++++++++++++++--- 2 files changed, 107 insertions(+), 24 deletions(-) diff --git a/arch/arm/crypto/chacha-glue.c b/arch/arm/crypto/chacha-glue.c index 59da6c0b63b6..7b5cf8430c6d 100644 --- a/arch/arm/crypto/chacha-glue.c +++ b/arch/arm/crypto/chacha-glue.c @@ -23,7 +23,7 @@ asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src, int nrounds); asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src, - int nrounds); + int nrounds, unsigned int nbytes); asmlinkage void hchacha_block_arm(const u32 *state, u32 *out, int nrounds); asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds); @@ -42,24 +42,24 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, { u8 buf[CHACHA_BLOCK_SIZE]; - while (bytes >= CHACHA_BLOCK_SIZE * 4) { - chacha_4block_xor_neon(state, dst, src, nrounds); - bytes -= CHACHA_BLOCK_SIZE * 4; - src += CHACHA_BLOCK_SIZE * 4; - dst += CHACHA_BLOCK_SIZE * 4; - state[12] += 4; - } - while (bytes >= CHACHA_BLOCK_SIZE) { - chacha_block_xor_neon(state, dst, src, nrounds); - bytes -= CHACHA_BLOCK_SIZE; - src += CHACHA_BLOCK_SIZE; - dst += CHACHA_BLOCK_SIZE; - state[12]++; + while (bytes > CHACHA_BLOCK_SIZE) { + unsigned int l = min(bytes, CHACHA_BLOCK_SIZE * 4U); + + chacha_4block_xor_neon(state, dst, src, nrounds, l); + bytes -= l; + src += l; + dst += l; + state[12] += DIV_ROUND_UP(l, CHACHA_BLOCK_SIZE); } if (bytes) { - memcpy(buf, src, bytes); - chacha_block_xor_neon(state, buf, buf, nrounds); - memcpy(dst, buf, bytes); + const u8 *s = src; + u8 *d = dst; + + if (bytes != CHACHA_BLOCK_SIZE) + s = d = memcpy(buf, src, bytes); + chacha_block_xor_neon(state, d, s, nrounds); + if (d != dst) + memcpy(dst, buf, bytes); } } diff --git a/arch/arm/crypto/chacha-neon-core.S b/arch/arm/crypto/chacha-neon-core.S index eb22926d4912..13d12f672656 100644 --- a/arch/arm/crypto/chacha-neon-core.S +++ b/arch/arm/crypto/chacha-neon-core.S @@ -47,6 +47,7 @@ */ #include +#include .text .fpu neon @@ -205,7 +206,7 @@ ENDPROC(hchacha_block_neon) .align 5 ENTRY(chacha_4block_xor_neon) - push {r4-r5} + push {r4, lr} mov r4, sp // preserve the stack pointer sub ip, sp, #0x20 // allocate a 32 byte buffer bic ip, ip, #0x1f // aligned to 32 bytes @@ -229,10 +230,10 @@ ENTRY(chacha_4block_xor_neon) vld1.32 {q0-q1}, [r0] vld1.32 {q2-q3}, [ip] - adr r5, .Lctrinc + adr lr, .Lctrinc vdup.32 q15, d7[1] vdup.32 q14, d7[0] - vld1.32 {q4}, [r5, :128] + vld1.32 {q4}, [lr, :128] vdup.32 q13, d6[1] vdup.32 q12, d6[0] vdup.32 q11, d5[1] @@ -455,7 +456,7 @@ ENTRY(chacha_4block_xor_neon) // Re-interleave the words in the first two rows of each block (x0..7). // Also add the counter values 0-3 to x12[0-3]. - vld1.32 {q8}, [r5, :128] // load counter values 0-3 + vld1.32 {q8}, [lr, :128] // load counter values 0-3 vzip.32 q0, q1 // => (0 1 0 1) (0 1 0 1) vzip.32 q2, q3 // => (2 3 2 3) (2 3 2 3) vzip.32 q4, q5 // => (4 5 4 5) (4 5 4 5) @@ -493,6 +494,8 @@ ENTRY(chacha_4block_xor_neon) // Re-interleave the words in the last two rows of each block (x8..15). vld1.32 {q8-q9}, [sp, :256] + mov sp, r4 // restore original stack pointer + ldr r4, [r4, #8] // load number of bytes vzip.32 q12, q13 // => (12 13 12 13) (12 13 12 13) vzip.32 q14, q15 // => (14 15 14 15) (14 15 14 15) vzip.32 q8, q9 // => (8 9 8 9) (8 9 8 9) @@ -520,41 +523,121 @@ ENTRY(chacha_4block_xor_neon) // XOR the rest of the data with the keystream vld1.8 {q0-q1}, [r2]! + subs r4, r4, #96 veor q0, q0, q8 veor q1, q1, q12 + ble .Lle96 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q2 veor q1, q1, q6 + ble .Lle128 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q10 veor q1, q1, q14 + ble .Lle160 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q4 veor q1, q1, q5 + ble .Lle192 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q9 veor q1, q1, q13 + ble .Lle224 vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2]! + subs r4, r4, #32 veor q0, q0, q3 veor q1, q1, q7 + blt .Llt256 +.Lout: vst1.8 {q0-q1}, [r1]! vld1.8 {q0-q1}, [r2] - mov sp, r4 // restore original stack pointer veor q0, q0, q11 veor q1, q1, q15 vst1.8 {q0-q1}, [r1] - pop {r4-r5} - bx lr + pop {r4, pc} + +.Lle192: + vmov q4, q9 + vmov q5, q13 + +.Lle160: + // nothing to do + +.Lfinalblock: + // Process the final block if processing less than 4 full blocks. + // Entered with 32 bytes of ChaCha cipher stream in q4-q5, and the + // previous 32 byte output block that still needs to be written at + // [r1] in q0-q1. + beq .Lfullblock + +.Lpartialblock: + adr lr, .Lpermute + 32 + add r2, r2, r4 + add lr, lr, r4 + add r4, r4, r1 + + vld1.8 {q2-q3}, [lr] + vld1.8 {q6-q7}, [r2] + + add r4, r4, #32 + + vtbl.8 d4, {q4-q5}, d4 + vtbl.8 d5, {q4-q5}, d5 + vtbl.8 d6, {q4-q5}, d6 + vtbl.8 d7, {q4-q5}, d7 + + veor q6, q6, q2 + veor q7, q7, q3 + + vst1.8 {q6-q7}, [r4] // overlapping stores + vst1.8 {q0-q1}, [r1] + pop {r4, pc} + +.Lfullblock: + vmov q11, q4 + vmov q15, q5 + b .Lout +.Lle96: + vmov q4, q2 + vmov q5, q6 + b .Lfinalblock +.Lle128: + vmov q4, q10 + vmov q5, q14 + b .Lfinalblock +.Lle224: + vmov q4, q3 + vmov q5, q7 + b .Lfinalblock +.Llt256: + vmov q4, q11 + vmov q5, q15 + b .Lpartialblock ENDPROC(chacha_4block_xor_neon) + + .align L1_CACHE_SHIFT +.Lpermute: + .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + .byte 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 + .byte 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f + .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + .byte 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17 + .byte 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f From 802c6c3ef3bc3eeca95d3ad55642fb7a973d185a Mon Sep 17 00:00:00 2001 From: Adam Guerin Date: Tue, 3 Nov 2020 17:29:36 +0000 Subject: [PATCH 123/360] crypto: qat - replace pci with PCI in comments Change all lower case pci in comments to be upper case PCI. Suggested-by: Andy Shevchenko Signed-off-by: Adam Guerin Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_dev_mgr.c | 4 ++-- drivers/crypto/qat/qat_common/adf_sriov.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c index 92ec035576df..29dc2e3d38ff 100644 --- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c +++ b/drivers/crypto/qat/qat_common/adf_dev_mgr.c @@ -285,9 +285,9 @@ struct adf_accel_dev *adf_devmgr_get_first(void) /** * adf_devmgr_pci_to_accel_dev() - Get accel_dev associated with the pci_dev. - * @pci_dev: Pointer to pci device. + * @pci_dev: Pointer to PCI device. * - * Function returns acceleration device associated with the given pci device. + * Function returns acceleration device associated with the given PCI device. * To be used by QAT device specific drivers. * * Return: pointer to accel_dev or NULL if not found. diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index d887640355d4..8c822c2861c2 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -122,13 +122,13 @@ EXPORT_SYMBOL_GPL(adf_disable_sriov); /** * adf_sriov_configure() - Enable SRIOV for the device - * @pdev: Pointer to pci device. + * @pdev: Pointer to PCI device. * @numvfs: Number of virtual functions (VFs) to enable. * * Note that the @numvfs parameter is ignored and all VFs supported by the * device are enabled due to the design of the hardware. * - * Function enables SRIOV for the pci device. + * Function enables SRIOV for the PCI device. * * Return: number of VFs enabled on success, error code otherwise. */ From cd078cb6a0f2e360b073b34f78a53b9410fa3e7a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 6 Nov 2020 17:53:52 +1100 Subject: [PATCH 124/360] crypto: cavium/nitrox - Fix sparse warnings This patch fixes all the sparse warnings in cavium/nitrox: - Fix endianness warnings by adding the correct markers to unions. - Add missing header inclusions for prototypes. - Move nitrox_sriov_configure prototype into the isr header file. Signed-off-by: Herbert Xu --- drivers/crypto/cavium/nitrox/nitrox_aead.c | 10 +++++----- drivers/crypto/cavium/nitrox/nitrox_debugfs.c | 1 + drivers/crypto/cavium/nitrox/nitrox_hal.c | 1 + drivers/crypto/cavium/nitrox/nitrox_isr.c | 1 + drivers/crypto/cavium/nitrox/nitrox_isr.h | 9 +++++++++ drivers/crypto/cavium/nitrox/nitrox_main.c | 13 ------------- drivers/crypto/cavium/nitrox/nitrox_mbx.c | 1 + drivers/crypto/cavium/nitrox/nitrox_req.h | 4 ++++ drivers/crypto/cavium/nitrox/nitrox_reqmgr.c | 7 ++++--- 9 files changed, 26 insertions(+), 21 deletions(-) diff --git a/drivers/crypto/cavium/nitrox/nitrox_aead.c b/drivers/crypto/cavium/nitrox/nitrox_aead.c index 1be2571363fe..e5d8607ecb1d 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_aead.c +++ b/drivers/crypto/cavium/nitrox/nitrox_aead.c @@ -45,9 +45,9 @@ static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, /* fill crypto context */ fctx = nctx->u.fctx; - flags.f = be64_to_cpu(fctx->flags.f); + flags.fu = be64_to_cpu(fctx->flags.f); flags.w0.aes_keylen = aes_keylen; - fctx->flags.f = cpu_to_be64(flags.f); + fctx->flags.f = cpu_to_be64(flags.fu); /* copy enc key to context */ memset(&fctx->crypto, 0, sizeof(fctx->crypto)); @@ -63,9 +63,9 @@ static int nitrox_aead_setauthsize(struct crypto_aead *aead, struct flexi_crypto_context *fctx = nctx->u.fctx; union fc_ctx_flags flags; - flags.f = be64_to_cpu(fctx->flags.f); + flags.fu = be64_to_cpu(fctx->flags.f); flags.w0.mac_len = authsize; - fctx->flags.f = cpu_to_be64(flags.f); + fctx->flags.f = cpu_to_be64(flags.fu); aead->authsize = authsize; @@ -319,7 +319,7 @@ static int nitrox_gcm_common_init(struct crypto_aead *aead) flags->w0.iv_source = IV_FROM_DPTR; /* ask microcode to calculate ipad/opad */ flags->w0.auth_input_type = 1; - flags->f = be64_to_cpu(flags->f); + flags->f = cpu_to_be64(flags->fu); return 0; } diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.c b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c index 16f7d0bd1303..741572a01995 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_debugfs.c +++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.c @@ -3,6 +3,7 @@ #include #include "nitrox_csr.h" +#include "nitrox_debugfs.h" #include "nitrox_dev.h" static int firmware_show(struct seq_file *s, void *v) diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c index 34a2f4f30a7e..13b137410b75 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_hal.c +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c @@ -3,6 +3,7 @@ #include "nitrox_dev.h" #include "nitrox_csr.h" +#include "nitrox_hal.h" #define PLL_REF_CLK 50 #define MAX_CSR_RETRIES 10 diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.c b/drivers/crypto/cavium/nitrox/nitrox_isr.c index 3dec570a190a..99b053094f5a 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_isr.c +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.c @@ -7,6 +7,7 @@ #include "nitrox_csr.h" #include "nitrox_common.h" #include "nitrox_hal.h" +#include "nitrox_isr.h" #include "nitrox_mbx.h" /** diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.h b/drivers/crypto/cavium/nitrox/nitrox_isr.h index 1062c9336c1f..2bb123cd2f1c 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_isr.h +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.h @@ -9,4 +9,13 @@ void nitrox_unregister_interrupts(struct nitrox_device *ndev); int nitrox_sriov_register_interupts(struct nitrox_device *ndev); void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev); +#ifdef CONFIG_PCI_IOV +int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs); +#else +static inline int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + return 0; +} +#endif + #endif /* __NITROX_ISR_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index 9d14be97e381..facc8e6bc580 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -49,15 +49,6 @@ static unsigned int qlen = DEFAULT_CMD_QLEN; module_param(qlen, uint, 0644); MODULE_PARM_DESC(qlen, "Command queue length - default 2048"); -#ifdef CONFIG_PCI_IOV -int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs); -#else -int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs) -{ - return 0; -} -#endif - /** * struct ucode - Firmware Header * @id: microcode ID @@ -555,10 +546,8 @@ static void nitrox_remove(struct pci_dev *pdev) nitrox_remove_from_devlist(ndev); -#ifdef CONFIG_PCI_IOV /* disable SR-IOV */ nitrox_sriov_configure(pdev, 0); -#endif nitrox_crypto_unregister(); nitrox_debugfs_exit(ndev); nitrox_pf_sw_cleanup(ndev); @@ -584,9 +573,7 @@ static struct pci_driver nitrox_driver = { .probe = nitrox_probe, .remove = nitrox_remove, .shutdown = nitrox_shutdown, -#ifdef CONFIG_PCI_IOV .sriov_configure = nitrox_sriov_configure, -#endif }; module_pci_driver(nitrox_driver); diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.c b/drivers/crypto/cavium/nitrox/nitrox_mbx.c index 73993f9e2311..c1af9d4fca6e 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_mbx.c +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.c @@ -4,6 +4,7 @@ #include "nitrox_csr.h" #include "nitrox_hal.h" #include "nitrox_dev.h" +#include "nitrox_mbx.h" #define RING_TO_VFNO(_x, _y) ((_x) / (_y)) diff --git a/drivers/crypto/cavium/nitrox/nitrox_req.h b/drivers/crypto/cavium/nitrox/nitrox_req.h index 12282c1b14f5..ed174883c8e3 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_req.h +++ b/drivers/crypto/cavium/nitrox/nitrox_req.h @@ -149,6 +149,7 @@ struct auth_keys { union fc_ctx_flags { __be64 f; + u64 fu; struct { #if defined(__BIG_ENDIAN_BITFIELD) u64 cipher_type : 4; @@ -280,6 +281,7 @@ struct nitrox_rfc4106_rctx { * - packet payload bytes */ union pkt_instr_hdr { + __be64 bev; u64 value; struct { #if defined(__BIG_ENDIAN_BITFIELD) @@ -324,6 +326,7 @@ union pkt_instr_hdr { * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases. */ union pkt_hdr { + __be64 bev[2]; u64 value[2]; struct { #if defined(__BIG_ENDIAN_BITFIELD) @@ -370,6 +373,7 @@ union pkt_hdr { * sglist components at [RPTR] on the remote host. */ union slc_store_info { + __be64 bev[2]; u64 value[2]; struct { #if defined(__BIG_ENDIAN_BITFIELD) diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c index 5826c2c98a50..53ef06792133 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c +++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c @@ -3,6 +3,7 @@ #include #include +#include "nitrox_common.h" #include "nitrox_dev.h" #include "nitrox_req.h" #include "nitrox_csr.h" @@ -448,7 +449,7 @@ int nitrox_process_se_request(struct nitrox_device *ndev, sr->instr.ih.s.ssz = sr->out.sgmap_cnt; sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr); sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes; - sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value); + sr->instr.ih.bev = cpu_to_be64(sr->instr.ih.value); /* word 2 */ sr->instr.irh.value[0] = 0; @@ -460,7 +461,7 @@ int nitrox_process_se_request(struct nitrox_device *ndev, sr->instr.irh.s.ctxc = req->ctrl.s.ctxc; sr->instr.irh.s.arg = req->ctrl.s.arg; sr->instr.irh.s.opcode = req->opcode; - sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]); + sr->instr.irh.bev[0] = cpu_to_be64(sr->instr.irh.value[0]); /* word 3 */ sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle); @@ -468,7 +469,7 @@ int nitrox_process_se_request(struct nitrox_device *ndev, /* word 4 */ sr->instr.slc.value[0] = 0; sr->instr.slc.s.ssz = sr->out.sgmap_cnt; - sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]); + sr->instr.slc.bev[0] = cpu_to_be64(sr->instr.slc.value[0]); /* word 5 */ sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma); From f21406b0511d53f9370d9de8713dea7c09f9f248 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Fri, 6 Nov 2020 19:27:39 +0800 Subject: [PATCH 125/360] crypto: qat - support for mof format in fw loader Implement infrastructure for the Multiple Object File (MOF) format in the firmware loader. This will allow to load a specific firmware image contained inside an MOF file. This patch is based on earlier work done by Pingchao Yang. Signed-off-by: Giovanni Cabiddu Reviewed-by: Jack Xu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_accel_engine.c | 2 +- .../crypto/qat/qat_common/adf_common_drv.h | 2 +- .../qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/icp_qat_uclo.h | 69 +++++ drivers/crypto/qat/qat_common/qat_uclo.c | 264 +++++++++++++++++- 5 files changed, 332 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c index c8ad85b882be..1da4176356ab 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_engine.c +++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c @@ -38,7 +38,7 @@ int adf_ae_fw_load(struct adf_accel_dev *accel_dev) dev_err(&GET_DEV(accel_dev), "Failed to load MMP\n"); goto out_err; } - if (qat_uclo_map_obj(loader_data->fw_loader, uof_addr, uof_size)) { + if (qat_uclo_map_obj(loader_data->fw_loader, uof_addr, uof_size, NULL)) { dev_err(&GET_DEV(accel_dev), "Failed to map FW\n"); goto out_err; } diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index f22342f612c1..8e6e346fd841 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -184,7 +184,7 @@ void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle); int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, int mem_size); int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, int mem_size); + void *addr_ptr, u32 mem_size, char *obj_name); #if defined(CONFIG_PCI_IOV) int adf_sriov_configure(struct pci_dev *pdev, int numvfs); void adf_disable_sriov(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 3e8e291cd122..7d44786a223a 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -27,6 +27,7 @@ struct icp_qat_fw_loader_handle { struct pci_dev *pci_dev; void *obj_handle; void *sobj_handle; + void *mobj_handle; bool fw_auth; void __iomem *hal_sram_addr_v; void __iomem *hal_cap_g_ctl_csr_addr_v; diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index 8fe1ec344fa2..101de1430896 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -31,6 +31,15 @@ #define ICP_QAT_SUOF_FID 0x53554f46 #define ICP_QAT_SUOF_MAJVER 0x0 #define ICP_QAT_SUOF_MINVER 0x1 +#define ICP_QAT_SUOF_OBJ_NAME_LEN 128 +#define ICP_QAT_MOF_OBJ_ID_LEN 8 +#define ICP_QAT_MOF_OBJ_CHUNKID_LEN 8 +#define ICP_QAT_MOF_FID 0x00666f6d +#define ICP_QAT_MOF_MAJVER 0x0 +#define ICP_QAT_MOF_MINVER 0x1 +#define ICP_QAT_MOF_SYM_OBJS "SYM_OBJS" +#define ICP_QAT_SUOF_OBJS "SUF_OBJS" +#define ICP_QAT_SUOF_IMAG "SUF_IMAG" #define ICP_QAT_SIMG_AE_INIT_SEQ_LEN (50 * sizeof(unsigned long long)) #define ICP_QAT_SIMG_AE_INSTS_LEN (0x4000 * sizeof(unsigned long long)) #define ICP_QAT_CSS_FWSK_MODULUS_LEN 256 @@ -481,4 +490,64 @@ struct icp_qat_suof_objhdr { unsigned int img_length; unsigned int reserved; }; + +struct icp_qat_mof_file_hdr { + unsigned int file_id; + unsigned int checksum; + char min_ver; + char maj_ver; + unsigned short reserved; + unsigned short max_chunks; + unsigned short num_chunks; +}; + +struct icp_qat_mof_chunkhdr { + char chunk_id[ICP_QAT_MOF_OBJ_ID_LEN]; + u64 offset; + u64 size; +}; + +struct icp_qat_mof_str_table { + unsigned int tab_len; + unsigned int strings; +}; + +struct icp_qat_mof_obj_hdr { + unsigned short max_chunks; + unsigned short num_chunks; + unsigned int reserved; +}; + +struct icp_qat_mof_obj_chunkhdr { + char chunk_id[ICP_QAT_MOF_OBJ_CHUNKID_LEN]; + u64 offset; + u64 size; + unsigned int name; + unsigned int reserved; +}; + +struct icp_qat_mof_objhdr { + char *obj_name; + char *obj_buf; + unsigned int obj_size; +}; + +struct icp_qat_mof_table { + unsigned int num_objs; + struct icp_qat_mof_objhdr *obj_hdr; +}; + +struct icp_qat_mof_handle { + unsigned int file_id; + unsigned int checksum; + char min_ver; + char maj_ver; + char *mof_buf; + u32 mof_size; + char *sym_str; + unsigned int sym_size; + char *uobjs_hdr; + char *sobjs_hdr; + struct icp_qat_mof_table obj_table; +}; #endif diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 5d1f28cd6680..b475f6bfb90b 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1437,18 +1437,272 @@ out_objbuf_err: return -ENOMEM; } -int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, int mem_size) +static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_mof_file_hdr *mof_ptr, + u32 mof_size) { + struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; + unsigned int min_ver_offset; + unsigned int checksum; + + mobj_handle->file_id = ICP_QAT_MOF_FID; + mobj_handle->mof_buf = (char *)mof_ptr; + mobj_handle->mof_size = mof_size; + + min_ver_offset = mof_size - offsetof(struct icp_qat_mof_file_hdr, + min_ver); + checksum = qat_uclo_calc_str_checksum(&mof_ptr->min_ver, + min_ver_offset); + if (checksum != mof_ptr->checksum) { + pr_err("QAT: incorrect MOF checksum\n"); + return -EINVAL; + } + + mobj_handle->checksum = mof_ptr->checksum; + mobj_handle->min_ver = mof_ptr->min_ver; + mobj_handle->maj_ver = mof_ptr->maj_ver; + return 0; +} + +static void qat_uclo_del_mof(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_mof_handle *mobj_handle = handle->mobj_handle; + + kfree(mobj_handle->obj_table.obj_hdr); + mobj_handle->obj_table.obj_hdr = NULL; + kfree(handle->mobj_handle); + handle->mobj_handle = NULL; +} + +static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle, + char *obj_name, char **obj_ptr, + unsigned int *obj_size) +{ + struct icp_qat_mof_objhdr *obj_hdr = mobj_handle->obj_table.obj_hdr; + unsigned int i; + + for (i = 0; i < mobj_handle->obj_table.num_objs; i++) { + if (!strncmp(obj_hdr[i].obj_name, obj_name, + ICP_QAT_SUOF_OBJ_NAME_LEN)) { + *obj_ptr = obj_hdr[i].obj_buf; + *obj_size = obj_hdr[i].obj_size; + return 0; + } + } + + pr_err("QAT: object %s is not found inside MOF\n", obj_name); + return -EINVAL; +} + +static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle, + struct icp_qat_mof_objhdr *mobj_hdr, + struct icp_qat_mof_obj_chunkhdr *obj_chunkhdr) +{ + u8 *obj; + + if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_UOF_IMAG, + ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { + obj = mobj_handle->uobjs_hdr + obj_chunkhdr->offset; + } else if (!strncmp(obj_chunkhdr->chunk_id, ICP_QAT_SUOF_IMAG, + ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { + obj = mobj_handle->sobjs_hdr + obj_chunkhdr->offset; + } else { + pr_err("QAT: unsupported chunk id\n"); + return -EINVAL; + } + mobj_hdr->obj_buf = obj; + mobj_hdr->obj_size = (unsigned int)obj_chunkhdr->size; + mobj_hdr->obj_name = obj_chunkhdr->name + mobj_handle->sym_str; + return 0; +} + +static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) +{ + struct icp_qat_mof_obj_chunkhdr *uobj_chunkhdr; + struct icp_qat_mof_obj_chunkhdr *sobj_chunkhdr; + struct icp_qat_mof_obj_hdr *uobj_hdr; + struct icp_qat_mof_obj_hdr *sobj_hdr; + struct icp_qat_mof_objhdr *mobj_hdr; + unsigned int uobj_chunk_num = 0; + unsigned int sobj_chunk_num = 0; + unsigned int *valid_chunk; + int ret, i; + + uobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->uobjs_hdr; + sobj_hdr = (struct icp_qat_mof_obj_hdr *)mobj_handle->sobjs_hdr; + if (uobj_hdr) + uobj_chunk_num = uobj_hdr->num_chunks; + if (sobj_hdr) + sobj_chunk_num = sobj_hdr->num_chunks; + + mobj_hdr = kzalloc((uobj_chunk_num + sobj_chunk_num) * + sizeof(*mobj_hdr), GFP_KERNEL); + if (!mobj_hdr) + return -ENOMEM; + + mobj_handle->obj_table.obj_hdr = mobj_hdr; + valid_chunk = &mobj_handle->obj_table.num_objs; + uobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) + ((uintptr_t)uobj_hdr + sizeof(*uobj_hdr)); + sobj_chunkhdr = (struct icp_qat_mof_obj_chunkhdr *) + ((uintptr_t)sobj_hdr + sizeof(*sobj_hdr)); + + /* map uof objects */ + for (i = 0; i < uobj_chunk_num; i++) { + ret = qat_uclo_map_obj_from_mof(mobj_handle, + &mobj_hdr[*valid_chunk], + &uobj_chunkhdr[i]); + if (ret) + return ret; + (*valid_chunk)++; + } + + /* map suof objects */ + for (i = 0; i < sobj_chunk_num; i++) { + ret = qat_uclo_map_obj_from_mof(mobj_handle, + &mobj_hdr[*valid_chunk], + &sobj_chunkhdr[i]); + if (ret) + return ret; + (*valid_chunk)++; + } + + if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunk) { + pr_err("QAT: inconsistent UOF/SUOF chunk amount\n"); + return -EINVAL; + } + return 0; +} + +static void qat_uclo_map_mof_symobjs(struct icp_qat_mof_handle *mobj_handle, + struct icp_qat_mof_chunkhdr *mof_chunkhdr) +{ + char **sym_str = (char **)&mobj_handle->sym_str; + unsigned int *sym_size = &mobj_handle->sym_size; + struct icp_qat_mof_str_table *str_table_obj; + + *sym_size = *(unsigned int *)(uintptr_t) + (mof_chunkhdr->offset + mobj_handle->mof_buf); + *sym_str = (char *)(uintptr_t) + (mobj_handle->mof_buf + mof_chunkhdr->offset + + sizeof(str_table_obj->tab_len)); +} + +static void qat_uclo_map_mof_chunk(struct icp_qat_mof_handle *mobj_handle, + struct icp_qat_mof_chunkhdr *mof_chunkhdr) +{ + char *chunk_id = mof_chunkhdr->chunk_id; + + if (!strncmp(chunk_id, ICP_QAT_MOF_SYM_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) + qat_uclo_map_mof_symobjs(mobj_handle, mof_chunkhdr); + else if (!strncmp(chunk_id, ICP_QAT_UOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) + mobj_handle->uobjs_hdr = mobj_handle->mof_buf + + mof_chunkhdr->offset; + else if (!strncmp(chunk_id, ICP_QAT_SUOF_OBJS, ICP_QAT_MOF_OBJ_ID_LEN)) + mobj_handle->sobjs_hdr = mobj_handle->mof_buf + + mof_chunkhdr->offset; +} + +static int qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr *mof_hdr) +{ + int maj = mof_hdr->maj_ver & 0xff; + int min = mof_hdr->min_ver & 0xff; + + if (mof_hdr->file_id != ICP_QAT_MOF_FID) { + pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id); + return -EINVAL; + } + + if (mof_hdr->num_chunks <= 0x1) { + pr_err("QAT: MOF chunk amount is incorrect\n"); + return -EINVAL; + } + if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) { + pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n", + maj, min); + return -EINVAL; + } + return 0; +} + +static int qat_uclo_map_mof_obj(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_mof_file_hdr *mof_ptr, + u32 mof_size, char *obj_name, char **obj_ptr, + unsigned int *obj_size) +{ + struct icp_qat_mof_chunkhdr *mof_chunkhdr; + unsigned int file_id = mof_ptr->file_id; + struct icp_qat_mof_handle *mobj_handle; + unsigned short chunks_num; + unsigned int i; + int ret; + + if (file_id == ICP_QAT_UOF_FID || file_id == ICP_QAT_SUOF_FID) { + if (obj_ptr) + *obj_ptr = (char *)mof_ptr; + if (obj_size) + *obj_size = mof_size; + return 0; + } + if (qat_uclo_check_mof_format(mof_ptr)) + return -EINVAL; + + mobj_handle = kzalloc(sizeof(*mobj_handle), GFP_KERNEL); + if (!mobj_handle) + return -ENOMEM; + + handle->mobj_handle = mobj_handle; + ret = qat_uclo_map_mof_file_hdr(handle, mof_ptr, mof_size); + if (ret) + return ret; + + mof_chunkhdr = (void *)mof_ptr + sizeof(*mof_ptr); + chunks_num = mof_ptr->num_chunks; + + /* Parse MOF file chunks */ + for (i = 0; i < chunks_num; i++) + qat_uclo_map_mof_chunk(mobj_handle, &mof_chunkhdr[i]); + + /* All sym_objs uobjs and sobjs should be available */ + if (!mobj_handle->sym_str || + (!mobj_handle->uobjs_hdr && !mobj_handle->sobjs_hdr)) + return -EINVAL; + + ret = qat_uclo_map_objs_from_mof(mobj_handle); + if (ret) + return ret; + + /* Seek specified uof object in MOF */ + return qat_uclo_seek_obj_inside_mof(mobj_handle, obj_name, + obj_ptr, obj_size); +} + +int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, u32 mem_size, char *obj_name) +{ + char *obj_addr; + u32 obj_size; + int ret; + BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >= (sizeof(handle->hal_handle->ae_mask) * 8)); if (!handle || !addr_ptr || mem_size < 24) return -EINVAL; + if (obj_name) { + ret = qat_uclo_map_mof_obj(handle, addr_ptr, mem_size, obj_name, + &obj_addr, &obj_size); + if (ret) + return ret; + } else { + obj_addr = addr_ptr; + obj_size = mem_size; + } + return (handle->fw_auth) ? - qat_uclo_map_suof_obj(handle, addr_ptr, mem_size) : - qat_uclo_map_uof_obj(handle, addr_ptr, mem_size); + qat_uclo_map_suof_obj(handle, obj_addr, obj_size) : + qat_uclo_map_uof_obj(handle, obj_addr, obj_size); } void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle) @@ -1456,6 +1710,8 @@ void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle) struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int a; + if (handle->mobj_handle) + qat_uclo_del_mof(handle); if (handle->sobj_handle) qat_uclo_del_suof(handle); if (!obj_handle) From 3b5c130fb2e4c045369791c33c83b59f6e84f7d6 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:40 +0800 Subject: [PATCH 126/360] crypto: qat - fix status check in qat_hal_put_rel_rd_xfer() The return value of qat_hal_rd_ae_csr() is always a CSR value and never a status and should not be stored in the status variable of qat_hal_put_rel_rd_xfer(). This removes the assignment as qat_hal_rd_ae_csr() is not expected to fail. A more comprehensive handling of the theoretical corner case which could result in a fail will be submitted in a separate patch. Fixes: 8c9478a400b7 ("crypto: qat - reduce stack size with KASAN") Signed-off-by: Jack Xu Reviewed-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_hal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index bc07199459e7..5da8475ed876 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -1149,7 +1149,7 @@ static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle, unsigned short mask; unsigned short dr_offset = 0x10; - status = ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + ctx_enables = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); if (CE_INUSE_CONTEXTS & ctx_enables) { if (ctx & 0x1) { pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx); From 49a6cccec4f89777a0ffd2a6d4c3b1af33636b3a Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:41 +0800 Subject: [PATCH 127/360] crypto: qat - fix CSR access Do not mask the AE number with the AE mask when accessing the AE local CSRs. Bit 12 of the local CSR address is the start of AE number so just take out the AE mask here. Signed-off-by: Jack Xu Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_hal.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index c0e9fc0c93dd..b48b313623fe 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -94,15 +94,13 @@ enum fcu_sts { #define SET_GLB_CSR(handle, csr, val) SET_CAP_CSR(handle, csr + GLOBAL_CSR, val) #define GET_GLB_CSR(handle, csr) GET_CAP_CSR(handle, GLOBAL_CSR + csr) #define AE_CSR(handle, ae) \ - ((char __iomem *)handle->hal_cap_ae_local_csr_addr_v + \ - ((ae & handle->hal_handle->ae_mask) << 12)) + ((char __iomem *)handle->hal_cap_ae_local_csr_addr_v + (ae << 12)) #define AE_CSR_ADDR(handle, ae, csr) (AE_CSR(handle, ae) + (0x3ff & csr)) #define SET_AE_CSR(handle, ae, csr, val) \ ADF_CSR_WR(AE_CSR_ADDR(handle, ae, csr), 0, val) #define GET_AE_CSR(handle, ae, csr) ADF_CSR_RD(AE_CSR_ADDR(handle, ae, csr), 0) #define AE_XFER(handle, ae) \ - ((char __iomem *)handle->hal_cap_ae_xfer_csr_addr_v + \ - ((ae & handle->hal_handle->ae_mask) << 12)) + ((char __iomem *)handle->hal_cap_ae_xfer_csr_addr_v + (ae << 12)) #define AE_XFER_ADDR(handle, ae, reg) (AE_XFER(handle, ae) + \ ((reg & 0xff) << 2)) #define SET_AE_XFER(handle, ae, reg, val) \ From ed8fc0c1b9c72f016130244b6a46483576fe8e5a Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:42 +0800 Subject: [PATCH 128/360] crypto: qat - fix error message Change message in error path of qat_uclo_check_image_compat() to report an incompatible firmware image that contains a neighbor register table. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index b475f6bfb90b..063af33c6ca6 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -536,7 +536,7 @@ qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, (encap_uof_obj->beg_uof + code_page->neigh_reg_tab_offset); if (neigh_reg_tab->entry_num) { - pr_err("QAT: UOF can't contain shared control store feature\n"); + pr_err("QAT: UOF can't contain neighbor register table\n"); return -EINVAL; } if (image->numpages > 1) { From 8ec0bee5d43e8582e6320dd9907d01c58844d3d3 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:43 +0800 Subject: [PATCH 129/360] crypto: qat - remove unnecessary parenthesis Remove unnecessary parenthesis across the firmware loader. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_hal.c | 6 +++--- drivers/crypto/qat/qat_common/qat_uclo.c | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 5da8475ed876..bbfb2b1b6fee 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -158,7 +158,7 @@ int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, { unsigned int csr, new_csr; - if ((mode != 4) && (mode != 8)) { + if (mode != 4 && mode != 8) { pr_err("QAT: bad ctx mode=%d\n", mode); return -EINVAL; } @@ -430,7 +430,7 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0); csr_val = ADF_CSR_RD(csr_addr, 0); } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--); - if ((times < 0)) { + if (times < 0) { pr_err("QAT: Fail to init eSram!\n"); return -EFAULT; } @@ -1128,7 +1128,7 @@ int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, plm_init = plm_init->next; } /* exec micro codes */ - if (micro_inst_arry && (micro_inst_num > 0)) { + if (micro_inst_arry && micro_inst_num > 0) { micro_inst_arry[micro_inst_num++] = 0x0E000010000ull; stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec, micro_inst_arry, diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 063af33c6ca6..4b2079353aa3 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -74,7 +74,7 @@ static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data) static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table, unsigned int str_offset) { - if ((!str_table->table_len) || (str_offset > str_table->table_len)) + if (!str_table->table_len || str_offset > str_table->table_len) return NULL; return (char *)(((uintptr_t)(str_table->strings)) + str_offset); } @@ -736,8 +736,8 @@ static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) return -EINVAL; } maj_ver = obj_handle->prod_rev & 0xff; - if ((obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver) || - (obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver)) { + if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver || + obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) { pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver); return -EINVAL; } @@ -1064,8 +1064,8 @@ static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle, return -EINVAL; } maj_ver = prod_rev & 0xff; - if ((maj_ver > img_ae_mode->devmax_ver) || - (maj_ver < img_ae_mode->devmin_ver)) { + if (maj_ver > img_ae_mode->devmax_ver || + maj_ver < img_ae_mode->devmin_ver) { pr_err("QAT: incompatible device majver 0x%x\n", maj_ver); return -EINVAL; } @@ -1108,7 +1108,7 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, unsigned int i = 0; struct icp_qat_suof_img_hdr img_header; - if (!suof_ptr || (suof_size == 0)) { + if (!suof_ptr || suof_size == 0) { pr_err("QAT: input parameter SUOF pointer/size is NULL\n"); return -EINVAL; } @@ -1199,7 +1199,7 @@ static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle, { dma_free_coherent(&handle->pci_dev->dev, (size_t)(dram_desc->dram_size), - (dram_desc->dram_base_addr_v), + dram_desc->dram_base_addr_v, dram_desc->dram_bus_addr); memset(dram_desc, 0, sizeof(*dram_desc)); } @@ -1851,7 +1851,7 @@ static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) if (qat_uclo_map_auth_fw(handle, (char *)simg_hdr[i].simg_buf, (unsigned int) - (simg_hdr[i].simg_len), + simg_hdr[i].simg_len, &desc)) goto wr_err; if (qat_uclo_auth_fw(handle, desc)) From 54fa5d4bf71246906efc3cd85ebddbc961af3498 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:44 +0800 Subject: [PATCH 130/360] crypto: qat - introduce additional parenthesis Introduce additional parenthesis to resolve a warninga reported by checkpatch. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_hal.h | 14 +++++++------- drivers/crypto/qat/qat_common/qat_hal.c | 6 +++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index b48b313623fe..5640bb278bb1 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -88,23 +88,23 @@ enum fcu_sts { #define FW_AUTH_MAX_RETRY 300 #define SET_CAP_CSR(handle, csr, val) \ - ADF_CSR_WR(handle->hal_cap_g_ctl_csr_addr_v, csr, val) + ADF_CSR_WR((handle)->hal_cap_g_ctl_csr_addr_v, csr, val) #define GET_CAP_CSR(handle, csr) \ - ADF_CSR_RD(handle->hal_cap_g_ctl_csr_addr_v, csr) + ADF_CSR_RD((handle)->hal_cap_g_ctl_csr_addr_v, csr) #define SET_GLB_CSR(handle, csr, val) SET_CAP_CSR(handle, csr + GLOBAL_CSR, val) #define GET_GLB_CSR(handle, csr) GET_CAP_CSR(handle, GLOBAL_CSR + csr) #define AE_CSR(handle, ae) \ - ((char __iomem *)handle->hal_cap_ae_local_csr_addr_v + (ae << 12)) -#define AE_CSR_ADDR(handle, ae, csr) (AE_CSR(handle, ae) + (0x3ff & csr)) + ((char __iomem *)(handle)->hal_cap_ae_local_csr_addr_v + ((ae) << 12)) +#define AE_CSR_ADDR(handle, ae, csr) (AE_CSR(handle, ae) + (0x3ff & (csr))) #define SET_AE_CSR(handle, ae, csr, val) \ ADF_CSR_WR(AE_CSR_ADDR(handle, ae, csr), 0, val) #define GET_AE_CSR(handle, ae, csr) ADF_CSR_RD(AE_CSR_ADDR(handle, ae, csr), 0) #define AE_XFER(handle, ae) \ - ((char __iomem *)handle->hal_cap_ae_xfer_csr_addr_v + (ae << 12)) + ((char __iomem *)(handle)->hal_cap_ae_xfer_csr_addr_v + ((ae) << 12)) #define AE_XFER_ADDR(handle, ae, reg) (AE_XFER(handle, ae) + \ - ((reg & 0xff) << 2)) + (((reg) & 0xff) << 2)) #define SET_AE_XFER(handle, ae, reg, val) \ ADF_CSR_WR(AE_XFER_ADDR(handle, ae, reg), 0, val) #define SRAM_WRITE(handle, addr, val) \ - ADF_CSR_WR(handle->hal_sram_addr_v, addr, val) + ADF_CSR_WR((handle)->hal_sram_addr_v, addr, val) #endif diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index bbfb2b1b6fee..c628ea30e3c2 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -33,7 +33,7 @@ ((((const_val) << 12) & 0x0FF00000ull) | \ (((const_val) << 0) & 0x000000FFull)))) -#define AE(handle, ae) handle->hal_handle->aes[ae] +#define AE(handle, ae) ((handle)->hal_handle->aes[ae]) static const u64 inst_4b[] = { 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull, @@ -150,8 +150,8 @@ static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle, return 0; } -#define CLR_BIT(wrd, bit) (wrd & ~(1 << bit)) -#define SET_BIT(wrd, bit) (wrd | 1 << bit) +#define CLR_BIT(wrd, bit) ((wrd) & ~(1 << (bit))) +#define SET_BIT(wrd, bit) ((wrd) | 1 << (bit)) int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned char mode) From ee103cf1f5af03ae300dcfdbe0f51ae12ce35dec Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:45 +0800 Subject: [PATCH 131/360] crypto: qat - rename qat_uclo_del_uof_obj() Rename the function qat_uclo_del_uof_obj() in qat_uclo_del_obj() since it frees the memory allocated for all firmware objects. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_engine.c | 2 +- drivers/crypto/qat/qat_common/adf_common_drv.h | 2 +- drivers/crypto/qat/qat_common/qat_uclo.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c index 1da4176356ab..2c4a8c7c736e 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_engine.c +++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c @@ -61,7 +61,7 @@ void adf_ae_fw_release(struct adf_accel_dev *accel_dev) if (!hw_device->fw_name) return; - qat_uclo_del_uof_obj(loader_data->fw_loader); + qat_uclo_del_obj(loader_data->fw_loader); qat_hal_deinit(loader_data->fw_loader); release_firmware(loader_data->uof_fw); release_firmware(loader_data->mmp_fw); diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 8e6e346fd841..22ac0517d15d 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -180,7 +180,7 @@ int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, int qat_hal_wr_lm(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned short lm_addr, unsigned int value); int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle); -void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle); +void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle); int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, int mem_size); int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 4b2079353aa3..dc2f2dcf21b8 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1705,7 +1705,7 @@ int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, qat_uclo_map_uof_obj(handle, obj_addr, obj_size); } -void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle) +void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int a; From 72b67d9507b954b5b5f4cecb74bcefff61a28d8c Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:46 +0800 Subject: [PATCH 132/360] crypto: qat - add support for relative FW ucode loading Improve the way micro instructions (FW code) are uploaded to Accelerator Engines (AEs). If code starts at PC zero (absolute addressing), read uwords with no relative address. Otherwise, use relative addressing to the page region. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index dc2f2dcf21b8..1c03205c7166 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1735,21 +1735,22 @@ static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle, u64 *uword, unsigned int addr_p, unsigned int raddr, u64 fill) { + unsigned int i, addr; u64 uwrd = 0; - unsigned int i; if (!encap_page) { *uword = fill; return; } + addr = (encap_page->page_region) ? raddr : addr_p; for (i = 0; i < encap_page->uwblock_num; i++) { - if (raddr >= encap_page->uwblock[i].start_addr && - raddr <= encap_page->uwblock[i].start_addr + + if (addr >= encap_page->uwblock[i].start_addr && + addr <= encap_page->uwblock[i].start_addr + encap_page->uwblock[i].words_num - 1) { - raddr -= encap_page->uwblock[i].start_addr; - raddr *= obj_handle->uword_in_bytes; + addr -= encap_page->uwblock[i].start_addr; + addr *= obj_handle->uword_in_bytes; memcpy(&uwrd, (void *)(((uintptr_t) - encap_page->uwblock[i].micro_words) + raddr), + encap_page->uwblock[i].micro_words) + addr), obj_handle->uword_in_bytes); uwrd = uwrd & 0xbffffffffffull; } From fe278bf35c59bbc9bc695dac58bfa7120ba125ba Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:47 +0800 Subject: [PATCH 133/360] crypto: qat - change type for ctx_mask Change type for ctx_mask from unsigned char to unsigned long to avoid type casting. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_common_drv.h | 8 ++++---- drivers/crypto/qat/qat_common/qat_hal.c | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 22ac0517d15d..8109e2ab4257 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -163,19 +163,19 @@ int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle, unsigned char ae, struct icp_qat_uof_batch_init *lm_init_header); int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata); int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata); int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata); int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, unsigned short reg_num, unsigned int regdata); int qat_hal_wr_lm(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned short lm_addr, unsigned int value); diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index c628ea30e3c2..a9243758a959 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -1274,7 +1274,7 @@ static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle } int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata) { @@ -1294,7 +1294,7 @@ int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, } else { reg = reg_num; type = reg_type; - if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + if (!test_bit(ctx, &ctx_mask)) continue; } stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata); @@ -1308,7 +1308,7 @@ int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle, } int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata) { @@ -1328,7 +1328,7 @@ int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, } else { reg = reg_num; type = reg_type; - if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + if (!test_bit(ctx, &ctx_mask)) continue; } stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg, @@ -1343,7 +1343,7 @@ int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle, } int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, enum icp_qat_uof_regtype reg_type, unsigned short reg_num, unsigned int regdata) { @@ -1363,7 +1363,7 @@ int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, } else { reg = reg_num; type = reg_type; - if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + if (!test_bit(ctx, &ctx_mask)) continue; } stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg, @@ -1378,7 +1378,7 @@ int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle, } int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, - unsigned char ae, unsigned char ctx_mask, + unsigned char ae, unsigned long ctx_mask, unsigned short reg_num, unsigned int regdata) { int stat = 0; @@ -1388,7 +1388,7 @@ int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, return -EINVAL; for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) { - if (!test_bit(ctx, (unsigned long *)&ctx_mask)) + if (!test_bit(ctx, &ctx_mask)) continue; stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata); if (stat) { From 82b3230658a90e60c36b426b4ca8d176c4d7ebc8 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:48 +0800 Subject: [PATCH 134/360] crypto: qat - change micro word data mask Change micro word data mask since the Acceleration Engine (AE) instruction codes have been changed in the new generation QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 1c03205c7166..6423b1ea7021 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1752,7 +1752,7 @@ static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle, memcpy(&uwrd, (void *)(((uintptr_t) encap_page->uwblock[i].micro_words) + addr), obj_handle->uword_in_bytes); - uwrd = uwrd & 0xbffffffffffull; + uwrd = uwrd & GENMASK_ULL(43, 0); } } *uword = uwrd; From 10fb050caef99d75895bf0978188090d3ed676c2 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:49 +0800 Subject: [PATCH 135/360] crypto: qat - refactor AE start Change the API and the behaviour of the qat_hal_start() function. With this change, the function starts under the hood all acceleration engines (AEs) and there is no longer need to call it for each engine. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_accel_engine.c | 9 ++----- .../crypto/qat/qat_common/adf_common_drv.h | 3 +-- drivers/crypto/qat/qat_common/qat_hal.c | 24 ++++++++++++------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c index 2c4a8c7c736e..08aaaf2b4659 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_engine.c +++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c @@ -74,17 +74,12 @@ int adf_ae_start(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - u32 ae_ctr, ae, max_aes = GET_MAX_ACCELENGINES(accel_dev); + u32 ae_ctr; if (!hw_data->fw_name) return 0; - for (ae = 0, ae_ctr = 0; ae < max_aes; ae++) { - if (hw_data->ae_mask & (1 << ae)) { - qat_hal_start(loader_data->fw_loader, ae, 0xFF); - ae_ctr++; - } - } + ae_ctr = qat_hal_start(loader_data->fw_loader); dev_info(&GET_DEV(accel_dev), "qat_dev%d started %d acceleration engines\n", accel_dev->accel_id, ae_ctr); diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 8109e2ab4257..945608b71937 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -133,8 +133,7 @@ void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev); int qat_hal_init(struct adf_accel_dev *accel_dev); void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle); -void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae, - unsigned int ctx_mask); +int qat_hal_start(struct icp_qat_fw_loader_handle *handle); void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned int ctx_mask); void qat_hal_reset(struct icp_qat_fw_loader_handle *handle); diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index a9243758a959..f127233eec17 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -742,26 +742,32 @@ void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) kfree(handle); } -void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae, - unsigned int ctx_mask) +int qat_hal_start(struct icp_qat_fw_loader_handle *handle) { + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned int fcu_sts; + unsigned char ae; + u32 ae_ctr = 0; int retry = 0; - unsigned int fcu_sts = 0; if (handle->fw_auth) { + ae_ctr = hweight32(ae_mask); SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_START); do { msleep(FW_AUTH_WAIT_PERIOD); fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1)) - return; + return ae_ctr; } while (retry++ < FW_AUTH_MAX_RETRY); - pr_err("QAT: start error (AE 0x%x FCU_STS = 0x%x)\n", ae, - fcu_sts); + pr_err("QAT: start error (FCU_STS = 0x%x)\n", fcu_sts); + return 0; } else { - qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) & - ICP_QAT_UCLO_AE_ALL_CTX, 0x10000); - qat_hal_enable_ctx(handle, ae, ctx_mask); + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + qat_hal_put_wakeup_event(handle, ae, 0, 0x10000); + qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX); + ae_ctr++; + } + return ae_ctr; } } From 8f87b6271ec85c09af6406665324c74bb79e3216 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:50 +0800 Subject: [PATCH 136/360] crypto: qat - remove global CSRs helpers Include the offset of GLOBAL_CSR directly into the enum hal_global_csr and remove the macros SET_GLB_CSR/GET_GLB_CSR to simplify the global CSR access. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_hal.h | 9 +++------ drivers/crypto/qat/qat_common/qat_hal.c | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index 5640bb278bb1..c2166dacdf5b 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -5,9 +5,9 @@ #include "icp_qat_fw_loader_handle.h" enum hal_global_csr { - MISC_CONTROL = 0x04, - ICP_RESET = 0x0c, - ICP_GLOBAL_CLK_ENABLE = 0x50 + MISC_CONTROL = 0xA04, + ICP_RESET = 0xA0c, + ICP_GLOBAL_CLK_ENABLE = 0xA50 }; enum hal_ae_csr { @@ -78,7 +78,6 @@ enum fcu_sts { #define XCWE_VOLUNTARY (0x1) #define LCS_STATUS (0x1) #define MMC_SHARE_CS_BITPOS 2 -#define GLOBAL_CSR 0xA00 #define FCU_CTRL_AE_POS 0x8 #define FCU_AUTH_STS_MASK 0x7 #define FCU_STS_DONE_POS 0x9 @@ -91,8 +90,6 @@ enum fcu_sts { ADF_CSR_WR((handle)->hal_cap_g_ctl_csr_addr_v, csr, val) #define GET_CAP_CSR(handle, csr) \ ADF_CSR_RD((handle)->hal_cap_g_ctl_csr_addr_v, csr) -#define SET_GLB_CSR(handle, csr, val) SET_CAP_CSR(handle, csr + GLOBAL_CSR, val) -#define GET_GLB_CSR(handle, csr) GET_CAP_CSR(handle, GLOBAL_CSR + csr) #define AE_CSR(handle, ae) \ ((char __iomem *)(handle)->hal_cap_ae_local_csr_addr_v + ((ae) << 12)) #define AE_CSR_ADDR(handle, ae, csr) (AE_CSR(handle, ae) + (0x3ff & (csr))) diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index f127233eec17..15ebb57ea14a 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -273,10 +273,10 @@ void qat_hal_reset(struct icp_qat_fw_loader_handle *handle) { unsigned int ae_reset_csr; - ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET); + ae_reset_csr = GET_CAP_CSR(handle, ICP_RESET); ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB; ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB; - SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr); + SET_CAP_CSR(handle, ICP_RESET, ae_reset_csr); } static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, @@ -390,9 +390,9 @@ static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) unsigned char ae; /* stop the timestamp timers */ - misc_ctl = GET_GLB_CSR(handle, MISC_CONTROL); + misc_ctl = GET_CAP_CSR(handle, MISC_CONTROL); if (misc_ctl & MC_TIMESTAMP_ENABLE) - SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl & + SET_CAP_CSR(handle, MISC_CONTROL, misc_ctl & (~MC_TIMESTAMP_ENABLE)); for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { @@ -400,7 +400,7 @@ static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); } /* start timestamp timers */ - SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE); + SET_CAP_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE); } #define ESRAM_AUTO_TINIT BIT(2) @@ -448,21 +448,21 @@ int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) unsigned int csr; /* write to the reset csr */ - ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET); + ae_reset_csr = GET_CAP_CSR(handle, ICP_RESET); ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB); ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB); do { - SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr); + SET_CAP_CSR(handle, ICP_RESET, ae_reset_csr); if (!(times--)) goto out_err; - csr = GET_GLB_CSR(handle, ICP_RESET); + csr = GET_CAP_CSR(handle, ICP_RESET); } while ((handle->hal_handle->ae_mask | (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr); /* enable clock */ - clk_csr = GET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE); + clk_csr = GET_CAP_CSR(handle, ICP_GLOBAL_CLK_ENABLE); clk_csr |= handle->hal_handle->ae_mask << 0; clk_csr |= handle->hal_handle->slice_mask << 20; - SET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr); + SET_CAP_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr); if (qat_hal_check_ae_alive(handle)) goto out_err; From 97b9840195307a9af1c47bed69a63880a13bbb0d Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:51 +0800 Subject: [PATCH 137/360] crypto: qat - move defines to header files Move the definition of ICP_QAT_AE_OFFSET, ICP_QAT_CAP_OFFSET, LOCAL_TO_XFER_REG_OFFSET and ICP_QAT_EP_OFFSET from qat_hal.c to icp_qat_hal.h to avoid the definition of generation specific constants in qat_hal.c. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_hal.h | 4 ++++ drivers/crypto/qat/qat_common/qat_hal.c | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index c2166dacdf5b..eff9a3811435 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -85,6 +85,10 @@ enum fcu_sts { #define FCU_LOADED_AE_POS 0x16 #define FW_AUTH_WAIT_PERIOD 10 #define FW_AUTH_MAX_RETRY 300 +#define ICP_QAT_AE_OFFSET 0x20000 +#define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000) +#define LOCAL_TO_XFER_REG_OFFSET 0x800 +#define ICP_QAT_EP_OFFSET 0x3a000 #define SET_CAP_CSR(handle, csr, val) \ ADF_CSR_WR((handle)->hal_cap_g_ctl_csr_addr_v, csr, val) diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 15ebb57ea14a..5bf42f01a3de 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -646,10 +646,6 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) return 0; } -#define ICP_QAT_AE_OFFSET 0x20000 -#define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000) -#define LOCAL_TO_XFER_REG_OFFSET 0x800 -#define ICP_QAT_EP_OFFSET 0x3a000 int qat_hal_init(struct adf_accel_dev *accel_dev) { unsigned char ae; From ecb917ad0fc2bcc6c9f9e3db005c08f6e2d22f88 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:52 +0800 Subject: [PATCH 138/360] crypto: qat - refactor qat_uclo_set_ae_mode() Refactor qat_uclo_set_ae_mode() by moving the logic that sets the AE modes to a separate function, qat_hal_set_modes(). Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 69 +++++++++++++++--------- 1 file changed, 43 insertions(+), 26 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 6423b1ea7021..095e1b469412 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -869,16 +869,52 @@ static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) return 0; } +static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_uclo_objhandle *obj_handle, + unsigned char ae, + struct icp_qat_uof_image *uof_image) +{ + unsigned char mode; + int ret; + + mode = ICP_QAT_CTX_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_ctx_mode(handle, ae, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); + return ret; + } + mode = ICP_QAT_NN_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_nn_mode(handle, ae, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); + return ret; + } + mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); + return ret; + } + mode = ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); + return ret; + } + return 0; +} + static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) { - unsigned char ae, nn_mode, s; struct icp_qat_uof_image *uof_image; struct icp_qat_uclo_aedata *ae_data; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned char ae, s; + int error; for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - if (!test_bit(ae, - (unsigned long *)&handle->hal_handle->ae_mask)) + if (!test_bit(ae, &ae_mask)) continue; ae_data = &obj_handle->ae_data[ae]; for (s = 0; s < min_t(unsigned int, ae_data->slice_num, @@ -886,29 +922,10 @@ static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) continue; uof_image = ae_data->ae_slices[s].encap_image->img_ptr; - if (qat_hal_set_ae_ctx_mode(handle, ae, - (char)ICP_QAT_CTX_MODE - (uof_image->ae_mode))) { - pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); - return -EFAULT; - } - nn_mode = ICP_QAT_NN_MODE(uof_image->ae_mode); - if (qat_hal_set_ae_nn_mode(handle, ae, nn_mode)) { - pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); - return -EFAULT; - } - if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, - (char)ICP_QAT_LOC_MEM0_MODE - (uof_image->ae_mode))) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); - return -EFAULT; - } - if (qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, - (char)ICP_QAT_LOC_MEM1_MODE - (uof_image->ae_mode))) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); - return -EFAULT; - } + error = qat_hal_set_modes(handle, obj_handle, ae, + uof_image); + if (error) + return error; } } return 0; From 58c173b9cb84aa40111c971d957980e4418a1c5e Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:53 +0800 Subject: [PATCH 139/360] crypto: qat - refactor long expressions Replace long expressions with local variables in the functions qat_uclo_wr_uimage_page(), qat_uclo_init_globals() and qat_uclo_init_umem_seg() to improve readability. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 41 +++++++++++++----------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 095e1b469412..8d08dac94ea9 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -324,6 +324,7 @@ static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int ae, ustore_size, uaddr, i; + struct icp_qat_uclo_aedata *aed; ustore_size = obj_handle->ustore_phy_size; if (qat_uclo_fetch_initmem_ae(handle, init_mem, ustore_size, &ae)) @@ -333,11 +334,10 @@ static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, return -EINVAL; /* set the highest ustore address referenced */ uaddr = (init_mem->addr + init_mem->num_in_bytes) >> 0x2; - for (i = 0; i < obj_handle->ae_data[ae].slice_num; i++) { - if (obj_handle->ae_data[ae].ae_slices[i]. - encap_image->uwords_num < uaddr) - obj_handle->ae_data[ae].ae_slices[i]. - encap_image->uwords_num = uaddr; + aed = &obj_handle->ae_data[ae]; + for (i = 0; i < aed->slice_num; i++) { + if (aed->ae_slices[i].encap_image->uwords_num < uaddr) + aed->ae_slices[i].encap_image->uwords_num = uaddr; } return 0; } @@ -845,6 +845,7 @@ static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + struct icp_qat_uclo_aedata *aed; unsigned int s, ae; if (obj_handle->global_inited) @@ -855,13 +856,13 @@ static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) return -EINVAL; } } + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - for (s = 0; s < obj_handle->ae_data[ae].slice_num; s++) { - if (!obj_handle->ae_data[ae].ae_slices[s].encap_image) + aed = &obj_handle->ae_data[ae]; + for (s = 0; s < aed->slice_num; s++) { + if (!aed->ae_slices[s].encap_image) continue; - if (qat_uclo_init_reg_sym(handle, ae, - obj_handle->ae_data[ae]. - ae_slices[s].encap_image)) + if (qat_uclo_init_reg_sym(handle, ae, aed->ae_slices[s].encap_image)) return -EINVAL; } } @@ -1820,6 +1821,8 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_image *image) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_assigned = image->ae_assigned; + struct icp_qat_uclo_aedata *aed; unsigned int ctx_mask, s; struct icp_qat_uclo_page *page; unsigned char ae; @@ -1832,24 +1835,26 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, /* load the default page and set assigned CTX PC * to the entrypoint address */ for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - if (!test_bit(ae, (unsigned long *)&image->ae_assigned)) + if (!test_bit(ae, &ae_assigned)) continue; + + aed = &obj_handle->ae_data[ae]; /* find the slice to which this image is assigned */ - for (s = 0; s < obj_handle->ae_data[ae].slice_num; s++) { - if (image->ctx_assigned & obj_handle->ae_data[ae]. - ae_slices[s].ctx_mask_assigned) + for (s = 0; s < aed->slice_num; s++) { + if (image->ctx_assigned & + aed->ae_slices[s].ctx_mask_assigned) break; } - if (s >= obj_handle->ae_data[ae].slice_num) + if (s >= aed->slice_num) continue; - page = obj_handle->ae_data[ae].ae_slices[s].page; + page = aed->ae_slices[s].page; if (!page->encap_page->def_page) continue; qat_uclo_wr_uimage_raw_page(handle, page->encap_page, ae); - page = obj_handle->ae_data[ae].ae_slices[s].page; + page = aed->ae_slices[s].page; for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) - obj_handle->ae_data[ae].ae_slices[s].cur_page[ctx] = + aed->ae_slices[s].cur_page[ctx] = (ctx_mask & (1 << ctx)) ? page : NULL; qat_hal_set_live_ctx(handle, (unsigned char)ae, image->ctx_assigned); From fc5f3f86e5afb4008c6dea054fe4df302edd84df Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:54 +0800 Subject: [PATCH 140/360] crypto: qat - introduce chip info structure Introduce the chip info structure which contains device specific information. The initialization path has been split between common and hardware specific in order to facilitate the introduction of the next generation hardware. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../qat/qat_common/icp_qat_fw_loader_handle.h | 7 +- drivers/crypto/qat/qat_common/qat_hal.c | 112 +++++++++++++----- drivers/crypto/qat/qat_common/qat_uclo.c | 6 +- 3 files changed, 89 insertions(+), 36 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 7d44786a223a..6b1ad629357b 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -22,13 +22,18 @@ struct icp_qat_fw_loader_hal_handle { unsigned int max_ustore; }; +struct icp_qat_fw_loader_chip_info { + bool sram_visible; + bool fw_auth; +}; + struct icp_qat_fw_loader_handle { struct icp_qat_fw_loader_hal_handle *hal_handle; + struct icp_qat_fw_loader_chip_info *chip_info; struct pci_dev *pci_dev; void *obj_handle; void *sobj_handle; void *mobj_handle; - bool fw_auth; void __iomem *hal_sram_addr_v; void __iomem *hal_cap_g_ctl_csr_addr_v; void __iomem *hal_cap_ae_xfer_csr_addr_v; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 5bf42f01a3de..2faf8638526c 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -646,23 +646,41 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) return 0; } -int qat_hal_init(struct adf_accel_dev *accel_dev) +static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, + struct adf_accel_dev *accel_dev) { - unsigned char ae; - unsigned int max_en_ae_id = 0; - struct icp_qat_fw_loader_handle *handle; struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct adf_bar *misc_bar = &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)]; - unsigned long ae_mask = hw_data->ae_mask; - unsigned int csr_val = 0; + unsigned int max_en_ae_id = 0; struct adf_bar *sram_bar; + unsigned int csr_val = 0; + unsigned long ae_mask; + unsigned char ae = 0; + int ret = 0; - handle = kzalloc(sizeof(*handle), GFP_KERNEL); - if (!handle) - return -ENOMEM; + handle->pci_dev = pci_info->pci_dev; + switch (handle->pci_dev->device) { + case PCI_DEVICE_ID_INTEL_QAT_C62X: + case PCI_DEVICE_ID_INTEL_QAT_C3XXX: + handle->chip_info->sram_visible = false; + handle->chip_info->fw_auth = true; + break; + case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: + handle->chip_info->sram_visible = true; + handle->chip_info->fw_auth = false; + break; + default: + ret = -EINVAL; + goto out_err; + } + if (handle->chip_info->sram_visible) { + sram_bar = + &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; + handle->hal_sram_addr_v = sram_bar->virt_addr; + } handle->hal_cap_g_ctl_csr_addr_v = (void __iomem *)((uintptr_t)misc_bar->virt_addr + ICP_QAT_CAP_OFFSET); @@ -676,22 +694,14 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET); handle->pci_dev = pci_info->pci_dev; - if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_DH895XCC) { - sram_bar = - &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; - handle->hal_sram_addr_v = sram_bar->virt_addr; - } - handle->fw_auth = (handle->pci_dev->device == - PCI_DEVICE_ID_INTEL_QAT_DH895XCC) ? false : true; - handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); - if (!handle->hal_handle) - goto out_hal_handle; handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; handle->hal_handle->ae_mask = hw_data->ae_mask; handle->hal_handle->slice_mask = hw_data->accel_mask; /* create AE objects */ handle->hal_handle->upc_mask = 0x1ffff; handle->hal_handle->max_ustore = 0x4000; + + ae_mask = handle->hal_handle->ae_mask; for_each_set_bit(ae, &ae_mask, ICP_QAT_UCLO_MAX_AE) { handle->hal_handle->aes[ae].free_addr = 0; handle->hal_handle->aes[ae].free_size = @@ -703,16 +713,6 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) max_en_ae_id = ae; } handle->hal_handle->ae_max_num = max_en_ae_id + 1; - /* take all AEs out of reset */ - if (qat_hal_clr_reset(handle)) { - dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n"); - goto out_err; - } - qat_hal_clear_xfer(handle); - if (!handle->fw_auth) { - if (qat_hal_clear_gpr(handle)) - goto out_err; - } /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { @@ -720,20 +720,68 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) csr_val |= 0x1; qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); } +out_err: + return ret; +} + +int qat_hal_init(struct adf_accel_dev *accel_dev) +{ + struct icp_qat_fw_loader_handle *handle; + int ret = 0; + + handle = kzalloc(sizeof(*handle), GFP_KERNEL); + if (!handle) + return -ENOMEM; + + handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); + if (!handle->hal_handle) { + ret = -ENOMEM; + goto out_hal_handle; + } + + handle->chip_info = kzalloc(sizeof(*handle->chip_info), GFP_KERNEL); + if (!handle->chip_info) { + ret = -ENOMEM; + goto out_chip_info; + } + + ret = qat_hal_chip_init(handle, accel_dev); + if (ret) { + dev_err(&GET_DEV(accel_dev), "qat_hal_chip_init error\n"); + goto out_err; + } + + /* take all AEs out of reset */ + ret = qat_hal_clr_reset(handle); + if (ret) { + dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n"); + goto out_err; + } + + qat_hal_clear_xfer(handle); + if (!handle->chip_info->fw_auth) { + ret = qat_hal_clear_gpr(handle); + if (ret) + goto out_err; + } + accel_dev->fw_loader->fw_loader = handle; return 0; out_err: + kfree(handle->chip_info); +out_chip_info: kfree(handle->hal_handle); out_hal_handle: kfree(handle); - return -EFAULT; + return ret; } void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) { if (!handle) return; + kfree(handle->chip_info); kfree(handle->hal_handle); kfree(handle); } @@ -746,7 +794,7 @@ int qat_hal_start(struct icp_qat_fw_loader_handle *handle) u32 ae_ctr = 0; int retry = 0; - if (handle->fw_auth) { + if (handle->chip_info->fw_auth) { ae_ctr = hweight32(ae_mask); SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_START); do { @@ -770,7 +818,7 @@ int qat_hal_start(struct icp_qat_fw_loader_handle *handle) void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned int ctx_mask) { - if (!handle->fw_auth) + if (!handle->chip_info->fw_auth) qat_hal_disable_ctx(handle, ae, ctx_mask); } diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 8d08dac94ea9..1533981baf3a 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1405,7 +1405,7 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc = NULL; int status = 0; - if (handle->fw_auth) { + if (handle->chip_info->fw_auth) { if (!qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc)) status = qat_uclo_auth_fw(handle, desc); qat_uclo_ummap_auth_fw(handle, &desc); @@ -1718,7 +1718,7 @@ int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, obj_size = mem_size; } - return (handle->fw_auth) ? + return (handle->chip_info->fw_auth) ? qat_uclo_map_suof_obj(handle, obj_addr, obj_size) : qat_uclo_map_uof_obj(handle, obj_addr, obj_size); } @@ -1909,6 +1909,6 @@ static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle) int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) { - return (handle->fw_auth) ? qat_uclo_wr_suof_img(handle) : + return (handle->chip_info->fw_auth) ? qat_uclo_wr_suof_img(handle) : qat_uclo_wr_uof_img(handle); } From 49c1327328ad2245bf06bffc2591d3eb99b51c60 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:55 +0800 Subject: [PATCH 141/360] crypto: qat - replace check based on DID Modify condition in qat_uclo_wr_mimage() to use a capability of the device (sram_visible), rather than the device ID, so the check is not specific to devices of the same type. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 1533981baf3a..5774916497bd 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1410,9 +1410,11 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, status = qat_uclo_auth_fw(handle, desc); qat_uclo_ummap_auth_fw(handle, &desc); } else { - if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_C3XXX) { - pr_err("QAT: C3XXX doesn't support unsigned MMP\n"); - return -EINVAL; + if (!handle->chip_info->sram_visible) { + dev_dbg(&handle->pci_dev->dev, + "QAT MMP fw not loaded for device 0x%x", + handle->pci_dev->device); + return status; } qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size); } From d25cf2c7a057d6c01e44d4696fde1d05bfec11bb Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:56 +0800 Subject: [PATCH 142/360] crypto: qat - add next neighbor to chip_info Introduce the next neighbor (NN) capability in chip_info as NN registers are not supported in certain SKUs of QAT. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/qat_hal.c | 11 ++++++++++- drivers/crypto/qat/qat_common/qat_uclo.c | 12 +++++++----- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 6b1ad629357b..8025be597d18 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -24,6 +24,7 @@ struct icp_qat_fw_loader_hal_handle { struct icp_qat_fw_loader_chip_info { bool sram_visible; + bool nn; bool fw_auth; }; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 2faf8638526c..e0d0ab43fd12 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -603,7 +603,9 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); csr_val = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); csr_val &= IGNORE_W1C_MASK; - csr_val |= CE_NN_MODE; + if (handle->chip_info->nn) + csr_val |= CE_NN_MODE; + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val); qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst), (u64 *)inst); @@ -665,10 +667,12 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, case PCI_DEVICE_ID_INTEL_QAT_C62X: case PCI_DEVICE_ID_INTEL_QAT_C3XXX: handle->chip_info->sram_visible = false; + handle->chip_info->nn = true; handle->chip_info->fw_auth = true; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: handle->chip_info->sram_visible = true; + handle->chip_info->nn = true; handle->chip_info->fw_auth = false; break; default: @@ -1433,6 +1437,11 @@ int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, { int stat = 0; unsigned char ctx; + if (!handle->chip_info->nn) { + dev_err(&handle->pci_dev->dev, "QAT: No next neigh in 0x%x\n", + handle->pci_dev->device); + return -EINVAL; + } if (ctx_mask == 0) return -EINVAL; diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 5774916497bd..fce075874962 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -884,11 +884,13 @@ static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); return ret; } - mode = ICP_QAT_NN_MODE(uof_image->ae_mode); - ret = qat_hal_set_ae_nn_mode(handle, ae, mode); - if (ret) { - pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); - return ret; + if (handle->chip_info->nn) { + mode = ICP_QAT_NN_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_nn_mode(handle, ae, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); + return ret; + } } mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode); From 8b487ae26ad58abd602ab2d0cda6988306a6da20 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:57 +0800 Subject: [PATCH 143/360] crypto: qat - add support for lm2 and lm3 Add support for local memory lm2 and lm3 which is introduced in the next generation of QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_common_drv.h | 2 + .../qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/icp_qat_hal.h | 9 +++ drivers/crypto/qat/qat_common/icp_qat_uclo.h | 5 ++ drivers/crypto/qat/qat_common/qat_hal.c | 70 ++++++++++++++++++- drivers/crypto/qat/qat_common/qat_uclo.c | 16 +++++ 6 files changed, 101 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 945608b71937..f4c90c701670 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -178,6 +178,8 @@ int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle, unsigned short reg_num, unsigned int regdata); int qat_hal_wr_lm(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned short lm_addr, unsigned int value); +void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode); int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle); void qat_uclo_del_obj(struct icp_qat_fw_loader_handle *handle); int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 8025be597d18..3c587105d09d 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -25,6 +25,7 @@ struct icp_qat_fw_loader_hal_handle { struct icp_qat_fw_loader_chip_info { bool sram_visible; bool nn; + bool lm2lm3; bool fw_auth; }; diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index eff9a3811435..82ac33a4500f 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -26,8 +26,14 @@ enum hal_ae_csr { CTX_WAKEUP_EVENTS_INDIRECT = 0x050, LM_ADDR_0_INDIRECT = 0x060, LM_ADDR_1_INDIRECT = 0x068, + LM_ADDR_2_INDIRECT = 0x0cc, + LM_ADDR_3_INDIRECT = 0x0d4, INDIRECT_LM_ADDR_0_BYTE_INDEX = 0x0e0, INDIRECT_LM_ADDR_1_BYTE_INDEX = 0x0e8, + INDIRECT_LM_ADDR_2_BYTE_INDEX = 0x10c, + INDIRECT_LM_ADDR_3_BYTE_INDEX = 0x114, + INDIRECT_T_INDEX = 0x0f8, + INDIRECT_T_INDEX_BYTE_INDEX = 0x0fc, FUTURE_COUNT_SIGNAL_INDIRECT = 0x078, TIMESTAMP_LOW = 0x0c0, TIMESTAMP_HIGH = 0x0c4, @@ -68,6 +74,9 @@ enum fcu_sts { #define CE_ENABLE_BITPOS 0x8 #define CE_LMADDR_0_GLOBAL_BITPOS 16 #define CE_LMADDR_1_GLOBAL_BITPOS 17 +#define CE_LMADDR_2_GLOBAL_BITPOS 22 +#define CE_LMADDR_3_GLOBAL_BITPOS 23 +#define CE_T_INDEX_GLOBAL_BITPOS 21 #define CE_NN_MODE_BITPOS 20 #define CE_REG_PAR_ERR_BITPOS 25 #define CE_BREAKPOINT_BITPOS 27 diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index 101de1430896..5728a81d9dea 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -69,6 +69,9 @@ #define ICP_QAT_LOC_MEM0_MODE(ae_mode) (((ae_mode) >> 0x8) & 0x1) #define ICP_QAT_LOC_MEM1_MODE(ae_mode) (((ae_mode) >> 0x9) & 0x1) +#define ICP_QAT_LOC_MEM2_MODE(ae_mode) (((ae_mode) >> 0x6) & 0x1) +#define ICP_QAT_LOC_MEM3_MODE(ae_mode) (((ae_mode) >> 0x7) & 0x1) +#define ICP_QAT_LOC_TINDEX_MODE(ae_mode) (((ae_mode) >> 0xe) & 0x1) enum icp_qat_uof_mem_region { ICP_QAT_UOF_SRAM_REGION = 0x0, @@ -98,6 +101,8 @@ enum icp_qat_uof_regtype { ICP_LMEM0 = 27, ICP_LMEM1 = 28, ICP_NEIGH_REL = 31, + ICP_LMEM2 = 61, + ICP_LMEM3 = 62, }; enum icp_qat_css_fwtype { diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index e0d0ab43fd12..70fc93f31e79 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -210,6 +210,16 @@ int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) : CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS); break; + case ICP_LMEM2: + new_csr = (mode) ? + SET_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_LMADDR_2_GLOBAL_BITPOS); + break; + case ICP_LMEM3: + new_csr = (mode) ? + SET_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_LMADDR_3_GLOBAL_BITPOS); + break; default: pr_err("QAT: lmType = 0x%x\n", lm_type); return -EINVAL; @@ -220,6 +230,20 @@ int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, return 0; } +void qat_hal_set_ae_tindex_mode(struct icp_qat_fw_loader_handle *handle, + unsigned char ae, unsigned char mode) +{ + unsigned int csr, new_csr; + + csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); + csr &= IGNORE_W1C_MASK; + new_csr = (mode) ? + SET_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS) : + CLR_BIT(csr, CE_T_INDEX_GLOBAL_BITPOS); + if (new_csr != csr) + qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr); +} + static unsigned short qat_hal_get_reg_addr(unsigned int type, unsigned short reg_num) { @@ -259,6 +283,12 @@ static unsigned short qat_hal_get_reg_addr(unsigned int type, case ICP_LMEM1: reg_addr = 0x220; break; + case ICP_LMEM2: + reg_addr = 0x2c0; + break; + case ICP_LMEM3: + reg_addr = 0x2e0; + break; case ICP_NO_DEST: reg_addr = 0x300 | (reg_num & 0xff); break; @@ -668,11 +698,13 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, case PCI_DEVICE_ID_INTEL_QAT_C3XXX: handle->chip_info->sram_visible = false; handle->chip_info->nn = true; + handle->chip_info->lm2lm3 = false; handle->chip_info->fw_auth = true; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: handle->chip_info->sram_visible = true; handle->chip_info->nn = true; + handle->chip_info->lm2lm3 = false; handle->chip_info->fw_auth = false; break; default: @@ -889,9 +921,12 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, int code_off, unsigned int max_cycle, unsigned int *endpc) { + unsigned int ind_lm_addr_byte0 = 0, ind_lm_addr_byte1 = 0; + unsigned int ind_lm_addr_byte2 = 0, ind_lm_addr_byte3 = 0; + unsigned int ind_t_index = 0, ind_t_index_byte = 0; + unsigned int ind_lm_addr0 = 0, ind_lm_addr1 = 0; + unsigned int ind_lm_addr2 = 0, ind_lm_addr3 = 0; u64 savuwords[MAX_EXEC_INST]; - unsigned int ind_lm_addr0, ind_lm_addr1; - unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1; unsigned int ind_cnt_sig; unsigned int ind_sig, act_sig; unsigned int csr_val = 0, newcsr_val; @@ -910,6 +945,20 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, INDIRECT_LM_ADDR_0_BYTE_INDEX); ind_lm_addr_byte1 = qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX); + if (handle->chip_info->lm2lm3) { + ind_lm_addr2 = qat_hal_rd_indr_csr(handle, ae, ctx, + LM_ADDR_2_INDIRECT); + ind_lm_addr3 = qat_hal_rd_indr_csr(handle, ae, ctx, + LM_ADDR_3_INDIRECT); + ind_lm_addr_byte2 = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_LM_ADDR_2_BYTE_INDEX); + ind_lm_addr_byte3 = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_LM_ADDR_3_BYTE_INDEX); + ind_t_index = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_T_INDEX); + ind_t_index_byte = qat_hal_rd_indr_csr(handle, ae, ctx, + INDIRECT_T_INDEX_BYTE_INDEX); + } if (inst_num <= MAX_EXEC_INST) qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords); qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events); @@ -967,6 +1016,23 @@ static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle, INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0); qat_hal_wr_indr_csr(handle, ae, (1 << ctx), INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1); + if (handle->chip_info->lm2lm3) { + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_2_INDIRECT, + ind_lm_addr2); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), LM_ADDR_3_INDIRECT, + ind_lm_addr3); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_LM_ADDR_2_BYTE_INDEX, + ind_lm_addr_byte2); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_LM_ADDR_3_BYTE_INDEX, + ind_lm_addr_byte3); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_T_INDEX, ind_t_index); + qat_hal_wr_indr_csr(handle, ae, BIT(ctx), + INDIRECT_T_INDEX_BYTE_INDEX, + ind_t_index_byte); + } qat_hal_wr_indr_csr(handle, ae, (1 << ctx), FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig); qat_hal_wr_indr_csr(handle, ae, (1 << ctx), diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index fce075874962..4a90b150199c 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -904,6 +904,22 @@ static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); return ret; } + if (handle->chip_info->lm2lm3) { + mode = ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n"); + return ret; + } + mode = ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode); + ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, mode); + if (ret) { + pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n"); + return ret; + } + mode = ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode); + qat_hal_set_ae_tindex_mode(handle, ae, mode); + } return 0; } From 4f07195d638e82a1a1f17f55d7aa8a74dacabfe9 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:58 +0800 Subject: [PATCH 144/360] crypto: qat - add local memory size to chip info Add the local memory size to the chip info since the size of this memory will be different in the next generation of QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/qat_hal.c | 2 ++ drivers/crypto/qat/qat_common/qat_uclo.c | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 3c587105d09d..0fa5c22fd9c0 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -26,6 +26,7 @@ struct icp_qat_fw_loader_chip_info { bool sram_visible; bool nn; bool lm2lm3; + u32 lm_size; bool fw_auth; }; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 70fc93f31e79..44cf797ace71 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -699,12 +699,14 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->sram_visible = false; handle->chip_info->nn = true; handle->chip_info->lm2lm3 = false; + handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->fw_auth = true; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: handle->chip_info->sram_visible = true; handle->chip_info->nn = true; handle->chip_info->lm2lm3 = false; + handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->fw_auth = false; break; default: diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 4a90b150199c..32c64a48926f 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -311,7 +311,7 @@ static int qat_uclo_init_lmem_seg(struct icp_qat_fw_loader_handle *handle, unsigned int ae; if (qat_uclo_fetch_initmem_ae(handle, init_mem, - ICP_QAT_UCLO_MAX_LMEM_REG, &ae)) + handle->chip_info->lm_size, &ae)) return -EINVAL; if (qat_uclo_create_batch_init_list(handle, init_mem, ae, &obj_handle->lm_init_tab[ae])) From cb439361a39bb8dcdd0856e9182e29678c8327af Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:27:59 +0800 Subject: [PATCH 145/360] crypto: qat - add reset CSR and mask to chip info Add reset CSR offset and mask to chip info since they are different in new QAT devices. This also simplifies the reset/clrReset functions by using the reset mask. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../qat/qat_common/icp_qat_fw_loader_handle.h | 2 + drivers/crypto/qat/qat_common/qat_hal.c | 39 +++++++++++-------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 0fa5c22fd9c0..5e2c0ef6d26c 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -27,6 +27,8 @@ struct icp_qat_fw_loader_chip_info { bool nn; bool lm2lm3; u32 lm_size; + u32 icp_rst_csr; + u32 icp_rst_mask; bool fw_auth; }; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 44cf797ace71..0d64e074fb44 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -301,12 +301,13 @@ static unsigned short qat_hal_get_reg_addr(unsigned int type, void qat_hal_reset(struct icp_qat_fw_loader_handle *handle) { - unsigned int ae_reset_csr; + unsigned int reset_mask = handle->chip_info->icp_rst_mask; + unsigned int reset_csr = handle->chip_info->icp_rst_csr; + unsigned int csr_val; - ae_reset_csr = GET_CAP_CSR(handle, ICP_RESET); - ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB; - ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB; - SET_CAP_CSR(handle, ICP_RESET, ae_reset_csr); + csr_val = GET_CAP_CSR(handle, reset_csr); + csr_val |= reset_mask; + SET_CAP_CSR(handle, reset_csr, csr_val); } static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle, @@ -470,28 +471,27 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) #define SHRAM_INIT_CYCLES 2060 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) { + unsigned int reset_mask = handle->chip_info->icp_rst_mask; + unsigned int reset_csr = handle->chip_info->icp_rst_csr; unsigned long ae_mask = handle->hal_handle->ae_mask; - unsigned int ae_reset_csr; - unsigned char ae; + unsigned char ae = 0; unsigned int clk_csr; unsigned int times = 100; - unsigned int csr; + unsigned int csr_val; /* write to the reset csr */ - ae_reset_csr = GET_CAP_CSR(handle, ICP_RESET); - ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB); - ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB); + csr_val = GET_CAP_CSR(handle, reset_csr); + csr_val &= ~reset_mask; do { - SET_CAP_CSR(handle, ICP_RESET, ae_reset_csr); + SET_CAP_CSR(handle, reset_csr, csr_val); if (!(times--)) goto out_err; - csr = GET_CAP_CSR(handle, ICP_RESET); - } while ((handle->hal_handle->ae_mask | - (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr); + csr_val = GET_CAP_CSR(handle, reset_csr); + csr_val &= reset_mask; + } while (csr_val); /* enable clock */ clk_csr = GET_CAP_CSR(handle, ICP_GLOBAL_CLK_ENABLE); - clk_csr |= handle->hal_handle->ae_mask << 0; - clk_csr |= handle->hal_handle->slice_mask << 20; + clk_csr |= reset_mask; SET_CAP_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr); if (qat_hal_check_ae_alive(handle)) goto out_err; @@ -700,6 +700,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->nn = true; handle->chip_info->lm2lm3 = false; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; + handle->chip_info->icp_rst_csr = ICP_RESET; handle->chip_info->fw_auth = true; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: @@ -707,6 +708,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->nn = true; handle->chip_info->lm2lm3 = false; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; + handle->chip_info->icp_rst_csr = ICP_RESET; handle->chip_info->fw_auth = false; break; default: @@ -719,6 +721,9 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; handle->hal_sram_addr_v = sram_bar->virt_addr; } + + handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | + (hw_data->accel_mask << RST_CSR_QAT_LSB); handle->hal_cap_g_ctl_csr_addr_v = (void __iomem *)((uintptr_t)misc_bar->virt_addr + ICP_QAT_CAP_OFFSET); From 767358119fca655056eb7340eb8a0ec4652bc888 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:00 +0800 Subject: [PATCH 146/360] crypto: qat - add clock enable CSR to chip info Add global clock enable CSR to the chip info since the CSR offset will be different in the next generation of QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/qat_hal.c | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 5e2c0ef6d26c..1d6ab3407dc9 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -29,6 +29,7 @@ struct icp_qat_fw_loader_chip_info { u32 lm_size; u32 icp_rst_csr; u32 icp_rst_mask; + u32 glb_clk_enable_csr; bool fw_auth; }; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 0d64e074fb44..6e6bca281ab7 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -471,11 +471,11 @@ static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) #define SHRAM_INIT_CYCLES 2060 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) { + unsigned int clk_csr = handle->chip_info->glb_clk_enable_csr; unsigned int reset_mask = handle->chip_info->icp_rst_mask; unsigned int reset_csr = handle->chip_info->icp_rst_csr; unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned char ae = 0; - unsigned int clk_csr; unsigned int times = 100; unsigned int csr_val; @@ -490,9 +490,9 @@ int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) csr_val &= reset_mask; } while (csr_val); /* enable clock */ - clk_csr = GET_CAP_CSR(handle, ICP_GLOBAL_CLK_ENABLE); - clk_csr |= reset_mask; - SET_CAP_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr); + csr_val = GET_CAP_CSR(handle, clk_csr); + csr_val |= reset_mask; + SET_CAP_CSR(handle, clk_csr, csr_val); if (qat_hal_check_ae_alive(handle)) goto out_err; @@ -701,6 +701,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm2lm3 = false; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; + handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; handle->chip_info->fw_auth = true; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: @@ -709,6 +710,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm2lm3 = false; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; + handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; handle->chip_info->fw_auth = false; break; default: From c4909d327cc3b001583de29fde988a22856a5b38 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:01 +0800 Subject: [PATCH 147/360] crypto: qat - add wake up event to chip info Add the wake up event to chip info since this value will be different in the next generation of QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/icp_qat_hal.h | 1 + drivers/crypto/qat/qat_common/qat_hal.c | 5 ++++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 1d6ab3407dc9..090c3e73938c 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -30,6 +30,7 @@ struct icp_qat_fw_loader_chip_info { u32 icp_rst_csr; u32 icp_rst_mask; u32 glb_clk_enable_csr; + u32 wakeup_event_val; bool fw_auth; }; diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index 82ac33a4500f..b3aa4c8a3ba8 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -87,6 +87,7 @@ enum fcu_sts { #define XCWE_VOLUNTARY (0x1) #define LCS_STATUS (0x1) #define MMC_SHARE_CS_BITPOS 2 +#define WAKEUP_EVENT 0x10000 #define FCU_CTRL_AE_POS 0x8 #define FCU_AUTH_STS_MASK 0x7 #define FCU_STS_DONE_POS 0x9 diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 6e6bca281ab7..c073e4e3e3ae 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -702,6 +702,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; + handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = true; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: @@ -711,6 +712,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; + handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = false; break; default: @@ -834,6 +836,7 @@ void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) int qat_hal_start(struct icp_qat_fw_loader_handle *handle) { unsigned long ae_mask = handle->hal_handle->ae_mask; + u32 wakeup_val = handle->chip_info->wakeup_event_val; unsigned int fcu_sts; unsigned char ae; u32 ae_ctr = 0; @@ -852,7 +855,7 @@ int qat_hal_start(struct icp_qat_fw_loader_handle *handle) return 0; } else { for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { - qat_hal_put_wakeup_event(handle, ae, 0, 0x10000); + qat_hal_put_wakeup_event(handle, ae, 0, wakeup_val); qat_hal_enable_ctx(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX); ae_ctr++; } From 4f1e941560e1c29299c3b1b9f90f4aea790d12dc Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:02 +0800 Subject: [PATCH 148/360] crypto: qat - add misc control CSR to chip info Add misc control CSR to chip info since the CSR offset will be different in the next generation of QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/qat_hal.c | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 090c3e73938c..81dba42248bf 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -30,6 +30,7 @@ struct icp_qat_fw_loader_chip_info { u32 icp_rst_csr; u32 icp_rst_mask; u32 glb_clk_enable_csr; + u32 misc_ctl_csr; u32 wakeup_event_val; bool fw_auth; }; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index c073e4e3e3ae..eae1a5e0efeb 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -417,13 +417,14 @@ int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) { unsigned long ae_mask = handle->hal_handle->ae_mask; - unsigned int misc_ctl; + unsigned int misc_ctl_csr, misc_ctl; unsigned char ae; + misc_ctl_csr = handle->chip_info->misc_ctl_csr; /* stop the timestamp timers */ - misc_ctl = GET_CAP_CSR(handle, MISC_CONTROL); + misc_ctl = GET_CAP_CSR(handle, misc_ctl_csr); if (misc_ctl & MC_TIMESTAMP_ENABLE) - SET_CAP_CSR(handle, MISC_CONTROL, misc_ctl & + SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl & (~MC_TIMESTAMP_ENABLE)); for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { @@ -431,7 +432,7 @@ static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); } /* start timestamp timers */ - SET_CAP_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE); + SET_CAP_CSR(handle, misc_ctl_csr, misc_ctl | MC_TIMESTAMP_ENABLE); } #define ESRAM_AUTO_TINIT BIT(2) @@ -702,6 +703,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; + handle->chip_info->misc_ctl_csr = MISC_CONTROL; handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = true; break; @@ -712,6 +714,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; + handle->chip_info->misc_ctl_csr = MISC_CONTROL; handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = false; break; From 912eebeb72b499de3b25cda9e2b9e3f5e91bed12 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:03 +0800 Subject: [PATCH 149/360] crypto: qat - add check for null pointer Add null pointer check when freeing the memory for firmware. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 28 ++++++++++++++---------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 32c64a48926f..7b02c4e165c6 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1233,11 +1233,15 @@ static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle, struct icp_firml_dram_desc *dram_desc) { - dma_free_coherent(&handle->pci_dev->dev, - (size_t)(dram_desc->dram_size), - dram_desc->dram_base_addr_v, - dram_desc->dram_bus_addr); - memset(dram_desc, 0, sizeof(*dram_desc)); + if (handle && dram_desc && dram_desc->dram_base_addr_v) { + dma_free_coherent(&handle->pci_dev->dev, + (size_t)(dram_desc->dram_size), + dram_desc->dram_base_addr_v, + dram_desc->dram_bus_addr); + } + + if (dram_desc) + memset(dram_desc, 0, sizeof(*dram_desc)); } static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, @@ -1245,12 +1249,14 @@ static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, { struct icp_firml_dram_desc dram_desc; - dram_desc.dram_base_addr_v = *desc; - dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *) - (*desc))->chunk_bus_addr; - dram_desc.dram_size = ((struct icp_qat_auth_chunk *) - (*desc))->chunk_size; - qat_uclo_simg_free(handle, &dram_desc); + if (*desc) { + dram_desc.dram_base_addr_v = *desc; + dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *) + (*desc))->chunk_bus_addr; + dram_desc.dram_size = ((struct icp_qat_auth_chunk *) + (*desc))->chunk_size; + qat_uclo_simg_free(handle, &dram_desc); + } } static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, From bf8313c71c887c3c8676c23a20b6f8eb1d56bd4f Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:04 +0800 Subject: [PATCH 150/360] crypto: qat - use ae_mask Use ae_mask to decide which Accelerator Engine (AE) to target in AE related operations, instead of a sequential loop, to skip AEs that are fused out. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_uclo.c | 26 +++++++++++++----------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 7b02c4e165c6..0b1cf0708e2e 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -373,6 +373,7 @@ static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, unsigned int ustore_size; unsigned int patt_pos; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; u64 *fill_data; uof_image = image->img_ptr; @@ -385,7 +386,7 @@ static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, sizeof(u64)); page = image->page; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (!test_bit(ae, (unsigned long *)&uof_image->ae_assigned)) continue; ustore_size = obj_handle->ae_data[ae].eff_ustore_size; @@ -406,6 +407,7 @@ static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) int i, ae; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; struct icp_qat_uof_initmem *initmem = obj_handle->init_mem_tab.init_mem; + unsigned long ae_mask = handle->hal_handle->ae_mask; for (i = 0; i < obj_handle->init_mem_tab.entry_num; i++) { if (initmem->num_in_bytes) { @@ -418,7 +420,8 @@ static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) (sizeof(struct icp_qat_uof_memvar_attr) * initmem->val_attr_num)); } - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (qat_hal_batch_wr_lm(handle, ae, obj_handle->lm_init_tab[ae])) { pr_err("QAT: fail to batch init lmem for AE %d\n", ae); @@ -649,11 +652,9 @@ static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) int i, ae; int mflag = 0; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; - for (ae = 0; ae < max_ae; ae++) { - if (!test_bit(ae, - (unsigned long *)&handle->hal_handle->ae_mask)) - continue; + for_each_set_bit(ae, &ae_mask, max_ae) { for (i = 0; i < obj_handle->uimage_num; i++) { if (!test_bit(ae, (unsigned long *) &obj_handle->ae_uimage[i].img_ptr->ae_assigned)) @@ -845,6 +846,7 @@ static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; struct icp_qat_uclo_aedata *aed; unsigned int s, ae; @@ -857,7 +859,7 @@ static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) } } - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { aed = &obj_handle->ae_data[ae]; for (s = 0; s < aed->slice_num; s++) { if (!aed->ae_slices[s].encap_image) @@ -932,9 +934,7 @@ static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) unsigned char ae, s; int error; - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - if (!test_bit(ae, &ae_mask)) - continue; + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { ae_data = &obj_handle->ae_data[ae]; for (s = 0; s < min_t(unsigned int, ae_data->slice_num, ICP_QAT_UCLO_MAX_CTX); s++) { @@ -1372,13 +1372,14 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, unsigned int fcu_sts; struct icp_qat_simg_ae_mode *virt_addr; unsigned int fcu_loaded_ae_pos = FCU_LOADED_AE_POS; + unsigned long ae_mask = handle->hal_handle->ae_mask; virt_addr = (void *)((uintptr_t)desc + sizeof(struct icp_qat_auth_chunk) + sizeof(struct icp_qat_css_hdr) + ICP_QAT_CSS_FWSK_PUB_LEN + ICP_QAT_CSS_SIGNATURE_LEN); - for (i = 0; i < handle->hal_handle->ae_max_num; i++) { + for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) { int retry = 0; if (!((virt_addr->ae_mask >> i) & 0x1)) @@ -1847,6 +1848,7 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_image *image) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; + unsigned long ae_mask = handle->hal_handle->ae_mask; unsigned long ae_assigned = image->ae_assigned; struct icp_qat_uclo_aedata *aed; unsigned int ctx_mask, s; @@ -1860,7 +1862,7 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, ctx_mask = 0x55; /* load the default page and set assigned CTX PC * to the entrypoint address */ - for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (!test_bit(ae, &ae_assigned)) continue; From 9e0f74b717e41811921e647e7a6121cc55987f41 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:05 +0800 Subject: [PATCH 151/360] crypto: qat - add CSS3K support Add support for CSS3K, which uses RSA3K as image signature algorithm, to support the next generation of QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/icp_qat_uclo.h | 54 +++++++++++++------ drivers/crypto/qat/qat_common/qat_hal.c | 2 + drivers/crypto/qat/qat_common/qat_uclo.c | 51 +++++++++--------- 4 files changed, 68 insertions(+), 40 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 81dba42248bf..29710e88e8b8 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -33,6 +33,7 @@ struct icp_qat_fw_loader_chip_info { u32 misc_ctl_csr; u32 wakeup_event_val; bool fw_auth; + bool css_3k; }; struct icp_qat_fw_loader_handle { diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index 5728a81d9dea..4315b4504c26 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -42,24 +42,48 @@ #define ICP_QAT_SUOF_IMAG "SUF_IMAG" #define ICP_QAT_SIMG_AE_INIT_SEQ_LEN (50 * sizeof(unsigned long long)) #define ICP_QAT_SIMG_AE_INSTS_LEN (0x4000 * sizeof(unsigned long long)) -#define ICP_QAT_CSS_FWSK_MODULUS_LEN 256 -#define ICP_QAT_CSS_FWSK_EXPONENT_LEN 4 -#define ICP_QAT_CSS_FWSK_PAD_LEN 252 -#define ICP_QAT_CSS_FWSK_PUB_LEN (ICP_QAT_CSS_FWSK_MODULUS_LEN + \ - ICP_QAT_CSS_FWSK_EXPONENT_LEN + \ - ICP_QAT_CSS_FWSK_PAD_LEN) -#define ICP_QAT_CSS_SIGNATURE_LEN 256 + +#define DSS_FWSK_MODULUS_LEN 384 /* RSA3K */ +#define DSS_FWSK_EXPONENT_LEN 4 +#define DSS_FWSK_PADDING_LEN 380 +#define DSS_SIGNATURE_LEN 384 /* RSA3K */ + +#define CSS_FWSK_MODULUS_LEN 256 /* RSA2K */ +#define CSS_FWSK_EXPONENT_LEN 4 +#define CSS_FWSK_PADDING_LEN 252 +#define CSS_SIGNATURE_LEN 256 /* RSA2K */ + +#define ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_FWSK_MODULUS_LEN : \ + CSS_FWSK_MODULUS_LEN) + +#define ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_FWSK_EXPONENT_LEN : \ + CSS_FWSK_EXPONENT_LEN) + +#define ICP_QAT_CSS_FWSK_PAD_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_FWSK_PADDING_LEN : \ + CSS_FWSK_PADDING_LEN) + +#define ICP_QAT_CSS_FWSK_PUB_LEN(handle) (ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + \ + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) + \ + ICP_QAT_CSS_FWSK_PAD_LEN(handle)) + +#define ICP_QAT_CSS_SIGNATURE_LEN(handle) ((handle)->chip_info->css_3k ? \ + DSS_SIGNATURE_LEN : \ + CSS_SIGNATURE_LEN) + #define ICP_QAT_CSS_AE_IMG_LEN (sizeof(struct icp_qat_simg_ae_mode) + \ ICP_QAT_SIMG_AE_INIT_SEQ_LEN + \ ICP_QAT_SIMG_AE_INSTS_LEN) -#define ICP_QAT_CSS_AE_SIMG_LEN (sizeof(struct icp_qat_css_hdr) + \ - ICP_QAT_CSS_FWSK_PUB_LEN + \ - ICP_QAT_CSS_SIGNATURE_LEN + \ - ICP_QAT_CSS_AE_IMG_LEN) -#define ICP_QAT_AE_IMG_OFFSET (sizeof(struct icp_qat_css_hdr) + \ - ICP_QAT_CSS_FWSK_MODULUS_LEN + \ - ICP_QAT_CSS_FWSK_EXPONENT_LEN + \ - ICP_QAT_CSS_SIGNATURE_LEN) +#define ICP_QAT_CSS_AE_SIMG_LEN(handle) (sizeof(struct icp_qat_css_hdr) + \ + ICP_QAT_CSS_FWSK_PUB_LEN(handle) + \ + ICP_QAT_CSS_SIGNATURE_LEN(handle) + \ + ICP_QAT_CSS_AE_IMG_LEN) +#define ICP_QAT_AE_IMG_OFFSET(handle) (sizeof(struct icp_qat_css_hdr) + \ + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + \ + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle) + \ + ICP_QAT_CSS_SIGNATURE_LEN(handle)) #define ICP_QAT_CSS_MAX_IMAGE_LEN 0x40000 #define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode) & 0xf) diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index eae1a5e0efeb..8470139bcfe8 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -706,6 +706,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->misc_ctl_csr = MISC_CONTROL; handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = true; + handle->chip_info->css_3k = false; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: handle->chip_info->sram_visible = true; @@ -717,6 +718,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->misc_ctl_csr = MISC_CONTROL; handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = false; + handle->chip_info->css_3k = false; break; default: ret = -EINVAL; diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 0b1cf0708e2e..933b6357971f 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1039,10 +1039,11 @@ static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, return 0; } -static void qat_uclo_map_simg(struct icp_qat_suof_handle *suof_handle, +static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_img_hdr *suof_img_hdr, struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) { + struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; struct icp_qat_simg_ae_mode *ae_mode; struct icp_qat_suof_objhdr *suof_objhdr; @@ -1057,10 +1058,10 @@ static void qat_uclo_map_simg(struct icp_qat_suof_handle *suof_handle, suof_img_hdr->css_key = (suof_img_hdr->css_header + sizeof(struct icp_qat_css_hdr)); suof_img_hdr->css_signature = suof_img_hdr->css_key + - ICP_QAT_CSS_FWSK_MODULUS_LEN + - ICP_QAT_CSS_FWSK_EXPONENT_LEN; + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); suof_img_hdr->css_simg = suof_img_hdr->css_signature + - ICP_QAT_CSS_SIGNATURE_LEN; + ICP_QAT_CSS_SIGNATURE_LEN(handle); ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg); suof_img_hdr->ae_mask = ae_mode->ae_mask; @@ -1169,7 +1170,7 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, } for (i = 0; i < suof_handle->img_table.num_simgs; i++) { - qat_uclo_map_simg(handle->sobj_handle, &suof_img_hdr[i], + qat_uclo_map_simg(handle, &suof_img_hdr[i], &suof_chunk_hdr[1 + i]); ret = qat_uclo_check_simg_compat(handle, &suof_img_hdr[i]); @@ -1270,13 +1271,13 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, unsigned int length, simg_offset = sizeof(*auth_chunk); struct icp_firml_dram_desc img_desc; - if (size > (ICP_QAT_AE_IMG_OFFSET + ICP_QAT_CSS_MAX_IMAGE_LEN)) { + if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_MAX_IMAGE_LEN)) { pr_err("QAT: error, input image size overflow %d\n", size); return -EINVAL; } length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ? - ICP_QAT_CSS_AE_SIMG_LEN + simg_offset : - size + ICP_QAT_CSS_FWSK_PAD_LEN + simg_offset; + ICP_QAT_CSS_AE_SIMG_LEN(handle) + simg_offset : + size + ICP_QAT_CSS_FWSK_PAD_LEN(handle) + simg_offset; if (qat_uclo_simg_alloc(handle, &img_desc, length)) { pr_err("QAT: error, allocate continuous dram fail\n"); return -ENOMEM; @@ -1303,42 +1304,42 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, memcpy((void *)(uintptr_t)virt_addr, (void *)(image + sizeof(*css_hdr)), - ICP_QAT_CSS_FWSK_MODULUS_LEN); + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); /* padding */ - memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN), - 0, ICP_QAT_CSS_FWSK_PAD_LEN); + memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), + 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); /* exponent */ - memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN + - ICP_QAT_CSS_FWSK_PAD_LEN), + memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_PAD_LEN(handle)), (void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN), + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), sizeof(unsigned int)); /* signature */ bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, auth_desc->fwsk_pub_low) + - ICP_QAT_CSS_FWSK_PUB_LEN; - virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN; + ICP_QAT_CSS_FWSK_PUB_LEN(handle); + virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle); auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->signature_low = (unsigned int)bus_addr; memcpy((void *)(uintptr_t)virt_addr, (void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN + - ICP_QAT_CSS_FWSK_EXPONENT_LEN), - ICP_QAT_CSS_SIGNATURE_LEN); + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)), + ICP_QAT_CSS_SIGNATURE_LEN(handle)); bus_addr = ADD_ADDR(auth_desc->signature_high, auth_desc->signature_low) + - ICP_QAT_CSS_SIGNATURE_LEN; - virt_addr += ICP_QAT_CSS_SIGNATURE_LEN; + ICP_QAT_CSS_SIGNATURE_LEN(handle); + virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); auth_desc->img_low = (unsigned int)bus_addr; - auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET; + auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle); memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + ICP_QAT_AE_IMG_OFFSET), + (void *)(image + ICP_QAT_AE_IMG_OFFSET(handle)), auth_desc->img_len); virt_addr = virt_base; /* AE firmware */ @@ -1377,8 +1378,8 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, virt_addr = (void *)((uintptr_t)desc + sizeof(struct icp_qat_auth_chunk) + sizeof(struct icp_qat_css_hdr) + - ICP_QAT_CSS_FWSK_PUB_LEN + - ICP_QAT_CSS_SIGNATURE_LEN); + ICP_QAT_CSS_FWSK_PUB_LEN(handle) + + ICP_QAT_CSS_SIGNATURE_LEN(handle)); for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) { int retry = 0; From d707d3f23e69181da71877b2a687560fdad81ad0 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:06 +0800 Subject: [PATCH 152/360] crypto: qat - add FCU CSRs to chip info Add firmware control unit (FCU) CSRs to chip info so the firmware authentication code is common between all devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../qat/qat_common/icp_qat_fw_loader_handle.h | 6 +++ drivers/crypto/qat/qat_common/qat_hal.c | 19 +++++++- drivers/crypto/qat/qat_common/qat_uclo.c | 44 +++++++++++++------ 3 files changed, 53 insertions(+), 16 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 29710e88e8b8..e280a077303f 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -34,6 +34,12 @@ struct icp_qat_fw_loader_chip_info { u32 wakeup_event_val; bool fw_auth; bool css_3k; + u32 fcu_ctl_csr; + u32 fcu_sts_csr; + u32 fcu_dram_addr_hi; + u32 fcu_dram_addr_lo; + u32 fcu_loaded_ae_csr; + u8 fcu_loaded_ae_pos; }; struct icp_qat_fw_loader_handle { diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 8470139bcfe8..da138fb11a63 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -707,6 +707,12 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = true; handle->chip_info->css_3k = false; + handle->chip_info->fcu_ctl_csr = FCU_CONTROL; + handle->chip_info->fcu_sts_csr = FCU_STATUS; + handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI; + handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO; + handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS; + handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS; break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: handle->chip_info->sram_visible = true; @@ -719,6 +725,12 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = false; handle->chip_info->css_3k = false; + handle->chip_info->fcu_ctl_csr = 0; + handle->chip_info->fcu_sts_csr = 0; + handle->chip_info->fcu_dram_addr_hi = 0; + handle->chip_info->fcu_dram_addr_lo = 0; + handle->chip_info->fcu_loaded_ae_csr = 0; + handle->chip_info->fcu_loaded_ae_pos = 0; break; default: ret = -EINVAL; @@ -842,17 +854,20 @@ int qat_hal_start(struct icp_qat_fw_loader_handle *handle) { unsigned long ae_mask = handle->hal_handle->ae_mask; u32 wakeup_val = handle->chip_info->wakeup_event_val; + u32 fcu_ctl_csr, fcu_sts_csr; unsigned int fcu_sts; unsigned char ae; u32 ae_ctr = 0; int retry = 0; if (handle->chip_info->fw_auth) { + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; ae_ctr = hweight32(ae_mask); - SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_START); + SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_START); do { msleep(FW_AUTH_WAIT_PERIOD); - fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1)) return ae_ctr; } while (retry++ < FW_AUTH_MAX_RETRY); diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 933b6357971f..3c5746d52756 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1190,18 +1190,26 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) { - unsigned int fcu_sts, retry = 0; + u32 fcu_sts, retry = 0; + u32 fcu_ctl_csr, fcu_sts_csr; + u32 fcu_dram_hi_csr, fcu_dram_lo_csr; u64 bus_addr; bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low) - sizeof(struct icp_qat_auth_chunk); - SET_CAP_CSR(handle, FCU_DRAM_ADDR_HI, (bus_addr >> BITS_IN_DWORD)); - SET_CAP_CSR(handle, FCU_DRAM_ADDR_LO, bus_addr); - SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_AUTH); + + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; + fcu_dram_hi_csr = handle->chip_info->fcu_dram_addr_hi; + fcu_dram_lo_csr = handle->chip_info->fcu_dram_addr_lo; + + SET_CAP_CSR(handle, fcu_dram_hi_csr, (bus_addr >> BITS_IN_DWORD)); + SET_CAP_CSR(handle, fcu_dram_lo_csr, bus_addr); + SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_AUTH); do { msleep(FW_AUTH_WAIT_PERIOD); - fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL) goto auth_fail; if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1)) @@ -1369,11 +1377,16 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) { - unsigned int i; - unsigned int fcu_sts; struct icp_qat_simg_ae_mode *virt_addr; - unsigned int fcu_loaded_ae_pos = FCU_LOADED_AE_POS; unsigned long ae_mask = handle->hal_handle->ae_mask; + u32 fcu_sts_csr, fcu_ctl_csr; + u32 loaded_aes, loaded_csr; + unsigned int i; + u32 fcu_sts; + + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; + loaded_csr = handle->chip_info->fcu_loaded_ae_csr; virt_addr = (void *)((uintptr_t)desc + sizeof(struct icp_qat_auth_chunk) + @@ -1389,16 +1402,19 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, pr_err("QAT: AE %d is active\n", i); return -EINVAL; } - SET_CAP_CSR(handle, FCU_CONTROL, + SET_CAP_CSR(handle, fcu_ctl_csr, (FCU_CTRL_CMD_LOAD | (i << FCU_CTRL_AE_POS))); do { msleep(FW_AUTH_WAIT_PERIOD); - fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); - if (((fcu_sts & FCU_AUTH_STS_MASK) == - FCU_STS_LOAD_DONE) && - ((fcu_sts >> fcu_loaded_ae_pos) & (1 << i))) - break; + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); + if ((fcu_sts & FCU_AUTH_STS_MASK) == + FCU_STS_LOAD_DONE) { + loaded_aes = GET_CAP_CSR(handle, loaded_csr); + loaded_aes >>= handle->chip_info->fcu_loaded_ae_pos; + if (loaded_aes & (1 << i)) + break; + } } while (retry++ < FW_AUTH_MAX_RETRY); if (retry > FW_AUTH_MAX_RETRY) { pr_err("QAT: firmware load failed timeout %x\n", retry); From 244f572cebec71f7a79a94706eded4d2213a4f1b Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:07 +0800 Subject: [PATCH 153/360] crypto: qat - allow to target specific AEs Introduce new API, qat_uclo_set_cfg_ae_mask(), to allow the load of the firmware image to a subset of Acceleration Engines (AEs). This is required by the next generation of QAT devices to be able to load different firmware images to the device. Signed-off-by: Jack Xu Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_common_drv.h | 2 + .../qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/icp_qat_hal.h | 2 + drivers/crypto/qat/qat_common/icp_qat_uclo.h | 2 +- drivers/crypto/qat/qat_common/qat_hal.c | 1 + drivers/crypto/qat/qat_common/qat_uclo.c | 42 +++++++++++++++---- 6 files changed, 42 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index f4c90c701670..c61476553728 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -186,6 +186,8 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, int mem_size); int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, u32 mem_size, char *obj_name); +int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, + unsigned int cfg_ae_mask); #if defined(CONFIG_PCI_IOV) int adf_sriov_configure(struct pci_dev *pdev, int numvfs); void adf_disable_sriov(struct adf_accel_dev *accel_dev); diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index e280a077303f..cc9b83d965af 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -49,6 +49,7 @@ struct icp_qat_fw_loader_handle { void *obj_handle; void *sobj_handle; void *mobj_handle; + unsigned int cfg_ae_mask; void __iomem *hal_sram_addr_v; void __iomem *hal_cap_g_ctl_csr_addr_v; void __iomem *hal_cap_ae_xfer_csr_addr_v; diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index b3aa4c8a3ba8..02476b2ceee1 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -68,6 +68,8 @@ enum fcu_sts { FCU_STS_LOAD_FAIL = 4, FCU_STS_BUSY = 5 }; + +#define ALL_AE_MASK 0xFFFFFFFF #define UA_ECS (0x1 << 31) #define ACS_ABO_BITPOS 31 #define ACS_ACNO 0x7 diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index 4315b4504c26..0ec8a5ab51b5 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -432,7 +432,7 @@ struct icp_qat_suof_handle { struct icp_qat_fw_auth_desc { unsigned int img_len; - unsigned int reserved; + unsigned int ae_mask; unsigned int css_hdr_high; unsigned int css_hdr_low; unsigned int img_high; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index da138fb11a63..94c0b04088b5 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -761,6 +761,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; handle->hal_handle->ae_mask = hw_data->ae_mask; handle->hal_handle->slice_mask = hw_data->accel_mask; + handle->cfg_ae_mask = ALL_AE_MASK; /* create AE objects */ handle->hal_handle->upc_mask = 0x1ffff; handle->hal_handle->max_ustore = 0x4000; diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 3c5746d52756..c6b309d107f3 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -374,6 +374,7 @@ static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, unsigned int patt_pos; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; u64 *fill_data; uof_image = image->img_ptr; @@ -389,6 +390,10 @@ static int qat_uclo_init_ustore(struct icp_qat_fw_loader_handle *handle, for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (!test_bit(ae, (unsigned long *)&uof_image->ae_assigned)) continue; + + if (!test_bit(ae, &cfg_ae_mask)) + continue; + ustore_size = obj_handle->ae_data[ae].eff_ustore_size; patt_pos = page->beg_addr_p + page->micro_words_num; @@ -653,8 +658,12 @@ static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) int mflag = 0; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; for_each_set_bit(ae, &ae_mask, max_ae) { + if (!test_bit(ae, &cfg_ae_mask)) + continue; + for (i = 0; i < obj_handle->uimage_num; i++) { if (!test_bit(ae, (unsigned long *) &obj_handle->ae_uimage[i].img_ptr->ae_assigned)) @@ -931,10 +940,14 @@ static int qat_uclo_set_ae_mode(struct icp_qat_fw_loader_handle *handle) struct icp_qat_uclo_aedata *ae_data; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; unsigned char ae, s; int error; for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + if (!test_bit(ae, &cfg_ae_mask)) + continue; + ae_data = &obj_handle->ae_data[ae]; for (s = 0; s < min_t(unsigned int, ae_data->slice_num, ICP_QAT_UCLO_MAX_CTX); s++) { @@ -1176,6 +1189,7 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, &suof_img_hdr[i]); if (ret) return ret; + suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask; if ((suof_img_hdr[i].ae_mask & 0x1) != 0) ae0_img = i; } @@ -1277,6 +1291,7 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_auth_chunk *auth_chunk; u64 virt_addr, bus_addr, virt_base; unsigned int length, simg_offset = sizeof(*auth_chunk); + struct icp_qat_simg_ae_mode *simg_ae_mode; struct icp_firml_dram_desc img_desc; if (size > (ICP_QAT_AE_IMG_OFFSET(handle) + ICP_QAT_CSS_MAX_IMAGE_LEN)) { @@ -1366,6 +1381,11 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, auth_desc->img_ae_insts_high = (unsigned int) (bus_addr >> BITS_IN_DWORD); auth_desc->img_ae_insts_low = (unsigned int)bus_addr; + virt_addr += sizeof(struct icp_qat_css_hdr); + virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle); + virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); + simg_ae_mode = (struct icp_qat_simg_ae_mode *)(uintptr_t)virt_addr; + auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask; } else { auth_desc->img_ae_insts_high = auth_desc->img_high; auth_desc->img_ae_insts_low = auth_desc->img_low; @@ -1377,7 +1397,6 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) { - struct icp_qat_simg_ae_mode *virt_addr; unsigned long ae_mask = handle->hal_handle->ae_mask; u32 fcu_sts_csr, fcu_ctl_csr; u32 loaded_aes, loaded_csr; @@ -1388,15 +1407,10 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, fcu_sts_csr = handle->chip_info->fcu_sts_csr; loaded_csr = handle->chip_info->fcu_loaded_ae_csr; - virt_addr = (void *)((uintptr_t)desc + - sizeof(struct icp_qat_auth_chunk) + - sizeof(struct icp_qat_css_hdr) + - ICP_QAT_CSS_FWSK_PUB_LEN(handle) + - ICP_QAT_CSS_SIGNATURE_LEN(handle)); for_each_set_bit(i, &ae_mask, handle->hal_handle->ae_max_num) { int retry = 0; - if (!((virt_addr->ae_mask >> i) & 0x1)) + if (!((desc->ae_mask >> i) & 0x1)) continue; if (qat_hal_check_ae_active(handle, i)) { pr_err("QAT: AE %d is active\n", i); @@ -1866,6 +1880,7 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long cfg_ae_mask = handle->cfg_ae_mask; unsigned long ae_assigned = image->ae_assigned; struct icp_qat_uclo_aedata *aed; unsigned int ctx_mask, s; @@ -1880,6 +1895,9 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, /* load the default page and set assigned CTX PC * to the entrypoint address */ for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + if (!test_bit(ae, &cfg_ae_mask)) + continue; + if (!test_bit(ae, &ae_assigned)) continue; @@ -1957,3 +1975,13 @@ int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) return (handle->chip_info->fw_auth) ? qat_uclo_wr_suof_img(handle) : qat_uclo_wr_uof_img(handle); } + +int qat_uclo_set_cfg_ae_mask(struct icp_qat_fw_loader_handle *handle, + unsigned int cfg_ae_mask) +{ + if (!cfg_ae_mask) + return -EINVAL; + + handle->cfg_ae_mask = cfg_ae_mask; + return 0; +} From bd684d83c789dd4882a539075e45d46011a95bdf Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:08 +0800 Subject: [PATCH 154/360] crypto: qat - add support for shared ustore Add support for shared ustore mode support. This is required by the next generation of QAT devices to share the same fw image across engines. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/qat_hal.c | 2 ++ drivers/crypto/qat/qat_common/qat_uclo.c | 29 ++++++++++--------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index cc9b83d965af..5b9f2e8c9451 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -34,6 +34,7 @@ struct icp_qat_fw_loader_chip_info { u32 wakeup_event_val; bool fw_auth; bool css_3k; + bool tgroup_share_ustore; u32 fcu_ctl_csr; u32 fcu_sts_csr; u32 fcu_dram_addr_hi; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 94c0b04088b5..6ccfb8cf3a07 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -707,6 +707,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = true; handle->chip_info->css_3k = false; + handle->chip_info->tgroup_share_ustore = false; handle->chip_info->fcu_ctl_csr = FCU_CONTROL; handle->chip_info->fcu_sts_csr = FCU_STATUS; handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI; @@ -725,6 +726,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->wakeup_event_val = WAKEUP_EVENT; handle->chip_info->fw_auth = false; handle->chip_info->css_3k = false; + handle->chip_info->tgroup_share_ustore = false; handle->chip_info->fcu_ctl_csr = 0; handle->chip_info->fcu_sts_csr = 0; handle->chip_info->fcu_dram_addr_hi = 0; diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index c6b309d107f3..b280fb0722c5 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1180,21 +1180,24 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, if (!suof_img_hdr) return -ENOMEM; suof_handle->img_table.simg_hdr = suof_img_hdr; - } - for (i = 0; i < suof_handle->img_table.num_simgs; i++) { - qat_uclo_map_simg(handle, &suof_img_hdr[i], - &suof_chunk_hdr[1 + i]); - ret = qat_uclo_check_simg_compat(handle, - &suof_img_hdr[i]); - if (ret) - return ret; - suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask; - if ((suof_img_hdr[i].ae_mask & 0x1) != 0) - ae0_img = i; + for (i = 0; i < suof_handle->img_table.num_simgs; i++) { + qat_uclo_map_simg(handle, &suof_img_hdr[i], + &suof_chunk_hdr[1 + i]); + ret = qat_uclo_check_simg_compat(handle, + &suof_img_hdr[i]); + if (ret) + return ret; + suof_img_hdr[i].ae_mask &= handle->cfg_ae_mask; + if ((suof_img_hdr[i].ae_mask & 0x1) != 0) + ae0_img = i; + } + + if (!handle->chip_info->tgroup_share_ustore) { + qat_uclo_tail_img(suof_img_hdr, ae0_img, + suof_handle->img_table.num_simgs); + } } - qat_uclo_tail_img(suof_img_hdr, ae0_img, - suof_handle->img_table.num_simgs); return 0; } From 2778d64cf3f5517642555781df5628488e7d8186 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:09 +0800 Subject: [PATCH 155/360] crypto: qat - add support for broadcasting mode Add support for broadcasting mode in firmware loader to enable the next generation of QAT devices. Signed-off-by: Jack Xu Co-developed-by: Wojciech Ziemba Signed-off-by: Wojciech Ziemba Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../qat/qat_common/icp_qat_fw_loader_handle.h | 1 + drivers/crypto/qat/qat_common/icp_qat_hal.h | 10 +++ drivers/crypto/qat/qat_common/qat_hal.c | 1 + drivers/crypto/qat/qat_common/qat_uclo.c | 90 ++++++++++++++++++- 4 files changed, 99 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 5b9f2e8c9451..b8f3463be6ef 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -15,6 +15,7 @@ struct icp_qat_fw_loader_ae_data { struct icp_qat_fw_loader_hal_handle { struct icp_qat_fw_loader_ae_data aes[ICP_QAT_UCLO_MAX_AE]; unsigned int ae_mask; + unsigned int admin_ae_mask; unsigned int slice_mask; unsigned int revision_id; unsigned int ae_max_num; diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index 02476b2ceee1..8372f18ebc80 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -53,6 +53,15 @@ enum fcu_csr { FCU_RAMBASE_ADDR_LO = 0x8d8 }; +enum fcu_csr_4xxx { + FCU_CONTROL_4XXX = 0x1000, + FCU_STATUS_4XXX = 0x1004, + FCU_ME_BROADCAST_MASK_TYPE = 0x1008, + FCU_AE_LOADED_4XXX = 0x1010, + FCU_DRAM_ADDR_LO_4XXX = 0x1014, + FCU_DRAM_ADDR_HI_4XXX = 0x1018, +}; + enum fcu_cmd { FCU_CTRL_CMD_NOOP = 0, FCU_CTRL_CMD_AUTH = 1, @@ -90,6 +99,7 @@ enum fcu_sts { #define LCS_STATUS (0x1) #define MMC_SHARE_CS_BITPOS 2 #define WAKEUP_EVENT 0x10000 +#define FCU_CTRL_BROADCAST_POS 0x4 #define FCU_CTRL_AE_POS 0x8 #define FCU_AUTH_STS_MASK 0x7 #define FCU_STS_DONE_POS 0x9 diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 6ccfb8cf3a07..a3c1f2163910 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -762,6 +762,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->pci_dev = pci_info->pci_dev; handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; handle->hal_handle->ae_mask = hw_data->ae_mask; + handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask; handle->hal_handle->slice_mask = hw_data->accel_mask; handle->cfg_ae_mask = ALL_AE_MASK; /* create AE objects */ diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index b280fb0722c5..c089c2709376 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -1239,6 +1239,83 @@ auth_fail: return -EINVAL; } +static bool qat_uclo_is_broadcast(struct icp_qat_fw_loader_handle *handle, + int imgid) +{ + struct icp_qat_suof_handle *sobj_handle; + + if (!handle->chip_info->tgroup_share_ustore) + return false; + + sobj_handle = (struct icp_qat_suof_handle *)handle->sobj_handle; + if (handle->hal_handle->admin_ae_mask & + sobj_handle->img_table.simg_hdr[imgid].ae_mask) + return false; + + return true; +} + +static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_fw_auth_desc *desc) +{ + unsigned long ae_mask = handle->hal_handle->ae_mask; + unsigned long desc_ae_mask = desc->ae_mask; + u32 fcu_sts, ae_broadcast_mask = 0; + u32 fcu_loaded_csr, ae_loaded; + u32 fcu_sts_csr, fcu_ctl_csr; + unsigned int ae, retry = 0; + + if (handle->chip_info->tgroup_share_ustore) { + fcu_ctl_csr = handle->chip_info->fcu_ctl_csr; + fcu_sts_csr = handle->chip_info->fcu_sts_csr; + fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr; + } else { + pr_err("Chip 0x%x doesn't support broadcast load\n", + handle->pci_dev->device); + return -EINVAL; + } + + for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { + if (qat_hal_check_ae_active(handle, (unsigned char)ae)) { + pr_err("QAT: Broadcast load failed. AE is not enabled or active.\n"); + return -EINVAL; + } + + if (test_bit(ae, &desc_ae_mask)) + ae_broadcast_mask |= 1 << ae; + } + + if (ae_broadcast_mask) { + SET_CAP_CSR(handle, FCU_ME_BROADCAST_MASK_TYPE, + ae_broadcast_mask); + + SET_CAP_CSR(handle, fcu_ctl_csr, FCU_CTRL_CMD_LOAD); + + do { + msleep(FW_AUTH_WAIT_PERIOD); + fcu_sts = GET_CAP_CSR(handle, fcu_sts_csr); + fcu_sts &= FCU_AUTH_STS_MASK; + + if (fcu_sts == FCU_STS_LOAD_FAIL) { + pr_err("Broadcast load failed: 0x%x)\n", fcu_sts); + return -EINVAL; + } else if (fcu_sts == FCU_STS_LOAD_DONE) { + ae_loaded = GET_CAP_CSR(handle, fcu_loaded_csr); + ae_loaded >>= handle->chip_info->fcu_loaded_ae_pos; + + if ((ae_loaded & ae_broadcast_mask) == ae_broadcast_mask) + break; + } + } while (retry++ < FW_AUTH_MAX_RETRY); + + if (retry > FW_AUTH_MAX_RETRY) { + pr_err("QAT: broadcast load failed timeout %d\n", retry); + return -EINVAL; + } + } + return 0; +} + static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, struct icp_firml_dram_desc *dram_desc, unsigned int size) @@ -1420,7 +1497,9 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, return -EINVAL; } SET_CAP_CSR(handle, fcu_ctl_csr, - (FCU_CTRL_CMD_LOAD | (i << FCU_CTRL_AE_POS))); + (FCU_CTRL_CMD_LOAD | + (1 << FCU_CTRL_BROADCAST_POS) | + (i << FCU_CTRL_AE_POS))); do { msleep(FW_AUTH_WAIT_PERIOD); @@ -1945,8 +2024,13 @@ static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) goto wr_err; if (qat_uclo_auth_fw(handle, desc)) goto wr_err; - if (qat_uclo_load_fw(handle, desc)) - goto wr_err; + if (qat_uclo_is_broadcast(handle, i)) { + if (qat_uclo_broadcast_load_fw(handle, desc)) + goto wr_err; + } else { + if (qat_uclo_load_fw(handle, desc)) + goto wr_err; + } qat_uclo_ummap_auth_fw(handle, &desc); } return 0; From 9c0cef2364750c00ab380cc8902dbbc91e230183 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Fri, 6 Nov 2020 19:28:10 +0800 Subject: [PATCH 156/360] crypto: qat - add gen4 firmware loader Add support for the QAT gen4 devices in the firmware loader. Signed-off-by: Jack Xu Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_accel_devices.h | 2 + drivers/crypto/qat/qat_common/icp_qat_hal.h | 12 ++- drivers/crypto/qat/qat_common/icp_qat_uclo.h | 2 + drivers/crypto/qat/qat_common/qat_hal.c | 77 +++++++++++++++---- drivers/crypto/qat/qat_common/qat_uclo.c | 2 + 5 files changed, 78 insertions(+), 17 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 996d25565b11..5694422ec66c 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -15,6 +15,8 @@ #define ADF_C62XVF_DEVICE_NAME "c6xxvf" #define ADF_C3XXX_DEVICE_NAME "c3xxx" #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" +#define ADF_4XXX_PCI_DEVICE_ID 0x4940 +#define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941 #define ADF_ERRSOU3 (0x3A000 + 0x0C) #define ADF_ERRSOU5 (0x3A000 + 0xD8) #define ADF_DEVICE_FUSECTL_OFFSET 0x40 diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index 8372f18ebc80..20b2ee1fc65a 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -10,6 +10,14 @@ enum hal_global_csr { ICP_GLOBAL_CLK_ENABLE = 0xA50 }; +enum { + MISC_CONTROL_C4XXX = 0xAA0, + ICP_RESET_CPP0 = 0x938, + ICP_RESET_CPP1 = 0x93c, + ICP_GLOBAL_CLK_ENABLE_CPP0 = 0x964, + ICP_GLOBAL_CLK_ENABLE_CPP1 = 0x968 +}; + enum hal_ae_csr { USTORE_ADDRESS = 0x000, USTORE_DATA_LOWER = 0x004, @@ -111,7 +119,9 @@ enum fcu_sts { #define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000) #define LOCAL_TO_XFER_REG_OFFSET 0x800 #define ICP_QAT_EP_OFFSET 0x3a000 - +#define ICP_QAT_EP_OFFSET_4XXX 0x200000 /* HI MMIO CSRs */ +#define ICP_QAT_AE_OFFSET_4XXX 0x600000 +#define ICP_QAT_CAP_OFFSET_4XXX 0x640000 #define SET_CAP_CSR(handle, csr, val) \ ADF_CSR_WR((handle)->hal_cap_g_ctl_csr_addr_v, csr, val) #define GET_CAP_CSR(handle, csr) \ diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index 0ec8a5ab51b5..4b36869bf460 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -6,6 +6,7 @@ #define ICP_QAT_AC_895XCC_DEV_TYPE 0x00400000 #define ICP_QAT_AC_C62X_DEV_TYPE 0x01000000 #define ICP_QAT_AC_C3XXX_DEV_TYPE 0x02000000 +#define ICP_QAT_AC_4XXX_A_DEV_TYPE 0x08000000 #define ICP_QAT_UCLO_MAX_AE 12 #define ICP_QAT_UCLO_MAX_CTX 8 #define ICP_QAT_UCLO_MAX_UIMAGE (ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX) @@ -13,6 +14,7 @@ #define ICP_QAT_UCLO_MAX_XFER_REG 128 #define ICP_QAT_UCLO_MAX_GPR_REG 128 #define ICP_QAT_UCLO_MAX_LMEM_REG 1024 +#define ICP_QAT_UCLO_MAX_LMEM_REG_2X 1280 #define ICP_QAT_UCLO_AE_ALL_CTX 0xff #define ICP_QAT_UOF_OBJID_LEN 8 #define ICP_QAT_UOF_FID 0xc6c2 diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index a3c1f2163910..bd3028126cbe 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -695,6 +695,39 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->pci_dev = pci_info->pci_dev; switch (handle->pci_dev->device) { + case ADF_4XXX_PCI_DEVICE_ID: + handle->chip_info->sram_visible = false; + handle->chip_info->nn = false; + handle->chip_info->lm2lm3 = true; + handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X; + handle->chip_info->icp_rst_csr = ICP_RESET_CPP0; + handle->chip_info->icp_rst_mask = 0x100015; + handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE_CPP0; + handle->chip_info->misc_ctl_csr = MISC_CONTROL_C4XXX; + handle->chip_info->wakeup_event_val = 0x80000000; + handle->chip_info->fw_auth = true; + handle->chip_info->css_3k = true; + handle->chip_info->tgroup_share_ustore = true; + handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX; + handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX; + handle->chip_info->fcu_dram_addr_hi = FCU_DRAM_ADDR_HI_4XXX; + handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO_4XXX; + handle->chip_info->fcu_loaded_ae_csr = FCU_AE_LOADED_4XXX; + handle->chip_info->fcu_loaded_ae_pos = 0; + + handle->hal_cap_g_ctl_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_CAP_OFFSET_4XXX); + handle->hal_cap_ae_xfer_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_AE_OFFSET_4XXX); + handle->hal_ep_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_EP_OFFSET_4XXX); + handle->hal_cap_ae_local_csr_addr_v = + (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + + LOCAL_TO_XFER_REG_OFFSET); + break; case PCI_DEVICE_ID_INTEL_QAT_C62X: case PCI_DEVICE_ID_INTEL_QAT_C3XXX: handle->chip_info->sram_visible = false; @@ -702,6 +735,8 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm2lm3 = false; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; + handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | + (hw_data->accel_mask << RST_CSR_QAT_LSB); handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; handle->chip_info->misc_ctl_csr = MISC_CONTROL; handle->chip_info->wakeup_event_val = WAKEUP_EVENT; @@ -714,6 +749,18 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->fcu_dram_addr_lo = FCU_DRAM_ADDR_LO; handle->chip_info->fcu_loaded_ae_csr = FCU_STATUS; handle->chip_info->fcu_loaded_ae_pos = FCU_LOADED_AE_POS; + handle->hal_cap_g_ctl_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_CAP_OFFSET); + handle->hal_cap_ae_xfer_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_AE_OFFSET); + handle->hal_ep_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_EP_OFFSET); + handle->hal_cap_ae_local_csr_addr_v = + (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + + LOCAL_TO_XFER_REG_OFFSET); break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: handle->chip_info->sram_visible = true; @@ -721,6 +768,8 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->lm2lm3 = false; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; handle->chip_info->icp_rst_csr = ICP_RESET; + handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | + (hw_data->accel_mask << RST_CSR_QAT_LSB); handle->chip_info->glb_clk_enable_csr = ICP_GLOBAL_CLK_ENABLE; handle->chip_info->misc_ctl_csr = MISC_CONTROL; handle->chip_info->wakeup_event_val = WAKEUP_EVENT; @@ -733,6 +782,18 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->fcu_dram_addr_lo = 0; handle->chip_info->fcu_loaded_ae_csr = 0; handle->chip_info->fcu_loaded_ae_pos = 0; + handle->hal_cap_g_ctl_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_CAP_OFFSET); + handle->hal_cap_ae_xfer_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_AE_OFFSET); + handle->hal_ep_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_EP_OFFSET); + handle->hal_cap_ae_local_csr_addr_v = + (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + + LOCAL_TO_XFER_REG_OFFSET); break; default: ret = -EINVAL; @@ -744,22 +805,6 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; handle->hal_sram_addr_v = sram_bar->virt_addr; } - - handle->chip_info->icp_rst_mask = (hw_data->ae_mask << RST_CSR_AE_LSB) | - (hw_data->accel_mask << RST_CSR_QAT_LSB); - handle->hal_cap_g_ctl_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_CAP_OFFSET); - handle->hal_cap_ae_xfer_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_AE_OFFSET); - handle->hal_ep_csr_addr_v = - (void __iomem *)((uintptr_t)misc_bar->virt_addr + - ICP_QAT_EP_OFFSET); - handle->hal_cap_ae_local_csr_addr_v = - (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + - LOCAL_TO_XFER_REG_OFFSET); - handle->pci_dev = pci_info->pci_dev; handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid; handle->hal_handle->ae_mask = hw_data->ae_mask; handle->hal_handle->admin_ae_mask = hw_data->admin_ae_mask; diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index c089c2709376..1fb5fc852f6b 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -728,6 +728,8 @@ qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) return ICP_QAT_AC_C62X_DEV_TYPE; case PCI_DEVICE_ID_INTEL_QAT_C3XXX: return ICP_QAT_AC_C3XXX_DEV_TYPE; + case ADF_4XXX_PCI_DEVICE_ID: + return ICP_QAT_AC_4XXX_A_DEV_TYPE; default: pr_err("QAT: unsupported device 0x%x\n", handle->pci_dev->device); From c4fc6328d6c67690a7e6e03f43a5a976a13120ef Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 6 Nov 2020 17:39:38 +0100 Subject: [PATCH 157/360] crypto: arm64/chacha - simplify tail block handling Based on lessons learnt from optimizing the 32-bit version of this driver, we can simplify the arm64 version considerably, by reordering the final two stores when the last block is not a multiple of 64 bytes. This removes the need to use permutation instructions to calculate the elements that are clobbered by the final overlapping store, given that the store of the penultimate block now follows it, and that one carries the correct values for those elements already. While at it, simplify the overlapping loads as well, by calculating the address of the final overlapping load upfront, and switching to this address for every load that would otherwise extend past the end of the source buffer. There is no impact on performance, but the resulting code is substantially smaller and easier to follow. Cc: Eric Biggers Cc: "Jason A . Donenfeld" Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/chacha-neon-core.S | 199 ++++++++++----------------- 1 file changed, 72 insertions(+), 127 deletions(-) diff --git a/arch/arm64/crypto/chacha-neon-core.S b/arch/arm64/crypto/chacha-neon-core.S index e90386a7db8e..b70ac76f2610 100644 --- a/arch/arm64/crypto/chacha-neon-core.S +++ b/arch/arm64/crypto/chacha-neon-core.S @@ -195,7 +195,6 @@ SYM_FUNC_START(chacha_4block_xor_neon) adr_l x10, .Lpermute and x5, x4, #63 add x10, x10, x5 - add x11, x10, #64 // // This function encrypts four consecutive ChaCha blocks by loading @@ -645,11 +644,11 @@ CPU_BE( rev a15, a15 ) zip2 v31.4s, v14.4s, v15.4s eor a15, a15, w9 - mov x3, #64 + add x3, x2, x4 + sub x3, x3, #128 // start of last block + subs x5, x4, #128 - add x6, x5, x2 - csel x3, x3, xzr, ge - csel x2, x2, x6, ge + csel x2, x2, x3, ge // interleave 64-bit words in state n, n+2 zip1 v0.2d, v16.2d, v18.2d @@ -658,13 +657,10 @@ CPU_BE( rev a15, a15 ) zip1 v8.2d, v17.2d, v19.2d zip2 v12.2d, v17.2d, v19.2d stp a2, a3, [x1, #-56] - ld1 {v16.16b-v19.16b}, [x2], x3 subs x6, x4, #192 - ccmp x3, xzr, #4, lt - add x7, x6, x2 - csel x3, x3, xzr, eq - csel x2, x2, x7, eq + ld1 {v16.16b-v19.16b}, [x2], #64 + csel x2, x2, x3, ge zip1 v1.2d, v20.2d, v22.2d zip2 v5.2d, v20.2d, v22.2d @@ -672,13 +668,10 @@ CPU_BE( rev a15, a15 ) zip1 v9.2d, v21.2d, v23.2d zip2 v13.2d, v21.2d, v23.2d stp a6, a7, [x1, #-40] - ld1 {v20.16b-v23.16b}, [x2], x3 subs x7, x4, #256 - ccmp x3, xzr, #4, lt - add x8, x7, x2 - csel x3, x3, xzr, eq - csel x2, x2, x8, eq + ld1 {v20.16b-v23.16b}, [x2], #64 + csel x2, x2, x3, ge zip1 v2.2d, v24.2d, v26.2d zip2 v6.2d, v24.2d, v26.2d @@ -686,12 +679,10 @@ CPU_BE( rev a15, a15 ) zip1 v10.2d, v25.2d, v27.2d zip2 v14.2d, v25.2d, v27.2d stp a10, a11, [x1, #-24] - ld1 {v24.16b-v27.16b}, [x2], x3 subs x8, x4, #320 - ccmp x3, xzr, #4, lt - add x9, x8, x2 - csel x2, x2, x9, eq + ld1 {v24.16b-v27.16b}, [x2], #64 + csel x2, x2, x3, ge zip1 v3.2d, v28.2d, v30.2d zip2 v7.2d, v28.2d, v30.2d @@ -699,151 +690,105 @@ CPU_BE( rev a15, a15 ) zip1 v11.2d, v29.2d, v31.2d zip2 v15.2d, v29.2d, v31.2d stp a14, a15, [x1, #-8] + + tbnz x5, #63, .Lt128 ld1 {v28.16b-v31.16b}, [x2] // xor with corresponding input, write to output - tbnz x5, #63, 0f eor v16.16b, v16.16b, v0.16b eor v17.16b, v17.16b, v1.16b eor v18.16b, v18.16b, v2.16b eor v19.16b, v19.16b, v3.16b - st1 {v16.16b-v19.16b}, [x1], #64 - cbz x5, .Lout - tbnz x6, #63, 1f + tbnz x6, #63, .Lt192 + eor v20.16b, v20.16b, v4.16b eor v21.16b, v21.16b, v5.16b eor v22.16b, v22.16b, v6.16b eor v23.16b, v23.16b, v7.16b - st1 {v20.16b-v23.16b}, [x1], #64 - cbz x6, .Lout - tbnz x7, #63, 2f + st1 {v16.16b-v19.16b}, [x1], #64 + tbnz x7, #63, .Lt256 + eor v24.16b, v24.16b, v8.16b eor v25.16b, v25.16b, v9.16b eor v26.16b, v26.16b, v10.16b eor v27.16b, v27.16b, v11.16b - st1 {v24.16b-v27.16b}, [x1], #64 - cbz x7, .Lout - tbnz x8, #63, 3f + st1 {v20.16b-v23.16b}, [x1], #64 + tbnz x8, #63, .Lt320 + eor v28.16b, v28.16b, v12.16b eor v29.16b, v29.16b, v13.16b eor v30.16b, v30.16b, v14.16b eor v31.16b, v31.16b, v15.16b + + st1 {v24.16b-v27.16b}, [x1], #64 st1 {v28.16b-v31.16b}, [x1] .Lout: frame_pop ret - // fewer than 128 bytes of in/output -0: ld1 {v8.16b}, [x10] - ld1 {v9.16b}, [x11] - movi v10.16b, #16 - sub x2, x1, #64 - add x1, x1, x5 - ld1 {v16.16b-v19.16b}, [x2] - tbl v4.16b, {v0.16b-v3.16b}, v8.16b - tbx v20.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v5.16b, {v0.16b-v3.16b}, v8.16b - tbx v21.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v6.16b, {v0.16b-v3.16b}, v8.16b - tbx v22.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v7.16b, {v0.16b-v3.16b}, v8.16b - tbx v23.16b, {v16.16b-v19.16b}, v9.16b - - eor v20.16b, v20.16b, v4.16b - eor v21.16b, v21.16b, v5.16b - eor v22.16b, v22.16b, v6.16b - eor v23.16b, v23.16b, v7.16b - st1 {v20.16b-v23.16b}, [x1] - b .Lout - // fewer than 192 bytes of in/output -1: ld1 {v8.16b}, [x10] - ld1 {v9.16b}, [x11] - movi v10.16b, #16 - add x1, x1, x6 - tbl v0.16b, {v4.16b-v7.16b}, v8.16b - tbx v20.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v1.16b, {v4.16b-v7.16b}, v8.16b - tbx v21.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v2.16b, {v4.16b-v7.16b}, v8.16b - tbx v22.16b, {v16.16b-v19.16b}, v9.16b - add v8.16b, v8.16b, v10.16b - add v9.16b, v9.16b, v10.16b - tbl v3.16b, {v4.16b-v7.16b}, v8.16b - tbx v23.16b, {v16.16b-v19.16b}, v9.16b +.Lt192: cbz x5, 1f // exactly 128 bytes? + ld1 {v28.16b-v31.16b}, [x10] + add x5, x5, x1 + tbl v28.16b, {v4.16b-v7.16b}, v28.16b + tbl v29.16b, {v4.16b-v7.16b}, v29.16b + tbl v30.16b, {v4.16b-v7.16b}, v30.16b + tbl v31.16b, {v4.16b-v7.16b}, v31.16b - eor v20.16b, v20.16b, v0.16b - eor v21.16b, v21.16b, v1.16b - eor v22.16b, v22.16b, v2.16b - eor v23.16b, v23.16b, v3.16b - st1 {v20.16b-v23.16b}, [x1] +0: eor v20.16b, v20.16b, v28.16b + eor v21.16b, v21.16b, v29.16b + eor v22.16b, v22.16b, v30.16b + eor v23.16b, v23.16b, v31.16b + st1 {v20.16b-v23.16b}, [x5] // overlapping stores +1: st1 {v16.16b-v19.16b}, [x1] b .Lout + // fewer than 128 bytes of in/output +.Lt128: ld1 {v28.16b-v31.16b}, [x10] + add x5, x5, x1 + sub x1, x1, #64 + tbl v28.16b, {v0.16b-v3.16b}, v28.16b + tbl v29.16b, {v0.16b-v3.16b}, v29.16b + tbl v30.16b, {v0.16b-v3.16b}, v30.16b + tbl v31.16b, {v0.16b-v3.16b}, v31.16b + ld1 {v16.16b-v19.16b}, [x1] // reload first output block + b 0b + // fewer than 256 bytes of in/output -2: ld1 {v4.16b}, [x10] - ld1 {v5.16b}, [x11] - movi v6.16b, #16 - add x1, x1, x7 +.Lt256: cbz x6, 2f // exactly 192 bytes? + ld1 {v4.16b-v7.16b}, [x10] + add x6, x6, x1 tbl v0.16b, {v8.16b-v11.16b}, v4.16b - tbx v24.16b, {v20.16b-v23.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v1.16b, {v8.16b-v11.16b}, v4.16b - tbx v25.16b, {v20.16b-v23.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v2.16b, {v8.16b-v11.16b}, v4.16b - tbx v26.16b, {v20.16b-v23.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v3.16b, {v8.16b-v11.16b}, v4.16b - tbx v27.16b, {v20.16b-v23.16b}, v5.16b - - eor v24.16b, v24.16b, v0.16b - eor v25.16b, v25.16b, v1.16b - eor v26.16b, v26.16b, v2.16b - eor v27.16b, v27.16b, v3.16b - st1 {v24.16b-v27.16b}, [x1] - b .Lout - - // fewer than 320 bytes of in/output -3: ld1 {v4.16b}, [x10] - ld1 {v5.16b}, [x11] - movi v6.16b, #16 - add x1, x1, x8 - tbl v0.16b, {v12.16b-v15.16b}, v4.16b - tbx v28.16b, {v24.16b-v27.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v1.16b, {v12.16b-v15.16b}, v4.16b - tbx v29.16b, {v24.16b-v27.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v2.16b, {v12.16b-v15.16b}, v4.16b - tbx v30.16b, {v24.16b-v27.16b}, v5.16b - add v4.16b, v4.16b, v6.16b - add v5.16b, v5.16b, v6.16b - tbl v3.16b, {v12.16b-v15.16b}, v4.16b - tbx v31.16b, {v24.16b-v27.16b}, v5.16b + tbl v1.16b, {v8.16b-v11.16b}, v5.16b + tbl v2.16b, {v8.16b-v11.16b}, v6.16b + tbl v3.16b, {v8.16b-v11.16b}, v7.16b eor v28.16b, v28.16b, v0.16b eor v29.16b, v29.16b, v1.16b eor v30.16b, v30.16b, v2.16b eor v31.16b, v31.16b, v3.16b - st1 {v28.16b-v31.16b}, [x1] + st1 {v28.16b-v31.16b}, [x6] // overlapping stores +2: st1 {v20.16b-v23.16b}, [x1] + b .Lout + + // fewer than 320 bytes of in/output +.Lt320: cbz x7, 3f // exactly 256 bytes? + ld1 {v4.16b-v7.16b}, [x10] + add x7, x7, x1 + tbl v0.16b, {v12.16b-v15.16b}, v4.16b + tbl v1.16b, {v12.16b-v15.16b}, v5.16b + tbl v2.16b, {v12.16b-v15.16b}, v6.16b + tbl v3.16b, {v12.16b-v15.16b}, v7.16b + + eor v28.16b, v28.16b, v0.16b + eor v29.16b, v29.16b, v1.16b + eor v30.16b, v30.16b, v2.16b + eor v31.16b, v31.16b, v3.16b + st1 {v28.16b-v31.16b}, [x7] // overlapping stores +3: st1 {v24.16b-v27.16b}, [x1] b .Lout SYM_FUNC_END(chacha_4block_xor_neon) @@ -851,7 +796,7 @@ SYM_FUNC_END(chacha_4block_xor_neon) .align L1_CACHE_SHIFT .Lpermute: .set .Li, 0 - .rept 192 + .rept 128 .byte (.Li - 64) .set .Li, .Li + 1 .endr From 95526cccc4863ea93c29917faa8bf4bc4d19d199 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 26 Oct 2020 13:25:49 +0100 Subject: [PATCH 158/360] m68k: defconfig: Update defconfigs for v5.10-rc1 - Enable modular build of SM2 crypto algorithm, - Drop CONFIG_CRYPTO_SM3=m (auto-enabled by CONFIG_CRYPTO_SM2), - Drop CONFIG_TEST_BITFIELD=m (converted to KUnit in commit d2585f5164c298aa ("lib: kunit: add bitfield test conversion to KUnit")), - Enable modular build of the freeing pages test module. Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20201026122549.3092526-1-geert@linux-m68k.org --- arch/m68k/configs/amiga_defconfig | 4 ++-- arch/m68k/configs/apollo_defconfig | 4 ++-- arch/m68k/configs/atari_defconfig | 4 ++-- arch/m68k/configs/bvme6000_defconfig | 4 ++-- arch/m68k/configs/hp300_defconfig | 4 ++-- arch/m68k/configs/mac_defconfig | 4 ++-- arch/m68k/configs/multi_defconfig | 4 ++-- arch/m68k/configs/mvme147_defconfig | 4 ++-- arch/m68k/configs/mvme16x_defconfig | 4 ++-- arch/m68k/configs/q40_defconfig | 4 ++-- arch/m68k/configs/sun3_defconfig | 4 ++-- arch/m68k/configs/sun3x_defconfig | 4 ++-- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index f9f4fa595e13..8fc77002f074 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -563,6 +563,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -583,7 +584,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -638,7 +638,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -659,3 +658,4 @@ CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index f4828e86d547..f84cd9b20b9d 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -519,6 +519,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -539,7 +540,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -594,7 +594,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -615,3 +614,4 @@ CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index e7911f141de1..9622c1d95c73 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -541,6 +541,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -561,7 +562,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -616,7 +616,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -637,3 +636,4 @@ CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index d574e438e6db..b5b6f25dbff3 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -512,6 +512,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -532,7 +533,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -587,7 +587,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -608,3 +607,4 @@ CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index c7ce206e6138..016f6708b851 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -521,6 +521,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -541,7 +542,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -596,7 +596,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -617,3 +616,4 @@ CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index 3cd76bfaee03..2e03fd20b79c 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -544,6 +544,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -564,7 +565,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -619,7 +619,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -640,3 +639,4 @@ CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index c3d6faa7894f..50c087e9e3fe 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -630,6 +630,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -650,7 +651,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -705,7 +705,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -726,3 +725,4 @@ CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 5568aa7d9d41..884b413dbbea 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -511,6 +511,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -531,7 +532,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -586,7 +586,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -607,3 +606,4 @@ CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 5b1e72ce53f8..45cccbb5f068 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -512,6 +512,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -532,7 +533,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -587,7 +587,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -608,3 +607,4 @@ CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index c3a3dcf30fb9..4b66b5b185bc 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -530,6 +530,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -550,7 +551,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -605,7 +605,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -626,3 +625,4 @@ CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 3c00e52f1bf0..243f5d5dfa44 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -514,6 +514,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -534,7 +535,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -588,7 +588,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -609,3 +608,4 @@ CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index 241242d73cbd..2553db10a7e9 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -513,6 +513,7 @@ CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECRDSA=m +CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m CONFIG_CRYPTO_CHACHA20POLY1305=m CONFIG_CRYPTO_AEGIS128=m @@ -533,7 +534,6 @@ CONFIG_CRYPTO_RMD160=m CONFIG_CRYPTO_RMD256=m CONFIG_CRYPTO_RMD320=m CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SM3=m CONFIG_CRYPTO_TGR192=m CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_AES=y @@ -588,7 +588,6 @@ CONFIG_TEST_STRSCPY=m CONFIG_TEST_KSTRTOX=m CONFIG_TEST_PRINTF=m CONFIG_TEST_BITMAP=m -CONFIG_TEST_BITFIELD=m CONFIG_TEST_UUID=m CONFIG_TEST_XARRAY=m CONFIG_TEST_OVERFLOW=m @@ -609,3 +608,4 @@ CONFIG_TEST_KMOD=m CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_STACKINIT=m CONFIG_TEST_MEMINIT=m +CONFIG_TEST_FREE_PAGES=m From 8ae0b65ac7e5b7a93b8d6abc520351ca11d91867 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Mon, 26 Oct 2020 13:26:22 +0100 Subject: [PATCH 159/360] m68k: defconfig: Enable KUnit tests Enable KUnit and all KUnit tests for modular builds, so they are available when needed, just like non-KUnit tests. Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/20201026122622.3092658-1-geert@linux-m68k.org --- arch/m68k/configs/amiga_defconfig | 5 +++++ arch/m68k/configs/apollo_defconfig | 5 +++++ arch/m68k/configs/atari_defconfig | 5 +++++ arch/m68k/configs/bvme6000_defconfig | 5 +++++ arch/m68k/configs/hp300_defconfig | 5 +++++ arch/m68k/configs/mac_defconfig | 5 +++++ arch/m68k/configs/multi_defconfig | 5 +++++ arch/m68k/configs/mvme147_defconfig | 5 +++++ arch/m68k/configs/mvme16x_defconfig | 5 +++++ arch/m68k/configs/q40_defconfig | 5 +++++ arch/m68k/configs/sun3_defconfig | 5 +++++ arch/m68k/configs/sun3x_defconfig | 5 +++++ 12 files changed, 60 insertions(+) diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index 8fc77002f074..19b40b6bc4b7 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -626,6 +626,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -652,6 +654,9 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index f84cd9b20b9d..07516abe0489 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -582,6 +582,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -608,6 +610,9 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 9622c1d95c73..cc901c4e9492 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -604,6 +604,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -630,6 +632,9 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index b5b6f25dbff3..fc9a94aa7d6b 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -575,6 +575,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -601,6 +603,9 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index 016f6708b851..260f1206c810 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -584,6 +584,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -610,6 +612,9 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index 2e03fd20b79c..f6d50b3fe8c2 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -607,6 +607,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -633,6 +635,9 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index 50c087e9e3fe..fbe000ca0003 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -693,6 +693,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -719,6 +721,9 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index 884b413dbbea..25ca836a5701 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -574,6 +574,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -600,6 +602,9 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 45cccbb5f068..5794e43a2acb 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -575,6 +575,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -601,6 +603,9 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index 4b66b5b185bc..dbfb18938e11 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -593,6 +593,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -619,6 +621,9 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 243f5d5dfa44..e6afbeee7c4a 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -576,6 +576,8 @@ CONFIG_STRING_SELFTEST=m CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -602,6 +604,9 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index 2553db10a7e9..5340507a9fff 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -576,6 +576,8 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_TEST_LOCKUP=m CONFIG_WW_MUTEX_SELFTEST=m CONFIG_EARLY_PRINTK=y +CONFIG_KUNIT=m +CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m @@ -602,6 +604,9 @@ CONFIG_TEST_BLACKHOLE_DEV=m CONFIG_FIND_BIT_BENCHMARK=m CONFIG_TEST_FIRMWARE=m CONFIG_TEST_SYSCTL=m +CONFIG_BITFIELD_KUNIT=m +CONFIG_LINEAR_RANGES_TEST=m +CONFIG_BITS_TEST=m CONFIG_TEST_UDELAY=m CONFIG_TEST_STATIC_KEYS=m CONFIG_TEST_KMOD=m From 098416e6986127f7e4c8ce4fd6bbbd80e55b0386 Mon Sep 17 00:00:00 2001 From: Tony Luck Date: Tue, 10 Nov 2020 16:39:54 -0800 Subject: [PATCH 160/360] x86/mce: Use "safe" MSR functions when enabling additional error logging Booting as a guest under KVM results in error messages about unchecked MSR access: unchecked MSR access error: RDMSR from 0x17f at rIP: 0xffffffff84483f16 (mce_intel_feature_init+0x156/0x270) because KVM doesn't provide emulation for random model specific registers. Switch to using rdmsrl_safe()/wrmsrl_safe() to avoid the message. Fixes: 68299a42f842 ("x86/mce: Enable additional error logging on certain Intel CPUs") Reported-by: Qian Cai Signed-off-by: Tony Luck Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201111003954.GA11878@agluck-desk2.amr.corp.intel.com --- arch/x86/kernel/cpu/mce/intel.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/mce/intel.c b/arch/x86/kernel/cpu/mce/intel.c index b47883e364b4..c2476fe0682e 100644 --- a/arch/x86/kernel/cpu/mce/intel.c +++ b/arch/x86/kernel/cpu/mce/intel.c @@ -521,9 +521,10 @@ static void intel_imc_init(struct cpuinfo_x86 *c) case INTEL_FAM6_SANDYBRIDGE_X: case INTEL_FAM6_IVYBRIDGE_X: case INTEL_FAM6_HASWELL_X: - rdmsrl(MSR_ERROR_CONTROL, error_control); + if (rdmsrl_safe(MSR_ERROR_CONTROL, &error_control)) + return; error_control |= 2; - wrmsrl(MSR_ERROR_CONTROL, error_control); + wrmsrl_safe(MSR_ERROR_CONTROL, error_control); break; } } From 8113ab20e850491b4144a1a64246f07a2d737a49 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Thu, 15 Oct 2020 12:28:58 +0200 Subject: [PATCH 161/360] tools/power/cpupower: Read energy_perf_bias from sysfs ... instead of poking at the MSR. For that, move the accessor functions to misc.c and add a sysfs-writing function too. There should be no functional changes resulting from this. Signed-off-by: Borislav Petkov Reviewed-by: Shuah Khan Cc: Thomas Renninger Link: https://lkml.kernel.org/r/20201029190259.3476-2-bp@alien8.de --- tools/power/cpupower/lib/cpupower.c | 23 +++++++++- tools/power/cpupower/lib/cpupower_intern.h | 5 ++ tools/power/cpupower/utils/cpupower-info.c | 2 +- tools/power/cpupower/utils/cpupower-set.c | 2 +- tools/power/cpupower/utils/helpers/helpers.h | 8 ++-- tools/power/cpupower/utils/helpers/misc.c | 48 ++++++++++++++++++++ tools/power/cpupower/utils/helpers/msr.c | 28 ------------ 7 files changed, 81 insertions(+), 35 deletions(-) diff --git a/tools/power/cpupower/lib/cpupower.c b/tools/power/cpupower/lib/cpupower.c index 3656e697537e..3f7d0c0c5067 100644 --- a/tools/power/cpupower/lib/cpupower.c +++ b/tools/power/cpupower/lib/cpupower.c @@ -16,8 +16,8 @@ unsigned int cpupower_read_sysfs(const char *path, char *buf, size_t buflen) { - int fd; ssize_t numread; + int fd; fd = open(path, O_RDONLY); if (fd == -1) @@ -35,6 +35,27 @@ unsigned int cpupower_read_sysfs(const char *path, char *buf, size_t buflen) return (unsigned int) numread; } +unsigned int cpupower_write_sysfs(const char *path, char *buf, size_t buflen) +{ + ssize_t numwritten; + int fd; + + fd = open(path, O_WRONLY); + if (fd == -1) + return 0; + + numwritten = write(fd, buf, buflen - 1); + if (numwritten < 1) { + perror(path); + close(fd); + return -1; + } + + close(fd); + + return (unsigned int) numwritten; +} + /* * Detect whether a CPU is online * diff --git a/tools/power/cpupower/lib/cpupower_intern.h b/tools/power/cpupower/lib/cpupower_intern.h index 4887c76d23f8..ac1112b956ec 100644 --- a/tools/power/cpupower/lib/cpupower_intern.h +++ b/tools/power/cpupower/lib/cpupower_intern.h @@ -1,6 +1,11 @@ /* SPDX-License-Identifier: GPL-2.0 */ #define PATH_TO_CPU "/sys/devices/system/cpu/" + +#ifndef MAX_LINE_LEN #define MAX_LINE_LEN 4096 +#endif + #define SYSFS_PATH_MAX 255 unsigned int cpupower_read_sysfs(const char *path, char *buf, size_t buflen); +unsigned int cpupower_write_sysfs(const char *path, char *buf, size_t buflen); diff --git a/tools/power/cpupower/utils/cpupower-info.c b/tools/power/cpupower/utils/cpupower-info.c index 0ba61a2c4d81..06345b543786 100644 --- a/tools/power/cpupower/utils/cpupower-info.c +++ b/tools/power/cpupower/utils/cpupower-info.c @@ -101,7 +101,7 @@ int cmd_info(int argc, char **argv) } if (params.perf_bias) { - ret = msr_intel_get_perf_bias(cpu); + ret = cpupower_intel_get_perf_bias(cpu); if (ret < 0) { fprintf(stderr, _("Could not read perf-bias value[%d]\n"), ret); diff --git a/tools/power/cpupower/utils/cpupower-set.c b/tools/power/cpupower/utils/cpupower-set.c index 052044d7e012..180d5ba877e6 100644 --- a/tools/power/cpupower/utils/cpupower-set.c +++ b/tools/power/cpupower/utils/cpupower-set.c @@ -95,7 +95,7 @@ int cmd_set(int argc, char **argv) } if (params.perf_bias) { - ret = msr_intel_set_perf_bias(cpu, perf_bias); + ret = cpupower_intel_set_perf_bias(cpu, perf_bias); if (ret) { fprintf(stderr, _("Error setting perf-bias " "value on CPU %d\n"), cpu); diff --git a/tools/power/cpupower/utils/helpers/helpers.h b/tools/power/cpupower/utils/helpers/helpers.h index c258eeccd05f..37dac161f3fe 100644 --- a/tools/power/cpupower/utils/helpers/helpers.h +++ b/tools/power/cpupower/utils/helpers/helpers.h @@ -105,8 +105,8 @@ extern struct cpupower_cpu_info cpupower_cpu_info; extern int read_msr(int cpu, unsigned int idx, unsigned long long *val); extern int write_msr(int cpu, unsigned int idx, unsigned long long val); -extern int msr_intel_set_perf_bias(unsigned int cpu, unsigned int val); -extern int msr_intel_get_perf_bias(unsigned int cpu); +extern int cpupower_intel_set_perf_bias(unsigned int cpu, unsigned int val); +extern int cpupower_intel_get_perf_bias(unsigned int cpu); extern unsigned long long msr_intel_get_turbo_ratio(unsigned int cpu); /* Read/Write msr ****************************/ @@ -150,9 +150,9 @@ static inline int read_msr(int cpu, unsigned int idx, unsigned long long *val) { return -1; }; static inline int write_msr(int cpu, unsigned int idx, unsigned long long val) { return -1; }; -static inline int msr_intel_set_perf_bias(unsigned int cpu, unsigned int val) +static inline int cpupower_intel_set_perf_bias(unsigned int cpu, unsigned int val) { return -1; }; -static inline int msr_intel_get_perf_bias(unsigned int cpu) +static inline int cpupower_intel_get_perf_bias(unsigned int cpu) { return -1; }; static inline unsigned long long msr_intel_get_turbo_ratio(unsigned int cpu) { return 0; }; diff --git a/tools/power/cpupower/utils/helpers/misc.c b/tools/power/cpupower/utils/helpers/misc.c index f406adc40bad..e8f8f643a627 100644 --- a/tools/power/cpupower/utils/helpers/misc.c +++ b/tools/power/cpupower/utils/helpers/misc.c @@ -1,7 +1,15 @@ // SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + #if defined(__i386__) || defined(__x86_64__) #include "helpers/helpers.h" +#include "helpers/sysfs.h" + +#include "cpupower_intern.h" #define MSR_AMD_HWCR 0xc0010015 @@ -40,4 +48,44 @@ int cpufreq_has_boost_support(unsigned int cpu, int *support, int *active, *support = *active = 1; return 0; } + +int cpupower_intel_get_perf_bias(unsigned int cpu) +{ + char linebuf[MAX_LINE_LEN]; + char path[SYSFS_PATH_MAX]; + unsigned long val; + char *endp; + + if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_PERF_BIAS)) + return -1; + + snprintf(path, sizeof(path), PATH_TO_CPU "cpu%u/power/energy_perf_bias", cpu); + + if (cpupower_read_sysfs(path, linebuf, MAX_LINE_LEN) == 0) + return -1; + + val = strtol(linebuf, &endp, 0); + if (endp == linebuf || errno == ERANGE) + return -1; + + return val; +} + +int cpupower_intel_set_perf_bias(unsigned int cpu, unsigned int val) +{ + char path[SYSFS_PATH_MAX]; + char linebuf[3] = {}; + + if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_PERF_BIAS)) + return -1; + + snprintf(path, sizeof(path), PATH_TO_CPU "cpu%u/power/energy_perf_bias", cpu); + snprintf(linebuf, sizeof(linebuf), "%d", val); + + if (cpupower_write_sysfs(path, linebuf, 3) <= 0) + return -1; + + return 0; +} + #endif /* #if defined(__i386__) || defined(__x86_64__) */ diff --git a/tools/power/cpupower/utils/helpers/msr.c b/tools/power/cpupower/utils/helpers/msr.c index ab9950748838..8b0b6be74bb8 100644 --- a/tools/power/cpupower/utils/helpers/msr.c +++ b/tools/power/cpupower/utils/helpers/msr.c @@ -11,7 +11,6 @@ /* Intel specific MSRs */ #define MSR_IA32_PERF_STATUS 0x198 #define MSR_IA32_MISC_ENABLES 0x1a0 -#define MSR_IA32_ENERGY_PERF_BIAS 0x1b0 #define MSR_NEHALEM_TURBO_RATIO_LIMIT 0x1ad /* @@ -73,33 +72,6 @@ int write_msr(int cpu, unsigned int idx, unsigned long long val) return -1; } -int msr_intel_get_perf_bias(unsigned int cpu) -{ - unsigned long long val; - int ret; - - if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_PERF_BIAS)) - return -1; - - ret = read_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, &val); - if (ret) - return ret; - return val; -} - -int msr_intel_set_perf_bias(unsigned int cpu, unsigned int val) -{ - int ret; - - if (!(cpupower_cpu_info.caps & CPUPOWER_CAP_PERF_BIAS)) - return -1; - - ret = write_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, val); - if (ret) - return ret; - return 0; -} - unsigned long long msr_intel_get_turbo_ratio(unsigned int cpu) { unsigned long long val; From 6d6501d912a9a5e1b73d7fbf419b90a8ec11ed7a Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Thu, 15 Oct 2020 14:50:16 +0200 Subject: [PATCH 162/360] tools/power/turbostat: Read energy_perf_bias from sysfs ... instead of poking at the MSR directly. Signed-off-by: Borislav Petkov Cc: Len Brown Cc: linux-pm@vger.kernel.org Link: https://lkml.kernel.org/r/20201029190259.3476-3-bp@alien8.de --- tools/power/x86/turbostat/turbostat.c | 29 ++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index 33b370865d16..0baec7ea1bbd 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -1721,6 +1721,25 @@ int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp) return 0; } +int get_epb(int cpu) +{ + char path[128 + PATH_BYTES]; + int ret, epb = -1; + FILE *fp; + + sprintf(path, "/sys/devices/system/cpu/cpu%d/power/energy_perf_bias", cpu); + + fp = fopen_or_die(path, "r"); + + ret = fscanf(fp, "%d", &epb); + if (ret != 1) + err(1, "%s(%s)", __func__, path); + + fclose(fp); + + return epb; +} + void get_apic_id(struct thread_data *t) { unsigned int eax, ebx, ecx, edx; @@ -3631,9 +3650,8 @@ dump_sysfs_pstate_config(void) */ int print_epb(struct thread_data *t, struct core_data *c, struct pkg_data *p) { - unsigned long long msr; char *epb_string; - int cpu; + int cpu, epb; if (!has_epb) return 0; @@ -3649,10 +3667,11 @@ int print_epb(struct thread_data *t, struct core_data *c, struct pkg_data *p) return -1; } - if (get_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, &msr)) + epb = get_epb(cpu); + if (epb < 0) return 0; - switch (msr & 0xF) { + switch (epb) { case ENERGY_PERF_BIAS_PERFORMANCE: epb_string = "performance"; break; @@ -3666,7 +3685,7 @@ int print_epb(struct thread_data *t, struct core_data *c, struct pkg_data *p) epb_string = "custom"; break; } - fprintf(outf, "cpu%d: MSR_IA32_ENERGY_PERF_BIAS: 0x%08llx (%s)\n", cpu, msr, epb_string); + fprintf(outf, "cpu%d: EPB: %d (%s)\n", cpu, epb, epb_string); return 0; } From fe0a5788624c8b8f113a35bbe4636e37f9321241 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Thu, 15 Oct 2020 14:58:48 +0200 Subject: [PATCH 163/360] tools/power/x86_energy_perf_policy: Read energy_perf_bias from sysfs ... and stop poking at the MSR directly. Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201029190259.3476-4-bp@alien8.de --- .../x86_energy_perf_policy.c | 109 ++++++++++++++++-- 1 file changed, 99 insertions(+), 10 deletions(-) diff --git a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c index 3fe1eed900d4..ad6aed1fabf8 100644 --- a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c +++ b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c @@ -91,6 +91,9 @@ unsigned int has_hwp_request_pkg; /* IA32_HWP_REQUEST_PKG */ unsigned int bdx_highest_ratio; +#define PATH_TO_CPU "/sys/devices/system/cpu/" +#define SYSFS_PATH_MAX 255 + /* * maintain compatibility with original implementation, but don't document it: */ @@ -668,6 +671,48 @@ int put_msr(int cpu, int offset, unsigned long long new_msr) return 0; } +static unsigned int read_sysfs(const char *path, char *buf, size_t buflen) +{ + ssize_t numread; + int fd; + + fd = open(path, O_RDONLY); + if (fd == -1) + return 0; + + numread = read(fd, buf, buflen - 1); + if (numread < 1) { + close(fd); + return 0; + } + + buf[numread] = '\0'; + close(fd); + + return (unsigned int) numread; +} + +static unsigned int write_sysfs(const char *path, char *buf, size_t buflen) +{ + ssize_t numwritten; + int fd; + + fd = open(path, O_WRONLY); + if (fd == -1) + return 0; + + numwritten = write(fd, buf, buflen - 1); + if (numwritten < 1) { + perror("write failed\n"); + close(fd); + return -1; + } + + close(fd); + + return (unsigned int) numwritten; +} + void print_hwp_cap(int cpu, struct msr_hwp_cap *cap, char *str) { if (cpu != -1) @@ -745,17 +790,61 @@ void write_hwp_request(int cpu, struct msr_hwp_request *hwp_req, unsigned int ms put_msr(cpu, msr_offset, msr); } +static int get_epb(int cpu) +{ + char path[SYSFS_PATH_MAX]; + char linebuf[3]; + char *endp; + long val; + + if (!has_epb) + return -1; + + snprintf(path, sizeof(path), PATH_TO_CPU "cpu%u/power/energy_perf_bias", cpu); + + if (!read_sysfs(path, linebuf, 3)) + return -1; + + val = strtol(linebuf, &endp, 0); + if (endp == linebuf || errno == ERANGE) + return -1; + + return (int)val; +} + +static int set_epb(int cpu, int val) +{ + char path[SYSFS_PATH_MAX]; + char linebuf[3]; + char *endp; + int ret; + + if (!has_epb) + return -1; + + snprintf(path, sizeof(path), PATH_TO_CPU "cpu%u/power/energy_perf_bias", cpu); + snprintf(linebuf, sizeof(linebuf), "%d", val); + + ret = write_sysfs(path, linebuf, 3); + if (ret <= 0) + return -1; + + val = strtol(linebuf, &endp, 0); + if (endp == linebuf || errno == ERANGE) + return -1; + + return (int)val; +} + int print_cpu_msrs(int cpu) { - unsigned long long msr; struct msr_hwp_request req; struct msr_hwp_cap cap; + int epb; - if (has_epb) { - get_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, &msr); - - printf("cpu%d: EPB %u\n", cpu, (unsigned int) msr); - } + epb = get_epb(cpu); + if (epb >= 0) + printf("cpu%d: EPB %u\n", cpu, (unsigned int) epb); if (!has_hwp) return 0; @@ -1038,15 +1127,15 @@ int enable_hwp_on_cpu(int cpu) int update_cpu_msrs(int cpu) { unsigned long long msr; - + int epb; if (update_epb) { - get_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, &msr); - put_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, new_epb); + epb = get_epb(cpu); + set_epb(cpu, new_epb); if (verbose) printf("cpu%d: ENERGY_PERF_BIAS old: %d new: %d\n", - cpu, (unsigned int) msr, (unsigned int) new_epb); + cpu, epb, (unsigned int) new_epb); } if (update_turbo) { From 18741a5251d018094536a2dffe284d269ebb07fe Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Thu, 15 Oct 2020 15:00:31 +0200 Subject: [PATCH 164/360] x86/msr: Do not allow writes to MSR_IA32_ENERGY_PERF_BIAS Now that all in-kernel-tree users are converted to using the sysfs file, remove the MSR from the "allowlist". Signed-off-by: Borislav Petkov Reviewed-by: Shuah Khan Link: https://lkml.kernel.org/r/20201029190259.3476-5-bp@alien8.de --- arch/x86/kernel/msr.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c index c0d409810658..b1147862730c 100644 --- a/arch/x86/kernel/msr.c +++ b/arch/x86/kernel/msr.c @@ -99,9 +99,6 @@ static int filter_write(u32 reg) if (!__ratelimit(&fw_rs)) return 0; - if (reg == MSR_IA32_ENERGY_PERF_BIAS) - return 0; - pr_err("Write to unrecognized MSR 0x%x by %s (pid: %d). Please report to x86@kernel.org.\n", reg, current->comm, current->pid); From 7bf765dd8442d2b482803f50613b9c2543491f4a Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 13 Nov 2020 13:19:15 -0800 Subject: [PATCH 165/360] fs-verity: remove filenames from file comments Embedding the file path inside kernel source code files isn't particularly useful as often files are moved around and the paths become incorrect. checkpatch.pl warns about this since v5.10-rc1. Acked-by: Luca Boccassi Link: https://lore.kernel.org/r/20201113211918.71883-2-ebiggers@kernel.org Signed-off-by: Eric Biggers --- fs/verity/enable.c | 2 +- fs/verity/hash_algs.c | 2 +- fs/verity/init.c | 2 +- fs/verity/measure.c | 2 +- fs/verity/open.c | 2 +- fs/verity/signature.c | 2 +- fs/verity/verify.c | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fs/verity/enable.c b/fs/verity/enable.c index 5ab3bbec8108..9c5b28c86522 100644 --- a/fs/verity/enable.c +++ b/fs/verity/enable.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * fs/verity/enable.c: ioctl to enable verity on a file + * Ioctl to enable verity on a file * * Copyright 2019 Google LLC */ diff --git a/fs/verity/hash_algs.c b/fs/verity/hash_algs.c index c37e186ebeb6..71d0fccb6d4c 100644 --- a/fs/verity/hash_algs.c +++ b/fs/verity/hash_algs.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * fs/verity/hash_algs.c: fs-verity hash algorithms + * fs-verity hash algorithms * * Copyright 2019 Google LLC */ diff --git a/fs/verity/init.c b/fs/verity/init.c index 94c104e00861..c98b7016f446 100644 --- a/fs/verity/init.c +++ b/fs/verity/init.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * fs/verity/init.c: fs-verity module initialization and logging + * fs-verity module initialization and logging * * Copyright 2019 Google LLC */ diff --git a/fs/verity/measure.c b/fs/verity/measure.c index df409a5682ed..5300b8d38537 100644 --- a/fs/verity/measure.c +++ b/fs/verity/measure.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * fs/verity/measure.c: ioctl to get a verity file's measurement + * Ioctl to get a verity file's measurement * * Copyright 2019 Google LLC */ diff --git a/fs/verity/open.c b/fs/verity/open.c index bfe0280c14e4..a28d5be78a09 100644 --- a/fs/verity/open.c +++ b/fs/verity/open.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * fs/verity/open.c: opening fs-verity files + * Opening fs-verity files * * Copyright 2019 Google LLC */ diff --git a/fs/verity/signature.c b/fs/verity/signature.c index b14ed96387ec..12794a4dd158 100644 --- a/fs/verity/signature.c +++ b/fs/verity/signature.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * fs/verity/signature.c: verification of builtin signatures + * Verification of builtin signatures * * Copyright 2019 Google LLC */ diff --git a/fs/verity/verify.c b/fs/verity/verify.c index a8b68c6f663d..0adb970f4e73 100644 --- a/fs/verity/verify.c +++ b/fs/verity/verify.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * fs/verity/verify.c: data verification functions, i.e. hooks for ->readpages() + * Data verification functions, i.e. hooks for ->readpages() * * Copyright 2019 Google LLC */ From 9e90f30e78572ecfc1c74c735a034c955d822ba6 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 13 Nov 2020 13:19:16 -0800 Subject: [PATCH 166/360] fs-verity: rename fsverity_signed_digest to fsverity_formatted_digest The name "struct fsverity_signed_digest" is causing confusion because it isn't actually a signed digest, but rather it's the way that the digest is formatted in order to be signed. Rename it to "struct fsverity_formatted_digest" to prevent this confusion. Also update the struct's comment to clarify that it's specific to the built-in signature verification support and isn't a requirement for all fs-verity users. I'll be renaming this struct in fsverity-utils too. Acked-by: Luca Boccassi Link: https://lore.kernel.org/r/20201113211918.71883-3-ebiggers@kernel.org Signed-off-by: Eric Biggers --- Documentation/filesystems/fsverity.rst | 2 +- fs/verity/fsverity_private.h | 17 ++++++++++++----- fs/verity/signature.c | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Documentation/filesystems/fsverity.rst b/Documentation/filesystems/fsverity.rst index 895e9711ed88..421b75498d49 100644 --- a/Documentation/filesystems/fsverity.rst +++ b/Documentation/filesystems/fsverity.rst @@ -372,7 +372,7 @@ kernel. Specifically, it adds support for: File measurements must be signed in the following format, which is similar to the structure used by `FS_IOC_MEASURE_VERITY`_:: - struct fsverity_signed_digest { + struct fsverity_formatted_digest { char magic[8]; /* must be "FSVerity" */ __le16 digest_algorithm; __le16 digest_size; diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index e96d99d5145e..75f8e18b44a5 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -101,12 +101,19 @@ struct fsverity_descriptor { sizeof(struct fsverity_descriptor)) /* - * Format in which verity file measurements are signed. This is the same as - * 'struct fsverity_digest', except here some magic bytes are prepended to - * provide some context about what is being signed in case the same key is used - * for non-fsverity purposes, and here the fields have fixed endianness. + * Format in which verity file measurements are signed in built-in signatures. + * This is the same as 'struct fsverity_digest', except here some magic bytes + * are prepended to provide some context about what is being signed in case the + * same key is used for non-fsverity purposes, and here the fields have fixed + * endianness. + * + * This struct is specific to the built-in signature verification support, which + * is optional. fs-verity users may also verify signatures in userspace, in + * which case userspace is responsible for deciding on what bytes are signed. + * This struct may still be used, but it doesn't have to be. For example, + * userspace could instead use a string like "sha256:$digest_as_hex_string". */ -struct fsverity_signed_digest { +struct fsverity_formatted_digest { char magic[8]; /* must be "FSVerity" */ __le16 digest_algorithm; __le16 digest_size; diff --git a/fs/verity/signature.c b/fs/verity/signature.c index 12794a4dd158..74ae10f04d21 100644 --- a/fs/verity/signature.c +++ b/fs/verity/signature.c @@ -44,7 +44,7 @@ int fsverity_verify_signature(const struct fsverity_info *vi, const struct inode *inode = vi->inode; const struct fsverity_hash_alg *hash_alg = vi->tree_params.hash_alg; const u32 sig_size = le32_to_cpu(desc->sig_size); - struct fsverity_signed_digest *d; + struct fsverity_formatted_digest *d; int err; if (sig_size == 0) { From ed45e201649344412445d6f65f0473a6112f0bcd Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 13 Nov 2020 13:19:17 -0800 Subject: [PATCH 167/360] fs-verity: rename "file measurement" to "file digest" I originally chose the name "file measurement" to refer to the fs-verity file digest to avoid confusion with traditional full-file digests or with the bare root hash of the Merkle tree. But the name "file measurement" hasn't caught on, and usually people are calling it something else, usually the "file digest". E.g. see "struct fsverity_digest" and "struct fsverity_formatted_digest", the libfsverity_compute_digest() and libfsverity_sign_digest() functions in libfsverity, and the "fsverity digest" command. Having multiple names for the same thing is always confusing. So to hopefully avoid confusion in the future, rename "fs-verity file measurement" to "fs-verity file digest". This leaves FS_IOC_MEASURE_VERITY as the only reference to "measure" in the kernel, which makes some amount of sense since the ioctl is actively "measuring" the file. I'll be renaming this in fsverity-utils too (though similarly the 'fsverity measure' command, which is a wrapper for FS_IOC_MEASURE_VERITY, will stay). Acked-by: Luca Boccassi Link: https://lore.kernel.org/r/20201113211918.71883-4-ebiggers@kernel.org Signed-off-by: Eric Biggers --- Documentation/filesystems/fsverity.rst | 60 +++++++++++++------------- fs/verity/enable.c | 6 +-- fs/verity/fsverity_private.h | 12 +++--- fs/verity/measure.c | 12 +++--- fs/verity/open.c | 22 +++++----- fs/verity/signature.c | 10 ++--- 6 files changed, 61 insertions(+), 61 deletions(-) diff --git a/Documentation/filesystems/fsverity.rst b/Documentation/filesystems/fsverity.rst index 421b75498d49..2eee558b7f5f 100644 --- a/Documentation/filesystems/fsverity.rst +++ b/Documentation/filesystems/fsverity.rst @@ -27,9 +27,9 @@ automatically verified against the file's Merkle tree. Reads of any corrupted data, including mmap reads, will fail. Userspace can use another ioctl to retrieve the root hash (actually -the "file measurement", which is a hash that includes the root hash) -that fs-verity is enforcing for the file. This ioctl executes in -constant time, regardless of the file size. +the "fs-verity file digest", which is a hash that includes the Merkle +tree root hash) that fs-verity is enforcing for the file. This ioctl +executes in constant time, regardless of the file size. fs-verity is essentially a way to hash a file in constant time, subject to the caveat that reads which would violate the hash will @@ -177,9 +177,10 @@ FS_IOC_ENABLE_VERITY can fail with the following errors: FS_IOC_MEASURE_VERITY --------------------- -The FS_IOC_MEASURE_VERITY ioctl retrieves the measurement of a verity -file. The file measurement is a digest that cryptographically -identifies the file contents that are being enforced on reads. +The FS_IOC_MEASURE_VERITY ioctl retrieves the digest of a verity file. +The fs-verity file digest is a cryptographic digest that identifies +the file contents that are being enforced on reads; it is computed via +a Merkle tree and is different from a traditional full-file digest. This ioctl takes in a pointer to a variable-length structure:: @@ -197,7 +198,7 @@ On success, 0 is returned and the kernel fills in the structure as follows: - ``digest_algorithm`` will be the hash algorithm used for the file - measurement. It will match ``fsverity_enable_arg::hash_algorithm``. + digest. It will match ``fsverity_enable_arg::hash_algorithm``. - ``digest_size`` will be the size of the digest in bytes, e.g. 32 for SHA-256. (This can be redundant with ``digest_algorithm``.) - ``digest`` will be the actual bytes of the digest. @@ -257,25 +258,24 @@ non-verity one, with the following exceptions: with EIO (for read()) or SIGBUS (for mmap() reads). - If the sysctl "fs.verity.require_signatures" is set to 1 and the - file's verity measurement is not signed by a key in the fs-verity - keyring, then opening the file will fail. See `Built-in signature - verification`_. + file is not signed by a key in the fs-verity keyring, then opening + the file will fail. See `Built-in signature verification`_. Direct access to the Merkle tree is not supported. Therefore, if a verity file is copied, or is backed up and restored, then it will lose its "verity"-ness. fs-verity is primarily meant for files like executables that are managed by a package manager. -File measurement computation -============================ +File digest computation +======================= This section describes how fs-verity hashes the file contents using a -Merkle tree to produce the "file measurement" which cryptographically -identifies the file contents. This algorithm is the same for all -filesystems that support fs-verity. +Merkle tree to produce the digest which cryptographically identifies +the file contents. This algorithm is the same for all filesystems +that support fs-verity. Userspace only needs to be aware of this algorithm if it needs to -compute the file measurement itself, e.g. in order to sign the file. +compute fs-verity file digests itself, e.g. in order to sign files. .. _fsverity_merkle_tree: @@ -325,9 +325,9 @@ can't a distinguish a large file from a small second file whose data is exactly the top-level hash block of the first file. Ambiguities also arise from the convention of padding to the next block boundary. -To solve this problem, the verity file measurement is actually -computed as a hash of the following structure, which contains the -Merkle tree root hash as well as other fields such as the file size:: +To solve this problem, the fs-verity file digest is actually computed +as a hash of the following structure, which contains the Merkle tree +root hash as well as other fields such as the file size:: struct fsverity_descriptor { __u8 version; /* must be 1 */ @@ -359,18 +359,18 @@ kernel. Specifically, it adds support for: certificates from being added. 2. `FS_IOC_ENABLE_VERITY`_ accepts a pointer to a PKCS#7 formatted - detached signature in DER format of the file measurement. On - success, this signature is persisted alongside the Merkle tree. + detached signature in DER format of the file's fs-verity digest. + On success, this signature is persisted alongside the Merkle tree. Then, any time the file is opened, the kernel will verify the - file's actual measurement against this signature, using the - certificates in the ".fs-verity" keyring. + file's actual digest against this signature, using the certificates + in the ".fs-verity" keyring. 3. A new sysctl "fs.verity.require_signatures" is made available. When set to 1, the kernel requires that all verity files have a - correctly signed file measurement as described in (2). + correctly signed digest as described in (2). -File measurements must be signed in the following format, which is -similar to the structure used by `FS_IOC_MEASURE_VERITY`_:: +fs-verity file digests must be signed in the following format, which +is similar to the structure used by `FS_IOC_MEASURE_VERITY`_:: struct fsverity_formatted_digest { char magic[8]; /* must be "FSVerity" */ @@ -421,8 +421,8 @@ can only be set by `FS_IOC_ENABLE_VERITY`_, and it cannot be cleared. ext4 also supports encryption, which can be used simultaneously with fs-verity. In this case, the plaintext data is verified rather than -the ciphertext. This is necessary in order to make the file -measurement meaningful, since every file is encrypted differently. +the ciphertext. This is necessary in order to make the fs-verity file +digest meaningful, since every file is encrypted differently. ext4 stores the verity metadata (Merkle tree and fsverity_descriptor) past the end of the file, starting at the first 64K boundary beyond @@ -592,8 +592,8 @@ weren't already directly answered in other parts of this document. :Q: Isn't fs-verity useless because the attacker can just modify the hashes in the Merkle tree, which is stored on-disk? :A: To verify the authenticity of an fs-verity file you must verify - the authenticity of the "file measurement", which is basically the - root hash of the Merkle tree. See `Use cases`_. + the authenticity of the "fs-verity file digest", which + incorporates the root hash of the Merkle tree. See `Use cases`_. :Q: Isn't fs-verity useless because the attacker can just replace a verity file with a non-verity one? diff --git a/fs/verity/enable.c b/fs/verity/enable.c index 9c5b28c86522..f7e997a01ad0 100644 --- a/fs/verity/enable.c +++ b/fs/verity/enable.c @@ -398,9 +398,9 @@ int fsverity_ioctl_enable(struct file *filp, const void __user *uarg) * Some pages of the file may have been evicted from pagecache after * being used in the Merkle tree construction, then read into pagecache * again by another process reading from the file concurrently. Since - * these pages didn't undergo verification against the file measurement - * which fs-verity now claims to be enforcing, we have to wipe the - * pagecache to ensure that all future reads are verified. + * these pages didn't undergo verification against the file digest which + * fs-verity now claims to be enforcing, we have to wipe the pagecache + * to ensure that all future reads are verified. */ filemap_write_and_wait(inode->i_mapping); invalidate_inode_pages2(inode->i_mapping); diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index 75f8e18b44a5..21e9930d65fb 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -67,19 +67,19 @@ struct merkle_tree_params { * When a verity file is first opened, an instance of this struct is allocated * and stored in ->i_verity_info; it remains until the inode is evicted. It * caches information about the Merkle tree that's needed to efficiently verify - * data read from the file. It also caches the file measurement. The Merkle - * tree pages themselves are not cached here, but the filesystem may cache them. + * data read from the file. It also caches the file digest. The Merkle tree + * pages themselves are not cached here, but the filesystem may cache them. */ struct fsverity_info { struct merkle_tree_params tree_params; u8 root_hash[FS_VERITY_MAX_DIGEST_SIZE]; - u8 measurement[FS_VERITY_MAX_DIGEST_SIZE]; + u8 file_digest[FS_VERITY_MAX_DIGEST_SIZE]; const struct inode *inode; }; /* - * Merkle tree properties. The file measurement is the hash of this structure - * excluding the signature and with the sig_size field set to 0. + * Merkle tree properties. The fs-verity file digest is the hash of this + * structure excluding the signature and with the sig_size field set to 0. */ struct fsverity_descriptor { __u8 version; /* must be 1 */ @@ -101,7 +101,7 @@ struct fsverity_descriptor { sizeof(struct fsverity_descriptor)) /* - * Format in which verity file measurements are signed in built-in signatures. + * Format in which fs-verity file digests are signed in built-in signatures. * This is the same as 'struct fsverity_digest', except here some magic bytes * are prepended to provide some context about what is being signed in case the * same key is used for non-fsverity purposes, and here the fields have fixed diff --git a/fs/verity/measure.c b/fs/verity/measure.c index 5300b8d38537..f0d7b30c62db 100644 --- a/fs/verity/measure.c +++ b/fs/verity/measure.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Ioctl to get a verity file's measurement + * Ioctl to get a verity file's digest * * Copyright 2019 Google LLC */ @@ -10,12 +10,12 @@ #include /** - * fsverity_ioctl_measure() - get a verity file's measurement - * @filp: file to get measurement of + * fsverity_ioctl_measure() - get a verity file's digest + * @filp: file to get digest of * @_uarg: user pointer to fsverity_digest * - * Retrieve the file measurement that the kernel is enforcing for reads from a - * verity file. See the "FS_IOC_MEASURE_VERITY" section of + * Retrieve the file digest that the kernel is enforcing for reads from a verity + * file. See the "FS_IOC_MEASURE_VERITY" section of * Documentation/filesystems/fsverity.rst for the documentation. * * Return: 0 on success, -errno on failure @@ -51,7 +51,7 @@ int fsverity_ioctl_measure(struct file *filp, void __user *_uarg) if (copy_to_user(uarg, &arg, sizeof(arg))) return -EFAULT; - if (copy_to_user(uarg->digest, vi->measurement, hash_alg->digest_size)) + if (copy_to_user(uarg->digest, vi->file_digest, hash_alg->digest_size)) return -EFAULT; return 0; diff --git a/fs/verity/open.c b/fs/verity/open.c index a28d5be78a09..228d0eca3e2e 100644 --- a/fs/verity/open.c +++ b/fs/verity/open.c @@ -124,18 +124,18 @@ out_err: } /* - * Compute the file measurement by hashing the fsverity_descriptor excluding the + * Compute the file digest by hashing the fsverity_descriptor excluding the * signature and with the sig_size field set to 0. */ -static int compute_file_measurement(struct fsverity_hash_alg *hash_alg, - struct fsverity_descriptor *desc, - u8 *measurement) +static int compute_file_digest(struct fsverity_hash_alg *hash_alg, + struct fsverity_descriptor *desc, + u8 *file_digest) { __le32 sig_size = desc->sig_size; int err; desc->sig_size = 0; - err = fsverity_hash_buffer(hash_alg, desc, sizeof(*desc), measurement); + err = fsverity_hash_buffer(hash_alg, desc, sizeof(*desc), file_digest); desc->sig_size = sig_size; return err; @@ -199,15 +199,15 @@ struct fsverity_info *fsverity_create_info(const struct inode *inode, memcpy(vi->root_hash, desc->root_hash, vi->tree_params.digest_size); - err = compute_file_measurement(vi->tree_params.hash_alg, desc, - vi->measurement); + err = compute_file_digest(vi->tree_params.hash_alg, desc, + vi->file_digest); if (err) { - fsverity_err(inode, "Error %d computing file measurement", err); + fsverity_err(inode, "Error %d computing file digest", err); goto out; } - pr_debug("Computed file measurement: %s:%*phN\n", + pr_debug("Computed file digest: %s:%*phN\n", vi->tree_params.hash_alg->name, - vi->tree_params.digest_size, vi->measurement); + vi->tree_params.digest_size, vi->file_digest); err = fsverity_verify_signature(vi, desc, desc_size); out: @@ -354,7 +354,7 @@ int __init fsverity_init_info_cache(void) { fsverity_info_cachep = KMEM_CACHE_USERCOPY(fsverity_info, SLAB_RECLAIM_ACCOUNT, - measurement); + file_digest); if (!fsverity_info_cachep) return -ENOMEM; return 0; diff --git a/fs/verity/signature.c b/fs/verity/signature.c index 74ae10f04d21..012468eda2a7 100644 --- a/fs/verity/signature.c +++ b/fs/verity/signature.c @@ -32,8 +32,8 @@ static struct key *fsverity_keyring; * @desc: the file's fsverity_descriptor * @desc_size: size of @desc * - * If the file's fs-verity descriptor includes a signature of the file - * measurement, verify it against the certificates in the fs-verity keyring. + * If the file's fs-verity descriptor includes a signature of the file digest, + * verify it against the certificates in the fs-verity keyring. * * Return: 0 on success (signature valid or not required); -errno on failure */ @@ -67,7 +67,7 @@ int fsverity_verify_signature(const struct fsverity_info *vi, memcpy(d->magic, "FSVerity", 8); d->digest_algorithm = cpu_to_le16(hash_alg - fsverity_hash_algs); d->digest_size = cpu_to_le16(hash_alg->digest_size); - memcpy(d->digest, vi->measurement, hash_alg->digest_size); + memcpy(d->digest, vi->file_digest, hash_alg->digest_size); err = verify_pkcs7_signature(d, sizeof(*d) + hash_alg->digest_size, desc->signature, sig_size, @@ -90,8 +90,8 @@ int fsverity_verify_signature(const struct fsverity_info *vi, return err; } - pr_debug("Valid signature for file measurement %s:%*phN\n", - hash_alg->name, hash_alg->digest_size, vi->measurement); + pr_debug("Valid signature for file digest %s:%*phN\n", + hash_alg->name, hash_alg->digest_size, vi->file_digest); return 0; } From 70d3b8ddcd20d3c859676f56c43c7b2360c70266 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 13 Nov 2020 00:01:12 +0200 Subject: [PATCH 168/360] x86/sgx: Add SGX architectural data structures Define the SGX architectural data structures used by various SGX functions. This is not an exhaustive representation of all SGX data structures but only those needed by the kernel. The goal is to sequester hardware structures in "sgx/arch.h" and keep them separate from kernel-internal or uapi structures. The data structures are described in Intel SDM section 37.6. Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-2-jarkko@kernel.org --- arch/x86/kernel/cpu/sgx/arch.h | 338 +++++++++++++++++++++++++++++++++ 1 file changed, 338 insertions(+) create mode 100644 arch/x86/kernel/cpu/sgx/arch.h diff --git a/arch/x86/kernel/cpu/sgx/arch.h b/arch/x86/kernel/cpu/sgx/arch.h new file mode 100644 index 000000000000..dd7602c44c72 --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/arch.h @@ -0,0 +1,338 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/** + * Copyright(c) 2016-20 Intel Corporation. + * + * Contains data structures defined by the SGX architecture. Data structures + * defined by the Linux software stack should not be placed here. + */ +#ifndef _ASM_X86_SGX_ARCH_H +#define _ASM_X86_SGX_ARCH_H + +#include +#include + +/* The SGX specific CPUID function. */ +#define SGX_CPUID 0x12 +/* EPC enumeration. */ +#define SGX_CPUID_EPC 2 +/* An invalid EPC section, i.e. the end marker. */ +#define SGX_CPUID_EPC_INVALID 0x0 +/* A valid EPC section. */ +#define SGX_CPUID_EPC_SECTION 0x1 +/* The bitmask for the EPC section type. */ +#define SGX_CPUID_EPC_MASK GENMASK(3, 0) + +/** + * enum sgx_return_code - The return code type for ENCLS, ENCLU and ENCLV + * %SGX_NOT_TRACKED: Previous ETRACK's shootdown sequence has not + * been completed yet. + * %SGX_INVALID_EINITTOKEN: EINITTOKEN is invalid and enclave signer's + * public key does not match IA32_SGXLEPUBKEYHASH. + * %SGX_UNMASKED_EVENT: An unmasked event, e.g. INTR, was received + */ +enum sgx_return_code { + SGX_NOT_TRACKED = 11, + SGX_INVALID_EINITTOKEN = 16, + SGX_UNMASKED_EVENT = 128, +}; + +/* The modulus size for 3072-bit RSA keys. */ +#define SGX_MODULUS_SIZE 384 + +/** + * enum sgx_miscselect - additional information to an SSA frame + * %SGX_MISC_EXINFO: Report #PF or #GP to the SSA frame. + * + * Save State Area (SSA) is a stack inside the enclave used to store processor + * state when an exception or interrupt occurs. This enum defines additional + * information stored to an SSA frame. + */ +enum sgx_miscselect { + SGX_MISC_EXINFO = BIT(0), +}; + +#define SGX_MISC_RESERVED_MASK GENMASK_ULL(63, 1) + +#define SGX_SSA_GPRS_SIZE 184 +#define SGX_SSA_MISC_EXINFO_SIZE 16 + +/** + * enum sgx_attributes - the attributes field in &struct sgx_secs + * %SGX_ATTR_INIT: Enclave can be entered (is initialized). + * %SGX_ATTR_DEBUG: Allow ENCLS(EDBGRD) and ENCLS(EDBGWR). + * %SGX_ATTR_MODE64BIT: Tell that this a 64-bit enclave. + * %SGX_ATTR_PROVISIONKEY: Allow to use provisioning keys for remote + * attestation. + * %SGX_ATTR_KSS: Allow to use key separation and sharing (KSS). + * %SGX_ATTR_EINITTOKENKEY: Allow to use token signing key that is used to + * sign cryptographic tokens that can be passed to + * EINIT as an authorization to run an enclave. + */ +enum sgx_attribute { + SGX_ATTR_INIT = BIT(0), + SGX_ATTR_DEBUG = BIT(1), + SGX_ATTR_MODE64BIT = BIT(2), + SGX_ATTR_PROVISIONKEY = BIT(4), + SGX_ATTR_EINITTOKENKEY = BIT(5), + SGX_ATTR_KSS = BIT(7), +}; + +#define SGX_ATTR_RESERVED_MASK (BIT_ULL(3) | BIT_ULL(6) | GENMASK_ULL(63, 8)) + +/** + * struct sgx_secs - SGX Enclave Control Structure (SECS) + * @size: size of the address space + * @base: base address of the address space + * @ssa_frame_size: size of an SSA frame + * @miscselect: additional information stored to an SSA frame + * @attributes: attributes for enclave + * @xfrm: XSave-Feature Request Mask (subset of XCR0) + * @mrenclave: SHA256-hash of the enclave contents + * @mrsigner: SHA256-hash of the public key used to sign the SIGSTRUCT + * @config_id: a user-defined value that is used in key derivation + * @isv_prod_id: a user-defined value that is used in key derivation + * @isv_svn: a user-defined value that is used in key derivation + * @config_svn: a user-defined value that is used in key derivation + * + * SGX Enclave Control Structure (SECS) is a special enclave page that is not + * visible in the address space. In fact, this structure defines the address + * range and other global attributes for the enclave and it is the first EPC + * page created for any enclave. It is moved from a temporary buffer to an EPC + * by the means of ENCLS[ECREATE] function. + */ +struct sgx_secs { + u64 size; + u64 base; + u32 ssa_frame_size; + u32 miscselect; + u8 reserved1[24]; + u64 attributes; + u64 xfrm; + u32 mrenclave[8]; + u8 reserved2[32]; + u32 mrsigner[8]; + u8 reserved3[32]; + u32 config_id[16]; + u16 isv_prod_id; + u16 isv_svn; + u16 config_svn; + u8 reserved4[3834]; +} __packed; + +/** + * enum sgx_tcs_flags - execution flags for TCS + * %SGX_TCS_DBGOPTIN: If enabled allows single-stepping and breakpoints + * inside an enclave. It is cleared by EADD but can + * be set later with EDBGWR. + */ +enum sgx_tcs_flags { + SGX_TCS_DBGOPTIN = 0x01, +}; + +#define SGX_TCS_RESERVED_MASK GENMASK_ULL(63, 1) +#define SGX_TCS_RESERVED_SIZE 4024 + +/** + * struct sgx_tcs - Thread Control Structure (TCS) + * @state: used to mark an entered TCS + * @flags: execution flags (cleared by EADD) + * @ssa_offset: SSA stack offset relative to the enclave base + * @ssa_index: the current SSA frame index (cleard by EADD) + * @nr_ssa_frames: the number of frame in the SSA stack + * @entry_offset: entry point offset relative to the enclave base + * @exit_addr: address outside the enclave to exit on an exception or + * interrupt + * @fs_offset: offset relative to the enclave base to become FS + * segment inside the enclave + * @gs_offset: offset relative to the enclave base to become GS + * segment inside the enclave + * @fs_limit: size to become a new FS-limit (only 32-bit enclaves) + * @gs_limit: size to become a new GS-limit (only 32-bit enclaves) + * + * Thread Control Structure (TCS) is an enclave page visible in its address + * space that defines an entry point inside the enclave. A thread enters inside + * an enclave by supplying address of TCS to ENCLU(EENTER). A TCS can be entered + * by only one thread at a time. + */ +struct sgx_tcs { + u64 state; + u64 flags; + u64 ssa_offset; + u32 ssa_index; + u32 nr_ssa_frames; + u64 entry_offset; + u64 exit_addr; + u64 fs_offset; + u64 gs_offset; + u32 fs_limit; + u32 gs_limit; + u8 reserved[SGX_TCS_RESERVED_SIZE]; +} __packed; + +/** + * struct sgx_pageinfo - an enclave page descriptor + * @addr: address of the enclave page + * @contents: pointer to the page contents + * @metadata: pointer either to a SECINFO or PCMD instance + * @secs: address of the SECS page + */ +struct sgx_pageinfo { + u64 addr; + u64 contents; + u64 metadata; + u64 secs; +} __packed __aligned(32); + + +/** + * enum sgx_page_type - bits in the SECINFO flags defining the page type + * %SGX_PAGE_TYPE_SECS: a SECS page + * %SGX_PAGE_TYPE_TCS: a TCS page + * %SGX_PAGE_TYPE_REG: a regular page + * %SGX_PAGE_TYPE_VA: a VA page + * %SGX_PAGE_TYPE_TRIM: a page in trimmed state + */ +enum sgx_page_type { + SGX_PAGE_TYPE_SECS, + SGX_PAGE_TYPE_TCS, + SGX_PAGE_TYPE_REG, + SGX_PAGE_TYPE_VA, + SGX_PAGE_TYPE_TRIM, +}; + +#define SGX_NR_PAGE_TYPES 5 +#define SGX_PAGE_TYPE_MASK GENMASK(7, 0) + +/** + * enum sgx_secinfo_flags - the flags field in &struct sgx_secinfo + * %SGX_SECINFO_R: allow read + * %SGX_SECINFO_W: allow write + * %SGX_SECINFO_X: allow execution + * %SGX_SECINFO_SECS: a SECS page + * %SGX_SECINFO_TCS: a TCS page + * %SGX_SECINFO_REG: a regular page + * %SGX_SECINFO_VA: a VA page + * %SGX_SECINFO_TRIM: a page in trimmed state + */ +enum sgx_secinfo_flags { + SGX_SECINFO_R = BIT(0), + SGX_SECINFO_W = BIT(1), + SGX_SECINFO_X = BIT(2), + SGX_SECINFO_SECS = (SGX_PAGE_TYPE_SECS << 8), + SGX_SECINFO_TCS = (SGX_PAGE_TYPE_TCS << 8), + SGX_SECINFO_REG = (SGX_PAGE_TYPE_REG << 8), + SGX_SECINFO_VA = (SGX_PAGE_TYPE_VA << 8), + SGX_SECINFO_TRIM = (SGX_PAGE_TYPE_TRIM << 8), +}; + +#define SGX_SECINFO_PERMISSION_MASK GENMASK_ULL(2, 0) +#define SGX_SECINFO_PAGE_TYPE_MASK (SGX_PAGE_TYPE_MASK << 8) +#define SGX_SECINFO_RESERVED_MASK ~(SGX_SECINFO_PERMISSION_MASK | \ + SGX_SECINFO_PAGE_TYPE_MASK) + +/** + * struct sgx_secinfo - describes attributes of an EPC page + * @flags: permissions and type + * + * Used together with ENCLS leaves that add or modify an EPC page to an + * enclave to define page permissions and type. + */ +struct sgx_secinfo { + u64 flags; + u8 reserved[56]; +} __packed __aligned(64); + +#define SGX_PCMD_RESERVED_SIZE 40 + +/** + * struct sgx_pcmd - Paging Crypto Metadata (PCMD) + * @enclave_id: enclave identifier + * @mac: MAC over PCMD, page contents and isvsvn + * + * PCMD is stored for every swapped page to the regular memory. When ELDU loads + * the page back it recalculates the MAC by using a isvsvn number stored in a + * VA page. Together these two structures bring integrity and rollback + * protection. + */ +struct sgx_pcmd { + struct sgx_secinfo secinfo; + u64 enclave_id; + u8 reserved[SGX_PCMD_RESERVED_SIZE]; + u8 mac[16]; +} __packed __aligned(128); + +#define SGX_SIGSTRUCT_RESERVED1_SIZE 84 +#define SGX_SIGSTRUCT_RESERVED2_SIZE 20 +#define SGX_SIGSTRUCT_RESERVED3_SIZE 32 +#define SGX_SIGSTRUCT_RESERVED4_SIZE 12 + +/** + * struct sgx_sigstruct_header - defines author of the enclave + * @header1: constant byte string + * @vendor: must be either 0x0000 or 0x8086 + * @date: YYYYMMDD in BCD + * @header2: costant byte string + * @swdefined: software defined value + */ +struct sgx_sigstruct_header { + u64 header1[2]; + u32 vendor; + u32 date; + u64 header2[2]; + u32 swdefined; + u8 reserved1[84]; +} __packed; + +/** + * struct sgx_sigstruct_body - defines contents of the enclave + * @miscselect: additional information stored to an SSA frame + * @misc_mask: required miscselect in SECS + * @attributes: attributes for enclave + * @xfrm: XSave-Feature Request Mask (subset of XCR0) + * @attributes_mask: required attributes in SECS + * @xfrm_mask: required XFRM in SECS + * @mrenclave: SHA256-hash of the enclave contents + * @isvprodid: a user-defined value that is used in key derivation + * @isvsvn: a user-defined value that is used in key derivation + */ +struct sgx_sigstruct_body { + u32 miscselect; + u32 misc_mask; + u8 reserved2[20]; + u64 attributes; + u64 xfrm; + u64 attributes_mask; + u64 xfrm_mask; + u8 mrenclave[32]; + u8 reserved3[32]; + u16 isvprodid; + u16 isvsvn; +} __packed; + +/** + * struct sgx_sigstruct - an enclave signature + * @header: defines author of the enclave + * @modulus: the modulus of the public key + * @exponent: the exponent of the public key + * @signature: the signature calculated over the fields except modulus, + * @body: defines contents of the enclave + * @q1: a value used in RSA signature verification + * @q2: a value used in RSA signature verification + * + * Header and body are the parts that are actual signed. The remaining fields + * define the signature of the enclave. + */ +struct sgx_sigstruct { + struct sgx_sigstruct_header header; + u8 modulus[SGX_MODULUS_SIZE]; + u32 exponent; + u8 signature[SGX_MODULUS_SIZE]; + struct sgx_sigstruct_body body; + u8 reserved4[12]; + u8 q1[SGX_MODULUS_SIZE]; + u8 q2[SGX_MODULUS_SIZE]; +} __packed; + +#define SGX_LAUNCH_TOKEN_SIZE 304 + +#endif /* _ASM_X86_SGX_ARCH_H */ From 2c273671d0dfcf89c9c8a319ed093406e3ff665c Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 13 Nov 2020 00:01:13 +0200 Subject: [PATCH 169/360] x86/sgx: Add wrappers for ENCLS functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ENCLS is the userspace instruction which wraps virtually all unprivileged SGX functionality for managing enclaves. It is essentially the ioctl() of instructions with each function implementing different SGX-related functionality. Add macros to wrap the ENCLS functionality. There are two main groups, one for functions which do not return error codes and a “ret_” set for those that do. ENCLS functions are documented in Intel SDM section 36.6. Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-3-jarkko@kernel.org --- arch/x86/kernel/cpu/sgx/encls.h | 231 ++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 arch/x86/kernel/cpu/sgx/encls.h diff --git a/arch/x86/kernel/cpu/sgx/encls.h b/arch/x86/kernel/cpu/sgx/encls.h new file mode 100644 index 000000000000..443188fe7e70 --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/encls.h @@ -0,0 +1,231 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _X86_ENCLS_H +#define _X86_ENCLS_H + +#include +#include +#include +#include +#include +#include +#include +#include "sgx.h" + +enum sgx_encls_function { + ECREATE = 0x00, + EADD = 0x01, + EINIT = 0x02, + EREMOVE = 0x03, + EDGBRD = 0x04, + EDGBWR = 0x05, + EEXTEND = 0x06, + ELDU = 0x08, + EBLOCK = 0x09, + EPA = 0x0A, + EWB = 0x0B, + ETRACK = 0x0C, +}; + +/** + * ENCLS_FAULT_FLAG - flag signifying an ENCLS return code is a trapnr + * + * ENCLS has its own (positive value) error codes and also generates + * ENCLS specific #GP and #PF faults. And the ENCLS values get munged + * with system error codes as everything percolates back up the stack. + * Unfortunately (for us), we need to precisely identify each unique + * error code, e.g. the action taken if EWB fails varies based on the + * type of fault and on the exact SGX error code, i.e. we can't simply + * convert all faults to -EFAULT. + * + * To make all three error types coexist, we set bit 30 to identify an + * ENCLS fault. Bit 31 (technically bits N:31) is used to differentiate + * between positive (faults and SGX error codes) and negative (system + * error codes) values. + */ +#define ENCLS_FAULT_FLAG 0x40000000 + +/* Retrieve the encoded trapnr from the specified return code. */ +#define ENCLS_TRAPNR(r) ((r) & ~ENCLS_FAULT_FLAG) + +/* Issue a WARN() about an ENCLS function. */ +#define ENCLS_WARN(r, name) { \ + do { \ + int _r = (r); \ + WARN_ONCE(_r, "%s returned %d (0x%x)\n", (name), _r, _r); \ + } while (0); \ +} + +/** + * encls_failed() - Check if an ENCLS function failed + * @ret: the return value of an ENCLS function call + * + * Check if an ENCLS function failed. This happens when the function causes a + * fault that is not caused by an EPCM conflict or when the function returns a + * non-zero value. + */ +static inline bool encls_failed(int ret) +{ + if (ret & ENCLS_FAULT_FLAG) + return ENCLS_TRAPNR(ret) != X86_TRAP_PF; + + return !!ret; +} + +/** + * __encls_ret_N - encode an ENCLS function that returns an error code in EAX + * @rax: function number + * @inputs: asm inputs for the function + * + * Emit assembly for an ENCLS function that returns an error code, e.g. EREMOVE. + * And because SGX isn't complex enough as it is, function that return an error + * code also modify flags. + * + * Return: + * 0 on success, + * SGX error code on failure + */ +#define __encls_ret_N(rax, inputs...) \ + ({ \ + int ret; \ + asm volatile( \ + "1: .byte 0x0f, 0x01, 0xcf;\n\t" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: orl $"__stringify(ENCLS_FAULT_FLAG)",%%eax\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE_FAULT(1b, 3b) \ + : "=a"(ret) \ + : "a"(rax), inputs \ + : "memory", "cc"); \ + ret; \ + }) + +#define __encls_ret_1(rax, rcx) \ + ({ \ + __encls_ret_N(rax, "c"(rcx)); \ + }) + +#define __encls_ret_2(rax, rbx, rcx) \ + ({ \ + __encls_ret_N(rax, "b"(rbx), "c"(rcx)); \ + }) + +#define __encls_ret_3(rax, rbx, rcx, rdx) \ + ({ \ + __encls_ret_N(rax, "b"(rbx), "c"(rcx), "d"(rdx)); \ + }) + +/** + * __encls_N - encode an ENCLS function that doesn't return an error code + * @rax: function number + * @rbx_out: optional output variable + * @inputs: asm inputs for the function + * + * Emit assembly for an ENCLS function that does not return an error code, e.g. + * ECREATE. Leaves without error codes either succeed or fault. @rbx_out is an + * optional parameter for use by EDGBRD, which returns the requested value in + * RBX. + * + * Return: + * 0 on success, + * trapnr with ENCLS_FAULT_FLAG set on fault + */ +#define __encls_N(rax, rbx_out, inputs...) \ + ({ \ + int ret; \ + asm volatile( \ + "1: .byte 0x0f, 0x01, 0xcf;\n\t" \ + " xor %%eax,%%eax;\n" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: orl $"__stringify(ENCLS_FAULT_FLAG)",%%eax\n" \ + " jmp 2b\n" \ + ".previous\n" \ + _ASM_EXTABLE_FAULT(1b, 3b) \ + : "=a"(ret), "=b"(rbx_out) \ + : "a"(rax), inputs \ + : "memory"); \ + ret; \ + }) + +#define __encls_2(rax, rbx, rcx) \ + ({ \ + unsigned long ign_rbx_out; \ + __encls_N(rax, ign_rbx_out, "b"(rbx), "c"(rcx)); \ + }) + +#define __encls_1_1(rax, data, rcx) \ + ({ \ + unsigned long rbx_out; \ + int ret = __encls_N(rax, rbx_out, "c"(rcx)); \ + if (!ret) \ + data = rbx_out; \ + ret; \ + }) + +static inline int __ecreate(struct sgx_pageinfo *pginfo, void *secs) +{ + return __encls_2(ECREATE, pginfo, secs); +} + +static inline int __eextend(void *secs, void *addr) +{ + return __encls_2(EEXTEND, secs, addr); +} + +static inline int __eadd(struct sgx_pageinfo *pginfo, void *addr) +{ + return __encls_2(EADD, pginfo, addr); +} + +static inline int __einit(void *sigstruct, void *token, void *secs) +{ + return __encls_ret_3(EINIT, sigstruct, secs, token); +} + +static inline int __eremove(void *addr) +{ + return __encls_ret_1(EREMOVE, addr); +} + +static inline int __edbgwr(void *addr, unsigned long *data) +{ + return __encls_2(EDGBWR, *data, addr); +} + +static inline int __edbgrd(void *addr, unsigned long *data) +{ + return __encls_1_1(EDGBRD, *data, addr); +} + +static inline int __etrack(void *addr) +{ + return __encls_ret_1(ETRACK, addr); +} + +static inline int __eldu(struct sgx_pageinfo *pginfo, void *addr, + void *va) +{ + return __encls_ret_3(ELDU, pginfo, addr, va); +} + +static inline int __eblock(void *addr) +{ + return __encls_ret_1(EBLOCK, addr); +} + +static inline int __epa(void *addr) +{ + unsigned long rbx = SGX_PAGE_TYPE_VA; + + return __encls_2(EPA, rbx, addr); +} + +static inline int __ewb(struct sgx_pageinfo *pginfo, void *addr, + void *va) +{ + return __encls_ret_3(EWB, pginfo, addr, va); +} + +#endif /* _X86_ENCLS_H */ From e7b6385b01d8e9fb7a97887c3ea649abb95bb8c8 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Nov 2020 00:01:14 +0200 Subject: [PATCH 170/360] x86/cpufeatures: Add Intel SGX hardware bits Populate X86_FEATURE_SGX feature from CPUID and tie it to the Kconfig option with disabled-features.h. IA32_FEATURE_CONTROL.SGX_ENABLE must be examined in addition to the CPUID bits to enable full SGX support. The BIOS must both set this bit and lock IA32_FEATURE_CONTROL for SGX to be supported (Intel SDM section 36.7.1). The setting or clearing of this bit has no impact on the CPUID bits above, which is why it needs to be detected separately. Signed-off-by: Sean Christopherson Co-developed-by: Jarkko Sakkinen Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-4-jarkko@kernel.org --- arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/include/asm/disabled-features.h | 8 +++++++- arch/x86/include/asm/msr-index.h | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index dad350d42ecf..1181f5c7bbef 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -241,6 +241,7 @@ /* Intel-defined CPU features, CPUID level 0x00000007:0 (EBX), word 9 */ #define X86_FEATURE_FSGSBASE ( 9*32+ 0) /* RDFSBASE, WRFSBASE, RDGSBASE, WRGSBASE instructions*/ #define X86_FEATURE_TSC_ADJUST ( 9*32+ 1) /* TSC adjustment MSR 0x3B */ +#define X86_FEATURE_SGX ( 9*32+ 2) /* Software Guard Extensions */ #define X86_FEATURE_BMI1 ( 9*32+ 3) /* 1st group bit manipulation extensions */ #define X86_FEATURE_HLE ( 9*32+ 4) /* Hardware Lock Elision */ #define X86_FEATURE_AVX2 ( 9*32+ 5) /* AVX2 instructions */ diff --git a/arch/x86/include/asm/disabled-features.h b/arch/x86/include/asm/disabled-features.h index 5861d34f9771..7947cb1782da 100644 --- a/arch/x86/include/asm/disabled-features.h +++ b/arch/x86/include/asm/disabled-features.h @@ -62,6 +62,12 @@ # define DISABLE_ENQCMD (1 << (X86_FEATURE_ENQCMD & 31)) #endif +#ifdef CONFIG_X86_SGX +# define DISABLE_SGX 0 +#else +# define DISABLE_SGX (1 << (X86_FEATURE_SGX & 31)) +#endif + /* * Make sure to add features to the correct mask */ @@ -74,7 +80,7 @@ #define DISABLED_MASK6 0 #define DISABLED_MASK7 (DISABLE_PTI) #define DISABLED_MASK8 0 -#define DISABLED_MASK9 (DISABLE_SMAP) +#define DISABLED_MASK9 (DISABLE_SMAP|DISABLE_SGX) #define DISABLED_MASK10 0 #define DISABLED_MASK11 0 #define DISABLED_MASK12 0 diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 972a34d93505..258d555d22f2 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -609,6 +609,7 @@ #define FEAT_CTL_LOCKED BIT(0) #define FEAT_CTL_VMX_ENABLED_INSIDE_SMX BIT(1) #define FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX BIT(2) +#define FEAT_CTL_SGX_ENABLED BIT(18) #define FEAT_CTL_LMCE_ENABLED BIT(20) #define MSR_IA32_TSC_ADJUST 0x0000003b From d205e0f1426e0f99e2b4f387c49f2d8b66e129dd Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Nov 2020 00:01:15 +0200 Subject: [PATCH 171/360] x86/{cpufeatures,msr}: Add Intel SGX Launch Control hardware bits The SGX Launch Control hardware helps restrict which enclaves the hardware will run. Launch control is intended to restrict what software can run with enclave protections, which helps protect the overall system from bad enclaves. For the kernel's purposes, there are effectively two modes in which the launch control hardware can operate: rigid and flexible. In its rigid mode, an entity other than the kernel has ultimate authority over which enclaves can be run (firmware, Intel, etc...). In its flexible mode, the kernel has ultimate authority over which enclaves can run. Enable X86_FEATURE_SGX_LC to enumerate when the CPU supports SGX Launch Control in general. Add MSR_IA32_SGXLEPUBKEYHASH{0, 1, 2, 3}, which when combined contain a SHA256 hash of a 3072-bit RSA public key. The hardware allows SGX enclaves signed with this public key to initialize and run [*]. Enclaves not signed with this key can not initialize and run. Add FEAT_CTL_SGX_LC_ENABLED, which informs whether the SGXLEPUBKEYHASH MSRs can be written by the kernel. If the MSRs do not exist or are read-only, the launch control hardware is operating in rigid mode. Linux does not and will not support creating enclaves when hardware is configured in rigid mode because it takes away the authority for launch decisions from the kernel. Note, this does not preclude KVM from virtualizing/exposing SGX to a KVM guest when launch control hardware is operating in rigid mode. [*] Intel SDM: 38.1.4 Intel SGX Launch Control Configuration Signed-off-by: Sean Christopherson Co-developed-by: Jarkko Sakkinen Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-5-jarkko@kernel.org --- arch/x86/include/asm/cpufeatures.h | 1 + arch/x86/include/asm/msr-index.h | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 1181f5c7bbef..f5ef2d5b9231 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -357,6 +357,7 @@ #define X86_FEATURE_MOVDIRI (16*32+27) /* MOVDIRI instruction */ #define X86_FEATURE_MOVDIR64B (16*32+28) /* MOVDIR64B instruction */ #define X86_FEATURE_ENQCMD (16*32+29) /* ENQCMD and ENQCMDS instructions */ +#define X86_FEATURE_SGX_LC (16*32+30) /* Software Guard Extensions Launch Control */ /* AMD-defined CPU features, CPUID level 0x80000007 (EBX), word 17 */ #define X86_FEATURE_OVERFLOW_RECOV (17*32+ 0) /* MCA overflow recovery support */ diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 258d555d22f2..d0c6cfff5b55 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -609,6 +609,7 @@ #define FEAT_CTL_LOCKED BIT(0) #define FEAT_CTL_VMX_ENABLED_INSIDE_SMX BIT(1) #define FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX BIT(2) +#define FEAT_CTL_SGX_LC_ENABLED BIT(17) #define FEAT_CTL_SGX_ENABLED BIT(18) #define FEAT_CTL_LMCE_ENABLED BIT(20) @@ -629,6 +630,12 @@ #define MSR_IA32_UCODE_WRITE 0x00000079 #define MSR_IA32_UCODE_REV 0x0000008b +/* Intel SGX Launch Enclave Public Key Hash MSRs */ +#define MSR_IA32_SGXLEPUBKEYHASH0 0x0000008C +#define MSR_IA32_SGXLEPUBKEYHASH1 0x0000008D +#define MSR_IA32_SGXLEPUBKEYHASH2 0x0000008E +#define MSR_IA32_SGXLEPUBKEYHASH3 0x0000008F + #define MSR_IA32_SMM_MONITOR_CTL 0x0000009b #define MSR_IA32_SMBASE 0x0000009e From e7e0545299d8cb0fd6fe3ba50401b7f5c3937362 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Nov 2020 00:01:16 +0200 Subject: [PATCH 172/360] x86/sgx: Initialize metadata for Enclave Page Cache (EPC) sections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although carved out of normal DRAM, enclave memory is marked in the system memory map as reserved and is not managed by the core mm. There may be several regions spread across the system. Each contiguous region is called an Enclave Page Cache (EPC) section. EPC sections are enumerated via CPUID Enclave pages can only be accessed when they are mapped as part of an enclave, by a hardware thread running inside the enclave. Parse CPUID data, create metadata for EPC pages and populate a simple EPC page allocator. Although much smaller, ‘struct sgx_epc_page’ metadata is the SGX analog of the core mm ‘struct page’. Similar to how the core mm’s page->flags encode zone and NUMA information, embed the EPC section index to the first eight bits of sgx_epc_page->desc. This allows a quick reverse lookup from EPC page to EPC section. Existing client hardware supports only a single section, while upcoming server hardware will support at most eight sections. Thus, eight bits should be enough for long term needs. Signed-off-by: Sean Christopherson Co-developed-by: Serge Ayoun Signed-off-by: Serge Ayoun Co-developed-by: Jarkko Sakkinen Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-6-jarkko@kernel.org --- arch/x86/Kconfig | 17 +++ arch/x86/kernel/cpu/Makefile | 1 + arch/x86/kernel/cpu/sgx/Makefile | 2 + arch/x86/kernel/cpu/sgx/main.c | 190 +++++++++++++++++++++++++++++++ arch/x86/kernel/cpu/sgx/sgx.h | 60 ++++++++++ 5 files changed, 270 insertions(+) create mode 100644 arch/x86/kernel/cpu/sgx/Makefile create mode 100644 arch/x86/kernel/cpu/sgx/main.c create mode 100644 arch/x86/kernel/cpu/sgx/sgx.h diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index f6946b81f74a..618d1aabccb8 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1930,6 +1930,23 @@ config X86_INTEL_TSX_MODE_AUTO side channel attacks- equals the tsx=auto command line parameter. endchoice +config X86_SGX + bool "Software Guard eXtensions (SGX)" + depends on X86_64 && CPU_SUP_INTEL + depends on CRYPTO=y + depends on CRYPTO_SHA256=y + select SRCU + select MMU_NOTIFIER + help + Intel(R) Software Guard eXtensions (SGX) is a set of CPU instructions + that can be used by applications to set aside private regions of code + and data, referred to as enclaves. An enclave's private memory can + only be accessed by code running within the enclave. Accesses from + outside the enclave, including other enclaves, are disallowed by + hardware. + + If unsure, say N. + config EFI bool "EFI runtime service support" depends on ACPI diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index 93792b457b81..637b499450d1 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -48,6 +48,7 @@ obj-$(CONFIG_X86_MCE) += mce/ obj-$(CONFIG_MTRR) += mtrr/ obj-$(CONFIG_MICROCODE) += microcode/ obj-$(CONFIG_X86_CPU_RESCTRL) += resctrl/ +obj-$(CONFIG_X86_SGX) += sgx/ obj-$(CONFIG_X86_LOCAL_APIC) += perfctr-watchdog.o diff --git a/arch/x86/kernel/cpu/sgx/Makefile b/arch/x86/kernel/cpu/sgx/Makefile new file mode 100644 index 000000000000..79510ce01b3b --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/Makefile @@ -0,0 +1,2 @@ +obj-y += \ + main.o diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c new file mode 100644 index 000000000000..187a237eec38 --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "encls.h" + +struct sgx_epc_section sgx_epc_sections[SGX_MAX_EPC_SECTIONS]; +static int sgx_nr_epc_sections; +static struct task_struct *ksgxd_tsk; + +/* + * Reset dirty EPC pages to uninitialized state. Laundry can be left with SECS + * pages whose child pages blocked EREMOVE. + */ +static void sgx_sanitize_section(struct sgx_epc_section *section) +{ + struct sgx_epc_page *page; + LIST_HEAD(dirty); + int ret; + + while (!list_empty(§ion->laundry_list)) { + if (kthread_should_stop()) + return; + + spin_lock(§ion->lock); + + page = list_first_entry(§ion->laundry_list, + struct sgx_epc_page, list); + + ret = __eremove(sgx_get_epc_virt_addr(page)); + if (!ret) + list_move(&page->list, §ion->page_list); + else + list_move_tail(&page->list, &dirty); + + spin_unlock(§ion->lock); + + cond_resched(); + } + + list_splice(&dirty, §ion->laundry_list); +} + +static int ksgxd(void *p) +{ + int i; + + set_freezable(); + + /* + * Sanitize pages in order to recover from kexec(). The 2nd pass is + * required for SECS pages, whose child pages blocked EREMOVE. + */ + for (i = 0; i < sgx_nr_epc_sections; i++) + sgx_sanitize_section(&sgx_epc_sections[i]); + + for (i = 0; i < sgx_nr_epc_sections; i++) { + sgx_sanitize_section(&sgx_epc_sections[i]); + + /* Should never happen. */ + if (!list_empty(&sgx_epc_sections[i].laundry_list)) + WARN(1, "EPC section %d has unsanitized pages.\n", i); + } + + return 0; +} + +static bool __init sgx_page_reclaimer_init(void) +{ + struct task_struct *tsk; + + tsk = kthread_run(ksgxd, NULL, "ksgxd"); + if (IS_ERR(tsk)) + return false; + + ksgxd_tsk = tsk; + + return true; +} + +static bool __init sgx_setup_epc_section(u64 phys_addr, u64 size, + unsigned long index, + struct sgx_epc_section *section) +{ + unsigned long nr_pages = size >> PAGE_SHIFT; + unsigned long i; + + section->virt_addr = memremap(phys_addr, size, MEMREMAP_WB); + if (!section->virt_addr) + return false; + + section->pages = vmalloc(nr_pages * sizeof(struct sgx_epc_page)); + if (!section->pages) { + memunmap(section->virt_addr); + return false; + } + + section->phys_addr = phys_addr; + spin_lock_init(§ion->lock); + INIT_LIST_HEAD(§ion->page_list); + INIT_LIST_HEAD(§ion->laundry_list); + + for (i = 0; i < nr_pages; i++) { + section->pages[i].section = index; + list_add_tail(§ion->pages[i].list, §ion->laundry_list); + } + + return true; +} + +/** + * A section metric is concatenated in a way that @low bits 12-31 define the + * bits 12-31 of the metric and @high bits 0-19 define the bits 32-51 of the + * metric. + */ +static inline u64 __init sgx_calc_section_metric(u64 low, u64 high) +{ + return (low & GENMASK_ULL(31, 12)) + + ((high & GENMASK_ULL(19, 0)) << 32); +} + +static bool __init sgx_page_cache_init(void) +{ + u32 eax, ebx, ecx, edx, type; + u64 pa, size; + int i; + + for (i = 0; i < ARRAY_SIZE(sgx_epc_sections); i++) { + cpuid_count(SGX_CPUID, i + SGX_CPUID_EPC, &eax, &ebx, &ecx, &edx); + + type = eax & SGX_CPUID_EPC_MASK; + if (type == SGX_CPUID_EPC_INVALID) + break; + + if (type != SGX_CPUID_EPC_SECTION) { + pr_err_once("Unknown EPC section type: %u\n", type); + break; + } + + pa = sgx_calc_section_metric(eax, ebx); + size = sgx_calc_section_metric(ecx, edx); + + pr_info("EPC section 0x%llx-0x%llx\n", pa, pa + size - 1); + + if (!sgx_setup_epc_section(pa, size, i, &sgx_epc_sections[i])) { + pr_err("No free memory for an EPC section\n"); + break; + } + + sgx_nr_epc_sections++; + } + + if (!sgx_nr_epc_sections) { + pr_err("There are zero EPC sections.\n"); + return false; + } + + return true; +} + +static void __init sgx_init(void) +{ + int i; + + if (!boot_cpu_has(X86_FEATURE_SGX)) + return; + + if (!sgx_page_cache_init()) + return; + + if (!sgx_page_reclaimer_init()) + goto err_page_cache; + + return; + +err_page_cache: + for (i = 0; i < sgx_nr_epc_sections; i++) { + vfree(sgx_epc_sections[i].pages); + memunmap(sgx_epc_sections[i].virt_addr); + } +} + +device_initcall(sgx_init); diff --git a/arch/x86/kernel/cpu/sgx/sgx.h b/arch/x86/kernel/cpu/sgx/sgx.h new file mode 100644 index 000000000000..02afa84dd8fd --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/sgx.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _X86_SGX_H +#define _X86_SGX_H + +#include +#include +#include +#include +#include +#include +#include "arch.h" + +#undef pr_fmt +#define pr_fmt(fmt) "sgx: " fmt + +#define SGX_MAX_EPC_SECTIONS 8 + +struct sgx_epc_page { + unsigned int section; + struct list_head list; +}; + +/* + * The firmware can define multiple chunks of EPC to the different areas of the + * physical memory e.g. for memory areas of the each node. This structure is + * used to store EPC pages for one EPC section and virtual memory area where + * the pages have been mapped. + */ +struct sgx_epc_section { + unsigned long phys_addr; + void *virt_addr; + struct list_head page_list; + struct list_head laundry_list; + struct sgx_epc_page *pages; + spinlock_t lock; +}; + +extern struct sgx_epc_section sgx_epc_sections[SGX_MAX_EPC_SECTIONS]; + +static inline unsigned long sgx_get_epc_phys_addr(struct sgx_epc_page *page) +{ + struct sgx_epc_section *section = &sgx_epc_sections[page->section]; + unsigned long index; + + index = ((unsigned long)page - (unsigned long)section->pages) / sizeof(*page); + + return section->phys_addr + index * PAGE_SIZE; +} + +static inline void *sgx_get_epc_virt_addr(struct sgx_epc_page *page) +{ + struct sgx_epc_section *section = &sgx_epc_sections[page->section]; + unsigned long index; + + index = ((unsigned long)page - (unsigned long)section->pages) / sizeof(*page); + + return section->virt_addr + index * PAGE_SIZE; +} + +#endif /* _X86_SGX_H */ From 74faeee06db81a06add0def6a394210c8fef0ab7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Nov 2020 00:01:17 +0200 Subject: [PATCH 173/360] x86/mm: Signal SIGSEGV with PF_SGX The x86 architecture has a set of page fault error codes. These indicate things like whether the fault occurred from a write, or whether it originated in userspace. The SGX hardware architecture has its own per-page memory management metadata (EPCM) [*] and hardware which is separate from the normal x86 MMU. The architecture has a new page fault error code: PF_SGX. This new error code bit is set whenever a page fault occurs as the result of the SGX MMU. These faults occur for a variety of reasons. For instance, an access attempt to enclave memory from outside the enclave causes a PF_SGX fault. PF_SGX would also be set for permission conflicts, such as if a write to an enclave page occurs and the page is marked read-write in the x86 page tables but is read-only in the EPCM. These faults do not always indicate errors, though. SGX pages are encrypted with a key that is destroyed at hardware reset, including suspend. Throwing a SIGSEGV allows user space software to react and recover when these events occur. Include PF_SGX in the PF error codes list and throw SIGSEGV when it is encountered. [*] Intel SDM: 36.5.1 Enclave Page Cache Map (EPCM) [ bp: Add bit 15 to the comment above enum x86_pf_error_code too. ] Signed-off-by: Sean Christopherson Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-7-jarkko@kernel.org --- arch/x86/include/asm/trap_pf.h | 2 ++ arch/x86/mm/fault.c | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/arch/x86/include/asm/trap_pf.h b/arch/x86/include/asm/trap_pf.h index 305bc1214aef..10b1de500ab1 100644 --- a/arch/x86/include/asm/trap_pf.h +++ b/arch/x86/include/asm/trap_pf.h @@ -11,6 +11,7 @@ * bit 3 == 1: use of reserved bit detected * bit 4 == 1: fault was an instruction fetch * bit 5 == 1: protection keys block access + * bit 15 == 1: SGX MMU page-fault */ enum x86_pf_error_code { X86_PF_PROT = 1 << 0, @@ -19,6 +20,7 @@ enum x86_pf_error_code { X86_PF_RSVD = 1 << 3, X86_PF_INSTR = 1 << 4, X86_PF_PK = 1 << 5, + X86_PF_SGX = 1 << 15, }; #endif /* _ASM_X86_TRAP_PF_H */ diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 82bf37a5c9ec..9339fee83784 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1101,6 +1101,18 @@ access_error(unsigned long error_code, struct vm_area_struct *vma) if (error_code & X86_PF_PK) return 1; + /* + * SGX hardware blocked the access. This usually happens + * when the enclave memory contents have been destroyed, like + * after a suspend/resume cycle. In any case, the kernel can't + * fix the cause of the fault. Handle the fault as an access + * error even in cases where no actual access violation + * occurred. This allows userspace to rebuild the enclave in + * response to the signal. + */ + if (unlikely(error_code & X86_PF_SGX)) + return 1; + /* * Make sure to check the VMA so that we do not perform * faults just to hit a X86_PF_PK as soon as we fill in a From 224ab3527f89f69ae57dc53555826667ac46a3cc Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Nov 2020 00:01:18 +0200 Subject: [PATCH 174/360] x86/cpu/intel: Detect SGX support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kernel support for SGX is ultimately decided by the state of the launch control bits in the feature control MSR (MSR_IA32_FEAT_CTL). If the hardware supports SGX, but neglects to support flexible launch control, the kernel will not enable SGX. Enable SGX at feature control MSR initialization and update the associated X86_FEATURE flags accordingly. Disable X86_FEATURE_SGX (and all derivatives) if the kernel is not able to establish itself as the authority over SGX Launch Control. All checks are performed for each logical CPU (not just boot CPU) in order to verify that MSR_IA32_FEATURE_CONTROL is correctly configured on all CPUs. All SGX code in this series expects the same configuration from all CPUs. This differs from VMX where X86_FEATURE_VMX is intentionally cleared only for the current CPU so that KVM can provide additional information if KVM fails to load like which CPU doesn't support VMX. There’s not much the kernel or an administrator can do to fix the situation, so SGX neglects to convey additional details about these kinds of failures if they occur. Signed-off-by: Sean Christopherson Co-developed-by: Jarkko Sakkinen Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-8-jarkko@kernel.org --- arch/x86/kernel/cpu/feat_ctl.c | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/feat_ctl.c b/arch/x86/kernel/cpu/feat_ctl.c index 29a3bedabd06..d38e97325018 100644 --- a/arch/x86/kernel/cpu/feat_ctl.c +++ b/arch/x86/kernel/cpu/feat_ctl.c @@ -93,16 +93,32 @@ static void init_vmx_capabilities(struct cpuinfo_x86 *c) } #endif /* CONFIG_X86_VMX_FEATURE_NAMES */ +static void clear_sgx_caps(void) +{ + setup_clear_cpu_cap(X86_FEATURE_SGX); + setup_clear_cpu_cap(X86_FEATURE_SGX_LC); +} + void init_ia32_feat_ctl(struct cpuinfo_x86 *c) { bool tboot = tboot_enabled(); + bool enable_sgx; u64 msr; if (rdmsrl_safe(MSR_IA32_FEAT_CTL, &msr)) { clear_cpu_cap(c, X86_FEATURE_VMX); + clear_sgx_caps(); return; } + /* + * Enable SGX if and only if the kernel supports SGX and Launch Control + * is supported, i.e. disable SGX if the LE hash MSRs can't be written. + */ + enable_sgx = cpu_has(c, X86_FEATURE_SGX) && + cpu_has(c, X86_FEATURE_SGX_LC) && + IS_ENABLED(CONFIG_X86_SGX); + if (msr & FEAT_CTL_LOCKED) goto update_caps; @@ -124,13 +140,16 @@ void init_ia32_feat_ctl(struct cpuinfo_x86 *c) msr |= FEAT_CTL_VMX_ENABLED_INSIDE_SMX; } + if (enable_sgx) + msr |= FEAT_CTL_SGX_ENABLED | FEAT_CTL_SGX_LC_ENABLED; + wrmsrl(MSR_IA32_FEAT_CTL, msr); update_caps: set_cpu_cap(c, X86_FEATURE_MSR_IA32_FEAT_CTL); if (!cpu_has(c, X86_FEATURE_VMX)) - return; + goto update_sgx; if ( (tboot && !(msr & FEAT_CTL_VMX_ENABLED_INSIDE_SMX)) || (!tboot && !(msr & FEAT_CTL_VMX_ENABLED_OUTSIDE_SMX))) { @@ -143,4 +162,12 @@ update_caps: init_vmx_capabilities(c); #endif } + +update_sgx: + if (!(msr & FEAT_CTL_SGX_ENABLED) || + !(msr & FEAT_CTL_SGX_LC_ENABLED) || !enable_sgx) { + if (enable_sgx) + pr_err_once("SGX disabled by BIOS\n"); + clear_sgx_caps(); + } } From 38853a303982e3be3eccb1a1132399a5c5e2d806 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 13 Nov 2020 00:01:19 +0200 Subject: [PATCH 175/360] x86/cpu/intel: Add a nosgx kernel parameter Add a kernel parameter to disable SGX kernel support and document it. [ bp: Massage. ] Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Reviewed-by: Sean Christopherson Acked-by: Jethro Beekman Tested-by: Sean Christopherson Link: https://lkml.kernel.org/r/20201112220135.165028-9-jarkko@kernel.org --- Documentation/admin-guide/kernel-parameters.txt | 2 ++ arch/x86/kernel/cpu/feat_ctl.c | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 526d65d8573a..42d1528a5b60 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3368,6 +3368,8 @@ nosep [BUGS=X86-32] Disables x86 SYSENTER/SYSEXIT support. + nosgx [X86-64,SGX] Disables Intel SGX kernel support. + nosmp [SMP] Tells an SMP kernel to act as a UP kernel, and disable the IO APIC. legacy for "maxcpus=0". diff --git a/arch/x86/kernel/cpu/feat_ctl.c b/arch/x86/kernel/cpu/feat_ctl.c index d38e97325018..3b1b01f2b248 100644 --- a/arch/x86/kernel/cpu/feat_ctl.c +++ b/arch/x86/kernel/cpu/feat_ctl.c @@ -99,6 +99,15 @@ static void clear_sgx_caps(void) setup_clear_cpu_cap(X86_FEATURE_SGX_LC); } +static int __init nosgx(char *str) +{ + clear_sgx_caps(); + + return 0; +} + +early_param("nosgx", nosgx); + void init_ia32_feat_ctl(struct cpuinfo_x86 *c) { bool tboot = tboot_enabled(); From d2285493bef310b66b56dfe4eb75c1e2f431ea5c Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 13 Nov 2020 00:01:20 +0200 Subject: [PATCH 176/360] x86/sgx: Add SGX page allocator functions Add functions for runtime allocation and free. This allocator and its algorithms are as simple as it gets. They do a linear search across all EPC sections and find the first free page. They are not NUMA-aware and only hand out individual pages. The SGX hardware does not support large pages, so something more complicated like a buddy allocator is unwarranted. The free function (sgx_free_epc_page()) implicitly calls ENCLS[EREMOVE], which returns the page to the uninitialized state. This ensures that the page is ready for use at the next allocation. Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-10-jarkko@kernel.org --- arch/x86/kernel/cpu/sgx/main.c | 65 ++++++++++++++++++++++++++++++++++ arch/x86/kernel/cpu/sgx/sgx.h | 3 ++ 2 files changed, 68 insertions(+) diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index 187a237eec38..2e53afc288a4 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -85,6 +85,71 @@ static bool __init sgx_page_reclaimer_init(void) return true; } +static struct sgx_epc_page *__sgx_alloc_epc_page_from_section(struct sgx_epc_section *section) +{ + struct sgx_epc_page *page; + + spin_lock(§ion->lock); + + if (list_empty(§ion->page_list)) { + spin_unlock(§ion->lock); + return NULL; + } + + page = list_first_entry(§ion->page_list, struct sgx_epc_page, list); + list_del_init(&page->list); + + spin_unlock(§ion->lock); + return page; +} + +/** + * __sgx_alloc_epc_page() - Allocate an EPC page + * + * Iterate through EPC sections and borrow a free EPC page to the caller. When a + * page is no longer needed it must be released with sgx_free_epc_page(). + * + * Return: + * an EPC page, + * -errno on error + */ +struct sgx_epc_page *__sgx_alloc_epc_page(void) +{ + struct sgx_epc_section *section; + struct sgx_epc_page *page; + int i; + + for (i = 0; i < sgx_nr_epc_sections; i++) { + section = &sgx_epc_sections[i]; + + page = __sgx_alloc_epc_page_from_section(section); + if (page) + return page; + } + + return ERR_PTR(-ENOMEM); +} + +/** + * sgx_free_epc_page() - Free an EPC page + * @page: an EPC page + * + * Call EREMOVE for an EPC page and insert it back to the list of free pages. + */ +void sgx_free_epc_page(struct sgx_epc_page *page) +{ + struct sgx_epc_section *section = &sgx_epc_sections[page->section]; + int ret; + + ret = __eremove(sgx_get_epc_virt_addr(page)); + if (WARN_ONCE(ret, "EREMOVE returned %d (0x%x)", ret, ret)) + return; + + spin_lock(§ion->lock); + list_add_tail(&page->list, §ion->page_list); + spin_unlock(§ion->lock); +} + static bool __init sgx_setup_epc_section(u64 phys_addr, u64 size, unsigned long index, struct sgx_epc_section *section) diff --git a/arch/x86/kernel/cpu/sgx/sgx.h b/arch/x86/kernel/cpu/sgx/sgx.h index 02afa84dd8fd..bd9dcb1ffcfa 100644 --- a/arch/x86/kernel/cpu/sgx/sgx.h +++ b/arch/x86/kernel/cpu/sgx/sgx.h @@ -57,4 +57,7 @@ static inline void *sgx_get_epc_virt_addr(struct sgx_epc_page *page) return section->virt_addr + index * PAGE_SIZE; } +struct sgx_epc_page *__sgx_alloc_epc_page(void); +void sgx_free_epc_page(struct sgx_epc_page *page); + #endif /* _X86_SGX_H */ From 95bb7c42ac8a94ce3d0eb059ad64430390351ccb Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Nov 2020 00:01:21 +0200 Subject: [PATCH 177/360] mm: Add 'mprotect' hook to struct vm_operations_struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Background ========== 1. SGX enclave pages are populated with data by copying from normal memory via ioctl() (SGX_IOC_ENCLAVE_ADD_PAGES), which will be added later in this series. 2. It is desirable to be able to restrict those normal memory data sources. For instance, to ensure that the source data is executable before copying data to an executable enclave page. 3. Enclave page permissions are dynamic (just like normal permissions) and can be adjusted at runtime with mprotect(). This creates a problem because the original data source may have long since vanished at the time when enclave page permissions are established (mmap() or mprotect()). The solution (elsewhere in this series) is to force enclave creators to declare their paging permission *intent* up front to the ioctl(). This intent can be immediately compared to the source data’s mapping and rejected if necessary. The “intent” is also stashed off for later comparison with enclave PTEs. This ensures that any future mmap()/mprotect() operations performed by the enclave creator or done on behalf of the enclave can be compared with the earlier declared permissions. Problem ======= There is an existing mmap() hook which allows SGX to perform this permission comparison at mmap() time. However, there is no corresponding ->mprotect() hook. Solution ======== Add a vm_ops->mprotect() hook so that mprotect() operations which are inconsistent with any page's stashed intent can be rejected by the driver. Signed-off-by: Sean Christopherson Co-developed-by: Jarkko Sakkinen Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Acked-by: Dave Hansen Acked-by: Mel Gorman Acked-by: Hillf Danton Cc: linux-mm@kvack.org Link: https://lkml.kernel.org/r/20201112220135.165028-11-jarkko@kernel.org --- include/linux/mm.h | 7 +++++++ mm/mprotect.c | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/include/linux/mm.h b/include/linux/mm.h index db6ae4d3fb4e..1813fa86b981 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -559,6 +559,13 @@ struct vm_operations_struct { void (*close)(struct vm_area_struct * area); int (*split)(struct vm_area_struct * area, unsigned long addr); int (*mremap)(struct vm_area_struct * area); + /* + * Called by mprotect() to make driver-specific permission + * checks before mprotect() is finalised. The VMA must not + * be modified. Returns 0 if eprotect() can proceed. + */ + int (*mprotect)(struct vm_area_struct *vma, unsigned long start, + unsigned long end, unsigned long newflags); vm_fault_t (*fault)(struct vm_fault *vmf); vm_fault_t (*huge_fault)(struct vm_fault *vmf, enum page_entry_size pe_size); diff --git a/mm/mprotect.c b/mm/mprotect.c index 56c02beb6041..ab709023e9aa 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -616,9 +616,16 @@ static int do_mprotect_pkey(unsigned long start, size_t len, tmp = vma->vm_end; if (tmp > end) tmp = end; + + if (vma->vm_ops && vma->vm_ops->mprotect) + error = vma->vm_ops->mprotect(vma, nstart, tmp, newflags); + if (error) + goto out; + error = mprotect_fixup(vma, &prev, nstart, tmp, newflags); if (error) goto out; + nstart = tmp; if (nstart < prev->vm_end) From 09a217c10504bcaef911cf2af74e424338efe629 Mon Sep 17 00:00:00 2001 From: Hui Su Date: Fri, 13 Nov 2020 21:39:43 +0800 Subject: [PATCH 178/360] x86/dumpstack: Make show_trace_log_lvl() static show_trace_log_lvl() is not used by other compilation units so make it static and remove the declaration from the header file. Signed-off-by: Hui Su Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201113133943.GA136221@rlk --- arch/x86/include/asm/stacktrace.h | 3 --- arch/x86/kernel/dumpstack.c | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/arch/x86/include/asm/stacktrace.h b/arch/x86/include/asm/stacktrace.h index 49600643faba..f248eb2ac2d4 100644 --- a/arch/x86/include/asm/stacktrace.h +++ b/arch/x86/include/asm/stacktrace.h @@ -88,9 +88,6 @@ get_stack_pointer(struct task_struct *task, struct pt_regs *regs) return (unsigned long *)task->thread.sp; } -void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs, - unsigned long *stack, const char *log_lvl); - /* The form of the top of the frame on the stack */ struct stack_frame { struct stack_frame *next_frame; diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index 25c06b67e7e0..067de0d0fa4f 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c @@ -168,7 +168,7 @@ static void show_regs_if_on_stack(struct stack_info *info, struct pt_regs *regs, } } -void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs, +static void show_trace_log_lvl(struct task_struct *task, struct pt_regs *regs, unsigned long *stack, const char *log_lvl) { struct unwind_state state; From da78693e6e496ccd5cb6b0e9025007803e8f93c2 Mon Sep 17 00:00:00 2001 From: Niklas Schnelle Date: Mon, 26 Oct 2020 10:01:24 +0100 Subject: [PATCH 179/360] s390/pci: inform when missing required facilities when we're missing the necessary machine facilities zPCI can not function. Until now it would silently fail to be initialized, add an informational print. Signed-off-by: Niklas Schnelle Signed-off-by: Heiko Carstens --- arch/s390/pci/pci.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 570016ae8bcd..41df8fcfddde 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -851,8 +851,10 @@ static int __init pci_base_init(void) if (!s390_pci_probe) return 0; - if (!test_facility(69) || !test_facility(71)) + if (!test_facility(69) || !test_facility(71)) { + pr_info("PCI is not supported because CPU facilities 69 or 71 are not available\n"); return 0; + } if (test_facility(153) && !s390_pci_no_mio) { static_branch_enable(&have_mio); From 08ab919d0dccc4ed6fb12231b20758cef112bd26 Mon Sep 17 00:00:00 2001 From: Sumanth Korikkar Date: Sat, 7 Nov 2020 22:55:51 -0600 Subject: [PATCH 180/360] s390/sclp: use memblock for early read cpu info sclp early read cpu info is used to detect the number of configured cpus, which is utilized by smp_detect_cpus() in early startup. * For read cpu info, the sccb block should be below 2gb. * smp_detect_cpus() utilizes read cpu info early, but after memblock initialization. Thus use memblock_allow_low() instead. * Avoid copy of sclp_core_info structure. * sclp_early_init_core_info(), sclp_early_core_info and sclp_early_core_info_valid initdata are no longer required. * smp_get_core_info() is called only once during early stage. Hence for early sclp_get_core_info(), directly call read cpu command. No need to maintain sclp_early_core_info_valid. Signed-off-by: Sumanth Korikkar Reviewed-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- drivers/s390/char/sclp_early.c | 50 +++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index cc5e84b80c69..62e0d35ea1a7 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -9,9 +9,11 @@ #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt #include +#include #include #include #include +#include #include "sclp_sdias.h" #include "sclp.h" @@ -107,29 +109,34 @@ void __init sclp_early_get_ipl_info(struct sclp_ipl_info *info) *info = sclp_ipl_info; } -static struct sclp_core_info sclp_early_core_info __initdata; -static int sclp_early_core_info_valid __initdata; - -static void __init sclp_early_init_core_info(struct read_cpu_info_sccb *sccb) -{ - if (!SCLP_HAS_CPU_INFO) - return; - memset(sccb, 0, sizeof(*sccb)); - sccb->header.length = sizeof(*sccb); - if (sclp_early_cmd(SCLP_CMDW_READ_CPU_INFO, sccb)) - return; - if (sccb->header.response_code != 0x0010) - return; - sclp_fill_core_info(&sclp_early_core_info, sccb); - sclp_early_core_info_valid = 1; -} - int __init sclp_early_get_core_info(struct sclp_core_info *info) { - if (!sclp_early_core_info_valid) - return -EIO; - *info = sclp_early_core_info; - return 0; + struct read_cpu_info_sccb *sccb; + int length = PAGE_SIZE; + int rc = 0; + + if (!SCLP_HAS_CPU_INFO) + return -EOPNOTSUPP; + + sccb = memblock_alloc_low(length, PAGE_SIZE); + if (!sccb) + return -ENOMEM; + + memset(sccb, 0, length); + sccb->header.length = length; + sccb->header.control_mask[2] = 0x80; + if (sclp_early_cmd(SCLP_CMDW_READ_CPU_INFO, sccb)) { + rc = -EIO; + goto out; + } + if (sccb->header.response_code != 0x0010) { + rc = -EIO; + goto out; + } + sclp_fill_core_info(info, sccb); +out: + memblock_free_early((unsigned long)sccb, length); + return rc; } static void __init sclp_early_console_detect(struct init_sccb *sccb) @@ -149,7 +156,6 @@ void __init sclp_early_detect(void) void *sccb = sclp_early_sccb; sclp_early_facilities_detect(sccb); - sclp_early_init_core_info(sccb); /* * Turn off SCLP event notifications. Also save remote masks in the From d25d23e134a43457759fa602a15c1e9fce741727 Mon Sep 17 00:00:00 2001 From: Sumanth Korikkar Date: Sun, 8 Nov 2020 14:21:14 -0600 Subject: [PATCH 181/360] s390/sclp: avoid copy of sclp_info_sccb For extended sccb support, sccb size could be up to 3 pages. Hence avoid copy of sclp_info_sccb. Signed-off-by: Sumanth Korikkar Reviewed-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- drivers/s390/char/sclp.h | 2 +- drivers/s390/char/sclp_early.c | 8 +++++--- drivers/s390/char/sclp_early_core.c | 7 +++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/drivers/s390/char/sclp.h b/drivers/s390/char/sclp.h index 69d9cde9ff5a..448ed25bf6b8 100644 --- a/drivers/s390/char/sclp.h +++ b/drivers/s390/char/sclp.h @@ -328,7 +328,7 @@ unsigned int sclp_early_con_check_vt220(struct init_sccb *sccb); int sclp_early_set_event_mask(struct init_sccb *sccb, sccb_mask_t receive_mask, sccb_mask_t send_mask); -int sclp_early_get_info(struct read_info_sccb *info); +struct read_info_sccb * __init sclp_early_get_info(void); /* useful inlines */ diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index 62e0d35ea1a7..cc78b538acbf 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -22,12 +22,14 @@ static struct sclp_ipl_info sclp_ipl_info; struct sclp_info sclp; EXPORT_SYMBOL(sclp); -static void __init sclp_early_facilities_detect(struct read_info_sccb *sccb) +static void __init sclp_early_facilities_detect(void) { struct sclp_core_entry *cpue; + struct read_info_sccb *sccb; u16 boot_cpu_address, cpu; - if (sclp_early_get_info(sccb)) + sccb = sclp_early_get_info(); + if (!sccb) return; sclp.facilities = sccb->facilities; @@ -155,7 +157,7 @@ void __init sclp_early_detect(void) { void *sccb = sclp_early_sccb; - sclp_early_facilities_detect(sccb); + sclp_early_facilities_detect(); /* * Turn off SCLP event notifications. Also save remote masks in the diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index a960afa974bf..d4fb61c10d7c 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -258,13 +258,12 @@ int __init sclp_early_read_info(void) return -EIO; } -int __init sclp_early_get_info(struct read_info_sccb *info) +struct read_info_sccb * __init sclp_early_get_info(void) { if (!sclp_info_sccb_valid) - return -EIO; + return NULL; - *info = sclp_info_sccb; - return 0; + return &sclp_info_sccb; } int __init sclp_early_get_memsize(unsigned long *mem) From b971cbd03ee0a24f7af47b681e8f911794c69780 Mon Sep 17 00:00:00 2001 From: Sumanth Korikkar Date: Wed, 11 Nov 2020 01:03:02 -0600 Subject: [PATCH 182/360] s390/sclp: provide extended sccb support As the number of cpus increases, the sccb response can exceed 4k for read cpu and read scp info sclp commands. Hence, all cpu info entries cant be embedded within a sccb response Solution: To overcome this limitation, extended sccb facility is provided by sclp. 1. Check if the extended sccb facility is installed. 2. If extended sccb is installed, perform the read scp and read cpu command considering a max sccb length of three page size. This max length is based on factors like max cpus, sccb header. 3. If extended sccb is not installed, perform the read scp and read cpu sclp command considering a max sccb length of one page size. Signed-off-by: Sumanth Korikkar Reviewed-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/include/asm/sclp.h | 7 ++++++- arch/s390/include/asm/setup.h | 2 -- drivers/s390/char/sclp.h | 8 ++++++-- drivers/s390/char/sclp_cmd.c | 10 +++++++--- drivers/s390/char/sclp_early.c | 3 ++- drivers/s390/char/sclp_early_core.c | 6 ++++-- 6 files changed, 25 insertions(+), 11 deletions(-) diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index a7bdd128d85b..5763769a39b6 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -12,7 +12,12 @@ #include #define SCLP_CHP_INFO_MASK_SIZE 32 -#define SCLP_MAX_CORES 256 +#define EARLY_SCCB_SIZE PAGE_SIZE +#define SCLP_MAX_CORES 512 +/* 144 + 16 * SCLP_MAX_CORES + 2 * (SCLP_MAX_CORES - 1) */ +#define EXT_SCCB_READ_SCP (3 * PAGE_SIZE) +/* 24 + 16 * SCLP_MAX_CORES */ +#define EXT_SCCB_READ_CPU (3 * PAGE_SIZE) struct sclp_chp_info { u8 recognized[SCLP_CHP_INFO_MASK_SIZE]; diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index bdb242a1544e..1be6a064f317 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -16,8 +16,6 @@ #define EARLY_SCCB_OFFSET 0x11000 #define HEAD_END 0x12000 -#define EARLY_SCCB_SIZE PAGE_SIZE - /* * Machine features detected in early.c */ diff --git a/drivers/s390/char/sclp.h b/drivers/s390/char/sclp.h index 448ed25bf6b8..6de919944a39 100644 --- a/drivers/s390/char/sclp.h +++ b/drivers/s390/char/sclp.h @@ -156,7 +156,11 @@ struct read_cpu_info_sccb { u16 offset_configured; u16 nr_standby; u16 offset_standby; - u8 reserved[4096 - 16]; + /* + * Without ext sccb, struct size is PAGE_SIZE. + * With ext sccb, struct size is EXT_SCCB_READ_CPU. + */ + u8 reserved[]; } __attribute__((packed, aligned(PAGE_SIZE))); struct read_info_sccb { @@ -199,7 +203,7 @@ struct read_info_sccb { u8 byte_134; /* 134 */ u8 cpudirq; /* 135 */ u16 cbl; /* 136-137 */ - u8 _pad_138[4096 - 138]; /* 138-4095 */ + u8 _pad_138[EXT_SCCB_READ_SCP - 138]; } __packed __aligned(PAGE_SIZE); struct read_storage_sccb { diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c index f6e97f0830f6..7ebe89890a9b 100644 --- a/drivers/s390/char/sclp_cmd.c +++ b/drivers/s390/char/sclp_cmd.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "sclp.h" @@ -87,14 +88,17 @@ out: int _sclp_get_core_info(struct sclp_core_info *info) { int rc; + int length = test_facility(140) ? EXT_SCCB_READ_CPU : PAGE_SIZE; struct read_cpu_info_sccb *sccb; if (!SCLP_HAS_CPU_INFO) return -EOPNOTSUPP; - sccb = (void *) get_zeroed_page(GFP_KERNEL | GFP_DMA); + + sccb = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA | __GFP_ZERO, get_order(length)); if (!sccb) return -ENOMEM; - sccb->header.length = sizeof(*sccb); + sccb->header.length = length; + sccb->header.control_mask[2] = 0x80; rc = sclp_sync_request_timeout(SCLP_CMDW_READ_CPU_INFO, sccb, SCLP_QUEUE_INTERVAL); if (rc) @@ -107,7 +111,7 @@ int _sclp_get_core_info(struct sclp_core_info *info) } sclp_fill_core_info(info, sccb); out: - free_page((unsigned long) sccb); + free_pages((unsigned long) sccb, get_order(length)); return rc; } diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index cc78b538acbf..2f3515fa242a 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "sclp_sdias.h" #include "sclp.h" @@ -114,7 +115,7 @@ void __init sclp_early_get_ipl_info(struct sclp_ipl_info *info) int __init sclp_early_get_core_info(struct sclp_core_info *info) { struct read_cpu_info_sccb *sccb; - int length = PAGE_SIZE; + int length = test_facility(140) ? EXT_SCCB_READ_CPU : PAGE_SIZE; int rc = 0; if (!SCLP_HAS_CPU_INFO) diff --git a/drivers/s390/char/sclp_early_core.c b/drivers/s390/char/sclp_early_core.c index d4fb61c10d7c..ec9f8ad5341c 100644 --- a/drivers/s390/char/sclp_early_core.c +++ b/drivers/s390/char/sclp_early_core.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "sclp.h" #include "sclp_rw.h" @@ -237,13 +238,14 @@ void sclp_early_printk(const char *str) int __init sclp_early_read_info(void) { int i; + int length = test_facility(140) ? EXT_SCCB_READ_SCP : PAGE_SIZE; struct read_info_sccb *sccb = &sclp_info_sccb; sclp_cmdw_t commands[] = {SCLP_CMDW_READ_SCP_INFO_FORCED, SCLP_CMDW_READ_SCP_INFO}; for (i = 0; i < ARRAY_SIZE(commands); i++) { - memset(sccb, 0, sizeof(*sccb)); - sccb->header.length = sizeof(*sccb); + memset(sccb, 0, length); + sccb->header.length = length; sccb->header.function_code = 0x80; sccb->header.control_mask[2] = 0x80; if (sclp_early_cmd(commands[i], sccb)) From 907f8eb8e0eb2b3312b292e67dc4dbc493424747 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 17 Nov 2020 21:23:35 +0100 Subject: [PATCH 183/360] x86/uaccess: Document copy_from_user_nmi() Document the functionality of copy_from_user_nmi() to avoid further confusion. Fix the typo in the existing comment while at it. Requested-by: Borislav Petkov Signed-off-by: Thomas Gleixner Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201117202753.806376613@linutronix.de --- arch/x86/lib/usercopy.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/arch/x86/lib/usercopy.c b/arch/x86/lib/usercopy.c index 3f435d7fca5e..c3e8a62ca561 100644 --- a/arch/x86/lib/usercopy.c +++ b/arch/x86/lib/usercopy.c @@ -9,9 +9,23 @@ #include -/* - * We rely on the nested NMI work to allow atomic faults from the NMI path; the - * nested NMI paths are careful to preserve CR2. +/** + * copy_from_user_nmi - NMI safe copy from user + * @to: Pointer to the destination buffer + * @from: Pointer to a user space address of the current task + * @n: Number of bytes to copy + * + * Returns: The number of not copied bytes. 0 is success, i.e. all bytes copied + * + * Contrary to other copy_from_user() variants this function can be called + * from NMI context. Despite the name it is not restricted to be called + * from NMI context. It is safe to be called from any other context as + * well. It disables pagefaults across the copy which means a fault will + * abort the copy. + * + * For NMI context invocations this relies on the nested NMI work to allow + * atomic faults from the NMI path; the nested NMI paths are careful to + * preserve CR2. */ unsigned long copy_from_user_nmi(void *to, const void __user *from, unsigned long n) @@ -27,7 +41,7 @@ copy_from_user_nmi(void *to, const void __user *from, unsigned long n) /* * Even though this function is typically called from NMI/IRQ context * disable pagefaults so that its behaviour is consistent even when - * called form other contexts. + * called from other contexts. */ pagefault_disable(); ret = __copy_from_user_inatomic(to, from, n); From 0ac317e89791b76055ef11b952625ef77a1d2eba Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Mon, 5 Oct 2020 11:12:07 -0400 Subject: [PATCH 184/360] x86/boot: Remove unused finalize_identity_maps() Commit 8570978ea030 ("x86/boot/compressed/64: Don't pre-map memory in KASLR code") removed all the references to finalize_identity_maps(), but neglected to delete the actual function. Remove it. Signed-off-by: Arvind Sankar Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201005151208.2212886-2-nivedita@alum.mit.edu --- arch/x86/boot/compressed/ident_map_64.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/arch/x86/boot/compressed/ident_map_64.c b/arch/x86/boot/compressed/ident_map_64.c index a5e5db6ada3c..6bf20223dc0f 100644 --- a/arch/x86/boot/compressed/ident_map_64.c +++ b/arch/x86/boot/compressed/ident_map_64.c @@ -167,16 +167,6 @@ void initialize_identity_maps(void *rmode) write_cr3(top_level_pgt); } -/* - * This switches the page tables to the new level4 that has been built - * via calls to add_identity_map() above. If booted via startup_32(), - * this is effectively a no-op. - */ -void finalize_identity_maps(void) -{ - write_cr3(top_level_pgt); -} - static pte_t *split_large_pmd(struct x86_mapping_info *info, pmd_t *pmdp, unsigned long __address) { From 3fe0778edac8628637e2fd23835996523b1a3372 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 13 Nov 2020 00:01:22 +0200 Subject: [PATCH 185/360] x86/sgx: Add an SGX misc driver interface Intel(R) SGX is a new hardware functionality that can be used by applications to set aside private regions of code and data called enclaves. New hardware protects enclave code and data from outside access and modification. Add a driver that presents a device file and ioctl API to build and manage enclaves. [ bp: Small touchups, remove unused encl variable in sgx_encl_find() as Reported-by: kernel test robot ] Signed-off-by: Jarkko Sakkinen Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Signed-off-by: Borislav Petkov Tested-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-12-jarkko@kernel.org --- arch/x86/kernel/cpu/sgx/Makefile | 2 + arch/x86/kernel/cpu/sgx/driver.c | 112 ++++++++++++++++++++++++ arch/x86/kernel/cpu/sgx/driver.h | 16 ++++ arch/x86/kernel/cpu/sgx/encl.c | 146 +++++++++++++++++++++++++++++++ arch/x86/kernel/cpu/sgx/encl.h | 58 ++++++++++++ arch/x86/kernel/cpu/sgx/main.c | 12 ++- 6 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 arch/x86/kernel/cpu/sgx/driver.c create mode 100644 arch/x86/kernel/cpu/sgx/driver.h create mode 100644 arch/x86/kernel/cpu/sgx/encl.c create mode 100644 arch/x86/kernel/cpu/sgx/encl.h diff --git a/arch/x86/kernel/cpu/sgx/Makefile b/arch/x86/kernel/cpu/sgx/Makefile index 79510ce01b3b..3fc451120735 100644 --- a/arch/x86/kernel/cpu/sgx/Makefile +++ b/arch/x86/kernel/cpu/sgx/Makefile @@ -1,2 +1,4 @@ obj-y += \ + driver.o \ + encl.o \ main.o diff --git a/arch/x86/kernel/cpu/sgx/driver.c b/arch/x86/kernel/cpu/sgx/driver.c new file mode 100644 index 000000000000..c2810e1c7cf1 --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/driver.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#include +#include +#include +#include +#include +#include +#include "driver.h" +#include "encl.h" + +static int sgx_open(struct inode *inode, struct file *file) +{ + struct sgx_encl *encl; + + encl = kzalloc(sizeof(*encl), GFP_KERNEL); + if (!encl) + return -ENOMEM; + + xa_init(&encl->page_array); + mutex_init(&encl->lock); + + file->private_data = encl; + + return 0; +} + +static int sgx_release(struct inode *inode, struct file *file) +{ + struct sgx_encl *encl = file->private_data; + struct sgx_encl_page *entry; + unsigned long index; + + xa_for_each(&encl->page_array, index, entry) { + if (entry->epc_page) { + sgx_free_epc_page(entry->epc_page); + encl->secs_child_cnt--; + entry->epc_page = NULL; + } + + kfree(entry); + } + + xa_destroy(&encl->page_array); + + if (!encl->secs_child_cnt && encl->secs.epc_page) { + sgx_free_epc_page(encl->secs.epc_page); + encl->secs.epc_page = NULL; + } + + /* Detect EPC page leaks. */ + WARN_ON_ONCE(encl->secs_child_cnt); + WARN_ON_ONCE(encl->secs.epc_page); + + kfree(encl); + return 0; +} + +static int sgx_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct sgx_encl *encl = file->private_data; + int ret; + + ret = sgx_encl_may_map(encl, vma->vm_start, vma->vm_end, vma->vm_flags); + if (ret) + return ret; + + vma->vm_ops = &sgx_vm_ops; + vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; + vma->vm_private_data = encl; + + return 0; +} + +static unsigned long sgx_get_unmapped_area(struct file *file, + unsigned long addr, + unsigned long len, + unsigned long pgoff, + unsigned long flags) +{ + if ((flags & MAP_TYPE) == MAP_PRIVATE) + return -EINVAL; + + if (flags & MAP_FIXED) + return addr; + + return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); +} + +static const struct file_operations sgx_encl_fops = { + .owner = THIS_MODULE, + .open = sgx_open, + .release = sgx_release, + .mmap = sgx_mmap, + .get_unmapped_area = sgx_get_unmapped_area, +}; + +static struct miscdevice sgx_dev_enclave = { + .minor = MISC_DYNAMIC_MINOR, + .name = "sgx_enclave", + .nodename = "sgx_enclave", + .fops = &sgx_encl_fops, +}; + +int __init sgx_drv_init(void) +{ + if (!cpu_feature_enabled(X86_FEATURE_SGX_LC)) + return -ENODEV; + + return misc_register(&sgx_dev_enclave); +} diff --git a/arch/x86/kernel/cpu/sgx/driver.h b/arch/x86/kernel/cpu/sgx/driver.h new file mode 100644 index 000000000000..cda9c43b7543 --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/driver.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ARCH_SGX_DRIVER_H__ +#define __ARCH_SGX_DRIVER_H__ + +#include +#include +#include +#include +#include +#include +#include +#include "sgx.h" + +int sgx_drv_init(void); + +#endif /* __ARCH_X86_SGX_DRIVER_H__ */ diff --git a/arch/x86/kernel/cpu/sgx/encl.c b/arch/x86/kernel/cpu/sgx/encl.c new file mode 100644 index 000000000000..b9d445db7ff1 --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/encl.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#include +#include +#include +#include +#include +#include +#include "arch.h" +#include "encl.h" +#include "encls.h" +#include "sgx.h" + +static struct sgx_encl_page *sgx_encl_load_page(struct sgx_encl *encl, + unsigned long addr, + unsigned long vm_flags) +{ + unsigned long vm_prot_bits = vm_flags & (VM_READ | VM_WRITE | VM_EXEC); + struct sgx_encl_page *entry; + + entry = xa_load(&encl->page_array, PFN_DOWN(addr)); + if (!entry) + return ERR_PTR(-EFAULT); + + /* + * Verify that the faulted page has equal or higher build time + * permissions than the VMA permissions (i.e. the subset of {VM_READ, + * VM_WRITE, VM_EXECUTE} in vma->vm_flags). + */ + if ((entry->vm_max_prot_bits & vm_prot_bits) != vm_prot_bits) + return ERR_PTR(-EFAULT); + + /* No page found. */ + if (!entry->epc_page) + return ERR_PTR(-EFAULT); + + /* Entry successfully located. */ + return entry; +} + +static vm_fault_t sgx_vma_fault(struct vm_fault *vmf) +{ + unsigned long addr = (unsigned long)vmf->address; + struct vm_area_struct *vma = vmf->vma; + struct sgx_encl_page *entry; + unsigned long phys_addr; + struct sgx_encl *encl; + vm_fault_t ret; + + encl = vma->vm_private_data; + + mutex_lock(&encl->lock); + + entry = sgx_encl_load_page(encl, addr, vma->vm_flags); + if (IS_ERR(entry)) { + mutex_unlock(&encl->lock); + + return VM_FAULT_SIGBUS; + } + + phys_addr = sgx_get_epc_phys_addr(entry->epc_page); + + ret = vmf_insert_pfn(vma, addr, PFN_DOWN(phys_addr)); + if (ret != VM_FAULT_NOPAGE) { + mutex_unlock(&encl->lock); + + return VM_FAULT_SIGBUS; + } + + mutex_unlock(&encl->lock); + + return VM_FAULT_NOPAGE; +} + +/** + * sgx_encl_may_map() - Check if a requested VMA mapping is allowed + * @encl: an enclave pointer + * @start: lower bound of the address range, inclusive + * @end: upper bound of the address range, exclusive + * @vm_flags: VMA flags + * + * Iterate through the enclave pages contained within [@start, @end) to verify + * that the permissions requested by a subset of {VM_READ, VM_WRITE, VM_EXEC} + * do not contain any permissions that are not contained in the build time + * permissions of any of the enclave pages within the given address range. + * + * An enclave creator must declare the strongest permissions that will be + * needed for each enclave page. This ensures that mappings have the identical + * or weaker permissions than the earlier declared permissions. + * + * Return: 0 on success, -EACCES otherwise + */ +int sgx_encl_may_map(struct sgx_encl *encl, unsigned long start, + unsigned long end, unsigned long vm_flags) +{ + unsigned long vm_prot_bits = vm_flags & (VM_READ | VM_WRITE | VM_EXEC); + struct sgx_encl_page *page; + unsigned long count = 0; + int ret = 0; + + XA_STATE(xas, &encl->page_array, PFN_DOWN(start)); + + /* + * Disallow READ_IMPLIES_EXEC tasks as their VMA permissions might + * conflict with the enclave page permissions. + */ + if (current->personality & READ_IMPLIES_EXEC) + return -EACCES; + + mutex_lock(&encl->lock); + xas_lock(&xas); + xas_for_each(&xas, page, PFN_DOWN(end - 1)) { + if (~page->vm_max_prot_bits & vm_prot_bits) { + ret = -EACCES; + break; + } + + /* Reschedule on every XA_CHECK_SCHED iteration. */ + if (!(++count % XA_CHECK_SCHED)) { + xas_pause(&xas); + xas_unlock(&xas); + mutex_unlock(&encl->lock); + + cond_resched(); + + mutex_lock(&encl->lock); + xas_lock(&xas); + } + } + xas_unlock(&xas); + mutex_unlock(&encl->lock); + + return ret; +} + +static int sgx_vma_mprotect(struct vm_area_struct *vma, unsigned long start, + unsigned long end, unsigned long newflags) +{ + return sgx_encl_may_map(vma->vm_private_data, start, end, newflags); +} + +const struct vm_operations_struct sgx_vm_ops = { + .fault = sgx_vma_fault, + .mprotect = sgx_vma_mprotect, +}; diff --git a/arch/x86/kernel/cpu/sgx/encl.h b/arch/x86/kernel/cpu/sgx/encl.h new file mode 100644 index 000000000000..1df8011fa23d --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/encl.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/** + * Copyright(c) 2016-20 Intel Corporation. + * + * Contains the software defined data structures for enclaves. + */ +#ifndef _X86_ENCL_H +#define _X86_ENCL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "sgx.h" + +struct sgx_encl_page { + unsigned long desc; + unsigned long vm_max_prot_bits; + struct sgx_epc_page *epc_page; + struct sgx_encl *encl; +}; + +struct sgx_encl { + unsigned long base; + unsigned long size; + unsigned int page_cnt; + unsigned int secs_child_cnt; + struct mutex lock; + struct xarray page_array; + struct sgx_encl_page secs; +}; + +extern const struct vm_operations_struct sgx_vm_ops; + +static inline int sgx_encl_find(struct mm_struct *mm, unsigned long addr, + struct vm_area_struct **vma) +{ + struct vm_area_struct *result; + + result = find_vma(mm, addr); + if (!result || result->vm_ops != &sgx_vm_ops || addr < result->vm_start) + return -EINVAL; + + *vma = result; + + return 0; +} + +int sgx_encl_may_map(struct sgx_encl *encl, unsigned long start, + unsigned long end, unsigned long vm_flags); + +#endif /* _X86_ENCL_H */ diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index 2e53afc288a4..38f2e80cc31a 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -9,6 +9,8 @@ #include #include #include +#include "driver.h" +#include "encl.h" #include "encls.h" struct sgx_epc_section sgx_epc_sections[SGX_MAX_EPC_SECTIONS]; @@ -232,9 +234,10 @@ static bool __init sgx_page_cache_init(void) static void __init sgx_init(void) { + int ret; int i; - if (!boot_cpu_has(X86_FEATURE_SGX)) + if (!cpu_feature_enabled(X86_FEATURE_SGX)) return; if (!sgx_page_cache_init()) @@ -243,8 +246,15 @@ static void __init sgx_init(void) if (!sgx_page_reclaimer_init()) goto err_page_cache; + ret = sgx_drv_init(); + if (ret) + goto err_kthread; + return; +err_kthread: + kthread_stop(ksgxd_tsk); + err_page_cache: for (i = 0; i < sgx_nr_epc_sections; i++) { vfree(sgx_epc_sections[i].pages); From 888d249117876239593fe3039b6ead8ad6849035 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 13 Nov 2020 00:01:23 +0200 Subject: [PATCH 186/360] x86/sgx: Add SGX_IOC_ENCLAVE_CREATE Add an ioctl() that performs the ECREATE function of the ENCLS instruction, which creates an SGX Enclave Control Structure (SECS). Although the SECS is an in-memory data structure, it is present in enclave memory and is not directly accessible by software. Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Tested-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-13-jarkko@kernel.org --- .../userspace-api/ioctl/ioctl-number.rst | 1 + arch/x86/include/uapi/asm/sgx.h | 25 ++++ arch/x86/kernel/cpu/sgx/Makefile | 1 + arch/x86/kernel/cpu/sgx/driver.c | 12 ++ arch/x86/kernel/cpu/sgx/driver.h | 3 + arch/x86/kernel/cpu/sgx/encl.c | 8 ++ arch/x86/kernel/cpu/sgx/encl.h | 7 + arch/x86/kernel/cpu/sgx/ioctl.c | 123 ++++++++++++++++++ 8 files changed, 180 insertions(+) create mode 100644 arch/x86/include/uapi/asm/sgx.h create mode 100644 arch/x86/kernel/cpu/sgx/ioctl.c diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 55a2d9b2ce33..a4c75a28c839 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -323,6 +323,7 @@ Code Seq# Include File Comments 0xA3 90-9F linux/dtlk.h 0xA4 00-1F uapi/linux/tee.h Generic TEE subsystem +0xA4 00-1F uapi/asm/sgx.h 0xAA 00-3F linux/uapi/linux/userfaultfd.h 0xAB 00-1F linux/nbd.h 0xAC 00-1F linux/raw.h diff --git a/arch/x86/include/uapi/asm/sgx.h b/arch/x86/include/uapi/asm/sgx.h new file mode 100644 index 000000000000..f31bb17e27c3 --- /dev/null +++ b/arch/x86/include/uapi/asm/sgx.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Copyright(c) 2016-20 Intel Corporation. + */ +#ifndef _UAPI_ASM_X86_SGX_H +#define _UAPI_ASM_X86_SGX_H + +#include +#include + +#define SGX_MAGIC 0xA4 + +#define SGX_IOC_ENCLAVE_CREATE \ + _IOW(SGX_MAGIC, 0x00, struct sgx_enclave_create) + +/** + * struct sgx_enclave_create - parameter structure for the + * %SGX_IOC_ENCLAVE_CREATE ioctl + * @src: address for the SECS page data + */ +struct sgx_enclave_create { + __u64 src; +}; + +#endif /* _UAPI_ASM_X86_SGX_H */ diff --git a/arch/x86/kernel/cpu/sgx/Makefile b/arch/x86/kernel/cpu/sgx/Makefile index 3fc451120735..91d3dc784a29 100644 --- a/arch/x86/kernel/cpu/sgx/Makefile +++ b/arch/x86/kernel/cpu/sgx/Makefile @@ -1,4 +1,5 @@ obj-y += \ driver.o \ encl.o \ + ioctl.o \ main.o diff --git a/arch/x86/kernel/cpu/sgx/driver.c b/arch/x86/kernel/cpu/sgx/driver.c index c2810e1c7cf1..ee947b721d8d 100644 --- a/arch/x86/kernel/cpu/sgx/driver.c +++ b/arch/x86/kernel/cpu/sgx/driver.c @@ -88,10 +88,22 @@ static unsigned long sgx_get_unmapped_area(struct file *file, return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); } +#ifdef CONFIG_COMPAT +static long sgx_compat_ioctl(struct file *filep, unsigned int cmd, + unsigned long arg) +{ + return sgx_ioctl(filep, cmd, arg); +} +#endif + static const struct file_operations sgx_encl_fops = { .owner = THIS_MODULE, .open = sgx_open, .release = sgx_release, + .unlocked_ioctl = sgx_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = sgx_compat_ioctl, +#endif .mmap = sgx_mmap, .get_unmapped_area = sgx_get_unmapped_area, }; diff --git a/arch/x86/kernel/cpu/sgx/driver.h b/arch/x86/kernel/cpu/sgx/driver.h index cda9c43b7543..a728e8e848bd 100644 --- a/arch/x86/kernel/cpu/sgx/driver.h +++ b/arch/x86/kernel/cpu/sgx/driver.h @@ -9,8 +9,11 @@ #include #include #include +#include #include "sgx.h" +long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg); + int sgx_drv_init(void); #endif /* __ARCH_X86_SGX_DRIVER_H__ */ diff --git a/arch/x86/kernel/cpu/sgx/encl.c b/arch/x86/kernel/cpu/sgx/encl.c index b9d445db7ff1..57eff300f487 100644 --- a/arch/x86/kernel/cpu/sgx/encl.c +++ b/arch/x86/kernel/cpu/sgx/encl.c @@ -46,6 +46,7 @@ static vm_fault_t sgx_vma_fault(struct vm_fault *vmf) struct sgx_encl_page *entry; unsigned long phys_addr; struct sgx_encl *encl; + unsigned long pfn; vm_fault_t ret; encl = vma->vm_private_data; @@ -61,6 +62,13 @@ static vm_fault_t sgx_vma_fault(struct vm_fault *vmf) phys_addr = sgx_get_epc_phys_addr(entry->epc_page); + /* Check if another thread got here first to insert the PTE. */ + if (!follow_pfn(vma, addr, &pfn)) { + mutex_unlock(&encl->lock); + + return VM_FAULT_NOPAGE; + } + ret = vmf_insert_pfn(vma, addr, PFN_DOWN(phys_addr)); if (ret != VM_FAULT_NOPAGE) { mutex_unlock(&encl->lock); diff --git a/arch/x86/kernel/cpu/sgx/encl.h b/arch/x86/kernel/cpu/sgx/encl.h index 1df8011fa23d..7cc175825b01 100644 --- a/arch/x86/kernel/cpu/sgx/encl.h +++ b/arch/x86/kernel/cpu/sgx/encl.h @@ -26,9 +26,16 @@ struct sgx_encl_page { struct sgx_encl *encl; }; +enum sgx_encl_flags { + SGX_ENCL_IOCTL = BIT(0), + SGX_ENCL_DEBUG = BIT(1), + SGX_ENCL_CREATED = BIT(2), +}; + struct sgx_encl { unsigned long base; unsigned long size; + unsigned long flags; unsigned int page_cnt; unsigned int secs_child_cnt; struct mutex lock; diff --git a/arch/x86/kernel/cpu/sgx/ioctl.c b/arch/x86/kernel/cpu/sgx/ioctl.c new file mode 100644 index 000000000000..1355490843d1 --- /dev/null +++ b/arch/x86/kernel/cpu/sgx/ioctl.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "driver.h" +#include "encl.h" +#include "encls.h" + +static int sgx_encl_create(struct sgx_encl *encl, struct sgx_secs *secs) +{ + struct sgx_epc_page *secs_epc; + struct sgx_pageinfo pginfo; + struct sgx_secinfo secinfo; + unsigned long encl_size; + long ret; + + /* The extra page goes to SECS. */ + encl_size = secs->size + PAGE_SIZE; + + secs_epc = __sgx_alloc_epc_page(); + if (IS_ERR(secs_epc)) + return PTR_ERR(secs_epc); + + encl->secs.epc_page = secs_epc; + + pginfo.addr = 0; + pginfo.contents = (unsigned long)secs; + pginfo.metadata = (unsigned long)&secinfo; + pginfo.secs = 0; + memset(&secinfo, 0, sizeof(secinfo)); + + ret = __ecreate((void *)&pginfo, sgx_get_epc_virt_addr(secs_epc)); + if (ret) { + ret = -EIO; + goto err_out; + } + + if (secs->attributes & SGX_ATTR_DEBUG) + set_bit(SGX_ENCL_DEBUG, &encl->flags); + + encl->secs.encl = encl; + encl->base = secs->base; + encl->size = secs->size; + + /* Set only after completion, as encl->lock has not been taken. */ + set_bit(SGX_ENCL_CREATED, &encl->flags); + + return 0; + +err_out: + sgx_free_epc_page(encl->secs.epc_page); + encl->secs.epc_page = NULL; + + return ret; +} + +/** + * sgx_ioc_enclave_create() - handler for %SGX_IOC_ENCLAVE_CREATE + * @encl: An enclave pointer. + * @arg: The ioctl argument. + * + * Allocate kernel data structures for the enclave and invoke ECREATE. + * + * Return: + * - 0: Success. + * - -EIO: ECREATE failed. + * - -errno: POSIX error. + */ +static long sgx_ioc_enclave_create(struct sgx_encl *encl, void __user *arg) +{ + struct sgx_enclave_create create_arg; + void *secs; + int ret; + + if (test_bit(SGX_ENCL_CREATED, &encl->flags)) + return -EINVAL; + + if (copy_from_user(&create_arg, arg, sizeof(create_arg))) + return -EFAULT; + + secs = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!secs) + return -ENOMEM; + + if (copy_from_user(secs, (void __user *)create_arg.src, PAGE_SIZE)) + ret = -EFAULT; + else + ret = sgx_encl_create(encl, secs); + + kfree(secs); + return ret; +} + +long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) +{ + struct sgx_encl *encl = filep->private_data; + int ret; + + if (test_and_set_bit(SGX_ENCL_IOCTL, &encl->flags)) + return -EBUSY; + + switch (cmd) { + case SGX_IOC_ENCLAVE_CREATE: + ret = sgx_ioc_enclave_create(encl, (void __user *)arg); + break; + default: + ret = -ENOIOCTLCMD; + break; + } + + clear_bit(SGX_ENCL_IOCTL, &encl->flags); + return ret; +} From c6d26d370767fa227fc44b98a8bdad112efdf563 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 13 Nov 2020 00:01:24 +0200 Subject: [PATCH 187/360] x86/sgx: Add SGX_IOC_ENCLAVE_ADD_PAGES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SGX enclave pages are inaccessible to normal software. They must be populated with data by copying from normal memory with the help of the EADD and EEXTEND functions of the ENCLS instruction. Add an ioctl() which performs EADD that adds new data to an enclave, and optionally EEXTEND functions that hash the page contents and use the hash as part of enclave “measurement” to ensure enclave integrity. The enclave author gets to decide which pages will be included in the enclave measurement with EEXTEND. Measurement is very slow and has sometimes has very little value. For instance, an enclave _could_ measure every page of data and code, but would be slow to initialize. Or, it might just measure its code and then trust that code to initialize the bulk of its data after it starts running. Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Tested-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-14-jarkko@kernel.org --- arch/x86/include/uapi/asm/sgx.h | 30 ++++ arch/x86/kernel/cpu/sgx/ioctl.c | 284 ++++++++++++++++++++++++++++++++ arch/x86/kernel/cpu/sgx/sgx.h | 1 + 3 files changed, 315 insertions(+) diff --git a/arch/x86/include/uapi/asm/sgx.h b/arch/x86/include/uapi/asm/sgx.h index f31bb17e27c3..835f7e588f0d 100644 --- a/arch/x86/include/uapi/asm/sgx.h +++ b/arch/x86/include/uapi/asm/sgx.h @@ -8,10 +8,21 @@ #include #include +/** + * enum sgx_epage_flags - page control flags + * %SGX_PAGE_MEASURE: Measure the page contents with a sequence of + * ENCLS[EEXTEND] operations. + */ +enum sgx_page_flags { + SGX_PAGE_MEASURE = 0x01, +}; + #define SGX_MAGIC 0xA4 #define SGX_IOC_ENCLAVE_CREATE \ _IOW(SGX_MAGIC, 0x00, struct sgx_enclave_create) +#define SGX_IOC_ENCLAVE_ADD_PAGES \ + _IOWR(SGX_MAGIC, 0x01, struct sgx_enclave_add_pages) /** * struct sgx_enclave_create - parameter structure for the @@ -22,4 +33,23 @@ struct sgx_enclave_create { __u64 src; }; +/** + * struct sgx_enclave_add_pages - parameter structure for the + * %SGX_IOC_ENCLAVE_ADD_PAGE ioctl + * @src: start address for the page data + * @offset: starting page offset + * @length: length of the data (multiple of the page size) + * @secinfo: address for the SECINFO data + * @flags: page control flags + * @count: number of bytes added (multiple of the page size) + */ +struct sgx_enclave_add_pages { + __u64 src; + __u64 offset; + __u64 length; + __u64 secinfo; + __u64 flags; + __u64 count; +}; + #endif /* _UAPI_ASM_X86_SGX_H */ diff --git a/arch/x86/kernel/cpu/sgx/ioctl.c b/arch/x86/kernel/cpu/sgx/ioctl.c index 1355490843d1..82acff7bda60 100644 --- a/arch/x86/kernel/cpu/sgx/ioctl.c +++ b/arch/x86/kernel/cpu/sgx/ioctl.c @@ -101,6 +101,287 @@ static long sgx_ioc_enclave_create(struct sgx_encl *encl, void __user *arg) return ret; } +static struct sgx_encl_page *sgx_encl_page_alloc(struct sgx_encl *encl, + unsigned long offset, + u64 secinfo_flags) +{ + struct sgx_encl_page *encl_page; + unsigned long prot; + + encl_page = kzalloc(sizeof(*encl_page), GFP_KERNEL); + if (!encl_page) + return ERR_PTR(-ENOMEM); + + encl_page->desc = encl->base + offset; + encl_page->encl = encl; + + prot = _calc_vm_trans(secinfo_flags, SGX_SECINFO_R, PROT_READ) | + _calc_vm_trans(secinfo_flags, SGX_SECINFO_W, PROT_WRITE) | + _calc_vm_trans(secinfo_flags, SGX_SECINFO_X, PROT_EXEC); + + /* + * TCS pages must always RW set for CPU access while the SECINFO + * permissions are *always* zero - the CPU ignores the user provided + * values and silently overwrites them with zero permissions. + */ + if ((secinfo_flags & SGX_SECINFO_PAGE_TYPE_MASK) == SGX_SECINFO_TCS) + prot |= PROT_READ | PROT_WRITE; + + /* Calculate maximum of the VM flags for the page. */ + encl_page->vm_max_prot_bits = calc_vm_prot_bits(prot, 0); + + return encl_page; +} + +static int sgx_validate_secinfo(struct sgx_secinfo *secinfo) +{ + u64 perm = secinfo->flags & SGX_SECINFO_PERMISSION_MASK; + u64 pt = secinfo->flags & SGX_SECINFO_PAGE_TYPE_MASK; + + if (pt != SGX_SECINFO_REG && pt != SGX_SECINFO_TCS) + return -EINVAL; + + if ((perm & SGX_SECINFO_W) && !(perm & SGX_SECINFO_R)) + return -EINVAL; + + /* + * CPU will silently overwrite the permissions as zero, which means + * that we need to validate it ourselves. + */ + if (pt == SGX_SECINFO_TCS && perm) + return -EINVAL; + + if (secinfo->flags & SGX_SECINFO_RESERVED_MASK) + return -EINVAL; + + if (memchr_inv(secinfo->reserved, 0, sizeof(secinfo->reserved))) + return -EINVAL; + + return 0; +} + +static int __sgx_encl_add_page(struct sgx_encl *encl, + struct sgx_encl_page *encl_page, + struct sgx_epc_page *epc_page, + struct sgx_secinfo *secinfo, unsigned long src) +{ + struct sgx_pageinfo pginfo; + struct vm_area_struct *vma; + struct page *src_page; + int ret; + + /* Deny noexec. */ + vma = find_vma(current->mm, src); + if (!vma) + return -EFAULT; + + if (!(vma->vm_flags & VM_MAYEXEC)) + return -EACCES; + + ret = get_user_pages(src, 1, 0, &src_page, NULL); + if (ret < 1) + return -EFAULT; + + pginfo.secs = (unsigned long)sgx_get_epc_virt_addr(encl->secs.epc_page); + pginfo.addr = encl_page->desc & PAGE_MASK; + pginfo.metadata = (unsigned long)secinfo; + pginfo.contents = (unsigned long)kmap_atomic(src_page); + + ret = __eadd(&pginfo, sgx_get_epc_virt_addr(epc_page)); + + kunmap_atomic((void *)pginfo.contents); + put_page(src_page); + + return ret ? -EIO : 0; +} + +/* + * If the caller requires measurement of the page as a proof for the content, + * use EEXTEND to add a measurement for 256 bytes of the page. Repeat this + * operation until the entire page is measured." + */ +static int __sgx_encl_extend(struct sgx_encl *encl, + struct sgx_epc_page *epc_page) +{ + unsigned long offset; + int ret; + + for (offset = 0; offset < PAGE_SIZE; offset += SGX_EEXTEND_BLOCK_SIZE) { + ret = __eextend(sgx_get_epc_virt_addr(encl->secs.epc_page), + sgx_get_epc_virt_addr(epc_page) + offset); + if (ret) { + if (encls_failed(ret)) + ENCLS_WARN(ret, "EEXTEND"); + + return -EIO; + } + } + + return 0; +} + +static int sgx_encl_add_page(struct sgx_encl *encl, unsigned long src, + unsigned long offset, struct sgx_secinfo *secinfo, + unsigned long flags) +{ + struct sgx_encl_page *encl_page; + struct sgx_epc_page *epc_page; + int ret; + + encl_page = sgx_encl_page_alloc(encl, offset, secinfo->flags); + if (IS_ERR(encl_page)) + return PTR_ERR(encl_page); + + epc_page = __sgx_alloc_epc_page(); + if (IS_ERR(epc_page)) { + kfree(encl_page); + return PTR_ERR(epc_page); + } + + mmap_read_lock(current->mm); + mutex_lock(&encl->lock); + + /* + * Insert prior to EADD in case of OOM. EADD modifies MRENCLAVE, i.e. + * can't be gracefully unwound, while failure on EADD/EXTEND is limited + * to userspace errors (or kernel/hardware bugs). + */ + ret = xa_insert(&encl->page_array, PFN_DOWN(encl_page->desc), + encl_page, GFP_KERNEL); + if (ret) + goto err_out_unlock; + + ret = __sgx_encl_add_page(encl, encl_page, epc_page, secinfo, + src); + if (ret) + goto err_out; + + /* + * Complete the "add" before doing the "extend" so that the "add" + * isn't in a half-baked state in the extremely unlikely scenario + * the enclave will be destroyed in response to EEXTEND failure. + */ + encl_page->encl = encl; + encl_page->epc_page = epc_page; + encl->secs_child_cnt++; + + if (flags & SGX_PAGE_MEASURE) { + ret = __sgx_encl_extend(encl, epc_page); + if (ret) + goto err_out; + } + + mutex_unlock(&encl->lock); + mmap_read_unlock(current->mm); + return ret; + +err_out: + xa_erase(&encl->page_array, PFN_DOWN(encl_page->desc)); + +err_out_unlock: + mutex_unlock(&encl->lock); + mmap_read_unlock(current->mm); + + sgx_free_epc_page(epc_page); + kfree(encl_page); + + return ret; +} + +/** + * sgx_ioc_enclave_add_pages() - The handler for %SGX_IOC_ENCLAVE_ADD_PAGES + * @encl: an enclave pointer + * @arg: a user pointer to a struct sgx_enclave_add_pages instance + * + * Add one or more pages to an uninitialized enclave, and optionally extend the + * measurement with the contents of the page. The SECINFO and measurement mask + * are applied to all pages. + * + * A SECINFO for a TCS is required to always contain zero permissions because + * CPU silently zeros them. Allowing anything else would cause a mismatch in + * the measurement. + * + * mmap()'s protection bits are capped by the page permissions. For each page + * address, the maximum protection bits are computed with the following + * heuristics: + * + * 1. A regular page: PROT_R, PROT_W and PROT_X match the SECINFO permissions. + * 2. A TCS page: PROT_R | PROT_W. + * + * mmap() is not allowed to surpass the minimum of the maximum protection bits + * within the given address range. + * + * The function deinitializes kernel data structures for enclave and returns + * -EIO in any of the following conditions: + * + * - Enclave Page Cache (EPC), the physical memory holding enclaves, has + * been invalidated. This will cause EADD and EEXTEND to fail. + * - If the source address is corrupted somehow when executing EADD. + * + * Return: + * - 0: Success. + * - -EACCES: The source page is located in a noexec partition. + * - -ENOMEM: Out of EPC pages. + * - -EINTR: The call was interrupted before data was processed. + * - -EIO: Either EADD or EEXTEND failed because invalid source address + * or power cycle. + * - -errno: POSIX error. + */ +static long sgx_ioc_enclave_add_pages(struct sgx_encl *encl, void __user *arg) +{ + struct sgx_enclave_add_pages add_arg; + struct sgx_secinfo secinfo; + unsigned long c; + int ret; + + if (!test_bit(SGX_ENCL_CREATED, &encl->flags)) + return -EINVAL; + + if (copy_from_user(&add_arg, arg, sizeof(add_arg))) + return -EFAULT; + + if (!IS_ALIGNED(add_arg.offset, PAGE_SIZE) || + !IS_ALIGNED(add_arg.src, PAGE_SIZE)) + return -EINVAL; + + if (add_arg.length & (PAGE_SIZE - 1)) + return -EINVAL; + + if (add_arg.offset + add_arg.length - PAGE_SIZE >= encl->size) + return -EINVAL; + + if (copy_from_user(&secinfo, (void __user *)add_arg.secinfo, + sizeof(secinfo))) + return -EFAULT; + + if (sgx_validate_secinfo(&secinfo)) + return -EINVAL; + + for (c = 0 ; c < add_arg.length; c += PAGE_SIZE) { + if (signal_pending(current)) { + if (!c) + ret = -EINTR; + + break; + } + + if (need_resched()) + cond_resched(); + + ret = sgx_encl_add_page(encl, add_arg.src + c, add_arg.offset + c, + &secinfo, add_arg.flags); + if (ret) + break; + } + + add_arg.count = c; + + if (copy_to_user(arg, &add_arg, sizeof(add_arg))) + return -EFAULT; + + return ret; +} + long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { struct sgx_encl *encl = filep->private_data; @@ -113,6 +394,9 @@ long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) case SGX_IOC_ENCLAVE_CREATE: ret = sgx_ioc_enclave_create(encl, (void __user *)arg); break; + case SGX_IOC_ENCLAVE_ADD_PAGES: + ret = sgx_ioc_enclave_add_pages(encl, (void __user *)arg); + break; default: ret = -ENOIOCTLCMD; break; diff --git a/arch/x86/kernel/cpu/sgx/sgx.h b/arch/x86/kernel/cpu/sgx/sgx.h index bd9dcb1ffcfa..91234f425b89 100644 --- a/arch/x86/kernel/cpu/sgx/sgx.h +++ b/arch/x86/kernel/cpu/sgx/sgx.h @@ -14,6 +14,7 @@ #define pr_fmt(fmt) "sgx: " fmt #define SGX_MAX_EPC_SECTIONS 8 +#define SGX_EEXTEND_BLOCK_SIZE 256 struct sgx_epc_page { unsigned int section; From 9d0c151b41fed7b879030f4e533143d098781701 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 13 Nov 2020 00:01:25 +0200 Subject: [PATCH 188/360] x86/sgx: Add SGX_IOC_ENCLAVE_INIT Enclaves have two basic states. They are either being built and are malleable and can be modified by doing things like adding pages. Or, they are locked down and not accepting changes. They can only be run after they have been locked down. The ENCLS[EINIT] function induces the transition from being malleable to locked-down. Add an ioctl() that performs ENCLS[EINIT]. After this, new pages can no longer be added with ENCLS[EADD]. This is also the time where the enclave can be measured to verify its integrity. Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Tested-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-15-jarkko@kernel.org --- arch/x86/include/uapi/asm/sgx.h | 11 ++ arch/x86/kernel/cpu/sgx/driver.c | 27 +++++ arch/x86/kernel/cpu/sgx/driver.h | 8 ++ arch/x86/kernel/cpu/sgx/encl.h | 3 + arch/x86/kernel/cpu/sgx/ioctl.c | 193 ++++++++++++++++++++++++++++++- 5 files changed, 241 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/uapi/asm/sgx.h b/arch/x86/include/uapi/asm/sgx.h index 835f7e588f0d..66f2d32cb4d7 100644 --- a/arch/x86/include/uapi/asm/sgx.h +++ b/arch/x86/include/uapi/asm/sgx.h @@ -23,6 +23,8 @@ enum sgx_page_flags { _IOW(SGX_MAGIC, 0x00, struct sgx_enclave_create) #define SGX_IOC_ENCLAVE_ADD_PAGES \ _IOWR(SGX_MAGIC, 0x01, struct sgx_enclave_add_pages) +#define SGX_IOC_ENCLAVE_INIT \ + _IOW(SGX_MAGIC, 0x02, struct sgx_enclave_init) /** * struct sgx_enclave_create - parameter structure for the @@ -52,4 +54,13 @@ struct sgx_enclave_add_pages { __u64 count; }; +/** + * struct sgx_enclave_init - parameter structure for the + * %SGX_IOC_ENCLAVE_INIT ioctl + * @sigstruct: address for the SIGSTRUCT data + */ +struct sgx_enclave_init { + __u64 sigstruct; +}; + #endif /* _UAPI_ASM_X86_SGX_H */ diff --git a/arch/x86/kernel/cpu/sgx/driver.c b/arch/x86/kernel/cpu/sgx/driver.c index ee947b721d8d..bf5c4a36a548 100644 --- a/arch/x86/kernel/cpu/sgx/driver.c +++ b/arch/x86/kernel/cpu/sgx/driver.c @@ -10,6 +10,10 @@ #include "driver.h" #include "encl.h" +u64 sgx_attributes_reserved_mask; +u64 sgx_xfrm_reserved_mask = ~0x3; +u32 sgx_misc_reserved_mask; + static int sgx_open(struct inode *inode, struct file *file) { struct sgx_encl *encl; @@ -117,8 +121,31 @@ static struct miscdevice sgx_dev_enclave = { int __init sgx_drv_init(void) { + unsigned int eax, ebx, ecx, edx; + u64 attr_mask; + u64 xfrm_mask; + if (!cpu_feature_enabled(X86_FEATURE_SGX_LC)) return -ENODEV; + cpuid_count(SGX_CPUID, 0, &eax, &ebx, &ecx, &edx); + + if (!(eax & 1)) { + pr_err("SGX disabled: SGX1 instruction support not available.\n"); + return -ENODEV; + } + + sgx_misc_reserved_mask = ~ebx | SGX_MISC_RESERVED_MASK; + + cpuid_count(SGX_CPUID, 1, &eax, &ebx, &ecx, &edx); + + attr_mask = (((u64)ebx) << 32) + (u64)eax; + sgx_attributes_reserved_mask = ~attr_mask | SGX_ATTR_RESERVED_MASK; + + if (cpu_feature_enabled(X86_FEATURE_OSXSAVE)) { + xfrm_mask = (((u64)edx) << 32) + (u64)ecx; + sgx_xfrm_reserved_mask = ~xfrm_mask; + } + return misc_register(&sgx_dev_enclave); } diff --git a/arch/x86/kernel/cpu/sgx/driver.h b/arch/x86/kernel/cpu/sgx/driver.h index a728e8e848bd..6b0063221659 100644 --- a/arch/x86/kernel/cpu/sgx/driver.h +++ b/arch/x86/kernel/cpu/sgx/driver.h @@ -12,6 +12,14 @@ #include #include "sgx.h" +#define SGX_EINIT_SPIN_COUNT 20 +#define SGX_EINIT_SLEEP_COUNT 50 +#define SGX_EINIT_SLEEP_TIME 20 + +extern u64 sgx_attributes_reserved_mask; +extern u64 sgx_xfrm_reserved_mask; +extern u32 sgx_misc_reserved_mask; + long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg); int sgx_drv_init(void); diff --git a/arch/x86/kernel/cpu/sgx/encl.h b/arch/x86/kernel/cpu/sgx/encl.h index 7cc175825b01..8a4d1edded68 100644 --- a/arch/x86/kernel/cpu/sgx/encl.h +++ b/arch/x86/kernel/cpu/sgx/encl.h @@ -30,6 +30,7 @@ enum sgx_encl_flags { SGX_ENCL_IOCTL = BIT(0), SGX_ENCL_DEBUG = BIT(1), SGX_ENCL_CREATED = BIT(2), + SGX_ENCL_INITIALIZED = BIT(3), }; struct sgx_encl { @@ -41,6 +42,8 @@ struct sgx_encl { struct mutex lock; struct xarray page_array; struct sgx_encl_page secs; + unsigned long attributes; + unsigned long attributes_mask; }; extern const struct vm_operations_struct sgx_vm_ops; diff --git a/arch/x86/kernel/cpu/sgx/ioctl.c b/arch/x86/kernel/cpu/sgx/ioctl.c index 82acff7bda60..e036819ea5c1 100644 --- a/arch/x86/kernel/cpu/sgx/ioctl.c +++ b/arch/x86/kernel/cpu/sgx/ioctl.c @@ -51,6 +51,8 @@ static int sgx_encl_create(struct sgx_encl *encl, struct sgx_secs *secs) encl->secs.encl = encl; encl->base = secs->base; encl->size = secs->size; + encl->attributes = secs->attributes; + encl->attributes_mask = SGX_ATTR_DEBUG | SGX_ATTR_MODE64BIT | SGX_ATTR_KSS; /* Set only after completion, as encl->lock has not been taken. */ set_bit(SGX_ENCL_CREATED, &encl->flags); @@ -334,7 +336,8 @@ static long sgx_ioc_enclave_add_pages(struct sgx_encl *encl, void __user *arg) unsigned long c; int ret; - if (!test_bit(SGX_ENCL_CREATED, &encl->flags)) + if (!test_bit(SGX_ENCL_CREATED, &encl->flags) || + test_bit(SGX_ENCL_INITIALIZED, &encl->flags)) return -EINVAL; if (copy_from_user(&add_arg, arg, sizeof(add_arg))) @@ -382,6 +385,191 @@ static long sgx_ioc_enclave_add_pages(struct sgx_encl *encl, void __user *arg) return ret; } +static int __sgx_get_key_hash(struct crypto_shash *tfm, const void *modulus, + void *hash) +{ + SHASH_DESC_ON_STACK(shash, tfm); + + shash->tfm = tfm; + + return crypto_shash_digest(shash, modulus, SGX_MODULUS_SIZE, hash); +} + +static int sgx_get_key_hash(const void *modulus, void *hash) +{ + struct crypto_shash *tfm; + int ret; + + tfm = crypto_alloc_shash("sha256", 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + + ret = __sgx_get_key_hash(tfm, modulus, hash); + + crypto_free_shash(tfm); + return ret; +} + +static int sgx_encl_init(struct sgx_encl *encl, struct sgx_sigstruct *sigstruct, + void *token) +{ + u64 mrsigner[4]; + int i, j, k; + void *addr; + int ret; + + /* + * Deny initializing enclaves with attributes (namely provisioning) + * that have not been explicitly allowed. + */ + if (encl->attributes & ~encl->attributes_mask) + return -EACCES; + + /* + * Attributes should not be enforced *only* against what's available on + * platform (done in sgx_encl_create) but checked and enforced against + * the mask for enforcement in sigstruct. For example an enclave could + * opt to sign with AVX bit in xfrm, but still be loadable on a platform + * without it if the sigstruct->body.attributes_mask does not turn that + * bit on. + */ + if (sigstruct->body.attributes & sigstruct->body.attributes_mask & + sgx_attributes_reserved_mask) + return -EINVAL; + + if (sigstruct->body.miscselect & sigstruct->body.misc_mask & + sgx_misc_reserved_mask) + return -EINVAL; + + if (sigstruct->body.xfrm & sigstruct->body.xfrm_mask & + sgx_xfrm_reserved_mask) + return -EINVAL; + + ret = sgx_get_key_hash(sigstruct->modulus, mrsigner); + if (ret) + return ret; + + mutex_lock(&encl->lock); + + /* + * ENCLS[EINIT] is interruptible because it has such a high latency, + * e.g. 50k+ cycles on success. If an IRQ/NMI/SMI becomes pending, + * EINIT may fail with SGX_UNMASKED_EVENT so that the event can be + * serviced. + */ + for (i = 0; i < SGX_EINIT_SLEEP_COUNT; i++) { + for (j = 0; j < SGX_EINIT_SPIN_COUNT; j++) { + addr = sgx_get_epc_virt_addr(encl->secs.epc_page); + + preempt_disable(); + + for (k = 0; k < 4; k++) + wrmsrl(MSR_IA32_SGXLEPUBKEYHASH0 + k, mrsigner[k]); + + ret = __einit(sigstruct, token, addr); + + preempt_enable(); + + if (ret == SGX_UNMASKED_EVENT) + continue; + else + break; + } + + if (ret != SGX_UNMASKED_EVENT) + break; + + msleep_interruptible(SGX_EINIT_SLEEP_TIME); + + if (signal_pending(current)) { + ret = -ERESTARTSYS; + goto err_out; + } + } + + if (ret & ENCLS_FAULT_FLAG) { + if (encls_failed(ret)) + ENCLS_WARN(ret, "EINIT"); + + ret = -EIO; + } else if (ret) { + pr_debug("EINIT returned %d\n", ret); + ret = -EPERM; + } else { + set_bit(SGX_ENCL_INITIALIZED, &encl->flags); + } + +err_out: + mutex_unlock(&encl->lock); + return ret; +} + +/** + * sgx_ioc_enclave_init() - handler for %SGX_IOC_ENCLAVE_INIT + * @encl: an enclave pointer + * @arg: userspace pointer to a struct sgx_enclave_init instance + * + * Flush any outstanding enqueued EADD operations and perform EINIT. The + * Launch Enclave Public Key Hash MSRs are rewritten as necessary to match + * the enclave's MRSIGNER, which is caculated from the provided sigstruct. + * + * Return: + * - 0: Success. + * - -EPERM: Invalid SIGSTRUCT. + * - -EIO: EINIT failed because of a power cycle. + * - -errno: POSIX error. + */ +static long sgx_ioc_enclave_init(struct sgx_encl *encl, void __user *arg) +{ + struct sgx_sigstruct *sigstruct; + struct sgx_enclave_init init_arg; + struct page *initp_page; + void *token; + int ret; + + if (!test_bit(SGX_ENCL_CREATED, &encl->flags) || + test_bit(SGX_ENCL_INITIALIZED, &encl->flags)) + return -EINVAL; + + if (copy_from_user(&init_arg, arg, sizeof(init_arg))) + return -EFAULT; + + initp_page = alloc_page(GFP_KERNEL); + if (!initp_page) + return -ENOMEM; + + sigstruct = kmap(initp_page); + token = (void *)((unsigned long)sigstruct + PAGE_SIZE / 2); + memset(token, 0, SGX_LAUNCH_TOKEN_SIZE); + + if (copy_from_user(sigstruct, (void __user *)init_arg.sigstruct, + sizeof(*sigstruct))) { + ret = -EFAULT; + goto out; + } + + /* + * A legacy field used with Intel signed enclaves. These used to mean + * regular and architectural enclaves. The CPU only accepts these values + * but they do not have any other meaning. + * + * Thus, reject any other values. + */ + if (sigstruct->header.vendor != 0x0000 && + sigstruct->header.vendor != 0x8086) { + ret = -EINVAL; + goto out; + } + + ret = sgx_encl_init(encl, sigstruct, token); + +out: + kunmap(initp_page); + __free_page(initp_page); + return ret; +} + + long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { struct sgx_encl *encl = filep->private_data; @@ -397,6 +585,9 @@ long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) case SGX_IOC_ENCLAVE_ADD_PAGES: ret = sgx_ioc_enclave_add_pages(encl, (void __user *)arg); break; + case SGX_IOC_ENCLAVE_INIT: + ret = sgx_ioc_enclave_init(encl, (void __user *)arg); + break; default: ret = -ENOIOCTLCMD; break; From c82c61865024b9981f00358433bebed92ca20c00 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 13 Nov 2020 00:01:26 +0200 Subject: [PATCH 189/360] x86/sgx: Add SGX_IOC_ENCLAVE_PROVISION MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole point of SGX is to create a hardware protected place to do “stuff”. But, before someone is willing to hand over the keys to the castle , an enclave must often prove that it is running on an SGX-protected processor. Provisioning enclaves play a key role in providing proof. There are actually three different enclaves in play in order to make this happen: 1. The application enclave. The familiar one we know and love that runs the actual code that’s doing real work. There can be many of these on a single system, or even in a single application. 2. The quoting enclave (QE). The QE is mentioned in lots of silly whitepapers, but, for the purposes of kernel enabling, just pretend they do not exist. 3. The provisioning enclave. There is typically only one of these enclaves per system. Provisioning enclaves have access to a special hardware key. They can use this key to help to generate certificates which serve as proof that enclaves are running on trusted SGX hardware. These certificates can be passed around without revealing the special key. Any user who can create a provisioning enclave can access the processor-unique Provisioning Certificate Key which has privacy and fingerprinting implications. Even if a user is permitted to create normal application enclaves (via /dev/sgx_enclave), they should not be able to create provisioning enclaves. That means a separate permissions scheme is needed to control provisioning enclave privileges. Implement a separate device file (/dev/sgx_provision) which allows creating provisioning enclaves. This device will typically have more strict permissions than the plain enclave device. The actual device “driver” is an empty stub. Open file descriptors for this device will represent a token which allows provisioning enclave duty. This file descriptor can be passed around and ultimately given as an argument to the /dev/sgx_enclave driver ioctl(). [ bp: Touchups. ] Suggested-by: Andy Lutomirski Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Cc: linux-security-module@vger.kernel.org Link: https://lkml.kernel.org/r/20201112220135.165028-16-jarkko@kernel.org --- arch/x86/include/uapi/asm/sgx.h | 11 ++++++++++ arch/x86/kernel/cpu/sgx/driver.c | 24 ++++++++++++++++++++- arch/x86/kernel/cpu/sgx/driver.h | 2 ++ arch/x86/kernel/cpu/sgx/ioctl.c | 37 ++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/uapi/asm/sgx.h b/arch/x86/include/uapi/asm/sgx.h index 66f2d32cb4d7..c32210235bf5 100644 --- a/arch/x86/include/uapi/asm/sgx.h +++ b/arch/x86/include/uapi/asm/sgx.h @@ -25,6 +25,8 @@ enum sgx_page_flags { _IOWR(SGX_MAGIC, 0x01, struct sgx_enclave_add_pages) #define SGX_IOC_ENCLAVE_INIT \ _IOW(SGX_MAGIC, 0x02, struct sgx_enclave_init) +#define SGX_IOC_ENCLAVE_PROVISION \ + _IOW(SGX_MAGIC, 0x03, struct sgx_enclave_provision) /** * struct sgx_enclave_create - parameter structure for the @@ -63,4 +65,13 @@ struct sgx_enclave_init { __u64 sigstruct; }; +/** + * struct sgx_enclave_provision - parameter structure for the + * %SGX_IOC_ENCLAVE_PROVISION ioctl + * @fd: file handle of /dev/sgx_provision + */ +struct sgx_enclave_provision { + __u64 fd; +}; + #endif /* _UAPI_ASM_X86_SGX_H */ diff --git a/arch/x86/kernel/cpu/sgx/driver.c b/arch/x86/kernel/cpu/sgx/driver.c index bf5c4a36a548..899c18499d1a 100644 --- a/arch/x86/kernel/cpu/sgx/driver.c +++ b/arch/x86/kernel/cpu/sgx/driver.c @@ -112,6 +112,10 @@ static const struct file_operations sgx_encl_fops = { .get_unmapped_area = sgx_get_unmapped_area, }; +const struct file_operations sgx_provision_fops = { + .owner = THIS_MODULE, +}; + static struct miscdevice sgx_dev_enclave = { .minor = MISC_DYNAMIC_MINOR, .name = "sgx_enclave", @@ -119,11 +123,19 @@ static struct miscdevice sgx_dev_enclave = { .fops = &sgx_encl_fops, }; +static struct miscdevice sgx_dev_provision = { + .minor = MISC_DYNAMIC_MINOR, + .name = "sgx_provision", + .nodename = "sgx_provision", + .fops = &sgx_provision_fops, +}; + int __init sgx_drv_init(void) { unsigned int eax, ebx, ecx, edx; u64 attr_mask; u64 xfrm_mask; + int ret; if (!cpu_feature_enabled(X86_FEATURE_SGX_LC)) return -ENODEV; @@ -147,5 +159,15 @@ int __init sgx_drv_init(void) sgx_xfrm_reserved_mask = ~xfrm_mask; } - return misc_register(&sgx_dev_enclave); + ret = misc_register(&sgx_dev_enclave); + if (ret) + return ret; + + ret = misc_register(&sgx_dev_provision); + if (ret) { + misc_deregister(&sgx_dev_enclave); + return ret; + } + + return 0; } diff --git a/arch/x86/kernel/cpu/sgx/driver.h b/arch/x86/kernel/cpu/sgx/driver.h index 6b0063221659..4eddb4d571ef 100644 --- a/arch/x86/kernel/cpu/sgx/driver.h +++ b/arch/x86/kernel/cpu/sgx/driver.h @@ -20,6 +20,8 @@ extern u64 sgx_attributes_reserved_mask; extern u64 sgx_xfrm_reserved_mask; extern u32 sgx_misc_reserved_mask; +extern const struct file_operations sgx_provision_fops; + long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg); int sgx_drv_init(void); diff --git a/arch/x86/kernel/cpu/sgx/ioctl.c b/arch/x86/kernel/cpu/sgx/ioctl.c index e036819ea5c1..0ba0e670e2f0 100644 --- a/arch/x86/kernel/cpu/sgx/ioctl.c +++ b/arch/x86/kernel/cpu/sgx/ioctl.c @@ -569,6 +569,40 @@ out: return ret; } +/** + * sgx_ioc_enclave_provision() - handler for %SGX_IOC_ENCLAVE_PROVISION + * @enclave: an enclave pointer + * @arg: userspace pointer to a struct sgx_enclave_provision instance + * + * Allow ATTRIBUTE.PROVISION_KEY for an enclave by providing a file handle to + * /dev/sgx_provision. + * + * Return: + * - 0: Success. + * - -errno: Otherwise. + */ +static long sgx_ioc_enclave_provision(struct sgx_encl *encl, void __user *arg) +{ + struct sgx_enclave_provision params; + struct file *file; + + if (copy_from_user(¶ms, arg, sizeof(params))) + return -EFAULT; + + file = fget(params.fd); + if (!file) + return -EINVAL; + + if (file->f_op != &sgx_provision_fops) { + fput(file); + return -EINVAL; + } + + encl->attributes_mask |= SGX_ATTR_PROVISIONKEY; + + fput(file); + return 0; +} long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { @@ -588,6 +622,9 @@ long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) case SGX_IOC_ENCLAVE_INIT: ret = sgx_ioc_enclave_init(encl, (void __user *)arg); break; + case SGX_IOC_ENCLAVE_PROVISION: + ret = sgx_ioc_enclave_provision(encl, (void __user *)arg); + break; default: ret = -ENOIOCTLCMD; break; From 8382c668ce4f367d902f4a340a1bfa9e46096ec1 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Nov 2020 00:01:27 +0200 Subject: [PATCH 190/360] x86/vdso: Add support for exception fixup in vDSO functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signals are a horrid little mechanism. They are especially nasty in multi-threaded environments because signal state like handlers is global across the entire process. But, signals are basically the only way that userspace can “gracefully” handle and recover from exceptions. The kernel generally does not like exceptions to occur during execution. But, exceptions are a fact of life and must be handled in some circumstances. The kernel handles them by keeping a list of individual instructions which may cause exceptions. Instead of truly handling the exception and returning to the instruction that caused it, the kernel instead restarts execution at a *different* instruction. This makes it obvious to that thread of execution that the exception occurred and lets *that* code handle the exception instead of the handler. This is not dissimilar to the try/catch exceptions mechanisms that some programming languages have, but applied *very* surgically to single instructions. It effectively changes the visible architecture of the instruction. Problem ======= SGX generates a lot of signals, and the code to enter and exit enclaves and muck with signal handling is truly horrid. At the same time, an approach like kernel exception fixup can not be easily applied to userspace instructions because it changes the visible instruction architecture. Solution ======== The vDSO is a special page of kernel-provided instructions that run in userspace. Any userspace calling into the vDSO knows that it is special. This allows the kernel a place to legitimately rewrite the user/kernel contract and change instruction behavior. Add support for fixing up exceptions that occur while executing in the vDSO. This replaces what could traditionally only be done with signal handling. This new mechanism will be used to replace previously direct use of SGX instructions by userspace. Just introduce the vDSO infrastructure. Later patches will actually replace signal generation with vDSO exception fixup. Suggested-by: Andy Lutomirski Signed-off-by: Sean Christopherson Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-17-jarkko@kernel.org --- arch/x86/entry/vdso/Makefile | 6 ++-- arch/x86/entry/vdso/extable.c | 46 ++++++++++++++++++++++++ arch/x86/entry/vdso/extable.h | 28 +++++++++++++++ arch/x86/entry/vdso/vdso-layout.lds.S | 9 ++++- arch/x86/entry/vdso/vdso2c.h | 50 ++++++++++++++++++++++++++- arch/x86/include/asm/vdso.h | 5 +++ 6 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 arch/x86/entry/vdso/extable.c create mode 100644 arch/x86/entry/vdso/extable.h diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index 21243747965d..2ad757fb3c23 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -29,7 +29,7 @@ vobjs32-y := vdso32/note.o vdso32/system_call.o vdso32/sigreturn.o vobjs32-y += vdso32/vclock_gettime.o # files to link into kernel -obj-y += vma.o +obj-y += vma.o extable.o KASAN_SANITIZE_vma.o := y UBSAN_SANITIZE_vma.o := y KCSAN_SANITIZE_vma.o := y @@ -128,8 +128,8 @@ $(obj)/%-x32.o: $(obj)/%.o FORCE targets += vdsox32.lds $(vobjx32s-y) -$(obj)/%.so: OBJCOPYFLAGS := -S -$(obj)/%.so: $(obj)/%.so.dbg FORCE +$(obj)/%.so: OBJCOPYFLAGS := -S --remove-section __ex_table +$(obj)/%.so: $(obj)/%.so.dbg $(call if_changed,objcopy) $(obj)/vdsox32.so.dbg: $(obj)/vdsox32.lds $(vobjx32s) FORCE diff --git a/arch/x86/entry/vdso/extable.c b/arch/x86/entry/vdso/extable.c new file mode 100644 index 000000000000..afcf5b65beef --- /dev/null +++ b/arch/x86/entry/vdso/extable.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +struct vdso_exception_table_entry { + int insn, fixup; +}; + +bool fixup_vdso_exception(struct pt_regs *regs, int trapnr, + unsigned long error_code, unsigned long fault_addr) +{ + const struct vdso_image *image = current->mm->context.vdso_image; + const struct vdso_exception_table_entry *extable; + unsigned int nr_entries, i; + unsigned long base; + + /* + * Do not attempt to fixup #DB or #BP. It's impossible to identify + * whether or not a #DB/#BP originated from within an SGX enclave and + * SGX enclaves are currently the only use case for vDSO fixup. + */ + if (trapnr == X86_TRAP_DB || trapnr == X86_TRAP_BP) + return false; + + if (!current->mm->context.vdso) + return false; + + base = (unsigned long)current->mm->context.vdso + image->extable_base; + nr_entries = image->extable_len / (sizeof(*extable)); + extable = image->extable; + + for (i = 0; i < nr_entries; i++) { + if (regs->ip == base + extable[i].insn) { + regs->ip = base + extable[i].fixup; + regs->di = trapnr; + regs->si = error_code; + regs->dx = fault_addr; + return true; + } + } + + return false; +} diff --git a/arch/x86/entry/vdso/extable.h b/arch/x86/entry/vdso/extable.h new file mode 100644 index 000000000000..b56f6b012941 --- /dev/null +++ b/arch/x86/entry/vdso/extable.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __VDSO_EXTABLE_H +#define __VDSO_EXTABLE_H + +/* + * Inject exception fixup for vDSO code. Unlike normal exception fixup, + * vDSO uses a dedicated handler the addresses are relative to the overall + * exception table, not each individual entry. + */ +#ifdef __ASSEMBLY__ +#define _ASM_VDSO_EXTABLE_HANDLE(from, to) \ + ASM_VDSO_EXTABLE_HANDLE from to + +.macro ASM_VDSO_EXTABLE_HANDLE from:req to:req + .pushsection __ex_table, "a" + .long (\from) - __ex_table + .long (\to) - __ex_table + .popsection +.endm +#else +#define _ASM_VDSO_EXTABLE_HANDLE(from, to) \ + ".pushsection __ex_table, \"a\"\n" \ + ".long (" #from ") - __ex_table\n" \ + ".long (" #to ") - __ex_table\n" \ + ".popsection\n" +#endif + +#endif /* __VDSO_EXTABLE_H */ diff --git a/arch/x86/entry/vdso/vdso-layout.lds.S b/arch/x86/entry/vdso/vdso-layout.lds.S index 4d152933547d..dc8da7695859 100644 --- a/arch/x86/entry/vdso/vdso-layout.lds.S +++ b/arch/x86/entry/vdso/vdso-layout.lds.S @@ -75,11 +75,18 @@ SECTIONS * stuff that isn't used at runtime in between. */ - .text : { *(.text*) } :text =0x90909090, + .text : { + *(.text*) + *(.fixup) + } :text =0x90909090, + + .altinstructions : { *(.altinstructions) } :text .altinstr_replacement : { *(.altinstr_replacement) } :text + __ex_table : { *(__ex_table) } :text + /DISCARD/ : { *(.discard) *(.discard.*) diff --git a/arch/x86/entry/vdso/vdso2c.h b/arch/x86/entry/vdso/vdso2c.h index 6f46e11ce539..1c7cfac7e64a 100644 --- a/arch/x86/entry/vdso/vdso2c.h +++ b/arch/x86/entry/vdso/vdso2c.h @@ -5,6 +5,41 @@ * are built for 32-bit userspace. */ +static void BITSFUNC(copy)(FILE *outfile, const unsigned char *data, size_t len) +{ + size_t i; + + for (i = 0; i < len; i++) { + if (i % 10 == 0) + fprintf(outfile, "\n\t"); + fprintf(outfile, "0x%02X, ", (int)(data)[i]); + } +} + + +/* + * Extract a section from the input data into a standalone blob. Used to + * capture kernel-only data that needs to persist indefinitely, e.g. the + * exception fixup tables, but only in the kernel, i.e. the section can + * be stripped from the final vDSO image. + */ +static void BITSFUNC(extract)(const unsigned char *data, size_t data_len, + FILE *outfile, ELF(Shdr) *sec, const char *name) +{ + unsigned long offset; + size_t len; + + offset = (unsigned long)GET_LE(&sec->sh_offset); + len = (size_t)GET_LE(&sec->sh_size); + + if (offset + len > data_len) + fail("section to extract overruns input data"); + + fprintf(outfile, "static const unsigned char %s[%lu] = {", name, len); + BITSFUNC(copy)(outfile, data + offset, len); + fprintf(outfile, "\n};\n\n"); +} + static void BITSFUNC(go)(void *raw_addr, size_t raw_len, void *stripped_addr, size_t stripped_len, FILE *outfile, const char *image_name) @@ -15,7 +50,7 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, ELF(Ehdr) *hdr = (ELF(Ehdr) *)raw_addr; unsigned long i, syms_nr; ELF(Shdr) *symtab_hdr = NULL, *strtab_hdr, *secstrings_hdr, - *alt_sec = NULL; + *alt_sec = NULL, *extable_sec = NULL; ELF(Dyn) *dyn = 0, *dyn_end = 0; const char *secstrings; INT_BITS syms[NSYMS] = {}; @@ -77,6 +112,8 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, if (!strcmp(secstrings + GET_LE(&sh->sh_name), ".altinstructions")) alt_sec = sh; + if (!strcmp(secstrings + GET_LE(&sh->sh_name), "__ex_table")) + extable_sec = sh; } if (!symtab_hdr) @@ -155,6 +192,9 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, (int)((unsigned char *)stripped_addr)[i]); } fprintf(outfile, "\n};\n\n"); + if (extable_sec) + BITSFUNC(extract)(raw_addr, raw_len, outfile, + extable_sec, "extable"); fprintf(outfile, "const struct vdso_image %s = {\n", image_name); fprintf(outfile, "\t.data = raw_data,\n"); @@ -165,6 +205,14 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, fprintf(outfile, "\t.alt_len = %lu,\n", (unsigned long)GET_LE(&alt_sec->sh_size)); } + if (extable_sec) { + fprintf(outfile, "\t.extable_base = %lu,\n", + (unsigned long)GET_LE(&extable_sec->sh_offset)); + fprintf(outfile, "\t.extable_len = %lu,\n", + (unsigned long)GET_LE(&extable_sec->sh_size)); + fprintf(outfile, "\t.extable = extable,\n"); + } + for (i = 0; i < NSYMS; i++) { if (required_syms[i].export && syms[i]) fprintf(outfile, "\t.sym_%s = %" PRIi64 ",\n", diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h index bbcdc7b8f963..b5d23470f56b 100644 --- a/arch/x86/include/asm/vdso.h +++ b/arch/x86/include/asm/vdso.h @@ -15,6 +15,8 @@ struct vdso_image { unsigned long size; /* Always a multiple of PAGE_SIZE */ unsigned long alt, alt_len; + unsigned long extable_base, extable_len; + const void *extable; long sym_vvar_start; /* Negative offset to the vvar area */ @@ -45,6 +47,9 @@ extern void __init init_vdso_image(const struct vdso_image *image); extern int map_vdso_once(const struct vdso_image *image, unsigned long addr); +extern bool fixup_vdso_exception(struct pt_regs *regs, int trapnr, + unsigned long error_code, + unsigned long fault_addr); #endif /* __ASSEMBLER__ */ #endif /* _ASM_X86_VDSO_H */ From cd072dab453a9b4a9f7927f9eddca5a156fbd87d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Nov 2020 00:01:28 +0200 Subject: [PATCH 191/360] x86/fault: Add a helper function to sanitize error code vDSO exception fixup is a replacement for signals in limited situations. Signals and vDSO exception fixup need to provide similar information to userspace, including the hardware error code. That hardware error code needs to be sanitized. For instance, if userspace accesses a kernel address, the error code could indicate to userspace whether the address had a Present=1 PTE. That can leak information about the kernel layout to userspace, which is bad. The existing signal code does this sanitization, but fairly late in the signal process. The vDSO exception code runs before the sanitization happens. Move error code sanitization out of the signal code and into a helper. Call the helper in the signal code. Signed-off-by: Sean Christopherson Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-18-jarkko@kernel.org --- arch/x86/mm/fault.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 9339fee83784..0161d4acf3ad 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -602,11 +602,9 @@ pgtable_bad(struct pt_regs *regs, unsigned long error_code, oops_end(flags, regs, sig); } -static void set_signal_archinfo(unsigned long address, - unsigned long error_code) +static void sanitize_error_code(unsigned long address, + unsigned long *error_code) { - struct task_struct *tsk = current; - /* * To avoid leaking information about the kernel page * table layout, pretend that user-mode accesses to @@ -617,7 +615,13 @@ static void set_signal_archinfo(unsigned long address, * information and does not appear to cause any problems. */ if (address >= TASK_SIZE_MAX) - error_code |= X86_PF_PROT; + *error_code |= X86_PF_PROT; +} + +static void set_signal_archinfo(unsigned long address, + unsigned long error_code) +{ + struct task_struct *tsk = current; tsk->thread.trap_nr = X86_TRAP_PF; tsk->thread.error_code = error_code | X86_PF_USER; @@ -658,6 +662,8 @@ no_context(struct pt_regs *regs, unsigned long error_code, * faulting through the emulate_vsyscall() logic. */ if (current->thread.sig_on_uaccess_err && signal) { + sanitize_error_code(address, &error_code); + set_signal_archinfo(address, error_code); /* XXX: hwpoison faults will set the wrong code. */ @@ -806,13 +812,7 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code, if (is_errata100(regs, address)) return; - /* - * To avoid leaking information about the kernel page table - * layout, pretend that user-mode accesses to kernel addresses - * are always protection faults. - */ - if (address >= TASK_SIZE_MAX) - error_code |= X86_PF_PROT; + sanitize_error_code(address, &error_code); if (likely(show_unhandled_signals)) show_signal_msg(regs, error_code, address, tsk); @@ -931,6 +931,8 @@ do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address, if (is_prefetch(regs, error_code, address)) return; + sanitize_error_code(address, &error_code); + set_signal_archinfo(address, error_code); #ifdef CONFIG_MEMORY_FAILURE From 334872a0919890a70cccd00b8e11931020a819be Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Nov 2020 00:01:29 +0200 Subject: [PATCH 192/360] x86/traps: Attempt to fixup exceptions in vDSO before signaling vDSO functions can now leverage an exception fixup mechanism similar to kernel exception fixup. For vDSO exception fixup, the initial user is Intel's Software Guard Extensions (SGX), which will wrap the low-level transitions to/from the enclave, i.e. EENTER and ERESUME instructions, in a vDSO function and leverage fixup to intercept exceptions that would otherwise generate a signal. This allows the vDSO wrapper to return the fault information directly to its caller, obviating the need for SGX applications and libraries to juggle signal handlers. Attempt to fixup vDSO exceptions immediately prior to populating and sending signal information. Except for the delivery mechanism, an exception in a vDSO function should be treated like any other exception in userspace, e.g. any fault that is successfully handled by the kernel should not be directly visible to userspace. Although it's debatable whether or not all exceptions are of interest to enclaves, defer to the vDSO fixup to decide whether to do fixup or generate a signal. Future users of vDSO fixup, if there ever are any, will undoubtedly have different requirements than SGX enclaves, e.g. the fixup vs. signal logic can be made function specific if/when necessary. Suggested-by: Andy Lutomirski Signed-off-by: Sean Christopherson Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-19-jarkko@kernel.org --- arch/x86/kernel/traps.c | 10 ++++++++++ arch/x86/mm/fault.c | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index e19df6cde35d..7798d862983f 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -60,6 +60,7 @@ #include #include #include +#include #ifdef CONFIG_X86_64 #include @@ -117,6 +118,9 @@ do_trap_no_signal(struct task_struct *tsk, int trapnr, const char *str, tsk->thread.error_code = error_code; tsk->thread.trap_nr = trapnr; die(str, regs, error_code); + } else { + if (fixup_vdso_exception(regs, trapnr, error_code, 0)) + return 0; } /* @@ -550,6 +554,9 @@ DEFINE_IDTENTRY_ERRORCODE(exc_general_protection) tsk->thread.error_code = error_code; tsk->thread.trap_nr = X86_TRAP_GP; + if (fixup_vdso_exception(regs, X86_TRAP_GP, error_code, 0)) + return; + show_signal(tsk, SIGSEGV, "", desc, regs, error_code); force_sig(SIGSEGV); goto exit; @@ -1048,6 +1055,9 @@ static void math_error(struct pt_regs *regs, int trapnr) if (!si_code) goto exit; + if (fixup_vdso_exception(regs, trapnr, 0, 0)) + return; + force_sig_fault(SIGFPE, si_code, (void __user *)uprobe_get_trap_addr(regs)); exit: diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 0161d4acf3ad..f1f1b5a0956a 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -30,6 +30,7 @@ #include /* exception stack */ #include /* VMALLOC_START, ... */ #include /* kvm_handle_async_pf */ +#include /* fixup_vdso_exception() */ #define CREATE_TRACE_POINTS #include @@ -814,6 +815,9 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code, sanitize_error_code(address, &error_code); + if (fixup_vdso_exception(regs, X86_TRAP_PF, error_code, address)) + return; + if (likely(show_unhandled_signals)) show_signal_msg(regs, error_code, address, tsk); @@ -933,6 +937,9 @@ do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address, sanitize_error_code(address, &error_code); + if (fixup_vdso_exception(regs, X86_TRAP_PF, error_code, address)) + return; + set_signal_archinfo(address, error_code); #ifdef CONFIG_MEMORY_FAILURE From 84664369520170f48546c55cbc1f3fbde9b1e140 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 13 Nov 2020 00:01:30 +0200 Subject: [PATCH 193/360] x86/vdso: Implement a vDSO for Intel SGX enclave call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enclaves encounter exceptions for lots of reasons: everything from enclave page faults to NULL pointer dereferences, to system calls that must be “proxied” to the kernel from outside the enclave. In addition to the code contained inside an enclave, there is also supporting code outside the enclave called an “SGX runtime”, which is virtually always implemented inside a shared library. The runtime helps build the enclave and handles things like *re*building the enclave if it got destroyed by something like a suspend/resume cycle. The rebuilding has traditionally been handled in SIGSEGV handlers, registered by the library. But, being process-wide, shared state, signal handling and shared libraries do not mix well. Introduce a vDSO function call that wraps the enclave entry functions (EENTER/ERESUME functions of the ENCLU instruciton) and returns information about any exceptions to the caller in the SGX runtime. Instead of generating a signal, the kernel places exception information in RDI, RSI and RDX. The kernel-provided userspace portion of the vDSO handler will place this information in a user-provided buffer or trigger a user-provided callback at the time of the exception. The vDSO function calling convention uses the standard RDI RSI, RDX, RCX, R8 and R9 registers. This makes it possible to declare the vDSO as a C prototype, but other than that there is no specific support for SystemV ABI. Things like storing XSAVE are the responsibility of the enclave and the runtime. [ bp: Change vsgx.o build dependency to CONFIG_X86_SGX. ] Suggested-by: Andy Lutomirski Signed-off-by: Sean Christopherson Co-developed-by: Cedric Xing Signed-off-by: Cedric Xing Co-developed-by: Jarkko Sakkinen Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Tested-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-20-jarkko@kernel.org --- arch/x86/entry/vdso/Makefile | 2 + arch/x86/entry/vdso/vdso.lds.S | 1 + arch/x86/entry/vdso/vsgx.S | 151 ++++++++++++++++++++++++++++++++ arch/x86/include/asm/enclu.h | 9 ++ arch/x86/include/uapi/asm/sgx.h | 91 +++++++++++++++++++ 5 files changed, 254 insertions(+) create mode 100644 arch/x86/entry/vdso/vsgx.S create mode 100644 arch/x86/include/asm/enclu.h diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index 2ad757fb3c23..02e3e42f380b 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -27,6 +27,7 @@ VDSO32-$(CONFIG_IA32_EMULATION) := y vobjs-y := vdso-note.o vclock_gettime.o vgetcpu.o vobjs32-y := vdso32/note.o vdso32/system_call.o vdso32/sigreturn.o vobjs32-y += vdso32/vclock_gettime.o +vobjs-$(CONFIG_X86_SGX) += vsgx.o # files to link into kernel obj-y += vma.o extable.o @@ -98,6 +99,7 @@ $(vobjs): KBUILD_CFLAGS := $(filter-out $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS CFLAGS_REMOVE_vclock_gettime.o = -pg CFLAGS_REMOVE_vdso32/vclock_gettime.o = -pg CFLAGS_REMOVE_vgetcpu.o = -pg +CFLAGS_REMOVE_vsgx.o = -pg # # X32 processes use x32 vDSO to access 64bit kernel data. diff --git a/arch/x86/entry/vdso/vdso.lds.S b/arch/x86/entry/vdso/vdso.lds.S index 36b644e16272..4bf48462fca7 100644 --- a/arch/x86/entry/vdso/vdso.lds.S +++ b/arch/x86/entry/vdso/vdso.lds.S @@ -27,6 +27,7 @@ VERSION { __vdso_time; clock_getres; __vdso_clock_getres; + __vdso_sgx_enter_enclave; local: *; }; } diff --git a/arch/x86/entry/vdso/vsgx.S b/arch/x86/entry/vdso/vsgx.S new file mode 100644 index 000000000000..86a0e94f68df --- /dev/null +++ b/arch/x86/entry/vdso/vsgx.S @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include +#include +#include +#include + +#include "extable.h" + +/* Relative to %rbp. */ +#define SGX_ENCLAVE_OFFSET_OF_RUN 16 + +/* The offsets relative to struct sgx_enclave_run. */ +#define SGX_ENCLAVE_RUN_TCS 0 +#define SGX_ENCLAVE_RUN_LEAF 8 +#define SGX_ENCLAVE_RUN_EXCEPTION_VECTOR 12 +#define SGX_ENCLAVE_RUN_EXCEPTION_ERROR_CODE 14 +#define SGX_ENCLAVE_RUN_EXCEPTION_ADDR 16 +#define SGX_ENCLAVE_RUN_USER_HANDLER 24 +#define SGX_ENCLAVE_RUN_USER_DATA 32 /* not used */ +#define SGX_ENCLAVE_RUN_RESERVED_START 40 +#define SGX_ENCLAVE_RUN_RESERVED_END 256 + +.code64 +.section .text, "ax" + +SYM_FUNC_START(__vdso_sgx_enter_enclave) + /* Prolog */ + .cfi_startproc + push %rbp + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset %rbp, 0 + mov %rsp, %rbp + .cfi_def_cfa_register %rbp + push %rbx + .cfi_rel_offset %rbx, -8 + + mov %ecx, %eax +.Lenter_enclave: + /* EENTER <= function <= ERESUME */ + cmp $EENTER, %eax + jb .Linvalid_input + cmp $ERESUME, %eax + ja .Linvalid_input + + mov SGX_ENCLAVE_OFFSET_OF_RUN(%rbp), %rcx + + /* Validate that the reserved area contains only zeros. */ + mov $SGX_ENCLAVE_RUN_RESERVED_START, %rbx +1: + cmpq $0, (%rcx, %rbx) + jne .Linvalid_input + add $8, %rbx + cmpq $SGX_ENCLAVE_RUN_RESERVED_END, %rbx + jne 1b + + /* Load TCS and AEP */ + mov SGX_ENCLAVE_RUN_TCS(%rcx), %rbx + lea .Lasync_exit_pointer(%rip), %rcx + + /* Single ENCLU serving as both EENTER and AEP (ERESUME) */ +.Lasync_exit_pointer: +.Lenclu_eenter_eresume: + enclu + + /* EEXIT jumps here unless the enclave is doing something fancy. */ + mov SGX_ENCLAVE_OFFSET_OF_RUN(%rbp), %rbx + + /* Set exit_reason. */ + movl $EEXIT, SGX_ENCLAVE_RUN_LEAF(%rbx) + + /* Invoke userspace's exit handler if one was provided. */ +.Lhandle_exit: + cmpq $0, SGX_ENCLAVE_RUN_USER_HANDLER(%rbx) + jne .Linvoke_userspace_handler + + /* Success, in the sense that ENCLU was attempted. */ + xor %eax, %eax + +.Lout: + pop %rbx + leave + .cfi_def_cfa %rsp, 8 + ret + + /* The out-of-line code runs with the pre-leave stack frame. */ + .cfi_def_cfa %rbp, 16 + +.Linvalid_input: + mov $(-EINVAL), %eax + jmp .Lout + +.Lhandle_exception: + mov SGX_ENCLAVE_OFFSET_OF_RUN(%rbp), %rbx + + /* Set the exception info. */ + mov %eax, (SGX_ENCLAVE_RUN_LEAF)(%rbx) + mov %di, (SGX_ENCLAVE_RUN_EXCEPTION_VECTOR)(%rbx) + mov %si, (SGX_ENCLAVE_RUN_EXCEPTION_ERROR_CODE)(%rbx) + mov %rdx, (SGX_ENCLAVE_RUN_EXCEPTION_ADDR)(%rbx) + jmp .Lhandle_exit + +.Linvoke_userspace_handler: + /* Pass the untrusted RSP (at exit) to the callback via %rcx. */ + mov %rsp, %rcx + + /* Save struct sgx_enclave_exception %rbx is about to be clobbered. */ + mov %rbx, %rax + + /* Save the untrusted RSP offset in %rbx (non-volatile register). */ + mov %rsp, %rbx + and $0xf, %rbx + + /* + * Align stack per x86_64 ABI. Note, %rsp needs to be 16-byte aligned + * _after_ pushing the parameters on the stack, hence the bonus push. + */ + and $-0x10, %rsp + push %rax + + /* Push struct sgx_enclave_exception as a param to the callback. */ + push %rax + + /* Clear RFLAGS.DF per x86_64 ABI */ + cld + + /* + * Load the callback pointer to %rax and lfence for LVI (load value + * injection) protection before making the call. + */ + mov SGX_ENCLAVE_RUN_USER_HANDLER(%rax), %rax + lfence + call *%rax + + /* Undo the post-exit %rsp adjustment. */ + lea 0x10(%rsp, %rbx), %rsp + + /* + * If the return from callback is zero or negative, return immediately, + * else re-execute ENCLU with the postive return value interpreted as + * the requested ENCLU function. + */ + cmp $0, %eax + jle .Lout + jmp .Lenter_enclave + + .cfi_endproc + +_ASM_VDSO_EXTABLE_HANDLE(.Lenclu_eenter_eresume, .Lhandle_exception) + +SYM_FUNC_END(__vdso_sgx_enter_enclave) diff --git a/arch/x86/include/asm/enclu.h b/arch/x86/include/asm/enclu.h new file mode 100644 index 000000000000..b1314e41a744 --- /dev/null +++ b/arch/x86/include/asm/enclu.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_ENCLU_H +#define _ASM_X86_ENCLU_H + +#define EENTER 0x02 +#define ERESUME 0x03 +#define EEXIT 0x04 + +#endif /* _ASM_X86_ENCLU_H */ diff --git a/arch/x86/include/uapi/asm/sgx.h b/arch/x86/include/uapi/asm/sgx.h index c32210235bf5..791e45334a4a 100644 --- a/arch/x86/include/uapi/asm/sgx.h +++ b/arch/x86/include/uapi/asm/sgx.h @@ -74,4 +74,95 @@ struct sgx_enclave_provision { __u64 fd; }; +struct sgx_enclave_run; + +/** + * typedef sgx_enclave_user_handler_t - Exit handler function accepted by + * __vdso_sgx_enter_enclave() + * @run: The run instance given by the caller + * + * The register parameters contain the snapshot of their values at enclave + * exit. An invalid ENCLU function number will cause -EINVAL to be returned + * to the caller. + * + * Return: + * - <= 0: The given value is returned back to the caller. + * - > 0: ENCLU function to invoke, either EENTER or ERESUME. + */ +typedef int (*sgx_enclave_user_handler_t)(long rdi, long rsi, long rdx, + long rsp, long r8, long r9, + struct sgx_enclave_run *run); + +/** + * struct sgx_enclave_run - the execution context of __vdso_sgx_enter_enclave() + * @tcs: TCS used to enter the enclave + * @function: The last seen ENCLU function (EENTER, ERESUME or EEXIT) + * @exception_vector: The interrupt vector of the exception + * @exception_error_code: The exception error code pulled out of the stack + * @exception_addr: The address that triggered the exception + * @user_handler: User provided callback run on exception + * @user_data: Data passed to the user handler + * @reserved Reserved for future extensions + * + * If @user_handler is provided, the handler will be invoked on all return paths + * of the normal flow. The user handler may transfer control, e.g. via a + * longjmp() call or a C++ exception, without returning to + * __vdso_sgx_enter_enclave(). + */ +struct sgx_enclave_run { + __u64 tcs; + __u32 function; + __u16 exception_vector; + __u16 exception_error_code; + __u64 exception_addr; + __u64 user_handler; + __u64 user_data; + __u8 reserved[216]; +}; + +/** + * typedef vdso_sgx_enter_enclave_t - Prototype for __vdso_sgx_enter_enclave(), + * a vDSO function to enter an SGX enclave. + * @rdi: Pass-through value for RDI + * @rsi: Pass-through value for RSI + * @rdx: Pass-through value for RDX + * @function: ENCLU function, must be EENTER or ERESUME + * @r8: Pass-through value for R8 + * @r9: Pass-through value for R9 + * @run: struct sgx_enclave_run, must be non-NULL + * + * NOTE: __vdso_sgx_enter_enclave() does not ensure full compliance with the + * x86-64 ABI, e.g. doesn't handle XSAVE state. Except for non-volatile + * general purpose registers, EFLAGS.DF, and RSP alignment, preserving/setting + * state in accordance with the x86-64 ABI is the responsibility of the enclave + * and its runtime, i.e. __vdso_sgx_enter_enclave() cannot be called from C + * code without careful consideration by both the enclave and its runtime. + * + * All general purpose registers except RAX, RBX and RCX are passed as-is to the + * enclave. RAX, RBX and RCX are consumed by EENTER and ERESUME and are loaded + * with @function, asynchronous exit pointer, and @run.tcs respectively. + * + * RBP and the stack are used to anchor __vdso_sgx_enter_enclave() to the + * pre-enclave state, e.g. to retrieve @run.exception and @run.user_handler + * after an enclave exit. All other registers are available for use by the + * enclave and its runtime, e.g. an enclave can push additional data onto the + * stack (and modify RSP) to pass information to the optional user handler (see + * below). + * + * Most exceptions reported on ENCLU, including those that occur within the + * enclave, are fixed up and reported synchronously instead of being delivered + * via a standard signal. Debug Exceptions (#DB) and Breakpoints (#BP) are + * never fixed up and are always delivered via standard signals. On synchrously + * reported exceptions, -EFAULT is returned and details about the exception are + * recorded in @run.exception, the optional sgx_enclave_exception struct. + * + * Return: + * - 0: ENCLU function was successfully executed. + * - -EINVAL: Invalid ENCL number (neither EENTER nor ERESUME). + */ +typedef int (*vdso_sgx_enter_enclave_t)(unsigned long rdi, unsigned long rsi, + unsigned long rdx, unsigned int function, + unsigned long r8, unsigned long r9, + struct sgx_enclave_run *run); + #endif /* _UAPI_ASM_X86_SGX_H */ From 2adcba79e69d4a4c0ac3bb86f466d8b5df301608 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 13 Nov 2020 00:01:31 +0200 Subject: [PATCH 194/360] selftests/x86: Add a selftest for SGX Add a selftest for SGX. It is a trivial test where a simple enclave copies one 64-bit word of memory between two memory locations, but ensures that all SGX hardware and software infrastructure is functioning. Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Cc: linux-kselftest@vger.kernel.org Link: https://lkml.kernel.org/r/20201112220135.165028-21-jarkko@kernel.org --- tools/testing/selftests/Makefile | 1 + tools/testing/selftests/sgx/.gitignore | 2 + tools/testing/selftests/sgx/Makefile | 53 +++ tools/testing/selftests/sgx/call.S | 44 ++ tools/testing/selftests/sgx/defines.h | 21 + tools/testing/selftests/sgx/load.c | 277 +++++++++++++ tools/testing/selftests/sgx/main.c | 246 +++++++++++ tools/testing/selftests/sgx/main.h | 38 ++ tools/testing/selftests/sgx/sigstruct.c | 391 ++++++++++++++++++ tools/testing/selftests/sgx/test_encl.c | 20 + tools/testing/selftests/sgx/test_encl.lds | 40 ++ .../selftests/sgx/test_encl_bootstrap.S | 89 ++++ 12 files changed, 1222 insertions(+) create mode 100644 tools/testing/selftests/sgx/.gitignore create mode 100644 tools/testing/selftests/sgx/Makefile create mode 100644 tools/testing/selftests/sgx/call.S create mode 100644 tools/testing/selftests/sgx/defines.h create mode 100644 tools/testing/selftests/sgx/load.c create mode 100644 tools/testing/selftests/sgx/main.c create mode 100644 tools/testing/selftests/sgx/main.h create mode 100644 tools/testing/selftests/sgx/sigstruct.c create mode 100644 tools/testing/selftests/sgx/test_encl.c create mode 100644 tools/testing/selftests/sgx/test_encl.lds create mode 100644 tools/testing/selftests/sgx/test_encl_bootstrap.S diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index d9c283503159..2e20e30a6faa 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -50,6 +50,7 @@ TARGETS += openat2 TARGETS += rseq TARGETS += rtc TARGETS += seccomp +TARGETS += sgx TARGETS += sigaltstack TARGETS += size TARGETS += sparc64 diff --git a/tools/testing/selftests/sgx/.gitignore b/tools/testing/selftests/sgx/.gitignore new file mode 100644 index 000000000000..fbaf0bda9a92 --- /dev/null +++ b/tools/testing/selftests/sgx/.gitignore @@ -0,0 +1,2 @@ +test_sgx +test_encl.elf diff --git a/tools/testing/selftests/sgx/Makefile b/tools/testing/selftests/sgx/Makefile new file mode 100644 index 000000000000..d51c90663943 --- /dev/null +++ b/tools/testing/selftests/sgx/Makefile @@ -0,0 +1,53 @@ +top_srcdir = ../../../.. + +include ../lib.mk + +.PHONY: all clean + +CAN_BUILD_X86_64 := $(shell ../x86/check_cc.sh $(CC) \ + ../x86/trivial_64bit_program.c) + +ifndef OBJCOPY +OBJCOPY := $(CROSS_COMPILE)objcopy +endif + +INCLUDES := -I$(top_srcdir)/tools/include +HOST_CFLAGS := -Wall -Werror -g $(INCLUDES) -fPIC -z noexecstack +ENCL_CFLAGS := -Wall -Werror -static -nostdlib -nostartfiles -fPIC \ + -fno-stack-protector -mrdrnd $(INCLUDES) + +TEST_CUSTOM_PROGS := $(OUTPUT)/test_sgx + +ifeq ($(CAN_BUILD_X86_64), 1) +all: $(TEST_CUSTOM_PROGS) $(OUTPUT)/test_encl.elf +endif + +$(OUTPUT)/test_sgx: $(OUTPUT)/main.o \ + $(OUTPUT)/load.o \ + $(OUTPUT)/sigstruct.o \ + $(OUTPUT)/call.o + $(CC) $(HOST_CFLAGS) -o $@ $^ -lcrypto + +$(OUTPUT)/main.o: main.c + $(CC) $(HOST_CFLAGS) -c $< -o $@ + +$(OUTPUT)/load.o: load.c + $(CC) $(HOST_CFLAGS) -c $< -o $@ + +$(OUTPUT)/sigstruct.o: sigstruct.c + $(CC) $(HOST_CFLAGS) -c $< -o $@ + +$(OUTPUT)/call.o: call.S + $(CC) $(HOST_CFLAGS) -c $< -o $@ + +$(OUTPUT)/test_encl.elf: test_encl.lds test_encl.c test_encl_bootstrap.S + $(CC) $(ENCL_CFLAGS) -T $^ -o $@ + +EXTRA_CLEAN := \ + $(OUTPUT)/test_encl.elf \ + $(OUTPUT)/load.o \ + $(OUTPUT)/call.o \ + $(OUTPUT)/main.o \ + $(OUTPUT)/sigstruct.o \ + $(OUTPUT)/test_sgx \ + $(OUTPUT)/test_sgx.o \ diff --git a/tools/testing/selftests/sgx/call.S b/tools/testing/selftests/sgx/call.S new file mode 100644 index 000000000000..4ecadc7490f4 --- /dev/null +++ b/tools/testing/selftests/sgx/call.S @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/** +* Copyright(c) 2016-20 Intel Corporation. +*/ + + .text + + .global sgx_call_vdso +sgx_call_vdso: + .cfi_startproc + push %r15 + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset %r15, 0 + push %r14 + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset %r14, 0 + push %r13 + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset %r13, 0 + push %r12 + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset %r12, 0 + push %rbx + .cfi_adjust_cfa_offset 8 + .cfi_rel_offset %rbx, 0 + push $0 + .cfi_adjust_cfa_offset 8 + push 0x38(%rsp) + .cfi_adjust_cfa_offset 8 + call *eenter(%rip) + add $0x10, %rsp + .cfi_adjust_cfa_offset -0x10 + pop %rbx + .cfi_adjust_cfa_offset -8 + pop %r12 + .cfi_adjust_cfa_offset -8 + pop %r13 + .cfi_adjust_cfa_offset -8 + pop %r14 + .cfi_adjust_cfa_offset -8 + pop %r15 + .cfi_adjust_cfa_offset -8 + ret + .cfi_endproc diff --git a/tools/testing/selftests/sgx/defines.h b/tools/testing/selftests/sgx/defines.h new file mode 100644 index 000000000000..592c1ccf4576 --- /dev/null +++ b/tools/testing/selftests/sgx/defines.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright(c) 2016-20 Intel Corporation. + */ + +#ifndef DEFINES_H +#define DEFINES_H + +#include + +#define PAGE_SIZE 4096 +#define PAGE_MASK (~(PAGE_SIZE - 1)) + +#define __aligned(x) __attribute__((__aligned__(x))) +#define __packed __attribute__((packed)) + +#include "../../../../arch/x86/kernel/cpu/sgx/arch.h" +#include "../../../../arch/x86/include/asm/enclu.h" +#include "../../../../arch/x86/include/uapi/asm/sgx.h" + +#endif /* DEFINES_H */ diff --git a/tools/testing/selftests/sgx/load.c b/tools/testing/selftests/sgx/load.c new file mode 100644 index 000000000000..9d43b75aaa55 --- /dev/null +++ b/tools/testing/selftests/sgx/load.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "defines.h" +#include "main.h" + +void encl_delete(struct encl *encl) +{ + if (encl->encl_base) + munmap((void *)encl->encl_base, encl->encl_size); + + if (encl->bin) + munmap(encl->bin, encl->bin_size); + + if (encl->fd) + close(encl->fd); + + if (encl->segment_tbl) + free(encl->segment_tbl); + + memset(encl, 0, sizeof(*encl)); +} + +static bool encl_map_bin(const char *path, struct encl *encl) +{ + struct stat sb; + void *bin; + int ret; + int fd; + + fd = open(path, O_RDONLY); + if (fd == -1) { + perror("open()"); + return false; + } + + ret = stat(path, &sb); + if (ret) { + perror("stat()"); + goto err; + } + + bin = mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (bin == MAP_FAILED) { + perror("mmap()"); + goto err; + } + + encl->bin = bin; + encl->bin_size = sb.st_size; + + close(fd); + return true; + +err: + close(fd); + return false; +} + +static bool encl_ioc_create(struct encl *encl) +{ + struct sgx_secs *secs = &encl->secs; + struct sgx_enclave_create ioc; + int rc; + + assert(encl->encl_base != 0); + + memset(secs, 0, sizeof(*secs)); + secs->ssa_frame_size = 1; + secs->attributes = SGX_ATTR_MODE64BIT; + secs->xfrm = 3; + secs->base = encl->encl_base; + secs->size = encl->encl_size; + + ioc.src = (unsigned long)secs; + rc = ioctl(encl->fd, SGX_IOC_ENCLAVE_CREATE, &ioc); + if (rc) { + fprintf(stderr, "SGX_IOC_ENCLAVE_CREATE failed: errno=%d\n", + errno); + munmap((void *)secs->base, encl->encl_size); + return false; + } + + return true; +} + +static bool encl_ioc_add_pages(struct encl *encl, struct encl_segment *seg) +{ + struct sgx_enclave_add_pages ioc; + struct sgx_secinfo secinfo; + int rc; + + memset(&secinfo, 0, sizeof(secinfo)); + secinfo.flags = seg->flags; + + ioc.src = (uint64_t)encl->src + seg->offset; + ioc.offset = seg->offset; + ioc.length = seg->size; + ioc.secinfo = (unsigned long)&secinfo; + ioc.flags = SGX_PAGE_MEASURE; + + rc = ioctl(encl->fd, SGX_IOC_ENCLAVE_ADD_PAGES, &ioc); + if (rc < 0) { + fprintf(stderr, "SGX_IOC_ENCLAVE_ADD_PAGES failed: errno=%d.\n", + errno); + return false; + } + + return true; +} + +bool encl_load(const char *path, struct encl *encl) +{ + Elf64_Phdr *phdr_tbl; + off_t src_offset; + Elf64_Ehdr *ehdr; + int i, j; + int ret; + + memset(encl, 0, sizeof(*encl)); + + ret = open("/dev/sgx_enclave", O_RDWR); + if (ret < 0) { + fprintf(stderr, "Unable to open /dev/sgx_enclave\n"); + goto err; + } + + encl->fd = ret; + + if (!encl_map_bin(path, encl)) + goto err; + + ehdr = encl->bin; + phdr_tbl = encl->bin + ehdr->e_phoff; + + for (i = 0; i < ehdr->e_phnum; i++) { + Elf64_Phdr *phdr = &phdr_tbl[i]; + + if (phdr->p_type == PT_LOAD) + encl->nr_segments++; + } + + encl->segment_tbl = calloc(encl->nr_segments, + sizeof(struct encl_segment)); + if (!encl->segment_tbl) + goto err; + + for (i = 0, j = 0; i < ehdr->e_phnum; i++) { + Elf64_Phdr *phdr = &phdr_tbl[i]; + unsigned int flags = phdr->p_flags; + struct encl_segment *seg; + + if (phdr->p_type != PT_LOAD) + continue; + + seg = &encl->segment_tbl[j]; + + if (!!(flags & ~(PF_R | PF_W | PF_X))) { + fprintf(stderr, + "%d has invalid segment flags 0x%02x.\n", i, + phdr->p_flags); + goto err; + } + + if (j == 0 && flags != (PF_R | PF_W)) { + fprintf(stderr, + "TCS has invalid segment flags 0x%02x.\n", + phdr->p_flags); + goto err; + } + + if (j == 0) { + src_offset = phdr->p_offset & PAGE_MASK; + + seg->prot = PROT_READ | PROT_WRITE; + seg->flags = SGX_PAGE_TYPE_TCS << 8; + } else { + seg->prot = (phdr->p_flags & PF_R) ? PROT_READ : 0; + seg->prot |= (phdr->p_flags & PF_W) ? PROT_WRITE : 0; + seg->prot |= (phdr->p_flags & PF_X) ? PROT_EXEC : 0; + seg->flags = (SGX_PAGE_TYPE_REG << 8) | seg->prot; + } + + seg->offset = (phdr->p_offset & PAGE_MASK) - src_offset; + seg->size = (phdr->p_filesz + PAGE_SIZE - 1) & PAGE_MASK; + + printf("0x%016lx 0x%016lx 0x%02x\n", seg->offset, seg->size, + seg->prot); + + j++; + } + + assert(j == encl->nr_segments); + + encl->src = encl->bin + src_offset; + encl->src_size = encl->segment_tbl[j - 1].offset + + encl->segment_tbl[j - 1].size; + + for (encl->encl_size = 4096; encl->encl_size < encl->src_size; ) + encl->encl_size <<= 1; + + return true; + +err: + encl_delete(encl); + return false; +} + +static bool encl_map_area(struct encl *encl) +{ + size_t encl_size = encl->encl_size; + void *area; + + area = mmap(NULL, encl_size * 2, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (area == MAP_FAILED) { + perror("mmap"); + return false; + } + + encl->encl_base = ((uint64_t)area + encl_size - 1) & ~(encl_size - 1); + + munmap(area, encl->encl_base - (uint64_t)area); + munmap((void *)(encl->encl_base + encl_size), + (uint64_t)area + encl_size - encl->encl_base); + + return true; +} + +bool encl_build(struct encl *encl) +{ + struct sgx_enclave_init ioc; + int ret; + int i; + + if (!encl_map_area(encl)) + return false; + + if (!encl_ioc_create(encl)) + return false; + + /* + * Pages must be added before mapping VMAs because their permissions + * cap the VMA permissions. + */ + for (i = 0; i < encl->nr_segments; i++) { + struct encl_segment *seg = &encl->segment_tbl[i]; + + if (!encl_ioc_add_pages(encl, seg)) + return false; + } + + ioc.sigstruct = (uint64_t)&encl->sigstruct; + ret = ioctl(encl->fd, SGX_IOC_ENCLAVE_INIT, &ioc); + if (ret) { + fprintf(stderr, "SGX_IOC_ENCLAVE_INIT failed: errno=%d\n", + errno); + return false; + } + + return true; +} diff --git a/tools/testing/selftests/sgx/main.c b/tools/testing/selftests/sgx/main.c new file mode 100644 index 000000000000..724cec700926 --- /dev/null +++ b/tools/testing/selftests/sgx/main.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "defines.h" +#include "main.h" +#include "../kselftest.h" + +static const uint64_t MAGIC = 0x1122334455667788ULL; +vdso_sgx_enter_enclave_t eenter; + +struct vdso_symtab { + Elf64_Sym *elf_symtab; + const char *elf_symstrtab; + Elf64_Word *elf_hashtab; +}; + +static void *vdso_get_base_addr(char *envp[]) +{ + Elf64_auxv_t *auxv; + int i; + + for (i = 0; envp[i]; i++) + ; + + auxv = (Elf64_auxv_t *)&envp[i + 1]; + + for (i = 0; auxv[i].a_type != AT_NULL; i++) { + if (auxv[i].a_type == AT_SYSINFO_EHDR) + return (void *)auxv[i].a_un.a_val; + } + + return NULL; +} + +static Elf64_Dyn *vdso_get_dyntab(void *addr) +{ + Elf64_Ehdr *ehdr = addr; + Elf64_Phdr *phdrtab = addr + ehdr->e_phoff; + int i; + + for (i = 0; i < ehdr->e_phnum; i++) + if (phdrtab[i].p_type == PT_DYNAMIC) + return addr + phdrtab[i].p_offset; + + return NULL; +} + +static void *vdso_get_dyn(void *addr, Elf64_Dyn *dyntab, Elf64_Sxword tag) +{ + int i; + + for (i = 0; dyntab[i].d_tag != DT_NULL; i++) + if (dyntab[i].d_tag == tag) + return addr + dyntab[i].d_un.d_ptr; + + return NULL; +} + +static bool vdso_get_symtab(void *addr, struct vdso_symtab *symtab) +{ + Elf64_Dyn *dyntab = vdso_get_dyntab(addr); + + symtab->elf_symtab = vdso_get_dyn(addr, dyntab, DT_SYMTAB); + if (!symtab->elf_symtab) + return false; + + symtab->elf_symstrtab = vdso_get_dyn(addr, dyntab, DT_STRTAB); + if (!symtab->elf_symstrtab) + return false; + + symtab->elf_hashtab = vdso_get_dyn(addr, dyntab, DT_HASH); + if (!symtab->elf_hashtab) + return false; + + return true; +} + +static unsigned long elf_sym_hash(const char *name) +{ + unsigned long h = 0, high; + + while (*name) { + h = (h << 4) + *name++; + high = h & 0xf0000000; + + if (high) + h ^= high >> 24; + + h &= ~high; + } + + return h; +} + +static Elf64_Sym *vdso_symtab_get(struct vdso_symtab *symtab, const char *name) +{ + Elf64_Word bucketnum = symtab->elf_hashtab[0]; + Elf64_Word *buckettab = &symtab->elf_hashtab[2]; + Elf64_Word *chaintab = &symtab->elf_hashtab[2 + bucketnum]; + Elf64_Sym *sym; + Elf64_Word i; + + for (i = buckettab[elf_sym_hash(name) % bucketnum]; i != STN_UNDEF; + i = chaintab[i]) { + sym = &symtab->elf_symtab[i]; + if (!strcmp(name, &symtab->elf_symstrtab[sym->st_name])) + return sym; + } + + return NULL; +} + +bool report_results(struct sgx_enclave_run *run, int ret, uint64_t result, + const char *test) +{ + bool valid = true; + + if (ret) { + printf("FAIL: %s() returned: %d\n", test, ret); + valid = false; + } + + if (run->function != EEXIT) { + printf("FAIL: %s() function, expected: %u, got: %u\n", test, EEXIT, + run->function); + valid = false; + } + + if (result != MAGIC) { + printf("FAIL: %s(), expected: 0x%lx, got: 0x%lx\n", test, MAGIC, + result); + valid = false; + } + + if (run->user_data) { + printf("FAIL: %s() user data, expected: 0x0, got: 0x%llx\n", + test, run->user_data); + valid = false; + } + + return valid; +} + +static int user_handler(long rdi, long rsi, long rdx, long ursp, long r8, long r9, + struct sgx_enclave_run *run) +{ + run->user_data = 0; + return 0; +} + +int main(int argc, char *argv[], char *envp[]) +{ + struct sgx_enclave_run run; + struct vdso_symtab symtab; + Elf64_Sym *eenter_sym; + uint64_t result = 0; + struct encl encl; + unsigned int i; + void *addr; + int ret; + + memset(&run, 0, sizeof(run)); + + if (!encl_load("test_encl.elf", &encl)) { + encl_delete(&encl); + ksft_exit_skip("cannot load enclaves\n"); + } + + if (!encl_measure(&encl)) + goto err; + + if (!encl_build(&encl)) + goto err; + + /* + * An enclave consumer only must do this. + */ + for (i = 0; i < encl.nr_segments; i++) { + struct encl_segment *seg = &encl.segment_tbl[i]; + + addr = mmap((void *)encl.encl_base + seg->offset, seg->size, + seg->prot, MAP_SHARED | MAP_FIXED, encl.fd, 0); + if (addr == MAP_FAILED) { + fprintf(stderr, "mmap() failed, errno=%d.\n", errno); + exit(KSFT_FAIL); + } + } + + memset(&run, 0, sizeof(run)); + run.tcs = encl.encl_base; + + addr = vdso_get_base_addr(envp); + if (!addr) + goto err; + + if (!vdso_get_symtab(addr, &symtab)) + goto err; + + eenter_sym = vdso_symtab_get(&symtab, "__vdso_sgx_enter_enclave"); + if (!eenter_sym) + goto err; + + eenter = addr + eenter_sym->st_value; + + ret = sgx_call_vdso((void *)&MAGIC, &result, 0, EENTER, NULL, NULL, &run); + if (!report_results(&run, ret, result, "sgx_call_vdso")) + goto err; + + + /* Invoke the vDSO directly. */ + result = 0; + ret = eenter((unsigned long)&MAGIC, (unsigned long)&result, 0, EENTER, + 0, 0, &run); + if (!report_results(&run, ret, result, "eenter")) + goto err; + + /* And with an exit handler. */ + run.user_handler = (__u64)user_handler; + run.user_data = 0xdeadbeef; + ret = eenter((unsigned long)&MAGIC, (unsigned long)&result, 0, EENTER, + 0, 0, &run); + if (!report_results(&run, ret, result, "user_handler")) + goto err; + + printf("SUCCESS\n"); + encl_delete(&encl); + exit(KSFT_PASS); + +err: + encl_delete(&encl); + exit(KSFT_FAIL); +} diff --git a/tools/testing/selftests/sgx/main.h b/tools/testing/selftests/sgx/main.h new file mode 100644 index 000000000000..45e6ab65442a --- /dev/null +++ b/tools/testing/selftests/sgx/main.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright(c) 2016-20 Intel Corporation. + */ + +#ifndef MAIN_H +#define MAIN_H + +struct encl_segment { + off_t offset; + size_t size; + unsigned int prot; + unsigned int flags; +}; + +struct encl { + int fd; + void *bin; + off_t bin_size; + void *src; + size_t src_size; + size_t encl_size; + off_t encl_base; + unsigned int nr_segments; + struct encl_segment *segment_tbl; + struct sgx_secs secs; + struct sgx_sigstruct sigstruct; +}; + +void encl_delete(struct encl *ctx); +bool encl_load(const char *path, struct encl *encl); +bool encl_measure(struct encl *encl); +bool encl_build(struct encl *encl); + +int sgx_call_vdso(void *rdi, void *rsi, long rdx, u32 function, void *r8, void *r9, + struct sgx_enclave_run *run); + +#endif /* MAIN_H */ diff --git a/tools/testing/selftests/sgx/sigstruct.c b/tools/testing/selftests/sgx/sigstruct.c new file mode 100644 index 000000000000..cc06f108bae7 --- /dev/null +++ b/tools/testing/selftests/sgx/sigstruct.c @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "defines.h" +#include "main.h" + +struct q1q2_ctx { + BN_CTX *bn_ctx; + BIGNUM *m; + BIGNUM *s; + BIGNUM *q1; + BIGNUM *qr; + BIGNUM *q2; +}; + +static void free_q1q2_ctx(struct q1q2_ctx *ctx) +{ + BN_CTX_free(ctx->bn_ctx); + BN_free(ctx->m); + BN_free(ctx->s); + BN_free(ctx->q1); + BN_free(ctx->qr); + BN_free(ctx->q2); +} + +static bool alloc_q1q2_ctx(const uint8_t *s, const uint8_t *m, + struct q1q2_ctx *ctx) +{ + ctx->bn_ctx = BN_CTX_new(); + ctx->s = BN_bin2bn(s, SGX_MODULUS_SIZE, NULL); + ctx->m = BN_bin2bn(m, SGX_MODULUS_SIZE, NULL); + ctx->q1 = BN_new(); + ctx->qr = BN_new(); + ctx->q2 = BN_new(); + + if (!ctx->bn_ctx || !ctx->s || !ctx->m || !ctx->q1 || !ctx->qr || + !ctx->q2) { + free_q1q2_ctx(ctx); + return false; + } + + return true; +} + +static bool calc_q1q2(const uint8_t *s, const uint8_t *m, uint8_t *q1, + uint8_t *q2) +{ + struct q1q2_ctx ctx; + + if (!alloc_q1q2_ctx(s, m, &ctx)) { + fprintf(stderr, "Not enough memory for Q1Q2 calculation\n"); + return false; + } + + if (!BN_mul(ctx.q1, ctx.s, ctx.s, ctx.bn_ctx)) + goto out; + + if (!BN_div(ctx.q1, ctx.qr, ctx.q1, ctx.m, ctx.bn_ctx)) + goto out; + + if (BN_num_bytes(ctx.q1) > SGX_MODULUS_SIZE) { + fprintf(stderr, "Too large Q1 %d bytes\n", + BN_num_bytes(ctx.q1)); + goto out; + } + + if (!BN_mul(ctx.q2, ctx.s, ctx.qr, ctx.bn_ctx)) + goto out; + + if (!BN_div(ctx.q2, NULL, ctx.q2, ctx.m, ctx.bn_ctx)) + goto out; + + if (BN_num_bytes(ctx.q2) > SGX_MODULUS_SIZE) { + fprintf(stderr, "Too large Q2 %d bytes\n", + BN_num_bytes(ctx.q2)); + goto out; + } + + BN_bn2bin(ctx.q1, q1); + BN_bn2bin(ctx.q2, q2); + + free_q1q2_ctx(&ctx); + return true; +out: + free_q1q2_ctx(&ctx); + return false; +} + +struct sgx_sigstruct_payload { + struct sgx_sigstruct_header header; + struct sgx_sigstruct_body body; +}; + +static bool check_crypto_errors(void) +{ + int err; + bool had_errors = false; + const char *filename; + int line; + char str[256]; + + for ( ; ; ) { + if (ERR_peek_error() == 0) + break; + + had_errors = true; + err = ERR_get_error_line(&filename, &line); + ERR_error_string_n(err, str, sizeof(str)); + fprintf(stderr, "crypto: %s: %s:%d\n", str, filename, line); + } + + return had_errors; +} + +static inline const BIGNUM *get_modulus(RSA *key) +{ + const BIGNUM *n; + + RSA_get0_key(key, &n, NULL, NULL); + return n; +} + +static RSA *gen_sign_key(void) +{ + BIGNUM *e; + RSA *key; + int ret; + + e = BN_new(); + key = RSA_new(); + + if (!e || !key) + goto err; + + ret = BN_set_word(e, RSA_3); + if (ret != 1) + goto err; + + ret = RSA_generate_key_ex(key, 3072, e, NULL); + if (ret != 1) + goto err; + + BN_free(e); + + return key; + +err: + RSA_free(key); + BN_free(e); + + return NULL; +} + +static void reverse_bytes(void *data, int length) +{ + int i = 0; + int j = length - 1; + uint8_t temp; + uint8_t *ptr = data; + + while (i < j) { + temp = ptr[i]; + ptr[i] = ptr[j]; + ptr[j] = temp; + i++; + j--; + } +} + +enum mrtags { + MRECREATE = 0x0045544145524345, + MREADD = 0x0000000044444145, + MREEXTEND = 0x00444E4554584545, +}; + +static bool mrenclave_update(EVP_MD_CTX *ctx, const void *data) +{ + if (!EVP_DigestUpdate(ctx, data, 64)) { + fprintf(stderr, "digest update failed\n"); + return false; + } + + return true; +} + +static bool mrenclave_commit(EVP_MD_CTX *ctx, uint8_t *mrenclave) +{ + unsigned int size; + + if (!EVP_DigestFinal_ex(ctx, (unsigned char *)mrenclave, &size)) { + fprintf(stderr, "digest commit failed\n"); + return false; + } + + if (size != 32) { + fprintf(stderr, "invalid digest size = %u\n", size); + return false; + } + + return true; +} + +struct mrecreate { + uint64_t tag; + uint32_t ssaframesize; + uint64_t size; + uint8_t reserved[44]; +} __attribute__((__packed__)); + + +static bool mrenclave_ecreate(EVP_MD_CTX *ctx, uint64_t blob_size) +{ + struct mrecreate mrecreate; + uint64_t encl_size; + + for (encl_size = 0x1000; encl_size < blob_size; ) + encl_size <<= 1; + + memset(&mrecreate, 0, sizeof(mrecreate)); + mrecreate.tag = MRECREATE; + mrecreate.ssaframesize = 1; + mrecreate.size = encl_size; + + if (!EVP_DigestInit_ex(ctx, EVP_sha256(), NULL)) + return false; + + return mrenclave_update(ctx, &mrecreate); +} + +struct mreadd { + uint64_t tag; + uint64_t offset; + uint64_t flags; /* SECINFO flags */ + uint8_t reserved[40]; +} __attribute__((__packed__)); + +static bool mrenclave_eadd(EVP_MD_CTX *ctx, uint64_t offset, uint64_t flags) +{ + struct mreadd mreadd; + + memset(&mreadd, 0, sizeof(mreadd)); + mreadd.tag = MREADD; + mreadd.offset = offset; + mreadd.flags = flags; + + return mrenclave_update(ctx, &mreadd); +} + +struct mreextend { + uint64_t tag; + uint64_t offset; + uint8_t reserved[48]; +} __attribute__((__packed__)); + +static bool mrenclave_eextend(EVP_MD_CTX *ctx, uint64_t offset, + const uint8_t *data) +{ + struct mreextend mreextend; + int i; + + for (i = 0; i < 0x1000; i += 0x100) { + memset(&mreextend, 0, sizeof(mreextend)); + mreextend.tag = MREEXTEND; + mreextend.offset = offset + i; + + if (!mrenclave_update(ctx, &mreextend)) + return false; + + if (!mrenclave_update(ctx, &data[i + 0x00])) + return false; + + if (!mrenclave_update(ctx, &data[i + 0x40])) + return false; + + if (!mrenclave_update(ctx, &data[i + 0x80])) + return false; + + if (!mrenclave_update(ctx, &data[i + 0xC0])) + return false; + } + + return true; +} + +static bool mrenclave_segment(EVP_MD_CTX *ctx, struct encl *encl, + struct encl_segment *seg) +{ + uint64_t end = seg->offset + seg->size; + uint64_t offset; + + for (offset = seg->offset; offset < end; offset += PAGE_SIZE) { + if (!mrenclave_eadd(ctx, offset, seg->flags)) + return false; + + if (!mrenclave_eextend(ctx, offset, encl->src + offset)) + return false; + } + + return true; +} + +bool encl_measure(struct encl *encl) +{ + uint64_t header1[2] = {0x000000E100000006, 0x0000000000010000}; + uint64_t header2[2] = {0x0000006000000101, 0x0000000100000060}; + struct sgx_sigstruct *sigstruct = &encl->sigstruct; + struct sgx_sigstruct_payload payload; + uint8_t digest[SHA256_DIGEST_LENGTH]; + unsigned int siglen; + RSA *key = NULL; + EVP_MD_CTX *ctx; + int i; + + memset(sigstruct, 0, sizeof(*sigstruct)); + + sigstruct->header.header1[0] = header1[0]; + sigstruct->header.header1[1] = header1[1]; + sigstruct->header.header2[0] = header2[0]; + sigstruct->header.header2[1] = header2[1]; + sigstruct->exponent = 3; + sigstruct->body.attributes = SGX_ATTR_MODE64BIT; + sigstruct->body.xfrm = 3; + + /* sanity check */ + if (check_crypto_errors()) + goto err; + + key = gen_sign_key(); + if (!key) + goto err; + + BN_bn2bin(get_modulus(key), sigstruct->modulus); + + ctx = EVP_MD_CTX_create(); + if (!ctx) + goto err; + + if (!mrenclave_ecreate(ctx, encl->src_size)) + goto err; + + for (i = 0; i < encl->nr_segments; i++) { + struct encl_segment *seg = &encl->segment_tbl[i]; + + if (!mrenclave_segment(ctx, encl, seg)) + goto err; + } + + if (!mrenclave_commit(ctx, sigstruct->body.mrenclave)) + goto err; + + memcpy(&payload.header, &sigstruct->header, sizeof(sigstruct->header)); + memcpy(&payload.body, &sigstruct->body, sizeof(sigstruct->body)); + + SHA256((unsigned char *)&payload, sizeof(payload), digest); + + if (!RSA_sign(NID_sha256, digest, SHA256_DIGEST_LENGTH, + sigstruct->signature, &siglen, key)) + goto err; + + if (!calc_q1q2(sigstruct->signature, sigstruct->modulus, sigstruct->q1, + sigstruct->q2)) + goto err; + + /* BE -> LE */ + reverse_bytes(sigstruct->signature, SGX_MODULUS_SIZE); + reverse_bytes(sigstruct->modulus, SGX_MODULUS_SIZE); + reverse_bytes(sigstruct->q1, SGX_MODULUS_SIZE); + reverse_bytes(sigstruct->q2, SGX_MODULUS_SIZE); + + EVP_MD_CTX_destroy(ctx); + RSA_free(key); + return true; + +err: + EVP_MD_CTX_destroy(ctx); + RSA_free(key); + return false; +} diff --git a/tools/testing/selftests/sgx/test_encl.c b/tools/testing/selftests/sgx/test_encl.c new file mode 100644 index 000000000000..cf25b5dc1e03 --- /dev/null +++ b/tools/testing/selftests/sgx/test_encl.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2016-20 Intel Corporation. */ + +#include +#include "defines.h" + +static void *memcpy(void *dest, const void *src, size_t n) +{ + size_t i; + + for (i = 0; i < n; i++) + ((char *)dest)[i] = ((char *)src)[i]; + + return dest; +} + +void encl_body(void *rdi, void *rsi) +{ + memcpy(rsi, rdi, 8); +} diff --git a/tools/testing/selftests/sgx/test_encl.lds b/tools/testing/selftests/sgx/test_encl.lds new file mode 100644 index 000000000000..0fbbda7e665e --- /dev/null +++ b/tools/testing/selftests/sgx/test_encl.lds @@ -0,0 +1,40 @@ +OUTPUT_FORMAT(elf64-x86-64) + +PHDRS +{ + tcs PT_LOAD; + text PT_LOAD; + data PT_LOAD; +} + +SECTIONS +{ + . = 0; + .tcs : { + *(.tcs*) + } : tcs + + . = ALIGN(4096); + .text : { + *(.text*) + *(.rodata*) + } : text + + . = ALIGN(4096); + .data : { + *(.data*) + } : data + + /DISCARD/ : { + *(.comment*) + *(.note*) + *(.debug*) + *(.eh_frame*) + } +} + +ASSERT(!DEFINED(.altinstructions), "ALTERNATIVES are not supported in enclaves") +ASSERT(!DEFINED(.altinstr_replacement), "ALTERNATIVES are not supported in enclaves") +ASSERT(!DEFINED(.discard.retpoline_safe), "RETPOLINE ALTERNATIVES are not supported in enclaves") +ASSERT(!DEFINED(.discard.nospec), "RETPOLINE ALTERNATIVES are not supported in enclaves") +ASSERT(!DEFINED(.got.plt), "Libcalls are not supported in enclaves") diff --git a/tools/testing/selftests/sgx/test_encl_bootstrap.S b/tools/testing/selftests/sgx/test_encl_bootstrap.S new file mode 100644 index 000000000000..5d5680d4ea39 --- /dev/null +++ b/tools/testing/selftests/sgx/test_encl_bootstrap.S @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright(c) 2016-20 Intel Corporation. + */ + + .macro ENCLU + .byte 0x0f, 0x01, 0xd7 + .endm + + .section ".tcs", "aw" + .balign 4096 + + .fill 1, 8, 0 # STATE (set by CPU) + .fill 1, 8, 0 # FLAGS + .quad encl_ssa # OSSA + .fill 1, 4, 0 # CSSA (set by CPU) + .fill 1, 4, 1 # NSSA + .quad encl_entry # OENTRY + .fill 1, 8, 0 # AEP (set by EENTER and ERESUME) + .fill 1, 8, 0 # OFSBASE + .fill 1, 8, 0 # OGSBASE + .fill 1, 4, 0xFFFFFFFF # FSLIMIT + .fill 1, 4, 0xFFFFFFFF # GSLIMIT + .fill 4024, 1, 0 # Reserved + + # Identical to the previous TCS. + .fill 1, 8, 0 # STATE (set by CPU) + .fill 1, 8, 0 # FLAGS + .quad encl_ssa # OSSA + .fill 1, 4, 0 # CSSA (set by CPU) + .fill 1, 4, 1 # NSSA + .quad encl_entry # OENTRY + .fill 1, 8, 0 # AEP (set by EENTER and ERESUME) + .fill 1, 8, 0 # OFSBASE + .fill 1, 8, 0 # OGSBASE + .fill 1, 4, 0xFFFFFFFF # FSLIMIT + .fill 1, 4, 0xFFFFFFFF # GSLIMIT + .fill 4024, 1, 0 # Reserved + + .text + +encl_entry: + # RBX contains the base address for TCS, which is also the first address + # inside the enclave. By adding the value of le_stack_end to it, we get + # the absolute address for the stack. + lea (encl_stack)(%rbx), %rax + xchg %rsp, %rax + push %rax + + push %rcx # push the address after EENTER + push %rbx # push the enclave base address + + call encl_body + + pop %rbx # pop the enclave base address + + /* Clear volatile GPRs, except RAX (EEXIT function). */ + xor %rcx, %rcx + xor %rdx, %rdx + xor %rdi, %rdi + xor %rsi, %rsi + xor %r8, %r8 + xor %r9, %r9 + xor %r10, %r10 + xor %r11, %r11 + + # Reset status flags. + add %rdx, %rdx # OF = SF = AF = CF = 0; ZF = PF = 1 + + # Prepare EEXIT target by popping the address of the instruction after + # EENTER to RBX. + pop %rbx + + # Restore the caller stack. + pop %rax + mov %rax, %rsp + + # EEXIT + mov $4, %rax + enclu + + .section ".data", "aw" + +encl_ssa: + .space 4096 + + .balign 4096 + .space 8192 +encl_stack: From 1728ab54b4be94aed89276eeb8e750a345659765 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 13 Nov 2020 00:01:32 +0200 Subject: [PATCH 195/360] x86/sgx: Add a page reclaimer Just like normal RAM, there is a limited amount of enclave memory available and overcommitting it is a very valuable tool to reduce resource use. Introduce a simple reclaim mechanism for enclave pages. In contrast to normal page reclaim, the kernel cannot directly access enclave memory. To get around this, the SGX architecture provides a set of functions to help. Among other things, these functions copy enclave memory to and from normal memory, encrypting it and protecting its integrity in the process. Implement a page reclaimer by using these functions. Picks victim pages in LRU fashion from all the enclaves running in the system. A new kernel thread (ksgxswapd) reclaims pages in the background based on watermarks, similar to normal kswapd. All enclave pages can be reclaimed, architecturally. But, there are some limits to this, such as the special SECS metadata page which must be reclaimed last. The page version array (used to mitigate replaying old reclaimed pages) is also architecturally reclaimable, but not yet implemented. The end result is that the vast majority of enclave pages are currently reclaimable. Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-22-jarkko@kernel.org --- arch/x86/kernel/cpu/sgx/driver.c | 61 ++-- arch/x86/kernel/cpu/sgx/encl.c | 483 ++++++++++++++++++++++++++++++- arch/x86/kernel/cpu/sgx/encl.h | 51 ++++ arch/x86/kernel/cpu/sgx/ioctl.c | 89 +++++- arch/x86/kernel/cpu/sgx/main.c | 466 +++++++++++++++++++++++++++++ arch/x86/kernel/cpu/sgx/sgx.h | 13 + 6 files changed, 1135 insertions(+), 28 deletions(-) diff --git a/arch/x86/kernel/cpu/sgx/driver.c b/arch/x86/kernel/cpu/sgx/driver.c index 899c18499d1a..f2eac41bb4ff 100644 --- a/arch/x86/kernel/cpu/sgx/driver.c +++ b/arch/x86/kernel/cpu/sgx/driver.c @@ -17,13 +17,24 @@ u32 sgx_misc_reserved_mask; static int sgx_open(struct inode *inode, struct file *file) { struct sgx_encl *encl; + int ret; encl = kzalloc(sizeof(*encl), GFP_KERNEL); if (!encl) return -ENOMEM; + kref_init(&encl->refcount); xa_init(&encl->page_array); mutex_init(&encl->lock); + INIT_LIST_HEAD(&encl->va_pages); + INIT_LIST_HEAD(&encl->mm_list); + spin_lock_init(&encl->mm_lock); + + ret = init_srcu_struct(&encl->srcu); + if (ret) { + kfree(encl); + return ret; + } file->private_data = encl; @@ -33,31 +44,37 @@ static int sgx_open(struct inode *inode, struct file *file) static int sgx_release(struct inode *inode, struct file *file) { struct sgx_encl *encl = file->private_data; - struct sgx_encl_page *entry; - unsigned long index; + struct sgx_encl_mm *encl_mm; - xa_for_each(&encl->page_array, index, entry) { - if (entry->epc_page) { - sgx_free_epc_page(entry->epc_page); - encl->secs_child_cnt--; - entry->epc_page = NULL; + /* + * Drain the remaining mm_list entries. At this point the list contains + * entries for processes, which have closed the enclave file but have + * not exited yet. The processes, which have exited, are gone from the + * list by sgx_mmu_notifier_release(). + */ + for ( ; ; ) { + spin_lock(&encl->mm_lock); + + if (list_empty(&encl->mm_list)) { + encl_mm = NULL; + } else { + encl_mm = list_first_entry(&encl->mm_list, + struct sgx_encl_mm, list); + list_del_rcu(&encl_mm->list); } - kfree(entry); + spin_unlock(&encl->mm_lock); + + /* The enclave is no longer mapped by any mm. */ + if (!encl_mm) + break; + + synchronize_srcu(&encl->srcu); + mmu_notifier_unregister(&encl_mm->mmu_notifier, encl_mm->mm); + kfree(encl_mm); } - xa_destroy(&encl->page_array); - - if (!encl->secs_child_cnt && encl->secs.epc_page) { - sgx_free_epc_page(encl->secs.epc_page); - encl->secs.epc_page = NULL; - } - - /* Detect EPC page leaks. */ - WARN_ON_ONCE(encl->secs_child_cnt); - WARN_ON_ONCE(encl->secs.epc_page); - - kfree(encl); + kref_put(&encl->refcount, sgx_encl_release); return 0; } @@ -70,6 +87,10 @@ static int sgx_mmap(struct file *file, struct vm_area_struct *vma) if (ret) return ret; + ret = sgx_encl_mm_add(encl, vma->vm_mm); + if (ret) + return ret; + vma->vm_ops = &sgx_vm_ops; vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; vma->vm_private_data = encl; diff --git a/arch/x86/kernel/cpu/sgx/encl.c b/arch/x86/kernel/cpu/sgx/encl.c index 57eff300f487..b74dadf85989 100644 --- a/arch/x86/kernel/cpu/sgx/encl.c +++ b/arch/x86/kernel/cpu/sgx/encl.c @@ -12,11 +12,90 @@ #include "encls.h" #include "sgx.h" +/* + * ELDU: Load an EPC page as unblocked. For more info, see "OS Management of EPC + * Pages" in the SDM. + */ +static int __sgx_encl_eldu(struct sgx_encl_page *encl_page, + struct sgx_epc_page *epc_page, + struct sgx_epc_page *secs_page) +{ + unsigned long va_offset = encl_page->desc & SGX_ENCL_PAGE_VA_OFFSET_MASK; + struct sgx_encl *encl = encl_page->encl; + struct sgx_pageinfo pginfo; + struct sgx_backing b; + pgoff_t page_index; + int ret; + + if (secs_page) + page_index = PFN_DOWN(encl_page->desc - encl_page->encl->base); + else + page_index = PFN_DOWN(encl->size); + + ret = sgx_encl_get_backing(encl, page_index, &b); + if (ret) + return ret; + + pginfo.addr = encl_page->desc & PAGE_MASK; + pginfo.contents = (unsigned long)kmap_atomic(b.contents); + pginfo.metadata = (unsigned long)kmap_atomic(b.pcmd) + + b.pcmd_offset; + + if (secs_page) + pginfo.secs = (u64)sgx_get_epc_virt_addr(secs_page); + else + pginfo.secs = 0; + + ret = __eldu(&pginfo, sgx_get_epc_virt_addr(epc_page), + sgx_get_epc_virt_addr(encl_page->va_page->epc_page) + va_offset); + if (ret) { + if (encls_failed(ret)) + ENCLS_WARN(ret, "ELDU"); + + ret = -EFAULT; + } + + kunmap_atomic((void *)(unsigned long)(pginfo.metadata - b.pcmd_offset)); + kunmap_atomic((void *)(unsigned long)pginfo.contents); + + sgx_encl_put_backing(&b, false); + + return ret; +} + +static struct sgx_epc_page *sgx_encl_eldu(struct sgx_encl_page *encl_page, + struct sgx_epc_page *secs_page) +{ + + unsigned long va_offset = encl_page->desc & SGX_ENCL_PAGE_VA_OFFSET_MASK; + struct sgx_encl *encl = encl_page->encl; + struct sgx_epc_page *epc_page; + int ret; + + epc_page = sgx_alloc_epc_page(encl_page, false); + if (IS_ERR(epc_page)) + return epc_page; + + ret = __sgx_encl_eldu(encl_page, epc_page, secs_page); + if (ret) { + sgx_free_epc_page(epc_page); + return ERR_PTR(ret); + } + + sgx_free_va_slot(encl_page->va_page, va_offset); + list_move(&encl_page->va_page->list, &encl->va_pages); + encl_page->desc &= ~SGX_ENCL_PAGE_VA_OFFSET_MASK; + encl_page->epc_page = epc_page; + + return epc_page; +} + static struct sgx_encl_page *sgx_encl_load_page(struct sgx_encl *encl, unsigned long addr, unsigned long vm_flags) { unsigned long vm_prot_bits = vm_flags & (VM_READ | VM_WRITE | VM_EXEC); + struct sgx_epc_page *epc_page; struct sgx_encl_page *entry; entry = xa_load(&encl->page_array, PFN_DOWN(addr)); @@ -31,11 +110,27 @@ static struct sgx_encl_page *sgx_encl_load_page(struct sgx_encl *encl, if ((entry->vm_max_prot_bits & vm_prot_bits) != vm_prot_bits) return ERR_PTR(-EFAULT); - /* No page found. */ - if (!entry->epc_page) - return ERR_PTR(-EFAULT); - /* Entry successfully located. */ + if (entry->epc_page) { + if (entry->desc & SGX_ENCL_PAGE_BEING_RECLAIMED) + return ERR_PTR(-EBUSY); + + return entry; + } + + if (!(encl->secs.epc_page)) { + epc_page = sgx_encl_eldu(&encl->secs, NULL); + if (IS_ERR(epc_page)) + return ERR_CAST(epc_page); + } + + epc_page = sgx_encl_eldu(entry, encl->secs.epc_page); + if (IS_ERR(epc_page)) + return ERR_CAST(epc_page); + + encl->secs_child_cnt++; + sgx_mark_page_reclaimable(entry->epc_page); + return entry; } @@ -51,12 +146,23 @@ static vm_fault_t sgx_vma_fault(struct vm_fault *vmf) encl = vma->vm_private_data; + /* + * It's very unlikely but possible that allocating memory for the + * mm_list entry of a forked process failed in sgx_vma_open(). When + * this happens, vm_private_data is set to NULL. + */ + if (unlikely(!encl)) + return VM_FAULT_SIGBUS; + mutex_lock(&encl->lock); entry = sgx_encl_load_page(encl, addr, vma->vm_flags); if (IS_ERR(entry)) { mutex_unlock(&encl->lock); + if (PTR_ERR(entry) == -EBUSY) + return VM_FAULT_NOPAGE; + return VM_FAULT_SIGBUS; } @@ -76,11 +182,29 @@ static vm_fault_t sgx_vma_fault(struct vm_fault *vmf) return VM_FAULT_SIGBUS; } + sgx_encl_test_and_clear_young(vma->vm_mm, entry); mutex_unlock(&encl->lock); return VM_FAULT_NOPAGE; } +static void sgx_vma_open(struct vm_area_struct *vma) +{ + struct sgx_encl *encl = vma->vm_private_data; + + /* + * It's possible but unlikely that vm_private_data is NULL. This can + * happen in a grandchild of a process, when sgx_encl_mm_add() had + * failed to allocate memory in this callback. + */ + if (unlikely(!encl)) + return; + + if (sgx_encl_mm_add(encl, vma->vm_mm)) + vma->vm_private_data = NULL; +} + + /** * sgx_encl_may_map() - Check if a requested VMA mapping is allowed * @encl: an enclave pointer @@ -151,4 +275,355 @@ static int sgx_vma_mprotect(struct vm_area_struct *vma, unsigned long start, const struct vm_operations_struct sgx_vm_ops = { .fault = sgx_vma_fault, .mprotect = sgx_vma_mprotect, + .open = sgx_vma_open, }; + +/** + * sgx_encl_release - Destroy an enclave instance + * @kref: address of a kref inside &sgx_encl + * + * Used together with kref_put(). Frees all the resources associated with the + * enclave and the instance itself. + */ +void sgx_encl_release(struct kref *ref) +{ + struct sgx_encl *encl = container_of(ref, struct sgx_encl, refcount); + struct sgx_va_page *va_page; + struct sgx_encl_page *entry; + unsigned long index; + + xa_for_each(&encl->page_array, index, entry) { + if (entry->epc_page) { + /* + * The page and its radix tree entry cannot be freed + * if the page is being held by the reclaimer. + */ + if (sgx_unmark_page_reclaimable(entry->epc_page)) + continue; + + sgx_free_epc_page(entry->epc_page); + encl->secs_child_cnt--; + entry->epc_page = NULL; + } + + kfree(entry); + } + + xa_destroy(&encl->page_array); + + if (!encl->secs_child_cnt && encl->secs.epc_page) { + sgx_free_epc_page(encl->secs.epc_page); + encl->secs.epc_page = NULL; + } + + while (!list_empty(&encl->va_pages)) { + va_page = list_first_entry(&encl->va_pages, struct sgx_va_page, + list); + list_del(&va_page->list); + sgx_free_epc_page(va_page->epc_page); + kfree(va_page); + } + + if (encl->backing) + fput(encl->backing); + + cleanup_srcu_struct(&encl->srcu); + + WARN_ON_ONCE(!list_empty(&encl->mm_list)); + + /* Detect EPC page leak's. */ + WARN_ON_ONCE(encl->secs_child_cnt); + WARN_ON_ONCE(encl->secs.epc_page); + + kfree(encl); +} + +/* + * 'mm' is exiting and no longer needs mmu notifications. + */ +static void sgx_mmu_notifier_release(struct mmu_notifier *mn, + struct mm_struct *mm) +{ + struct sgx_encl_mm *encl_mm = container_of(mn, struct sgx_encl_mm, mmu_notifier); + struct sgx_encl_mm *tmp = NULL; + + /* + * The enclave itself can remove encl_mm. Note, objects can't be moved + * off an RCU protected list, but deletion is ok. + */ + spin_lock(&encl_mm->encl->mm_lock); + list_for_each_entry(tmp, &encl_mm->encl->mm_list, list) { + if (tmp == encl_mm) { + list_del_rcu(&encl_mm->list); + break; + } + } + spin_unlock(&encl_mm->encl->mm_lock); + + if (tmp == encl_mm) { + synchronize_srcu(&encl_mm->encl->srcu); + mmu_notifier_put(mn); + } +} + +static void sgx_mmu_notifier_free(struct mmu_notifier *mn) +{ + struct sgx_encl_mm *encl_mm = container_of(mn, struct sgx_encl_mm, mmu_notifier); + + kfree(encl_mm); +} + +static const struct mmu_notifier_ops sgx_mmu_notifier_ops = { + .release = sgx_mmu_notifier_release, + .free_notifier = sgx_mmu_notifier_free, +}; + +static struct sgx_encl_mm *sgx_encl_find_mm(struct sgx_encl *encl, + struct mm_struct *mm) +{ + struct sgx_encl_mm *encl_mm = NULL; + struct sgx_encl_mm *tmp; + int idx; + + idx = srcu_read_lock(&encl->srcu); + + list_for_each_entry_rcu(tmp, &encl->mm_list, list) { + if (tmp->mm == mm) { + encl_mm = tmp; + break; + } + } + + srcu_read_unlock(&encl->srcu, idx); + + return encl_mm; +} + +int sgx_encl_mm_add(struct sgx_encl *encl, struct mm_struct *mm) +{ + struct sgx_encl_mm *encl_mm; + int ret; + + /* + * Even though a single enclave may be mapped into an mm more than once, + * each 'mm' only appears once on encl->mm_list. This is guaranteed by + * holding the mm's mmap lock for write before an mm can be added or + * remove to an encl->mm_list. + */ + mmap_assert_write_locked(mm); + + /* + * It's possible that an entry already exists in the mm_list, because it + * is removed only on VFS release or process exit. + */ + if (sgx_encl_find_mm(encl, mm)) + return 0; + + encl_mm = kzalloc(sizeof(*encl_mm), GFP_KERNEL); + if (!encl_mm) + return -ENOMEM; + + encl_mm->encl = encl; + encl_mm->mm = mm; + encl_mm->mmu_notifier.ops = &sgx_mmu_notifier_ops; + + ret = __mmu_notifier_register(&encl_mm->mmu_notifier, mm); + if (ret) { + kfree(encl_mm); + return ret; + } + + spin_lock(&encl->mm_lock); + list_add_rcu(&encl_mm->list, &encl->mm_list); + /* Pairs with smp_rmb() in sgx_reclaimer_block(). */ + smp_wmb(); + encl->mm_list_version++; + spin_unlock(&encl->mm_lock); + + return 0; +} + +static struct page *sgx_encl_get_backing_page(struct sgx_encl *encl, + pgoff_t index) +{ + struct inode *inode = encl->backing->f_path.dentry->d_inode; + struct address_space *mapping = inode->i_mapping; + gfp_t gfpmask = mapping_gfp_mask(mapping); + + return shmem_read_mapping_page_gfp(mapping, index, gfpmask); +} + +/** + * sgx_encl_get_backing() - Pin the backing storage + * @encl: an enclave pointer + * @page_index: enclave page index + * @backing: data for accessing backing storage for the page + * + * Pin the backing storage pages for storing the encrypted contents and Paging + * Crypto MetaData (PCMD) of an enclave page. + * + * Return: + * 0 on success, + * -errno otherwise. + */ +int sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index, + struct sgx_backing *backing) +{ + pgoff_t pcmd_index = PFN_DOWN(encl->size) + 1 + (page_index >> 5); + struct page *contents; + struct page *pcmd; + + contents = sgx_encl_get_backing_page(encl, page_index); + if (IS_ERR(contents)) + return PTR_ERR(contents); + + pcmd = sgx_encl_get_backing_page(encl, pcmd_index); + if (IS_ERR(pcmd)) { + put_page(contents); + return PTR_ERR(pcmd); + } + + backing->page_index = page_index; + backing->contents = contents; + backing->pcmd = pcmd; + backing->pcmd_offset = + (page_index & (PAGE_SIZE / sizeof(struct sgx_pcmd) - 1)) * + sizeof(struct sgx_pcmd); + + return 0; +} + +/** + * sgx_encl_put_backing() - Unpin the backing storage + * @backing: data for accessing backing storage for the page + * @do_write: mark pages dirty + */ +void sgx_encl_put_backing(struct sgx_backing *backing, bool do_write) +{ + if (do_write) { + set_page_dirty(backing->pcmd); + set_page_dirty(backing->contents); + } + + put_page(backing->pcmd); + put_page(backing->contents); +} + +static int sgx_encl_test_and_clear_young_cb(pte_t *ptep, unsigned long addr, + void *data) +{ + pte_t pte; + int ret; + + ret = pte_young(*ptep); + if (ret) { + pte = pte_mkold(*ptep); + set_pte_at((struct mm_struct *)data, addr, ptep, pte); + } + + return ret; +} + +/** + * sgx_encl_test_and_clear_young() - Test and reset the accessed bit + * @mm: mm_struct that is checked + * @page: enclave page to be tested for recent access + * + * Checks the Access (A) bit from the PTE corresponding to the enclave page and + * clears it. + * + * Return: 1 if the page has been recently accessed and 0 if not. + */ +int sgx_encl_test_and_clear_young(struct mm_struct *mm, + struct sgx_encl_page *page) +{ + unsigned long addr = page->desc & PAGE_MASK; + struct sgx_encl *encl = page->encl; + struct vm_area_struct *vma; + int ret; + + ret = sgx_encl_find(mm, addr, &vma); + if (ret) + return 0; + + if (encl != vma->vm_private_data) + return 0; + + ret = apply_to_page_range(vma->vm_mm, addr, PAGE_SIZE, + sgx_encl_test_and_clear_young_cb, vma->vm_mm); + if (ret < 0) + return 0; + + return ret; +} + +/** + * sgx_alloc_va_page() - Allocate a Version Array (VA) page + * + * Allocate a free EPC page and convert it to a Version Array (VA) page. + * + * Return: + * a VA page, + * -errno otherwise + */ +struct sgx_epc_page *sgx_alloc_va_page(void) +{ + struct sgx_epc_page *epc_page; + int ret; + + epc_page = sgx_alloc_epc_page(NULL, true); + if (IS_ERR(epc_page)) + return ERR_CAST(epc_page); + + ret = __epa(sgx_get_epc_virt_addr(epc_page)); + if (ret) { + WARN_ONCE(1, "EPA returned %d (0x%x)", ret, ret); + sgx_free_epc_page(epc_page); + return ERR_PTR(-EFAULT); + } + + return epc_page; +} + +/** + * sgx_alloc_va_slot - allocate a VA slot + * @va_page: a &struct sgx_va_page instance + * + * Allocates a slot from a &struct sgx_va_page instance. + * + * Return: offset of the slot inside the VA page + */ +unsigned int sgx_alloc_va_slot(struct sgx_va_page *va_page) +{ + int slot = find_first_zero_bit(va_page->slots, SGX_VA_SLOT_COUNT); + + if (slot < SGX_VA_SLOT_COUNT) + set_bit(slot, va_page->slots); + + return slot << 3; +} + +/** + * sgx_free_va_slot - free a VA slot + * @va_page: a &struct sgx_va_page instance + * @offset: offset of the slot inside the VA page + * + * Frees a slot from a &struct sgx_va_page instance. + */ +void sgx_free_va_slot(struct sgx_va_page *va_page, unsigned int offset) +{ + clear_bit(offset >> 3, va_page->slots); +} + +/** + * sgx_va_page_full - is the VA page full? + * @va_page: a &struct sgx_va_page instance + * + * Return: true if all slots have been taken + */ +bool sgx_va_page_full(struct sgx_va_page *va_page) +{ + int slot = find_first_zero_bit(va_page->slots, SGX_VA_SLOT_COUNT); + + return slot == SGX_VA_SLOT_COUNT; +} diff --git a/arch/x86/kernel/cpu/sgx/encl.h b/arch/x86/kernel/cpu/sgx/encl.h index 8a4d1edded68..d8d30ccbef4c 100644 --- a/arch/x86/kernel/cpu/sgx/encl.h +++ b/arch/x86/kernel/cpu/sgx/encl.h @@ -19,11 +19,18 @@ #include #include "sgx.h" +/* 'desc' bits holding the offset in the VA (version array) page. */ +#define SGX_ENCL_PAGE_VA_OFFSET_MASK GENMASK_ULL(11, 3) + +/* 'desc' bit marking that the page is being reclaimed. */ +#define SGX_ENCL_PAGE_BEING_RECLAIMED BIT(3) + struct sgx_encl_page { unsigned long desc; unsigned long vm_max_prot_bits; struct sgx_epc_page *epc_page; struct sgx_encl *encl; + struct sgx_va_page *va_page; }; enum sgx_encl_flags { @@ -33,6 +40,13 @@ enum sgx_encl_flags { SGX_ENCL_INITIALIZED = BIT(3), }; +struct sgx_encl_mm { + struct sgx_encl *encl; + struct mm_struct *mm; + struct list_head list; + struct mmu_notifier mmu_notifier; +}; + struct sgx_encl { unsigned long base; unsigned long size; @@ -44,6 +58,30 @@ struct sgx_encl { struct sgx_encl_page secs; unsigned long attributes; unsigned long attributes_mask; + + cpumask_t cpumask; + struct file *backing; + struct kref refcount; + struct list_head va_pages; + unsigned long mm_list_version; + struct list_head mm_list; + spinlock_t mm_lock; + struct srcu_struct srcu; +}; + +#define SGX_VA_SLOT_COUNT 512 + +struct sgx_va_page { + struct sgx_epc_page *epc_page; + DECLARE_BITMAP(slots, SGX_VA_SLOT_COUNT); + struct list_head list; +}; + +struct sgx_backing { + pgoff_t page_index; + struct page *contents; + struct page *pcmd; + unsigned long pcmd_offset; }; extern const struct vm_operations_struct sgx_vm_ops; @@ -65,4 +103,17 @@ static inline int sgx_encl_find(struct mm_struct *mm, unsigned long addr, int sgx_encl_may_map(struct sgx_encl *encl, unsigned long start, unsigned long end, unsigned long vm_flags); +void sgx_encl_release(struct kref *ref); +int sgx_encl_mm_add(struct sgx_encl *encl, struct mm_struct *mm); +int sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index, + struct sgx_backing *backing); +void sgx_encl_put_backing(struct sgx_backing *backing, bool do_write); +int sgx_encl_test_and_clear_young(struct mm_struct *mm, + struct sgx_encl_page *page); + +struct sgx_epc_page *sgx_alloc_va_page(void); +unsigned int sgx_alloc_va_slot(struct sgx_va_page *va_page); +void sgx_free_va_slot(struct sgx_va_page *va_page, unsigned int offset); +bool sgx_va_page_full(struct sgx_va_page *va_page); + #endif /* _X86_ENCL_H */ diff --git a/arch/x86/kernel/cpu/sgx/ioctl.c b/arch/x86/kernel/cpu/sgx/ioctl.c index 0ba0e670e2f0..6d37117ac8a0 100644 --- a/arch/x86/kernel/cpu/sgx/ioctl.c +++ b/arch/x86/kernel/cpu/sgx/ioctl.c @@ -16,20 +16,77 @@ #include "encl.h" #include "encls.h" +static struct sgx_va_page *sgx_encl_grow(struct sgx_encl *encl) +{ + struct sgx_va_page *va_page = NULL; + void *err; + + BUILD_BUG_ON(SGX_VA_SLOT_COUNT != + (SGX_ENCL_PAGE_VA_OFFSET_MASK >> 3) + 1); + + if (!(encl->page_cnt % SGX_VA_SLOT_COUNT)) { + va_page = kzalloc(sizeof(*va_page), GFP_KERNEL); + if (!va_page) + return ERR_PTR(-ENOMEM); + + va_page->epc_page = sgx_alloc_va_page(); + if (IS_ERR(va_page->epc_page)) { + err = ERR_CAST(va_page->epc_page); + kfree(va_page); + return err; + } + + WARN_ON_ONCE(encl->page_cnt % SGX_VA_SLOT_COUNT); + } + encl->page_cnt++; + return va_page; +} + +static void sgx_encl_shrink(struct sgx_encl *encl, struct sgx_va_page *va_page) +{ + encl->page_cnt--; + + if (va_page) { + sgx_free_epc_page(va_page->epc_page); + list_del(&va_page->list); + kfree(va_page); + } +} + static int sgx_encl_create(struct sgx_encl *encl, struct sgx_secs *secs) { struct sgx_epc_page *secs_epc; + struct sgx_va_page *va_page; struct sgx_pageinfo pginfo; struct sgx_secinfo secinfo; unsigned long encl_size; + struct file *backing; long ret; + va_page = sgx_encl_grow(encl); + if (IS_ERR(va_page)) + return PTR_ERR(va_page); + else if (va_page) + list_add(&va_page->list, &encl->va_pages); + /* else the tail page of the VA page list had free slots. */ + /* The extra page goes to SECS. */ encl_size = secs->size + PAGE_SIZE; - secs_epc = __sgx_alloc_epc_page(); - if (IS_ERR(secs_epc)) - return PTR_ERR(secs_epc); + backing = shmem_file_setup("SGX backing", encl_size + (encl_size >> 5), + VM_NORESERVE); + if (IS_ERR(backing)) { + ret = PTR_ERR(backing); + goto err_out_shrink; + } + + encl->backing = backing; + + secs_epc = sgx_alloc_epc_page(&encl->secs, true); + if (IS_ERR(secs_epc)) { + ret = PTR_ERR(secs_epc); + goto err_out_backing; + } encl->secs.epc_page = secs_epc; @@ -63,6 +120,13 @@ err_out: sgx_free_epc_page(encl->secs.epc_page); encl->secs.epc_page = NULL; +err_out_backing: + fput(encl->backing); + encl->backing = NULL; + +err_out_shrink: + sgx_encl_shrink(encl, va_page); + return ret; } @@ -228,21 +292,35 @@ static int sgx_encl_add_page(struct sgx_encl *encl, unsigned long src, { struct sgx_encl_page *encl_page; struct sgx_epc_page *epc_page; + struct sgx_va_page *va_page; int ret; encl_page = sgx_encl_page_alloc(encl, offset, secinfo->flags); if (IS_ERR(encl_page)) return PTR_ERR(encl_page); - epc_page = __sgx_alloc_epc_page(); + epc_page = sgx_alloc_epc_page(encl_page, true); if (IS_ERR(epc_page)) { kfree(encl_page); return PTR_ERR(epc_page); } + va_page = sgx_encl_grow(encl); + if (IS_ERR(va_page)) { + ret = PTR_ERR(va_page); + goto err_out_free; + } + mmap_read_lock(current->mm); mutex_lock(&encl->lock); + /* + * Adding to encl->va_pages must be done under encl->lock. Ditto for + * deleting (via sgx_encl_shrink()) in the error path. + */ + if (va_page) + list_add(&va_page->list, &encl->va_pages); + /* * Insert prior to EADD in case of OOM. EADD modifies MRENCLAVE, i.e. * can't be gracefully unwound, while failure on EADD/EXTEND is limited @@ -273,6 +351,7 @@ static int sgx_encl_add_page(struct sgx_encl *encl, unsigned long src, goto err_out; } + sgx_mark_page_reclaimable(encl_page->epc_page); mutex_unlock(&encl->lock); mmap_read_unlock(current->mm); return ret; @@ -281,9 +360,11 @@ err_out: xa_erase(&encl->page_array, PFN_DOWN(encl_page->desc)); err_out_unlock: + sgx_encl_shrink(encl, va_page); mutex_unlock(&encl->lock); mmap_read_unlock(current->mm); +err_out_free: sgx_free_epc_page(epc_page); kfree(encl_page); diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index 38f2e80cc31a..3426785df457 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -16,6 +16,15 @@ struct sgx_epc_section sgx_epc_sections[SGX_MAX_EPC_SECTIONS]; static int sgx_nr_epc_sections; static struct task_struct *ksgxd_tsk; +static DECLARE_WAIT_QUEUE_HEAD(ksgxd_waitq); + +/* + * These variables are part of the state of the reclaimer, and must be accessed + * with sgx_reclaimer_lock acquired. + */ +static LIST_HEAD(sgx_active_page_list); + +static DEFINE_SPINLOCK(sgx_reclaimer_lock); /* * Reset dirty EPC pages to uninitialized state. Laundry can be left with SECS @@ -50,6 +59,348 @@ static void sgx_sanitize_section(struct sgx_epc_section *section) list_splice(&dirty, §ion->laundry_list); } +static bool sgx_reclaimer_age(struct sgx_epc_page *epc_page) +{ + struct sgx_encl_page *page = epc_page->owner; + struct sgx_encl *encl = page->encl; + struct sgx_encl_mm *encl_mm; + bool ret = true; + int idx; + + idx = srcu_read_lock(&encl->srcu); + + list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) { + if (!mmget_not_zero(encl_mm->mm)) + continue; + + mmap_read_lock(encl_mm->mm); + ret = !sgx_encl_test_and_clear_young(encl_mm->mm, page); + mmap_read_unlock(encl_mm->mm); + + mmput_async(encl_mm->mm); + + if (!ret) + break; + } + + srcu_read_unlock(&encl->srcu, idx); + + if (!ret) + return false; + + return true; +} + +static void sgx_reclaimer_block(struct sgx_epc_page *epc_page) +{ + struct sgx_encl_page *page = epc_page->owner; + unsigned long addr = page->desc & PAGE_MASK; + struct sgx_encl *encl = page->encl; + unsigned long mm_list_version; + struct sgx_encl_mm *encl_mm; + struct vm_area_struct *vma; + int idx, ret; + + do { + mm_list_version = encl->mm_list_version; + + /* Pairs with smp_rmb() in sgx_encl_mm_add(). */ + smp_rmb(); + + idx = srcu_read_lock(&encl->srcu); + + list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) { + if (!mmget_not_zero(encl_mm->mm)) + continue; + + mmap_read_lock(encl_mm->mm); + + ret = sgx_encl_find(encl_mm->mm, addr, &vma); + if (!ret && encl == vma->vm_private_data) + zap_vma_ptes(vma, addr, PAGE_SIZE); + + mmap_read_unlock(encl_mm->mm); + + mmput_async(encl_mm->mm); + } + + srcu_read_unlock(&encl->srcu, idx); + } while (unlikely(encl->mm_list_version != mm_list_version)); + + mutex_lock(&encl->lock); + + ret = __eblock(sgx_get_epc_virt_addr(epc_page)); + if (encls_failed(ret)) + ENCLS_WARN(ret, "EBLOCK"); + + mutex_unlock(&encl->lock); +} + +static int __sgx_encl_ewb(struct sgx_epc_page *epc_page, void *va_slot, + struct sgx_backing *backing) +{ + struct sgx_pageinfo pginfo; + int ret; + + pginfo.addr = 0; + pginfo.secs = 0; + + pginfo.contents = (unsigned long)kmap_atomic(backing->contents); + pginfo.metadata = (unsigned long)kmap_atomic(backing->pcmd) + + backing->pcmd_offset; + + ret = __ewb(&pginfo, sgx_get_epc_virt_addr(epc_page), va_slot); + + kunmap_atomic((void *)(unsigned long)(pginfo.metadata - + backing->pcmd_offset)); + kunmap_atomic((void *)(unsigned long)pginfo.contents); + + return ret; +} + +static void sgx_ipi_cb(void *info) +{ +} + +static const cpumask_t *sgx_encl_ewb_cpumask(struct sgx_encl *encl) +{ + cpumask_t *cpumask = &encl->cpumask; + struct sgx_encl_mm *encl_mm; + int idx; + + /* + * Can race with sgx_encl_mm_add(), but ETRACK has already been + * executed, which means that the CPUs running in the new mm will enter + * into the enclave with a fresh epoch. + */ + cpumask_clear(cpumask); + + idx = srcu_read_lock(&encl->srcu); + + list_for_each_entry_rcu(encl_mm, &encl->mm_list, list) { + if (!mmget_not_zero(encl_mm->mm)) + continue; + + cpumask_or(cpumask, cpumask, mm_cpumask(encl_mm->mm)); + + mmput_async(encl_mm->mm); + } + + srcu_read_unlock(&encl->srcu, idx); + + return cpumask; +} + +/* + * Swap page to the regular memory transformed to the blocked state by using + * EBLOCK, which means that it can no loger be referenced (no new TLB entries). + * + * The first trial just tries to write the page assuming that some other thread + * has reset the count for threads inside the enlave by using ETRACK, and + * previous thread count has been zeroed out. The second trial calls ETRACK + * before EWB. If that fails we kick all the HW threads out, and then do EWB, + * which should be guaranteed the succeed. + */ +static void sgx_encl_ewb(struct sgx_epc_page *epc_page, + struct sgx_backing *backing) +{ + struct sgx_encl_page *encl_page = epc_page->owner; + struct sgx_encl *encl = encl_page->encl; + struct sgx_va_page *va_page; + unsigned int va_offset; + void *va_slot; + int ret; + + encl_page->desc &= ~SGX_ENCL_PAGE_BEING_RECLAIMED; + + va_page = list_first_entry(&encl->va_pages, struct sgx_va_page, + list); + va_offset = sgx_alloc_va_slot(va_page); + va_slot = sgx_get_epc_virt_addr(va_page->epc_page) + va_offset; + if (sgx_va_page_full(va_page)) + list_move_tail(&va_page->list, &encl->va_pages); + + ret = __sgx_encl_ewb(epc_page, va_slot, backing); + if (ret == SGX_NOT_TRACKED) { + ret = __etrack(sgx_get_epc_virt_addr(encl->secs.epc_page)); + if (ret) { + if (encls_failed(ret)) + ENCLS_WARN(ret, "ETRACK"); + } + + ret = __sgx_encl_ewb(epc_page, va_slot, backing); + if (ret == SGX_NOT_TRACKED) { + /* + * Slow path, send IPIs to kick cpus out of the + * enclave. Note, it's imperative that the cpu + * mask is generated *after* ETRACK, else we'll + * miss cpus that entered the enclave between + * generating the mask and incrementing epoch. + */ + on_each_cpu_mask(sgx_encl_ewb_cpumask(encl), + sgx_ipi_cb, NULL, 1); + ret = __sgx_encl_ewb(epc_page, va_slot, backing); + } + } + + if (ret) { + if (encls_failed(ret)) + ENCLS_WARN(ret, "EWB"); + + sgx_free_va_slot(va_page, va_offset); + } else { + encl_page->desc |= va_offset; + encl_page->va_page = va_page; + } +} + +static void sgx_reclaimer_write(struct sgx_epc_page *epc_page, + struct sgx_backing *backing) +{ + struct sgx_encl_page *encl_page = epc_page->owner; + struct sgx_encl *encl = encl_page->encl; + struct sgx_backing secs_backing; + int ret; + + mutex_lock(&encl->lock); + + sgx_encl_ewb(epc_page, backing); + encl_page->epc_page = NULL; + encl->secs_child_cnt--; + + if (!encl->secs_child_cnt && test_bit(SGX_ENCL_INITIALIZED, &encl->flags)) { + ret = sgx_encl_get_backing(encl, PFN_DOWN(encl->size), + &secs_backing); + if (ret) + goto out; + + sgx_encl_ewb(encl->secs.epc_page, &secs_backing); + + sgx_free_epc_page(encl->secs.epc_page); + encl->secs.epc_page = NULL; + + sgx_encl_put_backing(&secs_backing, true); + } + +out: + mutex_unlock(&encl->lock); +} + +/* + * Take a fixed number of pages from the head of the active page pool and + * reclaim them to the enclave's private shmem files. Skip the pages, which have + * been accessed since the last scan. Move those pages to the tail of active + * page pool so that the pages get scanned in LRU like fashion. + * + * Batch process a chunk of pages (at the moment 16) in order to degrade amount + * of IPI's and ETRACK's potentially required. sgx_encl_ewb() does degrade a bit + * among the HW threads with three stage EWB pipeline (EWB, ETRACK + EWB and IPI + * + EWB) but not sufficiently. Reclaiming one page at a time would also be + * problematic as it would increase the lock contention too much, which would + * halt forward progress. + */ +static void sgx_reclaim_pages(void) +{ + struct sgx_epc_page *chunk[SGX_NR_TO_SCAN]; + struct sgx_backing backing[SGX_NR_TO_SCAN]; + struct sgx_epc_section *section; + struct sgx_encl_page *encl_page; + struct sgx_epc_page *epc_page; + pgoff_t page_index; + int cnt = 0; + int ret; + int i; + + spin_lock(&sgx_reclaimer_lock); + for (i = 0; i < SGX_NR_TO_SCAN; i++) { + if (list_empty(&sgx_active_page_list)) + break; + + epc_page = list_first_entry(&sgx_active_page_list, + struct sgx_epc_page, list); + list_del_init(&epc_page->list); + encl_page = epc_page->owner; + + if (kref_get_unless_zero(&encl_page->encl->refcount) != 0) + chunk[cnt++] = epc_page; + else + /* The owner is freeing the page. No need to add the + * page back to the list of reclaimable pages. + */ + epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED; + } + spin_unlock(&sgx_reclaimer_lock); + + for (i = 0; i < cnt; i++) { + epc_page = chunk[i]; + encl_page = epc_page->owner; + + if (!sgx_reclaimer_age(epc_page)) + goto skip; + + page_index = PFN_DOWN(encl_page->desc - encl_page->encl->base); + ret = sgx_encl_get_backing(encl_page->encl, page_index, &backing[i]); + if (ret) + goto skip; + + mutex_lock(&encl_page->encl->lock); + encl_page->desc |= SGX_ENCL_PAGE_BEING_RECLAIMED; + mutex_unlock(&encl_page->encl->lock); + continue; + +skip: + spin_lock(&sgx_reclaimer_lock); + list_add_tail(&epc_page->list, &sgx_active_page_list); + spin_unlock(&sgx_reclaimer_lock); + + kref_put(&encl_page->encl->refcount, sgx_encl_release); + + chunk[i] = NULL; + } + + for (i = 0; i < cnt; i++) { + epc_page = chunk[i]; + if (epc_page) + sgx_reclaimer_block(epc_page); + } + + for (i = 0; i < cnt; i++) { + epc_page = chunk[i]; + if (!epc_page) + continue; + + encl_page = epc_page->owner; + sgx_reclaimer_write(epc_page, &backing[i]); + sgx_encl_put_backing(&backing[i], true); + + kref_put(&encl_page->encl->refcount, sgx_encl_release); + epc_page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED; + + section = &sgx_epc_sections[epc_page->section]; + spin_lock(§ion->lock); + list_add_tail(&epc_page->list, §ion->page_list); + section->free_cnt++; + spin_unlock(§ion->lock); + } +} + +static unsigned long sgx_nr_free_pages(void) +{ + unsigned long cnt = 0; + int i; + + for (i = 0; i < sgx_nr_epc_sections; i++) + cnt += sgx_epc_sections[i].free_cnt; + + return cnt; +} + +static bool sgx_should_reclaim(unsigned long watermark) +{ + return sgx_nr_free_pages() < watermark && + !list_empty(&sgx_active_page_list); +} + static int ksgxd(void *p) { int i; @@ -71,6 +422,20 @@ static int ksgxd(void *p) WARN(1, "EPC section %d has unsanitized pages.\n", i); } + while (!kthread_should_stop()) { + if (try_to_freeze()) + continue; + + wait_event_freezable(ksgxd_waitq, + kthread_should_stop() || + sgx_should_reclaim(SGX_NR_HIGH_PAGES)); + + if (sgx_should_reclaim(SGX_NR_HIGH_PAGES)) + sgx_reclaim_pages(); + + cond_resched(); + } + return 0; } @@ -100,6 +465,7 @@ static struct sgx_epc_page *__sgx_alloc_epc_page_from_section(struct sgx_epc_sec page = list_first_entry(§ion->page_list, struct sgx_epc_page, list); list_del_init(&page->list); + section->free_cnt--; spin_unlock(§ion->lock); return page; @@ -132,6 +498,100 @@ struct sgx_epc_page *__sgx_alloc_epc_page(void) return ERR_PTR(-ENOMEM); } +/** + * sgx_mark_page_reclaimable() - Mark a page as reclaimable + * @page: EPC page + * + * Mark a page as reclaimable and add it to the active page list. Pages + * are automatically removed from the active list when freed. + */ +void sgx_mark_page_reclaimable(struct sgx_epc_page *page) +{ + spin_lock(&sgx_reclaimer_lock); + page->flags |= SGX_EPC_PAGE_RECLAIMER_TRACKED; + list_add_tail(&page->list, &sgx_active_page_list); + spin_unlock(&sgx_reclaimer_lock); +} + +/** + * sgx_unmark_page_reclaimable() - Remove a page from the reclaim list + * @page: EPC page + * + * Clear the reclaimable flag and remove the page from the active page list. + * + * Return: + * 0 on success, + * -EBUSY if the page is in the process of being reclaimed + */ +int sgx_unmark_page_reclaimable(struct sgx_epc_page *page) +{ + spin_lock(&sgx_reclaimer_lock); + if (page->flags & SGX_EPC_PAGE_RECLAIMER_TRACKED) { + /* The page is being reclaimed. */ + if (list_empty(&page->list)) { + spin_unlock(&sgx_reclaimer_lock); + return -EBUSY; + } + + list_del(&page->list); + page->flags &= ~SGX_EPC_PAGE_RECLAIMER_TRACKED; + } + spin_unlock(&sgx_reclaimer_lock); + + return 0; +} + +/** + * sgx_alloc_epc_page() - Allocate an EPC page + * @owner: the owner of the EPC page + * @reclaim: reclaim pages if necessary + * + * Iterate through EPC sections and borrow a free EPC page to the caller. When a + * page is no longer needed it must be released with sgx_free_epc_page(). If + * @reclaim is set to true, directly reclaim pages when we are out of pages. No + * mm's can be locked when @reclaim is set to true. + * + * Finally, wake up ksgxd when the number of pages goes below the watermark + * before returning back to the caller. + * + * Return: + * an EPC page, + * -errno on error + */ +struct sgx_epc_page *sgx_alloc_epc_page(void *owner, bool reclaim) +{ + struct sgx_epc_page *page; + + for ( ; ; ) { + page = __sgx_alloc_epc_page(); + if (!IS_ERR(page)) { + page->owner = owner; + break; + } + + if (list_empty(&sgx_active_page_list)) + return ERR_PTR(-ENOMEM); + + if (!reclaim) { + page = ERR_PTR(-EBUSY); + break; + } + + if (signal_pending(current)) { + page = ERR_PTR(-ERESTARTSYS); + break; + } + + sgx_reclaim_pages(); + cond_resched(); + } + + if (sgx_should_reclaim(SGX_NR_LOW_PAGES)) + wake_up(&ksgxd_waitq); + + return page; +} + /** * sgx_free_epc_page() - Free an EPC page * @page: an EPC page @@ -143,12 +603,15 @@ void sgx_free_epc_page(struct sgx_epc_page *page) struct sgx_epc_section *section = &sgx_epc_sections[page->section]; int ret; + WARN_ON_ONCE(page->flags & SGX_EPC_PAGE_RECLAIMER_TRACKED); + ret = __eremove(sgx_get_epc_virt_addr(page)); if (WARN_ONCE(ret, "EREMOVE returned %d (0x%x)", ret, ret)) return; spin_lock(§ion->lock); list_add_tail(&page->list, §ion->page_list); + section->free_cnt++; spin_unlock(§ion->lock); } @@ -176,9 +639,12 @@ static bool __init sgx_setup_epc_section(u64 phys_addr, u64 size, for (i = 0; i < nr_pages; i++) { section->pages[i].section = index; + section->pages[i].flags = 0; + section->pages[i].owner = NULL; list_add_tail(§ion->pages[i].list, §ion->laundry_list); } + section->free_cnt = nr_pages; return true; } diff --git a/arch/x86/kernel/cpu/sgx/sgx.h b/arch/x86/kernel/cpu/sgx/sgx.h index 91234f425b89..a188a683ffb6 100644 --- a/arch/x86/kernel/cpu/sgx/sgx.h +++ b/arch/x86/kernel/cpu/sgx/sgx.h @@ -15,9 +15,17 @@ #define SGX_MAX_EPC_SECTIONS 8 #define SGX_EEXTEND_BLOCK_SIZE 256 +#define SGX_NR_TO_SCAN 16 +#define SGX_NR_LOW_PAGES 32 +#define SGX_NR_HIGH_PAGES 64 + +/* Pages, which are being tracked by the page reclaimer. */ +#define SGX_EPC_PAGE_RECLAIMER_TRACKED BIT(0) struct sgx_epc_page { unsigned int section; + unsigned int flags; + struct sgx_encl_page *owner; struct list_head list; }; @@ -33,6 +41,7 @@ struct sgx_epc_section { struct list_head page_list; struct list_head laundry_list; struct sgx_epc_page *pages; + unsigned long free_cnt; spinlock_t lock; }; @@ -61,4 +70,8 @@ static inline void *sgx_get_epc_virt_addr(struct sgx_epc_page *page) struct sgx_epc_page *__sgx_alloc_epc_page(void); void sgx_free_epc_page(struct sgx_epc_page *page); +void sgx_mark_page_reclaimable(struct sgx_epc_page *page); +int sgx_unmark_page_reclaimable(struct sgx_epc_page *page); +struct sgx_epc_page *sgx_alloc_epc_page(void *owner, bool reclaim); + #endif /* _X86_SGX_H */ From 947c6e11fa4310b31c10016ae9816cdca3f1694e Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 13 Nov 2020 00:01:33 +0200 Subject: [PATCH 196/360] x86/sgx: Add ptrace() support for the SGX driver Enclave memory is normally inaccessible from outside the enclave. This makes enclaves hard to debug. However, enclaves can be put in a debug mode when they are being built. In that mode, enclave data *can* be read and/or written by using the ENCLS[EDBGRD] and ENCLS[EDBGWR] functions. This is obviously only for debugging and destroys all the protections present with normal enclaves. But, enclaves know their own debug status and can adjust their behavior appropriately. Add a vm_ops->access() implementation which can be used to read and write memory inside debug enclaves. This is typically used via ptrace() APIs. [ bp: Massage. ] Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Tested-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-23-jarkko@kernel.org --- arch/x86/kernel/cpu/sgx/encl.c | 111 +++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/arch/x86/kernel/cpu/sgx/encl.c b/arch/x86/kernel/cpu/sgx/encl.c index b74dadf85989..ee50a5010277 100644 --- a/arch/x86/kernel/cpu/sgx/encl.c +++ b/arch/x86/kernel/cpu/sgx/encl.c @@ -272,10 +272,121 @@ static int sgx_vma_mprotect(struct vm_area_struct *vma, unsigned long start, return sgx_encl_may_map(vma->vm_private_data, start, end, newflags); } +static int sgx_encl_debug_read(struct sgx_encl *encl, struct sgx_encl_page *page, + unsigned long addr, void *data) +{ + unsigned long offset = addr & ~PAGE_MASK; + int ret; + + + ret = __edbgrd(sgx_get_epc_virt_addr(page->epc_page) + offset, data); + if (ret) + return -EIO; + + return 0; +} + +static int sgx_encl_debug_write(struct sgx_encl *encl, struct sgx_encl_page *page, + unsigned long addr, void *data) +{ + unsigned long offset = addr & ~PAGE_MASK; + int ret; + + ret = __edbgwr(sgx_get_epc_virt_addr(page->epc_page) + offset, data); + if (ret) + return -EIO; + + return 0; +} + +/* + * Load an enclave page to EPC if required, and take encl->lock. + */ +static struct sgx_encl_page *sgx_encl_reserve_page(struct sgx_encl *encl, + unsigned long addr, + unsigned long vm_flags) +{ + struct sgx_encl_page *entry; + + for ( ; ; ) { + mutex_lock(&encl->lock); + + entry = sgx_encl_load_page(encl, addr, vm_flags); + if (PTR_ERR(entry) != -EBUSY) + break; + + mutex_unlock(&encl->lock); + } + + if (IS_ERR(entry)) + mutex_unlock(&encl->lock); + + return entry; +} + +static int sgx_vma_access(struct vm_area_struct *vma, unsigned long addr, + void *buf, int len, int write) +{ + struct sgx_encl *encl = vma->vm_private_data; + struct sgx_encl_page *entry = NULL; + char data[sizeof(unsigned long)]; + unsigned long align; + int offset; + int cnt; + int ret = 0; + int i; + + /* + * If process was forked, VMA is still there but vm_private_data is set + * to NULL. + */ + if (!encl) + return -EFAULT; + + if (!test_bit(SGX_ENCL_DEBUG, &encl->flags)) + return -EFAULT; + + for (i = 0; i < len; i += cnt) { + entry = sgx_encl_reserve_page(encl, (addr + i) & PAGE_MASK, + vma->vm_flags); + if (IS_ERR(entry)) { + ret = PTR_ERR(entry); + break; + } + + align = ALIGN_DOWN(addr + i, sizeof(unsigned long)); + offset = (addr + i) & (sizeof(unsigned long) - 1); + cnt = sizeof(unsigned long) - offset; + cnt = min(cnt, len - i); + + ret = sgx_encl_debug_read(encl, entry, align, data); + if (ret) + goto out; + + if (write) { + memcpy(data + offset, buf + i, cnt); + ret = sgx_encl_debug_write(encl, entry, align, data); + if (ret) + goto out; + } else { + memcpy(buf + i, data + offset, cnt); + } + +out: + mutex_unlock(&encl->lock); + + if (ret) + break; + } + + return ret < 0 ? ret : i; +} + const struct vm_operations_struct sgx_vm_ops = { .fault = sgx_vma_fault, .mprotect = sgx_vma_mprotect, .open = sgx_vma_open, + .access = sgx_vma_access, }; /** From 31d8546033053b98de00846ede8088bdbe38651d Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Mon, 5 Oct 2020 11:12:08 -0400 Subject: [PATCH 197/360] x86/head/64: Remove unused GET_CR2_INTO() macro Commit 4b47cdbda6f1 ("x86/head/64: Move early exception dispatch to C code") removed the usage of GET_CR2_INTO(). Drop the definition as well, and related definitions in paravirt.h and asm-offsets.h Signed-off-by: Arvind Sankar Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201005151208.2212886-3-nivedita@alum.mit.edu --- arch/x86/include/asm/paravirt.h | 11 ----------- arch/x86/kernel/asm-offsets.c | 1 - arch/x86/kernel/head_64.S | 9 --------- 3 files changed, 21 deletions(-) diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index d25cc6830e89..f8dce11d2bc1 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -812,17 +812,6 @@ extern void default_banner(void); #endif /* CONFIG_PARAVIRT_XXL */ #endif /* CONFIG_X86_64 */ -#ifdef CONFIG_PARAVIRT_XXL - -#define GET_CR2_INTO_AX \ - PARA_SITE(PARA_PATCH(PV_MMU_read_cr2), \ - ANNOTATE_RETPOLINE_SAFE; \ - call PARA_INDIRECT(pv_ops+PV_MMU_read_cr2); \ - ) - -#endif /* CONFIG_PARAVIRT_XXL */ - - #endif /* __ASSEMBLY__ */ #else /* CONFIG_PARAVIRT */ # define default_banner x86_init_noop diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c index 70b7154f4bdd..60b9f42ce3c1 100644 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -66,7 +66,6 @@ static void __used common(void) OFFSET(PV_IRQ_irq_disable, paravirt_patch_template, irq.irq_disable); OFFSET(PV_IRQ_irq_enable, paravirt_patch_template, irq.irq_enable); OFFSET(PV_CPU_iret, paravirt_patch_template, cpu.iret); - OFFSET(PV_MMU_read_cr2, paravirt_patch_template, mmu.read_cr2); #endif #ifdef CONFIG_XEN diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 7eb2a1c87969..2215d4cff38b 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -26,15 +26,6 @@ #include #include -#ifdef CONFIG_PARAVIRT_XXL -#include -#include -#define GET_CR2_INTO(reg) GET_CR2_INTO_AX ; _ASM_MOV %_ASM_AX, reg -#else -#define INTERRUPT_RETURN iretq -#define GET_CR2_INTO(reg) _ASM_MOV %cr2, reg -#endif - /* * We are not able to switch in one step to the final KERNEL ADDRESS SPACE * because we need identity-mapped pages. From 3fa97bf001262a1d88ec9b4ac5ae6abe0ed1356c Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 13 Nov 2020 00:01:34 +0200 Subject: [PATCH 198/360] Documentation/x86: Document SGX kernel architecture Document the Intel SGX kernel architecture. The fine-grained architecture details can be looked up from Intel SDM Volume 3D. Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Cc: linux-doc@vger.kernel.org Link: https://lkml.kernel.org/r/20201112220135.165028-24-jarkko@kernel.org --- Documentation/x86/index.rst | 1 + Documentation/x86/sgx.rst | 211 ++++++++++++++++++++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 Documentation/x86/sgx.rst diff --git a/Documentation/x86/index.rst b/Documentation/x86/index.rst index b224d12c880b..647a570d4931 100644 --- a/Documentation/x86/index.rst +++ b/Documentation/x86/index.rst @@ -33,3 +33,4 @@ x86-specific Documentation i386/index x86_64/index sva + sgx diff --git a/Documentation/x86/sgx.rst b/Documentation/x86/sgx.rst new file mode 100644 index 000000000000..eaee1368b4fd --- /dev/null +++ b/Documentation/x86/sgx.rst @@ -0,0 +1,211 @@ +.. SPDX-License-Identifier: GPL-2.0 + +=============================== +Software Guard eXtensions (SGX) +=============================== + +Overview +======== + +Software Guard eXtensions (SGX) hardware enables for user space applications +to set aside private memory regions of code and data: + +* Privileged (ring-0) ENCLS functions orchestrate the construction of the. + regions. +* Unprivileged (ring-3) ENCLU functions allow an application to enter and + execute inside the regions. + +These memory regions are called enclaves. An enclave can be only entered at a +fixed set of entry points. Each entry point can hold a single hardware thread +at a time. While the enclave is loaded from a regular binary file by using +ENCLS functions, only the threads inside the enclave can access its memory. The +region is denied from outside access by the CPU, and encrypted before it leaves +from LLC. + +The support can be determined by + + ``grep sgx /proc/cpuinfo`` + +SGX must both be supported in the processor and enabled by the BIOS. If SGX +appears to be unsupported on a system which has hardware support, ensure +support is enabled in the BIOS. If a BIOS presents a choice between "Enabled" +and "Software Enabled" modes for SGX, choose "Enabled". + +Enclave Page Cache +================== + +SGX utilizes an *Enclave Page Cache (EPC)* to store pages that are associated +with an enclave. It is contained in a BIOS-reserved region of physical memory. +Unlike pages used for regular memory, pages can only be accessed from outside of +the enclave during enclave construction with special, limited SGX instructions. + +Only a CPU executing inside an enclave can directly access enclave memory. +However, a CPU executing inside an enclave may access normal memory outside the +enclave. + +The kernel manages enclave memory similar to how it treats device memory. + +Enclave Page Types +------------------ + +**SGX Enclave Control Structure (SECS)** + Enclave's address range, attributes and other global data are defined + by this structure. + +**Regular (REG)** + Regular EPC pages contain the code and data of an enclave. + +**Thread Control Structure (TCS)** + Thread Control Structure pages define the entry points to an enclave and + track the execution state of an enclave thread. + +**Version Array (VA)** + Version Array pages contain 512 slots, each of which can contain a version + number for a page evicted from the EPC. + +Enclave Page Cache Map +---------------------- + +The processor tracks EPC pages in a hardware metadata structure called the +*Enclave Page Cache Map (EPCM)*. The EPCM contains an entry for each EPC page +which describes the owning enclave, access rights and page type among the other +things. + +EPCM permissions are separate from the normal page tables. This prevents the +kernel from, for instance, allowing writes to data which an enclave wishes to +remain read-only. EPCM permissions may only impose additional restrictions on +top of normal x86 page permissions. + +For all intents and purposes, the SGX architecture allows the processor to +invalidate all EPCM entries at will. This requires that software be prepared to +handle an EPCM fault at any time. In practice, this can happen on events like +power transitions when the ephemeral key that encrypts enclave memory is lost. + +Application interface +===================== + +Enclave build functions +----------------------- + +In addition to the traditional compiler and linker build process, SGX has a +separate enclave “build” process. Enclaves must be built before they can be +executed (entered). The first step in building an enclave is opening the +**/dev/sgx_enclave** device. Since enclave memory is protected from direct +access, special privileged instructions are Then used to copy data into enclave +pages and establish enclave page permissions. + +.. kernel-doc:: arch/x86/kernel/cpu/sgx/ioctl.c + :functions: sgx_ioc_enclave_create + sgx_ioc_enclave_add_pages + sgx_ioc_enclave_init + sgx_ioc_enclave_provision + +Enclave vDSO +------------ + +Entering an enclave can only be done through SGX-specific EENTER and ERESUME +functions, and is a non-trivial process. Because of the complexity of +transitioning to and from an enclave, enclaves typically utilize a library to +handle the actual transitions. This is roughly analogous to how glibc +implementations are used by most applications to wrap system calls. + +Another crucial characteristic of enclaves is that they can generate exceptions +as part of their normal operation that need to be handled in the enclave or are +unique to SGX. + +Instead of the traditional signal mechanism to handle these exceptions, SGX +can leverage special exception fixup provided by the vDSO. The kernel-provided +vDSO function wraps low-level transitions to/from the enclave like EENTER and +ERESUME. The vDSO function intercepts exceptions that would otherwise generate +a signal and return the fault information directly to its caller. This avoids +the need to juggle signal handlers. + +.. kernel-doc:: arch/x86/include/uapi/asm/sgx.h + :functions: vdso_sgx_enter_enclave_t + +ksgxd +===== + +SGX support includes a kernel thread called *ksgxwapd*. + +EPC sanitization +---------------- + +ksgxd is started when SGX initializes. Enclave memory is typically ready +For use when the processor powers on or resets. However, if SGX has been in +use since the reset, enclave pages may be in an inconsistent state. This might +occur after a crash and kexec() cycle, for instance. At boot, ksgxd +reinitializes all enclave pages so that they can be allocated and re-used. + +The sanitization is done by going through EPC address space and applying the +EREMOVE function to each physical page. Some enclave pages like SECS pages have +hardware dependencies on other pages which prevents EREMOVE from functioning. +Executing two EREMOVE passes removes the dependencies. + +Page reclaimer +-------------- + +Similar to the core kswapd, ksgxd, is responsible for managing the +overcommitment of enclave memory. If the system runs out of enclave memory, +*ksgxwapd* “swaps” enclave memory to normal memory. + +Launch Control +============== + +SGX provides a launch control mechanism. After all enclave pages have been +copied, kernel executes EINIT function, which initializes the enclave. Only after +this the CPU can execute inside the enclave. + +ENIT function takes an RSA-3072 signature of the enclave measurement. The function +checks that the measurement is correct and signature is signed with the key +hashed to the four **IA32_SGXLEPUBKEYHASH{0, 1, 2, 3}** MSRs representing the +SHA256 of a public key. + +Those MSRs can be configured by the BIOS to be either readable or writable. +Linux supports only writable configuration in order to give full control to the +kernel on launch control policy. Before calling EINIT function, the driver sets +the MSRs to match the enclave's signing key. + +Encryption engines +================== + +In order to conceal the enclave data while it is out of the CPU package, the +memory controller has an encryption engine to transparently encrypt and decrypt +enclave memory. + +In CPUs prior to Ice Lake, the Memory Encryption Engine (MEE) is used to +encrypt pages leaving the CPU caches. MEE uses a n-ary Merkle tree with root in +SRAM to maintain integrity of the encrypted data. This provides integrity and +anti-replay protection but does not scale to large memory sizes because the time +required to update the Merkle tree grows logarithmically in relation to the +memory size. + +CPUs starting from Icelake use Total Memory Encryption (TME) in the place of +MEE. TME-based SGX implementations do not have an integrity Merkle tree, which +means integrity and replay-attacks are not mitigated. B, it includes +additional changes to prevent cipher text from being returned and SW memory +aliases from being Created. + +DMA to enclave memory is blocked by range registers on both MEE and TME systems +(SDM section 41.10). + +Usage Models +============ + +Shared Library +-------------- + +Sensitive data and the code that acts on it is partitioned from the application +into a separate library. The library is then linked as a DSO which can be loaded +into an enclave. The application can then make individual function calls into +the enclave through special SGX instructions. A run-time within the enclave is +configured to marshal function parameters into and out of the enclave and to +call the correct library function. + +Application Container +--------------------- + +An application may be loaded into a container enclave which is specially +configured with a library OS and run-time which permits the application to run. +The enclave run-time and library OS work together to execute the application +when a thread enters the enclave. From bc4bac2ecef0e47fd5c02f9c6f9585fd477f9beb Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Fri, 13 Nov 2020 00:01:35 +0200 Subject: [PATCH 199/360] x86/sgx: Update MAINTAINERS Add the maintainer information for the SGX subsystem. Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Acked-by: Jethro Beekman Link: https://lkml.kernel.org/r/20201112220135.165028-25-jarkko@kernel.org --- MAINTAINERS | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index e451dcce054f..44c240c2b33c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9126,6 +9126,19 @@ F: Documentation/x86/intel_txt.rst F: arch/x86/kernel/tboot.c F: include/linux/tboot.h +INTEL SGX +M: Jarkko Sakkinen +L: linux-sgx@vger.kernel.org +S: Supported +Q: https://patchwork.kernel.org/project/intel-sgx/list/ +T: git git://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-sgx.git +F: Documentation/x86/sgx.rst +F: arch/x86/entry/vdso/vsgx.S +F: arch/x86/include/uapi/asm/sgx.h +F: arch/x86/kernel/cpu/sgx/* +F: tools/testing/selftests/sgx/* +K: \bSGX_ + INTERCONNECT API M: Georgi Djakov L: linux-pm@vger.kernel.org From 67655b57f8f59467506463055d9a8398d2836377 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Mon, 16 Nov 2020 14:25:31 -0800 Subject: [PATCH 200/360] x86/sgx: Clarify 'laundry_list' locking Short Version: The SGX section->laundry_list structure is effectively thread-local, but declared next to some shared structures. Its semantics are clear as mud. Fix that. No functional changes. Compile tested only. Long Version: The SGX hardware keeps per-page metadata. This can provide things like permissions, integrity and replay protection. It also prevents things like having an enclave page mapped multiple times or shared between enclaves. But, that presents a problem for kexec()'d kernels (or any other kernel that does not run immediately after a hardware reset). This is because the last kernel may have been rude and forgotten to reset pages, which would trigger the "shared page" sanity check. To fix this, the SGX code "launders" the pages by running the EREMOVE instruction on all pages at boot. This is slow and can take a long time, so it is performed off in the SGX-specific ksgxd instead of being synchronous at boot. The init code hands the list of pages to launder in a per-SGX-section list: ->laundry_list. The only code to touch this list is the init code and ksgxd. This means that no locking is necessary for ->laundry_list. However, a lock is required for section->page_list, which is accessed while creating enclaves and by ksgxd. This lock (section->lock) is acquired by ksgxd while also processing ->laundry_list. It is easy to confuse the purpose of the locking as being for ->laundry_list and ->page_list. Rename ->laundry_list to ->init_laundry_list to make it clear that this is not normally used at runtime. Also add some comments clarifying the locking, and reorganize 'sgx_epc_section' to put 'lock' near the things it protects. Note: init_laundry_list is 128 bytes of wasted space at runtime. It could theoretically be dynamically allocated and then freed after the laundering process. But it would take nearly 128 bytes of extra instructions to do that. Signed-off-by: Jarkko Sakkinen Signed-off-by: Dave Hansen Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201116222531.4834-1-dave.hansen@intel.com --- arch/x86/kernel/cpu/sgx/main.c | 14 ++++++++------ arch/x86/kernel/cpu/sgx/sgx.h | 15 ++++++++++++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c index 3426785df457..c519fc5f6948 100644 --- a/arch/x86/kernel/cpu/sgx/main.c +++ b/arch/x86/kernel/cpu/sgx/main.c @@ -36,13 +36,15 @@ static void sgx_sanitize_section(struct sgx_epc_section *section) LIST_HEAD(dirty); int ret; - while (!list_empty(§ion->laundry_list)) { + /* init_laundry_list is thread-local, no need for a lock: */ + while (!list_empty(§ion->init_laundry_list)) { if (kthread_should_stop()) return; + /* needed for access to ->page_list: */ spin_lock(§ion->lock); - page = list_first_entry(§ion->laundry_list, + page = list_first_entry(§ion->init_laundry_list, struct sgx_epc_page, list); ret = __eremove(sgx_get_epc_virt_addr(page)); @@ -56,7 +58,7 @@ static void sgx_sanitize_section(struct sgx_epc_section *section) cond_resched(); } - list_splice(&dirty, §ion->laundry_list); + list_splice(&dirty, §ion->init_laundry_list); } static bool sgx_reclaimer_age(struct sgx_epc_page *epc_page) @@ -418,7 +420,7 @@ static int ksgxd(void *p) sgx_sanitize_section(&sgx_epc_sections[i]); /* Should never happen. */ - if (!list_empty(&sgx_epc_sections[i].laundry_list)) + if (!list_empty(&sgx_epc_sections[i].init_laundry_list)) WARN(1, "EPC section %d has unsanitized pages.\n", i); } @@ -635,13 +637,13 @@ static bool __init sgx_setup_epc_section(u64 phys_addr, u64 size, section->phys_addr = phys_addr; spin_lock_init(§ion->lock); INIT_LIST_HEAD(§ion->page_list); - INIT_LIST_HEAD(§ion->laundry_list); + INIT_LIST_HEAD(§ion->init_laundry_list); for (i = 0; i < nr_pages; i++) { section->pages[i].section = index; section->pages[i].flags = 0; section->pages[i].owner = NULL; - list_add_tail(§ion->pages[i].list, §ion->laundry_list); + list_add_tail(§ion->pages[i].list, §ion->init_laundry_list); } section->free_cnt = nr_pages; diff --git a/arch/x86/kernel/cpu/sgx/sgx.h b/arch/x86/kernel/cpu/sgx/sgx.h index a188a683ffb6..5fa42d143feb 100644 --- a/arch/x86/kernel/cpu/sgx/sgx.h +++ b/arch/x86/kernel/cpu/sgx/sgx.h @@ -34,15 +34,24 @@ struct sgx_epc_page { * physical memory e.g. for memory areas of the each node. This structure is * used to store EPC pages for one EPC section and virtual memory area where * the pages have been mapped. + * + * 'lock' must be held before accessing 'page_list' or 'free_cnt'. */ struct sgx_epc_section { unsigned long phys_addr; void *virt_addr; - struct list_head page_list; - struct list_head laundry_list; struct sgx_epc_page *pages; - unsigned long free_cnt; + spinlock_t lock; + struct list_head page_list; + unsigned long free_cnt; + + /* + * Pages which need EREMOVE run on them before they can be + * used. Only safe to be accessed in ksgxd and init code. + * Not protected by locks. + */ + struct list_head init_laundry_list; }; extern struct sgx_epc_section sgx_epc_sections[SGX_MAX_EPC_SECTIONS]; From 0eaa8d153a1d573e53b8283c90db44057d1376f6 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Wed, 18 Nov 2020 19:06:40 +0200 Subject: [PATCH 201/360] selftests/sgx: Use a statically generated 3072-bit RSA key Use a statically generated key for signing the enclave, because generating keys on the fly can eat the kernel entropy pool. Another good reason for doing this is predictable builds. The RSA has been arbitrarily selected. It's contents do not matter. This also makes the selftest execute a lot quicker instead of the delay that it had before (because of slow key generation). [ bp: Disambiguate "static key" which means something else in the kernel, fix typos. ] Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Cc: linux-kselftest@vger.kernel.org Link: https://lkml.kernel.org/r/20201118170640.39629-1-jarkko@kernel.org --- tools/testing/selftests/sgx/Makefile | 6 +++- tools/testing/selftests/sgx/main.h | 3 ++ tools/testing/selftests/sgx/sign_key.S | 12 ++++++++ tools/testing/selftests/sgx/sign_key.pem | 39 ++++++++++++++++++++++++ tools/testing/selftests/sgx/sigstruct.c | 34 ++++++++------------- 5 files changed, 71 insertions(+), 23 deletions(-) create mode 100644 tools/testing/selftests/sgx/sign_key.S create mode 100644 tools/testing/selftests/sgx/sign_key.pem diff --git a/tools/testing/selftests/sgx/Makefile b/tools/testing/selftests/sgx/Makefile index d51c90663943..7f12d55b97f8 100644 --- a/tools/testing/selftests/sgx/Makefile +++ b/tools/testing/selftests/sgx/Makefile @@ -25,7 +25,8 @@ endif $(OUTPUT)/test_sgx: $(OUTPUT)/main.o \ $(OUTPUT)/load.o \ $(OUTPUT)/sigstruct.o \ - $(OUTPUT)/call.o + $(OUTPUT)/call.o \ + $(OUTPUT)/sign_key.o $(CC) $(HOST_CFLAGS) -o $@ $^ -lcrypto $(OUTPUT)/main.o: main.c @@ -40,6 +41,9 @@ $(OUTPUT)/sigstruct.o: sigstruct.c $(OUTPUT)/call.o: call.S $(CC) $(HOST_CFLAGS) -c $< -o $@ +$(OUTPUT)/sign_key.o: sign_key.S + $(CC) $(HOST_CFLAGS) -c $< -o $@ + $(OUTPUT)/test_encl.elf: test_encl.lds test_encl.c test_encl_bootstrap.S $(CC) $(ENCL_CFLAGS) -T $^ -o $@ diff --git a/tools/testing/selftests/sgx/main.h b/tools/testing/selftests/sgx/main.h index 45e6ab65442a..67211a708f04 100644 --- a/tools/testing/selftests/sgx/main.h +++ b/tools/testing/selftests/sgx/main.h @@ -27,6 +27,9 @@ struct encl { struct sgx_sigstruct sigstruct; }; +extern unsigned char sign_key[]; +extern unsigned char sign_key_end[]; + void encl_delete(struct encl *ctx); bool encl_load(const char *path, struct encl *encl); bool encl_measure(struct encl *encl); diff --git a/tools/testing/selftests/sgx/sign_key.S b/tools/testing/selftests/sgx/sign_key.S new file mode 100644 index 000000000000..e4fbe948444a --- /dev/null +++ b/tools/testing/selftests/sgx/sign_key.S @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/** +* Copyright(c) 2016-20 Intel Corporation. +*/ + + .section ".rodata", "a" + +sign_key: + .globl sign_key + .incbin "sign_key.pem" +sign_key_end: + .globl sign_key_end diff --git a/tools/testing/selftests/sgx/sign_key.pem b/tools/testing/selftests/sgx/sign_key.pem new file mode 100644 index 000000000000..d76f21f19187 --- /dev/null +++ b/tools/testing/selftests/sgx/sign_key.pem @@ -0,0 +1,39 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIG4wIBAAKCAYEApalGbq7Q+usM91CPtksu3D+b0Prc8gAFL6grM3mg85A5Bx8V +cfMXPgtrw8EYFwQxDAvzZWwl+9VfOX0ECrFRBkOHcOiG0SnADN8+FLj1UiNUQwbp +S6OzhNWuRcSbGraSOyUlVlV0yMQSvewyzGklOaXBe30AJqzIBc8QfdSxKuP8rs0Z +ga6k/Bl73osrYKByILJTUUeZqjLERsE6GebsdzbWgKn8qVqng4ZS4yMNg6LeRlH3 ++9CIPgg4jwpSLHcp7dq2qTIB9a0tGe9ayp+5FbucpB6U7ePold0EeRN6RlJGDF9k +L93v8P5ykz5G5gYZ2g0K1X2sHIWV4huxPgv5PXgdyQYbK+6olqj0d5rjYuwX57Ul +k6SroPS1U6UbdCjG5txM+BNGU0VpD0ZhrIRw0leQdnNcCO9sTJuInZrgYacSVJ7u +mtB+uCt+uzUesc+l+xPRYA+9e14lLkZp7AAmo9FvL816XDI09deehJ3i/LmHKCRN +tuqC5TprRjFwUr6dAgEDAoIBgG5w2Z8fNfycs0+LCnmHdJLVEotR6KFVWMpwHMz7 +wKJgJgS/Y6FMuilc8oKAuroCy11dTO5IGVKOP3uorVx2NgQtBPXwWeDGgAiU1A3Q +o4wXjYIEm4fCd63jyYPYZ2ckYXzDbjmOTdstYdPyzIhGGNEZK6eoqsRzMAPfYFPj +IMdCqHSIu6vJw1K7p+myHOsVoWshjODaZnF3LYSA0WaZ8vokjwBxUxuRxQJZjJds +s60XPtmL+qfgWtQFewoG4XL6GuD8FcXccynRRtzrLtFNPIl9BQfWfjBBhTC1/Te1 +0Z6XbZvpdUTD9OfLB7SbR2OUFNpKQgriO0iYVdbW3cr7uu38Zwp4W1TX73DPjoi6 +KNooP6SGWd4mRJW2+dUmSYS4QNG8eVVZswKcploEIXlAKRsOe4kzJJ1iETugIe85 +uX8nd1WYEp65xwoRUg8hqng0MeyveVbXqNKuJG6tzNDt9kgFYo+hmC/oouAW2Dtc +T9jdRAwKJXqA2Eg6OkgXCEv+kwKBwQDYaQiFMlFhsmLlqI+EzCUh7c941/cL7m6U +7j98+8ngl0HgCEcrc10iJVCKakQW3YbPzAx3XkKTaGjWazvvrFarXIGlOud64B8a +iWyQ7VdlnmZnNEdk+C83tI91OQeaTKqRLDGzKh29Ry/jL8Pcbazt+kDgxa0H7qJp +roADUanLQuNkYubpbhFBh3xpa2EExaVq6rF7nIVsD8W9TrbmPKA4LgH7z0iy544D +kVCNYsTjYDdUWP+WiSor8kCnnpjnN9sCgcEAw/eNezUD1UDf6OYFC9+5JZJFn4Tg +mZMyN93JKIb199ffwnjtHUSjcyiWeesXucpzwtGbTcwQnDisSW4oneYKLSEBlBaq +scqiUugyGZZOthFSCbdXYXMViK2vHrKlkse7GxVlROKcEhM/pRBrmjaGO8eWR+D4 +FO2wCXzVs3KgV6j779frw0vC54oHOxc9+Lu1rSHp4i+600koyvL/zF6U/5tZXIvN +YW2yoiQJnjCmVA1pwbwV6KAUTPDTMnBK+YjnAoHBAJBGBa4hi5Z27JkbCliIGMFJ +NPs6pLKe9GNJf6in2+sPgUAFhMeiPhbDiwbxgrnpBIqICE+ULGJFmzmc0p/IOceT +ARjR76dAFLxbnbXzj5kURETNhO36yiUjCk4mBRGIcbYddndxaSjaH+zKgpLzyJ6m +1esuc1qfFvEfAAI2cTIsl5hB70ZJYNZaUvDyQK3ZGPHxy6e9rkgKg9OJz0QoatAe +q/002yHvtAJg4F5B2JeVejg7VQ8GHB1MKxppu0TP5wKBwQCCpQj8zgKOKz/wmViy +lSYZDC5qWJW7t3bP6TDFr06lOpUsUJ4TgxeiGw778g/RMaKB4RIz3WBoJcgw9BsT +7rFza1ZiucchMcGMmswRDt8kC4wGejpA92Owc8oUdxkMhSdnY5jYlxK2t3/DYEe8 +JFl9L7mFQKVjSSAGUzkiTGrlG1Kf5UfXh9dFBq98uilQfSPIwUaWynyM23CHTKqI +Pw3/vOY9sojrnncWwrEUIG7is5vWfWPwargzSzd29YdRBe8CgcEAuRVewK/YeNOX +B7ZG6gKKsfsvrGtY7FPETzLZAHjoVXYNea4LVZ2kn4hBXXlvw/4HD+YqcTt4wmif +5JQlDvjNobUiKJZpzy7hklVhF7wZFl4pCF7Yh43q9iQ7gKTaeUG7MiaK+G8Zz8aY +HW9rsiihbdZkccMvnPfO9334XMxl3HtBRzLstjUlbLB7Sdh+7tZ3JQidCOFNs5pE +XyWwnASPu4tKfDahH1UUTp1uJcq/6716CSWg080avYxFcn75qqsb +-----END RSA PRIVATE KEY----- diff --git a/tools/testing/selftests/sgx/sigstruct.c b/tools/testing/selftests/sgx/sigstruct.c index cc06f108bae7..dee7a3d6c5a5 100644 --- a/tools/testing/selftests/sgx/sigstruct.c +++ b/tools/testing/selftests/sgx/sigstruct.c @@ -135,33 +135,21 @@ static inline const BIGNUM *get_modulus(RSA *key) static RSA *gen_sign_key(void) { - BIGNUM *e; + unsigned long sign_key_length; + BIO *bio; RSA *key; - int ret; - e = BN_new(); - key = RSA_new(); + sign_key_length = (unsigned long)&sign_key_end - + (unsigned long)&sign_key; - if (!e || !key) - goto err; + bio = BIO_new_mem_buf(&sign_key, sign_key_length); + if (!bio) + return NULL; - ret = BN_set_word(e, RSA_3); - if (ret != 1) - goto err; - - ret = RSA_generate_key_ex(key, 3072, e, NULL); - if (ret != 1) - goto err; - - BN_free(e); + key = PEM_read_bio_RSAPrivateKey(bio, NULL, NULL, NULL); + BIO_free(bio); return key; - -err: - RSA_free(key); - BN_free(e); - - return NULL; } static void reverse_bytes(void *data, int length) @@ -339,8 +327,10 @@ bool encl_measure(struct encl *encl) goto err; key = gen_sign_key(); - if (!key) + if (!key) { + ERR_print_errors_fp(stdout); goto err; + } BN_bn2bin(get_modulus(key), sigstruct->modulus); From 61d35648c06cac042d88c6d0b8df8f8c8c72a4d4 Mon Sep 17 00:00:00 2001 From: Zhang Xiaoxu Date: Mon, 16 Nov 2020 08:58:10 -0500 Subject: [PATCH 202/360] EDAC/synopsys: Return the correct value in mc_probe() Return the error value if the inject sysfs file creation fails, rather than returning 0, to signal to the upper layer that the ->probe function failed. [ bp: Massage. ] Signed-off-by: Zhang Xiaoxu Signed-off-by: Borislav Petkov Reviewed-by: Michal Simek Link: https://lkml.kernel.org/r/20201116135810.3130845-1-zhangxiaoxu5@huawei.com --- drivers/edac/synopsys_edac.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/edac/synopsys_edac.c b/drivers/edac/synopsys_edac.c index 12211dc040e8..7e7146b22c16 100644 --- a/drivers/edac/synopsys_edac.c +++ b/drivers/edac/synopsys_edac.c @@ -1344,7 +1344,8 @@ static int mc_probe(struct platform_device *pdev) #ifdef CONFIG_EDAC_DEBUG if (priv->p_data->quirks & DDR_ECC_DATA_POISON_SUPPORT) { - if (edac_create_sysfs_attributes(mci)) { + rc = edac_create_sysfs_attributes(mci); + if (rc) { edac_printk(KERN_ERR, EDAC_MC, "Failed to create sysfs entries\n"); goto free_edac_mc; From b023fd5f741f34d2cd90258ccc3f245924d2eadd Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Wed, 18 Nov 2020 13:34:07 +0100 Subject: [PATCH 203/360] x86/msr: Downgrade unrecognized MSR message It is a warning and not an error so use pr_warn(). Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201118123806.19672-1-bp@alien8.de --- arch/x86/kernel/msr.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c index b1147862730c..95e6b97b7d8b 100644 --- a/arch/x86/kernel/msr.c +++ b/arch/x86/kernel/msr.c @@ -99,8 +99,8 @@ static int filter_write(u32 reg) if (!__ratelimit(&fw_rs)) return 0; - pr_err("Write to unrecognized MSR 0x%x by %s (pid: %d). Please report to x86@kernel.org.\n", - reg, current->comm, current->pid); + pr_warn("Write to unrecognized MSR 0x%x by %s (pid: %d). Please report to x86@kernel.org.\n", + reg, current->comm, current->pid); return 0; } From 14132a5b807bb5caf778fe7ae1597e630971e949 Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Wed, 18 Nov 2020 23:39:32 +0200 Subject: [PATCH 204/360] x86/sgx: Return -ERESTARTSYS in sgx_ioc_enclave_add_pages() Return -ERESTARTSYS instead of -EINTR in sgx_ioc_enclave_add_pages() when interrupted before any pages have been processed. At this point ioctl can be obviously safely restarted. Reported-by: Haitao Huang Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201118213932.63341-1-jarkko@kernel.org --- arch/x86/kernel/cpu/sgx/ioctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/sgx/ioctl.c b/arch/x86/kernel/cpu/sgx/ioctl.c index 6d37117ac8a0..30aefc93a31d 100644 --- a/arch/x86/kernel/cpu/sgx/ioctl.c +++ b/arch/x86/kernel/cpu/sgx/ioctl.c @@ -444,7 +444,7 @@ static long sgx_ioc_enclave_add_pages(struct sgx_encl *encl, void __user *arg) for (c = 0 ; c < add_arg.length; c += PAGE_SIZE) { if (signal_pending(current)) { if (!c) - ret = -EINTR; + ret = -ERESTARTSYS; break; } From 028c221ed1904af9ac3c5162ee98f48966de6b3d Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Mon, 9 Nov 2020 21:06:56 +0000 Subject: [PATCH 205/360] x86/CPU/AMD: Save AMD NodeId as cpu_die_id AMD systems provide a "NodeId" value that represents a global ID indicating to which "Node" a logical CPU belongs. The "Node" is a physical structure equivalent to a Die, and it should not be confused with logical structures like NUMA nodes. Logical nodes can be adjusted based on firmware or other settings whereas the physical nodes/dies are fixed based on hardware topology. The NodeId value can be used when a physical ID is needed by software. Save the AMD NodeId to struct cpuinfo_x86.cpu_die_id. Use the value from CPUID or MSR as appropriate. Default to phys_proc_id otherwise. Do so for both AMD and Hygon systems. Drop the node_id parameter from cacheinfo_*_init_llc_id() as it is no longer needed. Update the x86 topology documentation. Suggested-by: Borislav Petkov Signed-off-by: Yazen Ghannam Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201109210659.754018-2-Yazen.Ghannam@amd.com --- Documentation/x86/topology.rst | 9 +++++++++ arch/x86/include/asm/cacheinfo.h | 4 ++-- arch/x86/kernel/cpu/amd.c | 11 +++++------ arch/x86/kernel/cpu/cacheinfo.c | 6 +++--- arch/x86/kernel/cpu/hygon.c | 11 +++++------ 5 files changed, 24 insertions(+), 17 deletions(-) diff --git a/Documentation/x86/topology.rst b/Documentation/x86/topology.rst index e29739904e37..7f58010ea86a 100644 --- a/Documentation/x86/topology.rst +++ b/Documentation/x86/topology.rst @@ -41,6 +41,8 @@ Package Packages contain a number of cores plus shared resources, e.g. DRAM controller, shared caches etc. +Modern systems may also use the term 'Die' for package. + AMD nomenclature for package is 'Node'. Package-related topology information in the kernel: @@ -53,11 +55,18 @@ Package-related topology information in the kernel: The number of dies in a package. This information is retrieved via CPUID. + - cpuinfo_x86.cpu_die_id: + + The physical ID of the die. This information is retrieved via CPUID. + - cpuinfo_x86.phys_proc_id: The physical ID of the package. This information is retrieved via CPUID and deduced from the APIC IDs of the cores in the package. + Modern systems use this value for the socket. There may be multiple + packages within a socket. This value may differ from cpu_die_id. + - cpuinfo_x86.logical_proc_id: The logical ID of the package. As we do not trust BIOSes to enumerate the diff --git a/arch/x86/include/asm/cacheinfo.h b/arch/x86/include/asm/cacheinfo.h index 86b63c7feab7..86b2e0dcc4bf 100644 --- a/arch/x86/include/asm/cacheinfo.h +++ b/arch/x86/include/asm/cacheinfo.h @@ -2,7 +2,7 @@ #ifndef _ASM_X86_CACHEINFO_H #define _ASM_X86_CACHEINFO_H -void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu, u8 node_id); -void cacheinfo_hygon_init_llc_id(struct cpuinfo_x86 *c, int cpu, u8 node_id); +void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu); +void cacheinfo_hygon_init_llc_id(struct cpuinfo_x86 *c, int cpu); #endif /* _ASM_X86_CACHEINFO_H */ diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 6062ce586b95..2f1fbd8150af 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -330,7 +330,6 @@ static void legacy_fixup_core_id(struct cpuinfo_x86 *c) */ static void amd_get_topology(struct cpuinfo_x86 *c) { - u8 node_id; int cpu = smp_processor_id(); /* get information required for multi-node processors */ @@ -340,7 +339,7 @@ static void amd_get_topology(struct cpuinfo_x86 *c) cpuid(0x8000001e, &eax, &ebx, &ecx, &edx); - node_id = ecx & 0xff; + c->cpu_die_id = ecx & 0xff; if (c->x86 == 0x15) c->cu_id = ebx & 0xff; @@ -360,15 +359,15 @@ static void amd_get_topology(struct cpuinfo_x86 *c) if (!err) c->x86_coreid_bits = get_count_order(c->x86_max_cores); - cacheinfo_amd_init_llc_id(c, cpu, node_id); + cacheinfo_amd_init_llc_id(c, cpu); } else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) { u64 value; rdmsrl(MSR_FAM10H_NODE_ID, value); - node_id = value & 7; + c->cpu_die_id = value & 7; - per_cpu(cpu_llc_id, cpu) = node_id; + per_cpu(cpu_llc_id, cpu) = c->cpu_die_id; } else return; @@ -393,7 +392,7 @@ static void amd_detect_cmp(struct cpuinfo_x86 *c) /* Convert the initial APIC ID into the socket ID */ c->phys_proc_id = c->initial_apicid >> bits; /* use socket ID also for last level cache */ - per_cpu(cpu_llc_id, cpu) = c->phys_proc_id; + per_cpu(cpu_llc_id, cpu) = c->cpu_die_id = c->phys_proc_id; } static void amd_detect_ppin(struct cpuinfo_x86 *c) diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index 57074cf3ad7c..f9ac682e75e7 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -646,7 +646,7 @@ static int find_num_cache_leaves(struct cpuinfo_x86 *c) return i; } -void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu, u8 node_id) +void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu) { /* * We may have multiple LLCs if L3 caches exist, so check if we @@ -657,7 +657,7 @@ void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu, u8 node_id) if (c->x86 < 0x17) { /* LLC is at the node level. */ - per_cpu(cpu_llc_id, cpu) = node_id; + per_cpu(cpu_llc_id, cpu) = c->cpu_die_id; } else if (c->x86 == 0x17 && c->x86_model <= 0x1F) { /* * LLC is at the core complex level. @@ -684,7 +684,7 @@ void cacheinfo_amd_init_llc_id(struct cpuinfo_x86 *c, int cpu, u8 node_id) } } -void cacheinfo_hygon_init_llc_id(struct cpuinfo_x86 *c, int cpu, u8 node_id) +void cacheinfo_hygon_init_llc_id(struct cpuinfo_x86 *c, int cpu) { /* * We may have multiple LLCs if L3 caches exist, so check if we diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c index ac6c30e5801d..dc0840aae26c 100644 --- a/arch/x86/kernel/cpu/hygon.c +++ b/arch/x86/kernel/cpu/hygon.c @@ -65,7 +65,6 @@ static void hygon_get_topology_early(struct cpuinfo_x86 *c) */ static void hygon_get_topology(struct cpuinfo_x86 *c) { - u8 node_id; int cpu = smp_processor_id(); /* get information required for multi-node processors */ @@ -75,7 +74,7 @@ static void hygon_get_topology(struct cpuinfo_x86 *c) cpuid(0x8000001e, &eax, &ebx, &ecx, &edx); - node_id = ecx & 0xff; + c->cpu_die_id = ecx & 0xff; c->cpu_core_id = ebx & 0xff; @@ -93,14 +92,14 @@ static void hygon_get_topology(struct cpuinfo_x86 *c) /* Socket ID is ApicId[6] for these processors. */ c->phys_proc_id = c->apicid >> APICID_SOCKET_ID_BIT; - cacheinfo_hygon_init_llc_id(c, cpu, node_id); + cacheinfo_hygon_init_llc_id(c, cpu); } else if (cpu_has(c, X86_FEATURE_NODEID_MSR)) { u64 value; rdmsrl(MSR_FAM10H_NODE_ID, value); - node_id = value & 7; + c->cpu_die_id = value & 7; - per_cpu(cpu_llc_id, cpu) = node_id; + per_cpu(cpu_llc_id, cpu) = c->cpu_die_id; } else return; @@ -123,7 +122,7 @@ static void hygon_detect_cmp(struct cpuinfo_x86 *c) /* Convert the initial APIC ID into the socket ID */ c->phys_proc_id = c->initial_apicid >> bits; /* use socket ID also for last level cache */ - per_cpu(cpu_llc_id, cpu) = c->phys_proc_id; + per_cpu(cpu_llc_id, cpu) = c->cpu_die_id = c->phys_proc_id; } static void srat_detect_node(struct cpuinfo_x86 *c) From db970bd231c2264a062e0de4dcf4ead5e6669e7a Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Mon, 9 Nov 2020 21:06:57 +0000 Subject: [PATCH 206/360] x86/CPU/AMD: Remove amd_get_nb_id() The Last Level Cache ID is returned by amd_get_nb_id(). In practice, this value is the same as the AMD NodeId for callers of this function. The NodeId is saved in struct cpuinfo_x86.cpu_die_id. Replace calls to amd_get_nb_id() with the logical CPU's cpu_die_id and remove the function. Signed-off-by: Yazen Ghannam Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201109210659.754018-3-Yazen.Ghannam@amd.com --- arch/x86/events/amd/core.c | 2 +- arch/x86/include/asm/processor.h | 2 -- arch/x86/kernel/amd_nb.c | 4 ++-- arch/x86/kernel/cpu/amd.c | 6 ------ arch/x86/kernel/cpu/cacheinfo.c | 2 +- arch/x86/kernel/cpu/mce/amd.c | 4 ++-- arch/x86/kernel/cpu/mce/inject.c | 4 ++-- drivers/edac/amd64_edac.c | 4 ++-- drivers/edac/mce_amd.c | 2 +- 9 files changed, 11 insertions(+), 19 deletions(-) diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c index 39eb276d0277..2c1791c4a518 100644 --- a/arch/x86/events/amd/core.c +++ b/arch/x86/events/amd/core.c @@ -538,7 +538,7 @@ static void amd_pmu_cpu_starting(int cpu) if (!x86_pmu.amd_nb_constraints) return; - nb_id = amd_get_nb_id(cpu); + nb_id = topology_die_id(cpu); WARN_ON_ONCE(nb_id == BAD_APICID); for_each_online_cpu(i) { diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 82a08b585818..c20a52b5534b 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -813,10 +813,8 @@ extern int set_tsc_mode(unsigned int val); DECLARE_PER_CPU(u64, msr_misc_features_shadow); #ifdef CONFIG_CPU_SUP_AMD -extern u16 amd_get_nb_id(int cpu); extern u32 amd_get_nodes_per_socket(void); #else -static inline u16 amd_get_nb_id(int cpu) { return 0; } static inline u32 amd_get_nodes_per_socket(void) { return 0; } #endif diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index 18f6b7c4bd79..b4396952c9a6 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c @@ -384,7 +384,7 @@ struct resource *amd_get_mmconfig_range(struct resource *res) int amd_get_subcaches(int cpu) { - struct pci_dev *link = node_to_amd_nb(amd_get_nb_id(cpu))->link; + struct pci_dev *link = node_to_amd_nb(topology_die_id(cpu))->link; unsigned int mask; if (!amd_nb_has_feature(AMD_NB_L3_PARTITIONING)) @@ -398,7 +398,7 @@ int amd_get_subcaches(int cpu) int amd_set_subcaches(int cpu, unsigned long mask) { static unsigned int reset, ban; - struct amd_northbridge *nb = node_to_amd_nb(amd_get_nb_id(cpu)); + struct amd_northbridge *nb = node_to_amd_nb(topology_die_id(cpu)); unsigned int reg; int cuid; diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 2f1fbd8150af..1f71c7616917 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -424,12 +424,6 @@ clear_ppin: clear_cpu_cap(c, X86_FEATURE_AMD_PPIN); } -u16 amd_get_nb_id(int cpu) -{ - return per_cpu(cpu_llc_id, cpu); -} -EXPORT_SYMBOL_GPL(amd_get_nb_id); - u32 amd_get_nodes_per_socket(void) { return nodes_per_socket; diff --git a/arch/x86/kernel/cpu/cacheinfo.c b/arch/x86/kernel/cpu/cacheinfo.c index f9ac682e75e7..3ca9be482a9e 100644 --- a/arch/x86/kernel/cpu/cacheinfo.c +++ b/arch/x86/kernel/cpu/cacheinfo.c @@ -580,7 +580,7 @@ static void amd_init_l3_cache(struct _cpuid4_info_regs *this_leaf, int index) if (index < 3) return; - node = amd_get_nb_id(smp_processor_id()); + node = topology_die_id(smp_processor_id()); this_leaf->nb = node_to_amd_nb(node); if (this_leaf->nb && !this_leaf->nb->l3_cache.indices) amd_calc_l3_indices(this_leaf->nb); diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index 0c6b02dd744c..e486f96b3cb3 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -1341,7 +1341,7 @@ static int threshold_create_bank(struct threshold_bank **bp, unsigned int cpu, return -ENODEV; if (is_shared_bank(bank)) { - nb = node_to_amd_nb(amd_get_nb_id(cpu)); + nb = node_to_amd_nb(topology_die_id(cpu)); /* threshold descriptor already initialized on this node? */ if (nb && nb->bank4) { @@ -1445,7 +1445,7 @@ static void threshold_remove_bank(struct threshold_bank *bank) * The last CPU on this node using the shared bank is going * away, remove that bank now. */ - nb = node_to_amd_nb(amd_get_nb_id(smp_processor_id())); + nb = node_to_amd_nb(topology_die_id(smp_processor_id())); nb->bank4 = NULL; } diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c index 3a44346f2276..7b360731fc2d 100644 --- a/arch/x86/kernel/cpu/mce/inject.c +++ b/arch/x86/kernel/cpu/mce/inject.c @@ -522,8 +522,8 @@ static void do_inject(void) if (boot_cpu_has(X86_FEATURE_AMD_DCM) && b == 4 && boot_cpu_data.x86 < 0x17) { - toggle_nb_mca_mst_cpu(amd_get_nb_id(cpu)); - cpu = get_nbc_for_node(amd_get_nb_id(cpu)); + toggle_nb_mca_mst_cpu(topology_die_id(cpu)); + cpu = get_nbc_for_node(topology_die_id(cpu)); } get_online_cpus(); diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 1362274d840b..39fb90049809 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1133,7 +1133,7 @@ static int k8_early_channel_count(struct amd64_pvt *pvt) /* On F10h and later ErrAddr is MC4_ADDR[47:1] */ static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m) { - u16 mce_nid = amd_get_nb_id(m->extcpu); + u16 mce_nid = topology_die_id(m->extcpu); struct mem_ctl_info *mci; u8 start_bit = 1; u8 end_bit = 47; @@ -3046,7 +3046,7 @@ static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid) int cpu; for_each_online_cpu(cpu) - if (amd_get_nb_id(cpu) == nid) + if (topology_die_id(cpu) == nid) cpumask_set_cpu(cpu, mask); } diff --git a/drivers/edac/mce_amd.c b/drivers/edac/mce_amd.c index 7f28edb070bd..85095e3902ec 100644 --- a/drivers/edac/mce_amd.c +++ b/drivers/edac/mce_amd.c @@ -869,7 +869,7 @@ static void decode_mc3_mce(struct mce *m) static void decode_mc4_mce(struct mce *m) { unsigned int fam = x86_family(m->cpuid); - int node_id = amd_get_nb_id(m->extcpu); + int node_id = topology_die_id(m->extcpu); u16 ec = EC(m->status); u8 xec = XEC(m->status, 0x1f); u8 offset = 0; From 8de0c9917cc1297bc5543b61992d5bdee4ce621a Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Mon, 9 Nov 2020 21:06:58 +0000 Subject: [PATCH 207/360] EDAC/mce_amd: Use struct cpuinfo_x86.cpu_die_id for AMD NodeId The edac_mce_amd module calls decode_dram_ecc() on AMD Family17h and later systems. This function is used in amd64_edac_mod to do system-specific decoding for DRAM ECC errors. The function takes a "NodeId" as a parameter. In AMD documentation, NodeId is used to identify a physical die in a system. This can be used to identify a node in the AMD_NB code and also it is used with umc_normaddr_to_sysaddr(). However, the input used for decode_dram_ecc() is currently the NUMA node of a logical CPU. In the default configuration, the NUMA node and physical die will be equivalent, so this doesn't have an impact. But the NUMA node configuration can be adjusted with optional memory interleaving modes. This will cause the NUMA node enumeration to not match the physical die enumeration. The mismatch will cause the address translation function to fail or report incorrect results. Use struct cpuinfo_x86.cpu_die_id for the node_id parameter to ensure the physical ID is used. Fixes: fbe63acf62f5 ("EDAC, mce_amd: Use cpu_to_node() to find the node ID") Signed-off-by: Yazen Ghannam Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201109210659.754018-4-Yazen.Ghannam@amd.com --- drivers/edac/mce_amd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/edac/mce_amd.c b/drivers/edac/mce_amd.c index 85095e3902ec..5dd905a3f30c 100644 --- a/drivers/edac/mce_amd.c +++ b/drivers/edac/mce_amd.c @@ -1003,7 +1003,7 @@ static void decode_smca_error(struct mce *m) pr_cont(", %s.\n", smca_mce_descs[bank_type].descs[xec]); if (bank_type == SMCA_UMC && xec == 0 && decode_dram_ecc) - decode_dram_ecc(cpu_to_node(m->extcpu), m); + decode_dram_ecc(topology_die_id(m->extcpu), m); } static inline void amd_decode_err_code(u16 ec) From cb09a379724d299c603a7a79f444f52a9a75b8d2 Mon Sep 17 00:00:00 2001 From: Yazen Ghannam Date: Mon, 9 Nov 2020 21:06:59 +0000 Subject: [PATCH 208/360] x86/topology: Set cpu_die_id only if DIE_TYPE found CPUID Leaf 0x1F defines a DIE_TYPE level (nb: ECX[8:15] level type == 0x5), but CPUID Leaf 0xB does not. However, detect_extended_topology() will set struct cpuinfo_x86.cpu_die_id regardless of whether a valid Die ID was found. Only set cpu_die_id if a DIE_TYPE level is found. CPU topology code may use another value for cpu_die_id, e.g. the AMD NodeId on AMD-based systems. Code ordering should be maintained so that the CPUID Leaf 0x1F Die ID value will take precedence on systems that may use another value. Suggested-by: Borislav Petkov Signed-off-by: Yazen Ghannam Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201109210659.754018-5-Yazen.Ghannam@amd.com --- arch/x86/kernel/cpu/topology.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/topology.c b/arch/x86/kernel/cpu/topology.c index d3a0791bc052..1068002c8532 100644 --- a/arch/x86/kernel/cpu/topology.c +++ b/arch/x86/kernel/cpu/topology.c @@ -96,6 +96,7 @@ int detect_extended_topology(struct cpuinfo_x86 *c) unsigned int ht_mask_width, core_plus_mask_width, die_plus_mask_width; unsigned int core_select_mask, core_level_siblings; unsigned int die_select_mask, die_level_siblings; + bool die_level_present = false; int leaf; leaf = detect_extended_topology_leaf(c); @@ -126,6 +127,7 @@ int detect_extended_topology(struct cpuinfo_x86 *c) die_plus_mask_width = BITS_SHIFT_NEXT_LEVEL(eax); } if (LEAFB_SUBTYPE(ecx) == DIE_TYPE) { + die_level_present = true; die_level_siblings = LEVEL_MAX_SIBLINGS(ebx); die_plus_mask_width = BITS_SHIFT_NEXT_LEVEL(eax); } @@ -139,8 +141,12 @@ int detect_extended_topology(struct cpuinfo_x86 *c) c->cpu_core_id = apic->phys_pkg_id(c->initial_apicid, ht_mask_width) & core_select_mask; - c->cpu_die_id = apic->phys_pkg_id(c->initial_apicid, - core_plus_mask_width) & die_select_mask; + + if (die_level_present) { + c->cpu_die_id = apic->phys_pkg_id(c->initial_apicid, + core_plus_mask_width) & die_select_mask; + } + c->phys_proc_id = apic->phys_pkg_id(c->initial_apicid, die_plus_mask_width); /* From 2002d2951398317d0f46e64ae6d8dd58ed541c6d Mon Sep 17 00:00:00 2001 From: Rikard Falkeborn Date: Wed, 11 Nov 2020 00:02:28 +0100 Subject: [PATCH 209/360] x86/resctrl: Constify kernfs_ops The only usage of the kf_ops field in the rftype struct is to pass it as argument to __kernfs_create_file(), which accepts a pointer to const. Make it a pointer to const. This makes it possible to make rdtgroup_kf_single_ops and kf_mondata_ops const, which allows the compiler to put them in read-only memory. Signed-off-by: Rikard Falkeborn Signed-off-by: Borislav Petkov Acked-by: Reinette Chatre Link: https://lkml.kernel.org/r/20201110230228.801785-1-rikard.falkeborn@gmail.com --- arch/x86/kernel/cpu/resctrl/internal.h | 2 +- arch/x86/kernel/cpu/resctrl/rdtgroup.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h index 6cb068fcf501..5540b025880c 100644 --- a/arch/x86/kernel/cpu/resctrl/internal.h +++ b/arch/x86/kernel/cpu/resctrl/internal.h @@ -264,7 +264,7 @@ void __exit rdtgroup_exit(void); struct rftype { char *name; umode_t mode; - struct kernfs_ops *kf_ops; + const struct kernfs_ops *kf_ops; unsigned long flags; unsigned long fflags; diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index af323e2e3100..78dcbb8e0172 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -240,13 +240,13 @@ static ssize_t rdtgroup_file_write(struct kernfs_open_file *of, char *buf, return -EINVAL; } -static struct kernfs_ops rdtgroup_kf_single_ops = { +static const struct kernfs_ops rdtgroup_kf_single_ops = { .atomic_write_len = PAGE_SIZE, .write = rdtgroup_file_write, .seq_show = rdtgroup_seqfile_show, }; -static struct kernfs_ops kf_mondata_ops = { +static const struct kernfs_ops kf_mondata_ops = { .atomic_write_len = PAGE_SIZE, .seq_show = rdtgroup_mondata_show, }; From 10590a9d4f23e0a519730d79d39331df60ad2079 Mon Sep 17 00:00:00 2001 From: Qiuxu Zhuo Date: Thu, 5 Nov 2020 15:49:14 +0800 Subject: [PATCH 210/360] EDAC/igen6: Add EDAC driver for Intel client SoCs using IBECC This driver supports Intel client SoC with integrated memory controller using In-Band ECC(IBECC). The memory correctable and uncorrectable errors are reported via NMIs. The driver handles the NMIs and decodes the memory error address to platform specific address. The first IBECC-supported SoC is Elkhart Lake. [Tony: s/#include /#include / to fix randconfig build] Signed-off-by: Qiuxu Zhuo Signed-off-by: Tony Luck --- drivers/edac/Kconfig | 9 + drivers/edac/Makefile | 1 + drivers/edac/igen6_edac.c | 918 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 928 insertions(+) create mode 100644 drivers/edac/igen6_edac.c diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig index 7a47680d6f07..fa0c3b5797e4 100644 --- a/drivers/edac/Kconfig +++ b/drivers/edac/Kconfig @@ -269,6 +269,15 @@ config EDAC_PND2 first used on the Apollo Lake platform and Denverton micro-server but may appear on others in the future. +config EDAC_IGEN6 + tristate "Intel client SoC Integrated MC" + depends on PCI && X86_64 && PCI_MMCONFIG && ARCH_HAVE_NMI_SAFE_CMPXCHG + help + Support for error detection and correction on the Intel + client SoC Integrated Memory Controller using In-Band ECC IP. + This In-Band ECC is first used on the Elkhart Lake SoC but + may appear on others in the future. + config EDAC_MPC85XX bool "Freescale MPC83xx / MPC85xx" depends on FSL_SOC && EDAC=y diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile index 3a849168780d..3cd1aeb0a916 100644 --- a/drivers/edac/Makefile +++ b/drivers/edac/Makefile @@ -32,6 +32,7 @@ obj-$(CONFIG_EDAC_I7300) += i7300_edac.o obj-$(CONFIG_EDAC_I7CORE) += i7core_edac.o obj-$(CONFIG_EDAC_SBRIDGE) += sb_edac.o obj-$(CONFIG_EDAC_PND2) += pnd2_edac.o +obj-$(CONFIG_EDAC_IGEN6) += igen6_edac.o obj-$(CONFIG_EDAC_E7XXX) += e7xxx_edac.o obj-$(CONFIG_EDAC_E752X) += e752x_edac.o obj-$(CONFIG_EDAC_I82443BXGX) += i82443bxgx_edac.o diff --git a/drivers/edac/igen6_edac.c b/drivers/edac/igen6_edac.c new file mode 100644 index 000000000000..318b9b67080f --- /dev/null +++ b/drivers/edac/igen6_edac.c @@ -0,0 +1,918 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for Intel client SoC with integrated memory controller using IBECC + * + * Copyright (C) 2020 Intel Corporation + * + * The In-Band ECC (IBECC) IP provides ECC protection to all or specific + * regions of the physical memory space. It's used for memory controllers + * that don't support the out-of-band ECC which often needs an additional + * storage device to each channel for storing ECC data. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "edac_mc.h" +#include "edac_module.h" + +#define IGEN6_REVISION "v2.4" + +#define EDAC_MOD_STR "igen6_edac" +#define IGEN6_NMI_NAME "igen6_ibecc" + +/* Debug macros */ +#define igen6_printk(level, fmt, arg...) \ + edac_printk(level, "igen6", fmt, ##arg) + +#define igen6_mc_printk(mci, level, fmt, arg...) \ + edac_mc_chipset_printk(mci, level, "igen6", fmt, ##arg) + +#define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo)) + +#define NUM_IMC 1 /* Max memory controllers */ +#define NUM_CHANNELS 2 /* Max channels */ +#define NUM_DIMMS 2 /* Max DIMMs per channel */ + +#define _4GB BIT_ULL(32) + +/* Size of physical memory */ +#define TOM_OFFSET 0xa0 +/* Top of low usable DRAM */ +#define TOLUD_OFFSET 0xbc +/* Capability register C */ +#define CAPID_C_OFFSET 0xec +#define CAPID_C_IBECC BIT(15) + +/* Error Status */ +#define ERRSTS_OFFSET 0xc8 +#define ERRSTS_CE BIT_ULL(6) +#define ERRSTS_UE BIT_ULL(7) + +/* Error Command */ +#define ERRCMD_OFFSET 0xca +#define ERRCMD_CE BIT_ULL(6) +#define ERRCMD_UE BIT_ULL(7) + +/* IBECC MMIO base address */ +#define IBECC_BASE (res_cfg->ibecc_base) +#define IBECC_ACTIVATE_OFFSET IBECC_BASE +#define IBECC_ACTIVATE_EN BIT(0) + +/* IBECC error log */ +#define ECC_ERROR_LOG_OFFSET (IBECC_BASE + 0x170) +#define ECC_ERROR_LOG_CE BIT_ULL(62) +#define ECC_ERROR_LOG_UE BIT_ULL(63) +#define ECC_ERROR_LOG_ADDR_SHIFT 5 +#define ECC_ERROR_LOG_ADDR(v) GET_BITFIELD(v, 5, 38) +#define ECC_ERROR_LOG_SYND(v) GET_BITFIELD(v, 46, 61) + +/* Host MMIO base address */ +#define MCHBAR_OFFSET 0x48 +#define MCHBAR_EN BIT_ULL(0) +#define MCHBAR_BASE(v) (GET_BITFIELD(v, 16, 38) << 16) +#define MCHBAR_SIZE 0x10000 + +/* Parameters for the channel decode stage */ +#define MAD_INTER_CHANNEL_OFFSET 0x5000 +#define MAD_INTER_CHANNEL_DDR_TYPE(v) GET_BITFIELD(v, 0, 2) +#define MAD_INTER_CHANNEL_ECHM(v) GET_BITFIELD(v, 3, 3) +#define MAD_INTER_CHANNEL_CH_L_MAP(v) GET_BITFIELD(v, 4, 4) +#define MAD_INTER_CHANNEL_CH_S_SIZE(v) ((u64)GET_BITFIELD(v, 12, 19) << 29) + +/* Parameters for DRAM decode stage */ +#define MAD_INTRA_CH0_OFFSET 0x5004 +#define MAD_INTRA_CH_DIMM_L_MAP(v) GET_BITFIELD(v, 0, 0) + +/* DIMM characteristics */ +#define MAD_DIMM_CH0_OFFSET 0x500c +#define MAD_DIMM_CH_DIMM_L_SIZE(v) ((u64)GET_BITFIELD(v, 0, 6) << 29) +#define MAD_DIMM_CH_DLW(v) GET_BITFIELD(v, 7, 8) +#define MAD_DIMM_CH_DIMM_S_SIZE(v) ((u64)GET_BITFIELD(v, 16, 22) << 29) +#define MAD_DIMM_CH_DSW(v) GET_BITFIELD(v, 24, 25) + +/* Hash for channel selection */ +#define CHANNEL_HASH_OFFSET 0X5024 +/* Hash for enhanced channel selection */ +#define CHANNEL_EHASH_OFFSET 0X5028 +#define CHANNEL_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6) +#define CHANNEL_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26) +#define CHANNEL_HASH_MODE(v) GET_BITFIELD(v, 28, 28) + +static struct res_config { + int num_imc; + u32 ibecc_base; + bool (*ibecc_available)(struct pci_dev *pdev); + /* Convert error address logged in IBECC to system physical address */ + u64 (*err_addr_to_sys_addr)(u64 eaddr); + /* Convert error address logged in IBECC to integrated memory controller address */ + u64 (*err_addr_to_imc_addr)(u64 eaddr); +} *res_cfg; + +struct igen6_imc { + int mc; + struct mem_ctl_info *mci; + struct pci_dev *pdev; + struct device dev; + void __iomem *window; + u64 ch_s_size; + int ch_l_map; + u64 dimm_s_size[NUM_CHANNELS]; + u64 dimm_l_size[NUM_CHANNELS]; + int dimm_l_map[NUM_CHANNELS]; +}; + +static struct igen6_pvt { + struct igen6_imc imc[NUM_IMC]; +} *igen6_pvt; + +/* The top of low usable DRAM */ +static u32 igen6_tolud; +/* The size of physical memory */ +static u64 igen6_tom; + +struct decoded_addr { + int mc; + u64 imc_addr; + u64 sys_addr; + int channel_idx; + u64 channel_addr; + int sub_channel_idx; + u64 sub_channel_addr; +}; + +struct ecclog_node { + struct llist_node llnode; + int mc; + u64 ecclog; +}; + +/* + * In the NMI handler, the driver uses the lock-less memory allocator + * to allocate memory to store the IBECC error logs and links the logs + * to the lock-less list. Delay printk() and the work of error reporting + * to EDAC core in a worker. + */ +#define ECCLOG_POOL_SIZE PAGE_SIZE +LLIST_HEAD(ecclog_llist); +static struct gen_pool *ecclog_pool; +static char ecclog_buf[ECCLOG_POOL_SIZE]; +static struct irq_work ecclog_irq_work; +static struct work_struct ecclog_work; + +/* Compute die IDs for Elkhart Lake with IBECC */ +#define DID_EHL_SKU5 0x4514 +#define DID_EHL_SKU6 0x4528 +#define DID_EHL_SKU7 0x452a +#define DID_EHL_SKU8 0x4516 +#define DID_EHL_SKU9 0x452c +#define DID_EHL_SKU10 0x452e +#define DID_EHL_SKU11 0x4532 +#define DID_EHL_SKU12 0x4518 +#define DID_EHL_SKU13 0x451a +#define DID_EHL_SKU14 0x4534 +#define DID_EHL_SKU15 0x4536 + +static bool ehl_ibecc_available(struct pci_dev *pdev) +{ + u32 v; + + if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v)) + return false; + + return !!(CAPID_C_IBECC & v); +} + +static u64 ehl_err_addr_to_sys_addr(u64 eaddr) +{ + return eaddr; +} + +static u64 ehl_err_addr_to_imc_addr(u64 eaddr) +{ + if (eaddr < igen6_tolud) + return eaddr; + + if (igen6_tom <= _4GB) + return eaddr + igen6_tolud - _4GB; + + if (eaddr < _4GB) + return eaddr + igen6_tolud - igen6_tom; + + return eaddr; +} + +static struct res_config ehl_cfg = { + .num_imc = 1, + .ibecc_base = 0xdc00, + .ibecc_available = ehl_ibecc_available, + .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr, + .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr, +}; + +static const struct pci_device_id igen6_pci_tbl[] = { + { PCI_VDEVICE(INTEL, DID_EHL_SKU5), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU6), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU7), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU8), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU9), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU10), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU11), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU12), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU13), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU14), (kernel_ulong_t)&ehl_cfg }, + { PCI_VDEVICE(INTEL, DID_EHL_SKU15), (kernel_ulong_t)&ehl_cfg }, + { }, +}; +MODULE_DEVICE_TABLE(pci, igen6_pci_tbl); + +static enum dev_type get_width(int dimm_l, u32 mad_dimm) +{ + u32 w = dimm_l ? MAD_DIMM_CH_DLW(mad_dimm) : + MAD_DIMM_CH_DSW(mad_dimm); + + switch (w) { + case 0: + return DEV_X8; + case 1: + return DEV_X16; + case 2: + return DEV_X32; + default: + return DEV_UNKNOWN; + } +} + +static enum mem_type get_memory_type(u32 mad_inter) +{ + u32 t = MAD_INTER_CHANNEL_DDR_TYPE(mad_inter); + + switch (t) { + case 0: + return MEM_DDR4; + case 1: + return MEM_DDR3; + case 2: + return MEM_LPDDR3; + case 3: + return MEM_LPDDR4; + case 4: + return MEM_WIO2; + default: + return MEM_UNKNOWN; + } +} + +static int decode_chan_idx(u64 addr, u64 mask, int intlv_bit) +{ + u64 hash_addr = addr & mask, hash = 0; + u64 intlv = (addr >> intlv_bit) & 1; + int i; + + for (i = 6; i < 20; i++) + hash ^= (hash_addr >> i) & 1; + + return (int)hash ^ intlv; +} + +static u64 decode_channel_addr(u64 addr, int intlv_bit) +{ + u64 channel_addr; + + /* Remove the interleave bit and shift upper part down to fill gap */ + channel_addr = GET_BITFIELD(addr, intlv_bit + 1, 63) << intlv_bit; + channel_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1); + + return channel_addr; +} + +static void decode_addr(u64 addr, u32 hash, u64 s_size, int l_map, + int *idx, u64 *sub_addr) +{ + int intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6; + + if (addr > 2 * s_size) { + *sub_addr = addr - s_size; + *idx = l_map; + return; + } + + if (CHANNEL_HASH_MODE(hash)) { + *sub_addr = decode_channel_addr(addr, intlv_bit); + *idx = decode_chan_idx(addr, CHANNEL_HASH_MASK(hash), intlv_bit); + } else { + *sub_addr = decode_channel_addr(addr, 6); + *idx = GET_BITFIELD(addr, 6, 6); + } +} + +static int igen6_decode(struct decoded_addr *res) +{ + struct igen6_imc *imc = &igen6_pvt->imc[res->mc]; + u64 addr = res->imc_addr, sub_addr, s_size; + int idx, l_map; + u32 hash; + + if (addr >= igen6_tom) { + edac_dbg(0, "Address 0x%llx out of range\n", addr); + return -EINVAL; + } + + /* Decode channel */ + hash = readl(imc->window + CHANNEL_HASH_OFFSET); + s_size = imc->ch_s_size; + l_map = imc->ch_l_map; + decode_addr(addr, hash, s_size, l_map, &idx, &sub_addr); + res->channel_idx = idx; + res->channel_addr = sub_addr; + + /* Decode sub-channel/DIMM */ + hash = readl(imc->window + CHANNEL_EHASH_OFFSET); + s_size = imc->dimm_s_size[idx]; + l_map = imc->dimm_l_map[idx]; + decode_addr(res->channel_addr, hash, s_size, l_map, &idx, &sub_addr); + res->sub_channel_idx = idx; + res->sub_channel_addr = sub_addr; + + return 0; +} + +static void igen6_output_error(struct decoded_addr *res, + struct mem_ctl_info *mci, u64 ecclog) +{ + enum hw_event_mc_err_type type = ecclog & ECC_ERROR_LOG_UE ? + HW_EVENT_ERR_UNCORRECTED : + HW_EVENT_ERR_CORRECTED; + + edac_mc_handle_error(type, mci, 1, + res->sys_addr >> PAGE_SHIFT, + res->sys_addr & ~PAGE_MASK, + ECC_ERROR_LOG_SYND(ecclog), + res->channel_idx, res->sub_channel_idx, + -1, "", ""); +} + +static struct gen_pool *ecclog_gen_pool_create(void) +{ + struct gen_pool *pool; + + pool = gen_pool_create(ilog2(sizeof(struct ecclog_node)), -1); + if (!pool) + return NULL; + + if (gen_pool_add(pool, (unsigned long)ecclog_buf, ECCLOG_POOL_SIZE, -1)) { + gen_pool_destroy(pool); + return NULL; + } + + return pool; +} + +static int ecclog_gen_pool_add(int mc, u64 ecclog) +{ + struct ecclog_node *node; + + node = (void *)gen_pool_alloc(ecclog_pool, sizeof(*node)); + if (!node) + return -ENOMEM; + + node->mc = mc; + node->ecclog = ecclog; + llist_add(&node->llnode, &ecclog_llist); + + return 0; +} + +/* + * Either the memory-mapped I/O status register ECC_ERROR_LOG or the PCI + * configuration space status register ERRSTS can indicate whether a + * correctable error or an uncorrectable error occurred. We only use the + * ECC_ERROR_LOG register to check error type, but need to clear both + * registers to enable future error events. + */ +static u64 ecclog_read_and_clear(struct igen6_imc *imc) +{ + u64 ecclog = readq(imc->window + ECC_ERROR_LOG_OFFSET); + + if (ecclog & (ECC_ERROR_LOG_CE | ECC_ERROR_LOG_UE)) { + /* Clear CE/UE bits by writing 1s */ + writeq(ecclog, imc->window + ECC_ERROR_LOG_OFFSET); + return ecclog; + } + + return 0; +} + +static void errsts_clear(struct igen6_imc *imc) +{ + u16 errsts; + + if (pci_read_config_word(imc->pdev, ERRSTS_OFFSET, &errsts)) { + igen6_printk(KERN_ERR, "Failed to read ERRSTS\n"); + return; + } + + /* Clear CE/UE bits by writing 1s */ + if (errsts & (ERRSTS_CE | ERRSTS_UE)) + pci_write_config_word(imc->pdev, ERRSTS_OFFSET, errsts); +} + +static int errcmd_enable_error_reporting(bool enable) +{ + struct igen6_imc *imc = &igen6_pvt->imc[0]; + u16 errcmd; + int rc; + + rc = pci_read_config_word(imc->pdev, ERRCMD_OFFSET, &errcmd); + if (rc) + return rc; + + if (enable) + errcmd |= ERRCMD_CE | ERRSTS_UE; + else + errcmd &= ~(ERRCMD_CE | ERRSTS_UE); + + rc = pci_write_config_word(imc->pdev, ERRCMD_OFFSET, errcmd); + if (rc) + return rc; + + return 0; +} + +static int ecclog_handler(void) +{ + struct igen6_imc *imc; + int i, n = 0; + u64 ecclog; + + for (i = 0; i < res_cfg->num_imc; i++) { + imc = &igen6_pvt->imc[i]; + + /* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */ + + ecclog = ecclog_read_and_clear(imc); + if (!ecclog) + continue; + + if (!ecclog_gen_pool_add(i, ecclog)) + irq_work_queue(&ecclog_irq_work); + + n++; + } + + return n; +} + +static void ecclog_work_cb(struct work_struct *work) +{ + struct ecclog_node *node, *tmp; + struct mem_ctl_info *mci; + struct llist_node *head; + struct decoded_addr res; + u64 eaddr; + + head = llist_del_all(&ecclog_llist); + if (!head) + return; + + llist_for_each_entry_safe(node, tmp, head, llnode) { + memset(&res, 0, sizeof(res)); + eaddr = ECC_ERROR_LOG_ADDR(node->ecclog) << + ECC_ERROR_LOG_ADDR_SHIFT; + res.mc = node->mc; + res.sys_addr = res_cfg->err_addr_to_sys_addr(eaddr); + res.imc_addr = res_cfg->err_addr_to_imc_addr(eaddr); + + mci = igen6_pvt->imc[res.mc].mci; + + edac_dbg(2, "MC %d, ecclog = 0x%llx\n", node->mc, node->ecclog); + igen6_mc_printk(mci, KERN_DEBUG, "HANDLING IBECC MEMORY ERROR\n"); + igen6_mc_printk(mci, KERN_DEBUG, "ADDR 0x%llx ", res.sys_addr); + + if (!igen6_decode(&res)) + igen6_output_error(&res, mci, node->ecclog); + + gen_pool_free(ecclog_pool, (unsigned long)node, sizeof(*node)); + } +} + +static void ecclog_irq_work_cb(struct irq_work *irq_work) +{ + int i; + + for (i = 0; i < res_cfg->num_imc; i++) + errsts_clear(&igen6_pvt->imc[i]); + + if (!llist_empty(&ecclog_llist)) + schedule_work(&ecclog_work); +} + +static int ecclog_nmi_handler(unsigned int cmd, struct pt_regs *regs) +{ + unsigned char reason; + + if (!ecclog_handler()) + return NMI_DONE; + + /* + * Both In-Band ECC correctable error and uncorrectable error are + * reported by SERR# NMI. The NMI generic code (see pci_serr_error()) + * doesn't clear the bit NMI_REASON_CLEAR_SERR (in port 0x61) to + * re-enable the SERR# NMI after NMI handling. So clear this bit here + * to re-enable SERR# NMI for receiving future In-Band ECC errors. + */ + reason = x86_platform.get_nmi_reason() & NMI_REASON_CLEAR_MASK; + reason |= NMI_REASON_CLEAR_SERR; + outb(reason, NMI_REASON_PORT); + reason &= ~NMI_REASON_CLEAR_SERR; + outb(reason, NMI_REASON_PORT); + + return NMI_HANDLED; +} + +static bool igen6_check_ecc(struct igen6_imc *imc) +{ + u32 activate = readl(imc->window + IBECC_ACTIVATE_OFFSET); + + return !!(activate & IBECC_ACTIVATE_EN); +} + +static int igen6_get_dimm_config(struct mem_ctl_info *mci) +{ + struct igen6_imc *imc = mci->pvt_info; + u32 mad_inter, mad_intra, mad_dimm; + int i, j, ndimms, mc = imc->mc; + struct dimm_info *dimm; + enum mem_type mtype; + enum dev_type dtype; + u64 dsize; + bool ecc; + + edac_dbg(2, "\n"); + + mad_inter = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); + mtype = get_memory_type(mad_inter); + ecc = igen6_check_ecc(imc); + imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(mad_inter); + imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(mad_inter); + + for (i = 0; i < NUM_CHANNELS; i++) { + mad_intra = readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4); + mad_dimm = readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4); + + imc->dimm_l_size[i] = MAD_DIMM_CH_DIMM_L_SIZE(mad_dimm); + imc->dimm_s_size[i] = MAD_DIMM_CH_DIMM_S_SIZE(mad_dimm); + imc->dimm_l_map[i] = MAD_INTRA_CH_DIMM_L_MAP(mad_intra); + ndimms = 0; + + for (j = 0; j < NUM_DIMMS; j++) { + dimm = edac_get_dimm(mci, i, j, 0); + + if (j ^ imc->dimm_l_map[i]) { + dtype = get_width(0, mad_dimm); + dsize = imc->dimm_s_size[i]; + } else { + dtype = get_width(1, mad_dimm); + dsize = imc->dimm_l_size[i]; + } + + if (!dsize) + continue; + + dimm->grain = 64; + dimm->mtype = mtype; + dimm->dtype = dtype; + dimm->nr_pages = MiB_TO_PAGES(dsize >> 20); + dimm->edac_mode = EDAC_SECDED; + snprintf(dimm->label, sizeof(dimm->label), + "MC#%d_Chan#%d_DIMM#%d", mc, i, j); + edac_dbg(0, "MC %d, Channel %d, DIMM %d, Size %llu MiB (%u pages)\n", + mc, i, j, dsize >> 20, dimm->nr_pages); + + ndimms++; + } + + if (ndimms && !ecc) { + igen6_printk(KERN_ERR, "MC%d In-Band ECC is disabled\n", mc); + return -ENODEV; + } + } + + return 0; +} + +#ifdef CONFIG_EDAC_DEBUG +static void igen6_reg_dump(struct igen6_imc *imc) +{ + int i; + + edac_dbg(2, "CHANNEL_HASH : 0x%x\n", + readl(imc->window + CHANNEL_HASH_OFFSET)); + edac_dbg(2, "CHANNEL_EHASH : 0x%x\n", + readl(imc->window + CHANNEL_EHASH_OFFSET)); + edac_dbg(2, "MAD_INTER_CHANNEL: 0x%x\n", + readl(imc->window + MAD_INTER_CHANNEL_OFFSET)); + edac_dbg(2, "ECC_ERROR_LOG : 0x%llx\n", + readq(imc->window + ECC_ERROR_LOG_OFFSET)); + + for (i = 0; i < NUM_CHANNELS; i++) { + edac_dbg(2, "MAD_INTRA_CH%d : 0x%x\n", i, + readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4)); + edac_dbg(2, "MAD_DIMM_CH%d : 0x%x\n", i, + readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4)); + } + edac_dbg(2, "TOLUD : 0x%x", igen6_tolud); + edac_dbg(2, "TOM : 0x%llx", igen6_tom); +} +#else +static void igen6_reg_dump(struct igen6_imc *imc) {} +#endif + +static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar) +{ + union { + u64 v; + struct { + u32 v_lo; + u32 v_hi; + }; + } u; + + edac_dbg(2, "\n"); + + if (!res_cfg->ibecc_available(pdev)) { + edac_dbg(2, "No In-Band ECC IP\n"); + goto fail; + } + + if (pci_read_config_dword(pdev, TOLUD_OFFSET, &igen6_tolud)) { + igen6_printk(KERN_ERR, "Failed to read TOLUD\n"); + goto fail; + } + + igen6_tolud &= GENMASK(31, 20); + + if (pci_read_config_dword(pdev, TOM_OFFSET, &u.v_lo)) { + igen6_printk(KERN_ERR, "Failed to read lower TOM\n"); + goto fail; + } + + if (pci_read_config_dword(pdev, TOM_OFFSET + 4, &u.v_hi)) { + igen6_printk(KERN_ERR, "Failed to read upper TOM\n"); + goto fail; + } + + igen6_tom = u.v & GENMASK_ULL(38, 20); + + if (pci_read_config_dword(pdev, MCHBAR_OFFSET, &u.v_lo)) { + igen6_printk(KERN_ERR, "Failed to read lower MCHBAR\n"); + goto fail; + } + + if (pci_read_config_dword(pdev, MCHBAR_OFFSET + 4, &u.v_hi)) { + igen6_printk(KERN_ERR, "Failed to read upper MCHBAR\n"); + goto fail; + } + + if (!(u.v & MCHBAR_EN)) { + igen6_printk(KERN_ERR, "MCHBAR is disabled\n"); + goto fail; + } + + *mchbar = MCHBAR_BASE(u.v); + + return 0; +fail: + return -ENODEV; +} + +static int igen6_register_mci(int mc, u64 mchbar, struct pci_dev *pdev) +{ + struct edac_mc_layer layers[2]; + struct mem_ctl_info *mci; + struct igen6_imc *imc; + void __iomem *window; + int rc; + + edac_dbg(2, "\n"); + + mchbar += mc * MCHBAR_SIZE; + window = ioremap(mchbar, MCHBAR_SIZE); + if (!window) { + igen6_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", mchbar); + return -ENODEV; + } + + layers[0].type = EDAC_MC_LAYER_CHANNEL; + layers[0].size = NUM_CHANNELS; + layers[0].is_virt_csrow = false; + layers[1].type = EDAC_MC_LAYER_SLOT; + layers[1].size = NUM_DIMMS; + layers[1].is_virt_csrow = true; + + mci = edac_mc_alloc(mc, ARRAY_SIZE(layers), layers, 0); + if (!mci) { + rc = -ENOMEM; + goto fail; + } + + mci->ctl_name = kasprintf(GFP_KERNEL, "Intel_client_SoC MC#%d", mc); + if (!mci->ctl_name) { + rc = -ENOMEM; + goto fail2; + } + + mci->mtype_cap = MEM_FLAG_LPDDR4 | MEM_FLAG_DDR4; + mci->edac_ctl_cap = EDAC_FLAG_SECDED; + mci->edac_cap = EDAC_FLAG_SECDED; + mci->mod_name = EDAC_MOD_STR; + mci->dev_name = pci_name(pdev); + mci->pvt_info = &igen6_pvt->imc[mc]; + + imc = mci->pvt_info; + device_initialize(&imc->dev); + /* + * EDAC core uses mci->pdev(pointer of structure device) as + * memory controller ID. The client SoCs attach one or more + * memory controllers to single pci_dev (single pci_dev->dev + * can be for multiple memory controllers). + * + * To make mci->pdev unique, assign pci_dev->dev to mci->pdev + * for the first memory controller and assign a unique imc->dev + * to mci->pdev for each non-first memory controller. + */ + mci->pdev = mc ? &imc->dev : &pdev->dev; + imc->mc = mc; + imc->pdev = pdev; + imc->window = window; + + igen6_reg_dump(imc); + + rc = igen6_get_dimm_config(mci); + if (rc) + goto fail3; + + rc = edac_mc_add_mc(mci); + if (rc) { + igen6_printk(KERN_ERR, "Failed to register mci#%d\n", mc); + goto fail3; + } + + imc->mci = mci; + return 0; +fail3: + kfree(mci->ctl_name); +fail2: + edac_mc_free(mci); +fail: + iounmap(window); + return rc; +} + +static void igen6_unregister_mcis(void) +{ + struct mem_ctl_info *mci; + struct igen6_imc *imc; + int i; + + edac_dbg(2, "\n"); + + for (i = 0; i < res_cfg->num_imc; i++) { + imc = &igen6_pvt->imc[i]; + mci = imc->mci; + if (!mci) + continue; + + edac_mc_del_mc(mci->pdev); + kfree(mci->ctl_name); + edac_mc_free(mci); + iounmap(imc->window); + } +} + +static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + u64 mchbar; + int i, rc; + + edac_dbg(2, "\n"); + + igen6_pvt = kzalloc(sizeof(*igen6_pvt), GFP_KERNEL); + if (!igen6_pvt) + return -ENOMEM; + + res_cfg = (struct res_config *)ent->driver_data; + + rc = igen6_pci_setup(pdev, &mchbar); + if (rc) + goto fail; + + for (i = 0; i < res_cfg->num_imc; i++) { + rc = igen6_register_mci(i, mchbar, pdev); + if (rc) + goto fail2; + } + + ecclog_pool = ecclog_gen_pool_create(); + if (!ecclog_pool) { + rc = -ENOMEM; + goto fail2; + } + + INIT_WORK(&ecclog_work, ecclog_work_cb); + init_irq_work(&ecclog_irq_work, ecclog_irq_work_cb); + + /* Check if any pending errors before registering the NMI handler */ + ecclog_handler(); + + rc = register_nmi_handler(NMI_SERR, ecclog_nmi_handler, + 0, IGEN6_NMI_NAME); + if (rc) { + igen6_printk(KERN_ERR, "Failed to register NMI handler\n"); + goto fail3; + } + + /* Enable error reporting */ + rc = errcmd_enable_error_reporting(true); + if (rc) { + igen6_printk(KERN_ERR, "Failed to enable error reporting\n"); + goto fail4; + } + + return 0; +fail4: + unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); +fail3: + gen_pool_destroy(ecclog_pool); +fail2: + igen6_unregister_mcis(); +fail: + kfree(igen6_pvt); + return rc; +} + +static void igen6_remove(struct pci_dev *pdev) +{ + edac_dbg(2, "\n"); + + errcmd_enable_error_reporting(false); + unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); + irq_work_sync(&ecclog_irq_work); + flush_work(&ecclog_work); + gen_pool_destroy(ecclog_pool); + igen6_unregister_mcis(); + kfree(igen6_pvt); +} + +static struct pci_driver igen6_driver = { + .name = EDAC_MOD_STR, + .probe = igen6_probe, + .remove = igen6_remove, + .id_table = igen6_pci_tbl, +}; + +static int __init igen6_init(void) +{ + const char *owner; + int rc; + + edac_dbg(2, "\n"); + + owner = edac_get_owner(); + if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) + return -ENODEV; + + edac_op_state = EDAC_OPSTATE_NMI; + + rc = pci_register_driver(&igen6_driver); + if (rc) + return rc; + + igen6_printk(KERN_INFO, "%s\n", IGEN6_REVISION); + + return 0; +} + +static void __exit igen6_exit(void) +{ + edac_dbg(2, "\n"); + + pci_unregister_driver(&igen6_driver); +} + +module_init(igen6_init); +module_exit(igen6_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Qiuxu Zhuo"); +MODULE_DESCRIPTION("MC Driver for Intel client SoC using In-Band ECC"); From 2223d8c781a0c1a8cf26b1d8f13aff84557ecbfc Mon Sep 17 00:00:00 2001 From: Qiuxu Zhuo Date: Thu, 5 Nov 2020 15:49:34 +0800 Subject: [PATCH 211/360] EDAC/igen6: Add debugfs interface for Intel client SoC EDAC driver Add debugfs support to fake memory correctable errors to test the error reporting path and the error address decoding logic in the igen6_edac driver. Please note that the fake errors are also reported to EDAC core and then the CE counter in EDAC sysfs is also increased. Signed-off-by: Qiuxu Zhuo Signed-off-by: Tony Luck --- drivers/edac/igen6_edac.c | 59 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/drivers/edac/igen6_edac.c b/drivers/edac/igen6_edac.c index 318b9b67080f..6c0039e1171f 100644 --- a/drivers/edac/igen6_edac.c +++ b/drivers/edac/igen6_edac.c @@ -612,6 +612,10 @@ static int igen6_get_dimm_config(struct mem_ctl_info *mci) } #ifdef CONFIG_EDAC_DEBUG +/* Top of upper usable DRAM */ +static u64 igen6_touud; +#define TOUUD_OFFSET 0xa8 + static void igen6_reg_dump(struct igen6_imc *imc) { int i; @@ -632,10 +636,54 @@ static void igen6_reg_dump(struct igen6_imc *imc) readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4)); } edac_dbg(2, "TOLUD : 0x%x", igen6_tolud); + edac_dbg(2, "TOUUD : 0x%llx", igen6_touud); edac_dbg(2, "TOM : 0x%llx", igen6_tom); } + +static struct dentry *igen6_test; + +static int debugfs_u64_set(void *data, u64 val) +{ + u64 ecclog; + + if ((val >= igen6_tolud && val < _4GB) || val >= igen6_touud) { + edac_dbg(0, "Address 0x%llx out of range\n", val); + return 0; + } + + pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val); + + val >>= ECC_ERROR_LOG_ADDR_SHIFT; + ecclog = (val << ECC_ERROR_LOG_ADDR_SHIFT) | ECC_ERROR_LOG_CE; + + if (!ecclog_gen_pool_add(0, ecclog)) + irq_work_queue(&ecclog_irq_work); + + return 0; +} +DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n"); + +static void igen6_debug_setup(void) +{ + igen6_test = edac_debugfs_create_dir("igen6_test"); + if (!igen6_test) + return; + + if (!edac_debugfs_create_file("addr", 0200, igen6_test, + NULL, &fops_u64_wo)) { + debugfs_remove(igen6_test); + igen6_test = NULL; + } +} + +static void igen6_debug_teardown(void) +{ + debugfs_remove_recursive(igen6_test); +} #else static void igen6_reg_dump(struct igen6_imc *imc) {} +static void igen6_debug_setup(void) {} +static void igen6_debug_teardown(void) {} #endif static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar) @@ -691,6 +739,15 @@ static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar) *mchbar = MCHBAR_BASE(u.v); +#ifdef CONFIG_EDAC_DEBUG + if (pci_read_config_dword(pdev, TOUUD_OFFSET, &u.v_lo)) + edac_dbg(2, "Failed to read lower TOUUD\n"); + else if (pci_read_config_dword(pdev, TOUUD_OFFSET + 4, &u.v_hi)) + edac_dbg(2, "Failed to read upper TOUUD\n"); + else + igen6_touud = u.v & GENMASK_ULL(38, 20); +#endif + return 0; fail: return -ENODEV; @@ -849,6 +906,7 @@ static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto fail4; } + igen6_debug_setup(); return 0; fail4: unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); @@ -865,6 +923,7 @@ static void igen6_remove(struct pci_dev *pdev) { edac_dbg(2, "\n"); + igen6_debug_teardown(); errcmd_enable_error_reporting(false); unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); irq_work_sync(&ecclog_irq_work); From 41545aabff1ef71f83efe56dd4fec91a7fe65027 Mon Sep 17 00:00:00 2001 From: Tony Luck Date: Thu, 5 Nov 2020 10:39:16 -0800 Subject: [PATCH 212/360] MAINTAINERS: Add entry for Intel IGEN6 EDAC driver New driver for "client" system on chip CPUs. Signed-off-by: Tony Luck --- MAINTAINERS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index e73636b75f29..86eb55697c8b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6353,6 +6353,13 @@ L: linux-edac@vger.kernel.org S: Maintained F: drivers/edac/ie31200_edac.c +EDAC-IGEN6 +M: Tony Luck +R: Qiuxu Zhuo +L: linux-edac@vger.kernel.org +S: Maintained +F: drivers/edac/igen6_edac.c + EDAC-MPC85XX M: Johannes Thumshirn L: linux-edac@vger.kernel.org From 83ff51c4e3fecf6b8587ce4d46f6eac59f5d7c5a Mon Sep 17 00:00:00 2001 From: Qiuxu Zhuo Date: Tue, 17 Nov 2020 20:49:51 +0800 Subject: [PATCH 213/360] EDAC/i10nm: Use readl() to access MMIO registers Instead of raw access, use readl() to access MMIO registers of memory controller to avoid possible compiler re-ordering. Fixes: d4dc89d069aa ("EDAC, i10nm: Add a driver for Intel 10nm server processors") Cc: Signed-off-by: Qiuxu Zhuo Signed-off-by: Tony Luck --- drivers/edac/i10nm_base.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c index c8d11da85bec..7b52691c45d2 100644 --- a/drivers/edac/i10nm_base.c +++ b/drivers/edac/i10nm_base.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -19,14 +20,16 @@ #define i10nm_printk(level, fmt, arg...) \ edac_printk(level, "i10nm", fmt, ##arg) -#define I10NM_GET_SCK_BAR(d, reg) \ +#define I10NM_GET_SCK_BAR(d, reg) \ pci_read_config_dword((d)->uracu, 0xd0, &(reg)) #define I10NM_GET_IMC_BAR(d, i, reg) \ pci_read_config_dword((d)->uracu, 0xd8 + (i) * 4, &(reg)) #define I10NM_GET_DIMMMTR(m, i, j) \ - (*(u32 *)((m)->mbase + 0x2080c + (i) * 0x4000 + (j) * 4)) + readl((m)->mbase + 0x2080c + (i) * 0x4000 + (j) * 4) #define I10NM_GET_MCDDRTCFG(m, i, j) \ - (*(u32 *)((m)->mbase + 0x20970 + (i) * 0x4000 + (j) * 4)) + readl((m)->mbase + 0x20970 + (i) * 0x4000 + (j) * 4) +#define I10NM_GET_MCMTR(m, i) \ + readl((m)->mbase + 0x20ef8 + (i) * 0x4000) #define I10NM_GET_SCK_MMIO_BASE(reg) (GET_BITFIELD(reg, 0, 28) << 23) #define I10NM_GET_IMC_MMIO_OFFSET(reg) (GET_BITFIELD(reg, 0, 10) << 12) @@ -148,7 +151,7 @@ static bool i10nm_check_ecc(struct skx_imc *imc, int chan) { u32 mcmtr; - mcmtr = *(u32 *)(imc->mbase + 0x20ef8 + chan * 0x4000); + mcmtr = I10NM_GET_MCMTR(imc, chan); edac_dbg(1, "ch%d mcmtr reg %x\n", chan, mcmtr); return !!GET_BITFIELD(mcmtr, 2, 2); From bc1c99a5971aa7571e8b9731c28fa32abe12cab8 Mon Sep 17 00:00:00 2001 From: Qiuxu Zhuo Date: Tue, 17 Nov 2020 20:49:52 +0800 Subject: [PATCH 214/360] EDAC: Add DDR5 new memory type Add a new entry to 'enum mem_type' and a new string to 'edac_mem_types[]' for DDR5 new memory type. Signed-off-by: Qiuxu Zhuo Signed-off-by: Tony Luck --- drivers/edac/edac_mc.c | 1 + include/linux/edac.h | 3 +++ 2 files changed, 4 insertions(+) diff --git a/drivers/edac/edac_mc.c b/drivers/edac/edac_mc.c index eef8724faae0..f6d462d0be2d 100644 --- a/drivers/edac/edac_mc.c +++ b/drivers/edac/edac_mc.c @@ -163,6 +163,7 @@ const char * const edac_mem_types[] = { [MEM_RDDR4] = "Registered-DDR4", [MEM_LPDDR4] = "Low-Power-DDR4-RAM", [MEM_LRDDR4] = "Load-Reduced-DDR4-RAM", + [MEM_DDR5] = "Unbuffered-DDR5", [MEM_NVDIMM] = "Non-volatile-RAM", [MEM_WIO2] = "Wide-IO-2", }; diff --git a/include/linux/edac.h b/include/linux/edac.h index 8f63245f7f7c..e64b73b556eb 100644 --- a/include/linux/edac.h +++ b/include/linux/edac.h @@ -181,6 +181,7 @@ static inline char *mc_event_error_type(const unsigned int err_type) * This is a variant of the DDR4 memories. * @MEM_LRDDR4: Load-Reduced DDR4 memory. * @MEM_LPDDR4: Low-Power DDR4 memory. + * @MEM_DDR5: Unbuffered DDR5 RAM * @MEM_NVDIMM: Non-volatile RAM * @MEM_WIO2: Wide I/O 2. */ @@ -208,6 +209,7 @@ enum mem_type { MEM_RDDR4, MEM_LRDDR4, MEM_LPDDR4, + MEM_DDR5, MEM_NVDIMM, MEM_WIO2, }; @@ -234,6 +236,7 @@ enum mem_type { #define MEM_FLAG_RDDR4 BIT(MEM_RDDR4) #define MEM_FLAG_LRDDR4 BIT(MEM_LRDDR4) #define MEM_FLAG_LPDDR4 BIT(MEM_LPDDR4) +#define MEM_FLAG_DDR5 BIT(MEM_DDR5) #define MEM_FLAG_NVDIMM BIT(MEM_NVDIMM) #define MEM_FLAG_WIO2 BIT(MEM_WIO2) From 479f58dda25bb46daeb937f124718e8b4aea6781 Mon Sep 17 00:00:00 2001 From: Qiuxu Zhuo Date: Tue, 17 Nov 2020 20:49:53 +0800 Subject: [PATCH 215/360] EDAC/i10nm: Add Intel Sapphire Rapids server support The Sapphire Rapids CPU model shares the same memory controller architecture with Ice Lake server. There are some configurations different from Ice Lake server as below: - The device ID for configuration agent. - The size for per channel memory-mapped I/O. - The DDR5 memory support. So add the above configurations and the Sapphire Rapids CPU model ID for EDAC support. Signed-off-by: Qiuxu Zhuo Signed-off-by: Tony Luck --- drivers/edac/i10nm_base.c | 34 +++++++++++++++++++++++++--------- drivers/edac/skx_base.c | 6 +++--- drivers/edac/skx_common.c | 23 ++++++++++++++++++----- drivers/edac/skx_common.h | 16 ++++++++++++---- 4 files changed, 58 insertions(+), 21 deletions(-) diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c index 7b52691c45d2..238a4ad1e526 100644 --- a/drivers/edac/i10nm_base.c +++ b/drivers/edac/i10nm_base.c @@ -13,7 +13,7 @@ #include "edac_module.h" #include "skx_common.h" -#define I10NM_REVISION "v0.0.3" +#define I10NM_REVISION "v0.0.4" #define EDAC_MOD_STR "i10nm_edac" /* Debug macros */ @@ -25,11 +25,13 @@ #define I10NM_GET_IMC_BAR(d, i, reg) \ pci_read_config_dword((d)->uracu, 0xd8 + (i) * 4, &(reg)) #define I10NM_GET_DIMMMTR(m, i, j) \ - readl((m)->mbase + 0x2080c + (i) * 0x4000 + (j) * 4) + readl((m)->mbase + 0x2080c + (i) * (m)->chan_mmio_sz + (j) * 4) #define I10NM_GET_MCDDRTCFG(m, i, j) \ - readl((m)->mbase + 0x20970 + (i) * 0x4000 + (j) * 4) + readl((m)->mbase + 0x20970 + (i) * (m)->chan_mmio_sz + (j) * 4) #define I10NM_GET_MCMTR(m, i) \ - readl((m)->mbase + 0x20ef8 + (i) * 0x4000) + readl((m)->mbase + 0x20ef8 + (i) * (m)->chan_mmio_sz) +#define I10NM_GET_AMAP(m, i) \ + readl((m)->mbase + 0x20814 + (i) * (m)->chan_mmio_sz) #define I10NM_GET_SCK_MMIO_BASE(reg) (GET_BITFIELD(reg, 0, 28) << 23) #define I10NM_GET_IMC_MMIO_OFFSET(reg) (GET_BITFIELD(reg, 0, 10) << 12) @@ -129,12 +131,22 @@ static struct res_config i10nm_cfg0 = { .type = I10NM, .decs_did = 0x3452, .busno_cfg_offset = 0xcc, + .ddr_chan_mmio_sz = 0x4000, }; static struct res_config i10nm_cfg1 = { .type = I10NM, .decs_did = 0x3452, .busno_cfg_offset = 0xd0, + .ddr_chan_mmio_sz = 0x4000, +}; + +static struct res_config spr_cfg = { + .type = SPR, + .decs_did = 0x3252, + .busno_cfg_offset = 0xd0, + .ddr_chan_mmio_sz = 0x8000, + .support_ddr5 = true, }; static const struct x86_cpu_id i10nm_cpuids[] = { @@ -143,6 +155,7 @@ static const struct x86_cpu_id i10nm_cpuids[] = { X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_X, X86_STEPPINGS(0x0, 0x3), &i10nm_cfg0), X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_X, X86_STEPPINGS(0x4, 0xf), &i10nm_cfg1), X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(ICELAKE_D, X86_STEPPINGS(0x0, 0xf), &i10nm_cfg1), + X86_MATCH_INTEL_FAM6_MODEL_STEPPINGS(SAPPHIRERAPIDS_X, X86_STEPPINGS(0x0, 0xf), &spr_cfg), {} }; MODULE_DEVICE_TABLE(x86cpu, i10nm_cpuids); @@ -157,12 +170,13 @@ static bool i10nm_check_ecc(struct skx_imc *imc, int chan) return !!GET_BITFIELD(mcmtr, 2, 2); } -static int i10nm_get_dimm_config(struct mem_ctl_info *mci) +static int i10nm_get_dimm_config(struct mem_ctl_info *mci, + struct res_config *cfg) { struct skx_pvt *pvt = mci->pvt_info; struct skx_imc *imc = pvt->imc; + u32 mtr, amap, mcddrtcfg; struct dimm_info *dimm; - u32 mtr, mcddrtcfg; int i, j, ndimms; for (i = 0; i < I10NM_NUM_CHANNELS; i++) { @@ -170,6 +184,7 @@ static int i10nm_get_dimm_config(struct mem_ctl_info *mci) continue; ndimms = 0; + amap = I10NM_GET_AMAP(imc, i); for (j = 0; j < I10NM_NUM_DIMMS; j++) { dimm = edac_get_dimm(mci, i, j, 0); mtr = I10NM_GET_DIMMMTR(imc, i, j); @@ -178,8 +193,8 @@ static int i10nm_get_dimm_config(struct mem_ctl_info *mci) mtr, mcddrtcfg, imc->mc, i, j); if (IS_DIMM_PRESENT(mtr)) - ndimms += skx_get_dimm_info(mtr, 0, 0, dimm, - imc, i, j); + ndimms += skx_get_dimm_info(mtr, 0, amap, dimm, + imc, i, j, cfg); else if (IS_NVDIMM_PRESENT(mcddrtcfg, j)) ndimms += skx_get_nvdimm_info(dimm, imc, i, j, EDAC_MOD_STR); @@ -303,10 +318,11 @@ static int __init i10nm_init(void) d->imc[i].lmc = i; d->imc[i].src_id = src_id; d->imc[i].node_id = node_id; + d->imc[i].chan_mmio_sz = cfg->ddr_chan_mmio_sz; rc = skx_register_mci(&d->imc[i], d->imc[i].mdev, "Intel_10nm Socket", EDAC_MOD_STR, - i10nm_get_dimm_config); + i10nm_get_dimm_config, cfg); if (rc < 0) goto fail; } diff --git a/drivers/edac/skx_base.c b/drivers/edac/skx_base.c index 2c7db95df326..6a4f0b27c654 100644 --- a/drivers/edac/skx_base.c +++ b/drivers/edac/skx_base.c @@ -174,7 +174,7 @@ static bool skx_check_ecc(u32 mcmtr) return !!GET_BITFIELD(mcmtr, 2, 2); } -static int skx_get_dimm_config(struct mem_ctl_info *mci) +static int skx_get_dimm_config(struct mem_ctl_info *mci, struct res_config *cfg) { struct skx_pvt *pvt = mci->pvt_info; u32 mtr, mcmtr, amap, mcddrtcfg; @@ -195,7 +195,7 @@ static int skx_get_dimm_config(struct mem_ctl_info *mci) pci_read_config_dword(imc->chan[i].cdev, 0x80 + 4 * j, &mtr); if (IS_DIMM_PRESENT(mtr)) { - ndimms += skx_get_dimm_info(mtr, mcmtr, amap, dimm, imc, i, j); + ndimms += skx_get_dimm_info(mtr, mcmtr, amap, dimm, imc, i, j, cfg); } else if (IS_NVDIMM_PRESENT(mcddrtcfg, j)) { ndimms += skx_get_nvdimm_info(dimm, imc, i, j, EDAC_MOD_STR); @@ -702,7 +702,7 @@ static int __init skx_init(void) d->imc[i].node_id = node_id; rc = skx_register_mci(&d->imc[i], d->imc[i].chan[0].cdev, "Skylake Socket", EDAC_MOD_STR, - skx_get_dimm_config); + skx_get_dimm_config, cfg); if (rc < 0) goto fail; } diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c index 2b4ce8e5ac2f..81c3e2ec6f56 100644 --- a/drivers/edac/skx_common.c +++ b/drivers/edac/skx_common.c @@ -304,15 +304,25 @@ static int skx_get_dimm_attr(u32 reg, int lobit, int hibit, int add, #define numcol(reg) skx_get_dimm_attr(reg, 0, 1, 10, 0, 2, "cols") int skx_get_dimm_info(u32 mtr, u32 mcmtr, u32 amap, struct dimm_info *dimm, - struct skx_imc *imc, int chan, int dimmno) + struct skx_imc *imc, int chan, int dimmno, + struct res_config *cfg) { - int banks = 16, ranks, rows, cols, npages; + int banks, ranks, rows, cols, npages; + enum mem_type mtype; u64 size; ranks = numrank(mtr); rows = numrow(mtr); cols = numcol(mtr); + if (cfg->support_ddr5 && (amap & 0x8)) { + banks = 32; + mtype = MEM_DDR5; + } else { + banks = 16; + mtype = MEM_DDR4; + } + /* * Compute size in 8-byte (2^3) words, then shift to MiB (2^20) */ @@ -332,7 +342,7 @@ int skx_get_dimm_info(u32 mtr, u32 mcmtr, u32 amap, struct dimm_info *dimm, dimm->nr_pages = npages; dimm->grain = 32; dimm->dtype = get_width(mtr); - dimm->mtype = MEM_DDR4; + dimm->mtype = mtype; dimm->edac_mode = EDAC_SECDED; /* likely better than this */ snprintf(dimm->label, sizeof(dimm->label), "CPU_SrcID#%u_MC#%u_Chan#%u_DIMM#%u", imc->src_id, imc->lmc, chan, dimmno); @@ -390,7 +400,8 @@ unknown_size: int skx_register_mci(struct skx_imc *imc, struct pci_dev *pdev, const char *ctl_name, const char *mod_str, - get_dimm_config_f get_dimm_config) + get_dimm_config_f get_dimm_config, + struct res_config *cfg) { struct mem_ctl_info *mci; struct edac_mc_layer layers[2]; @@ -425,13 +436,15 @@ int skx_register_mci(struct skx_imc *imc, struct pci_dev *pdev, } mci->mtype_cap = MEM_FLAG_DDR4 | MEM_FLAG_NVDIMM; + if (cfg->support_ddr5) + mci->mtype_cap |= MEM_FLAG_DDR5; mci->edac_ctl_cap = EDAC_FLAG_NONE; mci->edac_cap = EDAC_FLAG_NONE; mci->mod_name = mod_str; mci->dev_name = pci_name(pdev); mci->ctl_page_to_phys = NULL; - rc = get_dimm_config(mci); + rc = get_dimm_config(mci, cfg); if (rc < 0) goto fail; diff --git a/drivers/edac/skx_common.h b/drivers/edac/skx_common.h index 78f8c1de0b71..bf56bebff138 100644 --- a/drivers/edac/skx_common.h +++ b/drivers/edac/skx_common.h @@ -59,6 +59,7 @@ struct skx_dev { struct mem_ctl_info *mci; struct pci_dev *mdev; /* for i10nm CPU */ void __iomem *mbase; /* for i10nm CPU */ + int chan_mmio_sz; /* for i10nm CPU */ u8 mc; /* system wide mc# */ u8 lmc; /* socket relative mc# */ u8 src_id, node_id; @@ -82,7 +83,8 @@ struct skx_pvt { enum type { SKX, - I10NM + I10NM, + SPR }; enum { @@ -118,9 +120,13 @@ struct res_config { unsigned int decs_did; /* Default bus number configuration register offset */ int busno_cfg_offset; + /* Per DDR channel memory-mapped I/O size */ + int ddr_chan_mmio_sz; + bool support_ddr5; }; -typedef int (*get_dimm_config_f)(struct mem_ctl_info *mci); +typedef int (*get_dimm_config_f)(struct mem_ctl_info *mci, + struct res_config *cfg); typedef bool (*skx_decode_f)(struct decoded_addr *res); typedef void (*skx_show_retry_log_f)(struct decoded_addr *res, char *msg, int len); @@ -136,14 +142,16 @@ int skx_get_all_bus_mappings(struct res_config *cfg, struct list_head **list); int skx_get_hi_lo(unsigned int did, int off[], u64 *tolm, u64 *tohm); int skx_get_dimm_info(u32 mtr, u32 mcmtr, u32 amap, struct dimm_info *dimm, - struct skx_imc *imc, int chan, int dimmno); + struct skx_imc *imc, int chan, int dimmno, + struct res_config *cfg); int skx_get_nvdimm_info(struct dimm_info *dimm, struct skx_imc *imc, int chan, int dimmno, const char *mod_str); int skx_register_mci(struct skx_imc *imc, struct pci_dev *pdev, const char *ctl_name, const char *mod_str, - get_dimm_config_f get_dimm_config); + get_dimm_config_f get_dimm_config, + struct res_config *cfg); int skx_mce_check_error(struct notifier_block *nb, unsigned long val, void *data); From 5761498c4d074560b38a203000afc6e971ffaa7f Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Mon, 9 Nov 2020 17:00:27 +0800 Subject: [PATCH 216/360] crypto: hisilicon/sec2 - Fix aead authentication setting key error Fix aead auth setting key process error. if use soft shash function, driver need to use digest size replace of the user input key length. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/sec2/sec_crypto.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 87bc08afe567..891e04914615 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -857,7 +857,7 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, struct crypto_authenc_keys *keys) { struct crypto_shash *hash_tfm = ctx->hash_tfm; - int blocksize, ret; + int blocksize, digestsize, ret; if (!keys->authkeylen) { pr_err("hisi_sec2: aead auth key error!\n"); @@ -865,6 +865,7 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, } blocksize = crypto_shash_blocksize(hash_tfm); + digestsize = crypto_shash_digestsize(hash_tfm); if (keys->authkeylen > blocksize) { ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey, keys->authkeylen, ctx->a_key); @@ -872,7 +873,7 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, pr_err("hisi_sec2: aead auth digest error!\n"); return -EINVAL; } - ctx->a_key_len = blocksize; + ctx->a_key_len = digestsize; } else { memcpy(ctx->a_key, keys->authkey, keys->authkeylen); ctx->a_key_len = keys->authkeylen; From 3ad99c22cebee84034bc53e42f5115dbc97490e8 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 10 Nov 2020 10:10:42 +0100 Subject: [PATCH 217/360] crypto: arm64/gcm - move authentication tag check to SIMD domain Instead of copying the calculated authentication tag to memory and calling crypto_memneq() to verify it, use vector bytewise compare and min across vector instructions to decide whether the tag is valid. This is more efficient, and given that the tag is only transiently held in a NEON register, it is also safer, given that calculated tags for failed decryptions should be withheld. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm64/crypto/ghash-ce-core.S | 15 ++++++++++ arch/arm64/crypto/ghash-ce-glue.c | 46 +++++++++++++++++++------------ 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S index 6b958dcdf136..7868330dd54e 100644 --- a/arch/arm64/crypto/ghash-ce-core.S +++ b/arch/arm64/crypto/ghash-ce-core.S @@ -544,7 +544,22 @@ CPU_LE( rev w8, w8 ) ext XL.16b, XL.16b, XL.16b, #8 rev64 XL.16b, XL.16b eor XL.16b, XL.16b, KS0.16b + + .if \enc == 1 st1 {XL.16b}, [x10] // store tag + .else + ldp x11, x12, [sp, #40] // load tag pointer and authsize + adr_l x17, .Lpermute_table + ld1 {KS0.16b}, [x11] // load supplied tag + add x17, x17, x12 + ld1 {KS1.16b}, [x17] // load permute vector + + cmeq XL.16b, XL.16b, KS0.16b // compare tags + mvn XL.16b, XL.16b // -1 for fail, 0 for pass + tbl XL.16b, {XL.16b}, KS1.16b // keep authsize bytes only + sminv b0, XL.16b // signed minimum across XL + smov w0, v0.b[0] // return b0 + .endif 4: ldp x29, x30, [sp], #32 ret diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 2427e2f3a9a1..720cd3a58da3 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -55,10 +55,10 @@ asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[], u64 const h[][2], u64 dg[], u8 ctr[], u32 const rk[], int rounds, u8 tag[]); - -asmlinkage void pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], - u64 const h[][2], u64 dg[], u8 ctr[], - u32 const rk[], int rounds, u8 tag[]); +asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[], + u64 const h[][2], u64 dg[], u8 ctr[], + u32 const rk[], int rounds, const u8 l[], + const u8 tag[], u64 authsize); static int ghash_init(struct shash_desc *desc) { @@ -458,6 +458,7 @@ static int gcm_decrypt(struct aead_request *req) unsigned int authsize = crypto_aead_authsize(aead); int nrounds = num_rounds(&ctx->aes_key); struct skcipher_walk walk; + u8 otag[AES_BLOCK_SIZE]; u8 buf[AES_BLOCK_SIZE]; u8 iv[AES_BLOCK_SIZE]; u64 dg[2] = {}; @@ -474,9 +475,15 @@ static int gcm_decrypt(struct aead_request *req) memcpy(iv, req->iv, GCM_IV_SIZE); put_unaligned_be32(2, iv + GCM_IV_SIZE); + scatterwalk_map_and_copy(otag, req->src, + req->assoclen + req->cryptlen - authsize, + authsize, 0); + err = skcipher_walk_aead_decrypt(&walk, req, false); if (likely(crypto_simd_usable())) { + int ret; + do { const u8 *src = walk.src.virt.addr; u8 *dst = walk.dst.virt.addr; @@ -493,9 +500,10 @@ static int gcm_decrypt(struct aead_request *req) } kernel_neon_begin(); - pmull_gcm_decrypt(nbytes, dst, src, ctx->ghash_key.h, - dg, iv, ctx->aes_key.key_enc, nrounds, - tag); + ret = pmull_gcm_decrypt(nbytes, dst, src, + ctx->ghash_key.h, + dg, iv, ctx->aes_key.key_enc, + nrounds, tag, otag, authsize); kernel_neon_end(); if (unlikely(!nbytes)) @@ -507,6 +515,11 @@ static int gcm_decrypt(struct aead_request *req) err = skcipher_walk_done(&walk, walk.nbytes - nbytes); } while (walk.nbytes); + + if (err) + return err; + if (ret) + return -EBADMSG; } else { while (walk.nbytes >= AES_BLOCK_SIZE) { int blocks = walk.nbytes / AES_BLOCK_SIZE; @@ -548,23 +561,20 @@ static int gcm_decrypt(struct aead_request *req) err = skcipher_walk_done(&walk, 0); } + if (err) + return err; + put_unaligned_be64(dg[1], tag); put_unaligned_be64(dg[0], tag + 8); put_unaligned_be32(1, iv + GCM_IV_SIZE); aes_encrypt(&ctx->aes_key, iv, iv); crypto_xor(tag, iv, AES_BLOCK_SIZE); + + if (crypto_memneq(tag, otag, authsize)) { + memzero_explicit(tag, AES_BLOCK_SIZE); + return -EBADMSG; + } } - - if (err) - return err; - - /* compare calculated auth tag with the stored one */ - scatterwalk_map_and_copy(buf, req->src, - req->assoclen + req->cryptlen - authsize, - authsize, 0); - - if (crypto_memneq(tag, buf, authsize)) - return -EBADMSG; return 0; } From 0049a1329bb9569196a551bb658500394d98ed50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Horia=20Geant=C4=83?= Date: Thu, 12 Nov 2020 11:21:46 +0200 Subject: [PATCH 218/360] crypto: caam/qi - simplify error path for context allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wang Qing reports that IS_ERR_OR_NULL() should be matched with PTR_ERR_OR_ZERO(), not PTR_ERR(). As it turns out, the error path always returns an error code, i.e. NULL is never returned. Update the code accordingly - s/IS_ERR_OR_NULL/IS_ERR. Reported-by: Wang Qing Signed-off-by: Horia Geantă Signed-off-by: Herbert Xu --- drivers/crypto/caam/caamalg_qi.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index a24ae966df4a..189a7438b29c 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -852,7 +852,7 @@ static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, cpu = smp_processor_id(); drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc); - if (!IS_ERR_OR_NULL(drv_ctx)) + if (!IS_ERR(drv_ctx)) drv_ctx->op_type = type; ctx->drv_ctx[type] = drv_ctx; @@ -955,7 +955,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, struct caam_drv_ctx *drv_ctx; drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); - if (IS_ERR_OR_NULL(drv_ctx)) + if (IS_ERR(drv_ctx)) return (struct aead_edesc *)drv_ctx; /* allocate space for base edesc and hw desc commands, link tables */ @@ -1165,7 +1165,7 @@ static inline int aead_crypt(struct aead_request *req, bool encrypt) /* allocate extended descriptor */ edesc = aead_edesc_alloc(req, encrypt); - if (IS_ERR_OR_NULL(edesc)) + if (IS_ERR(edesc)) return PTR_ERR(edesc); /* Create and submit job descriptor */ @@ -1259,7 +1259,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, struct caam_drv_ctx *drv_ctx; drv_ctx = get_drv_ctx(ctx, encrypt ? ENCRYPT : DECRYPT); - if (IS_ERR_OR_NULL(drv_ctx)) + if (IS_ERR(drv_ctx)) return (struct skcipher_edesc *)drv_ctx; src_nents = sg_nents_for_len(req->src, req->cryptlen); From 5bdad829c31a09069fd508534f03c2ea1576ac75 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Thu, 12 Nov 2020 13:07:02 -0700 Subject: [PATCH 219/360] crypto: crypto4xx - Replace bitwise OR with logical OR in crypto4xx_build_pd Clang warns: drivers/crypto/amcc/crypto4xx_core.c:921:60: warning: operator '?:' has lower precedence than '|'; '|' will be evaluated first [-Wbitwise-conditional-parentheses] (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ^ drivers/crypto/amcc/crypto4xx_core.c:921:60: note: place parentheses around the '|' expression to silence this warning (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? ^ ) drivers/crypto/amcc/crypto4xx_core.c:921:60: note: place parentheses around the '?:' expression to evaluate it first (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? ^ ( 1 warning generated. It looks like this should have been a logical OR so that PD_CTL_HASH_FINAL gets added to the w bitmask if crypto_tfm_alg_type is either CRYPTO_ALG_TYPE_AHASH or CRYPTO_ALG_TYPE_AEAD. Change the operator so that everything works properly. Fixes: 4b5b79998af6 ("crypto: crypto4xx - fix stalls under heavy load") Link: https://github.com/ClangBuiltLinux/linux/issues/1198 Signed-off-by: Nathan Chancellor Reviewed-by: Christian Lamparter Signed-off-by: Herbert Xu --- drivers/crypto/amcc/crypto4xx_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 981de43ea5e2..2e3690f65786 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -917,7 +917,7 @@ int crypto4xx_build_pd(struct crypto_async_request *req, } pd->pd_ctl.w = PD_CTL_HOST_READY | - ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) | + ((crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AHASH) || (crypto_tfm_alg_type(req->tfm) == CRYPTO_ALG_TYPE_AEAD) ? PD_CTL_HASH_FINAL : 0); pd->pd_ctl_len.w = 0x00400000 | (assoclen + datalen); From a24d22b225ce158651378869a6b88105c4bdb887 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 12 Nov 2020 21:20:21 -0800 Subject: [PATCH 220/360] crypto: sha - split sha.h into sha1.h and sha2.h Currently contains declarations for both SHA-1 and SHA-2, and contains declarations for SHA-3. This organization is inconsistent, but more importantly SHA-1 is no longer considered to be cryptographically secure. So to the extent possible, SHA-1 shouldn't be grouped together with any of the other SHA versions, and usage of it should be phased out. Therefore, split into two headers and , and make everyone explicitly specify whether they want the declarations for SHA-1, SHA-2, or both. This avoids making the SHA-1 declarations visible to files that don't want anything to do with SHA-1. It also prepares for potentially moving sha1.h into a new insecure/ or dangerous/ directory. Signed-off-by: Eric Biggers Acked-by: Ard Biesheuvel Acked-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- arch/arm/crypto/sha1-ce-glue.c | 2 +- arch/arm/crypto/sha1.h | 2 +- arch/arm/crypto/sha1_glue.c | 2 +- arch/arm/crypto/sha1_neon_glue.c | 2 +- arch/arm/crypto/sha2-ce-glue.c | 2 +- arch/arm/crypto/sha256_glue.c | 2 +- arch/arm/crypto/sha256_neon_glue.c | 2 +- arch/arm/crypto/sha512-glue.c | 2 +- arch/arm/crypto/sha512-neon-glue.c | 2 +- arch/arm64/crypto/aes-glue.c | 2 +- arch/arm64/crypto/sha1-ce-glue.c | 2 +- arch/arm64/crypto/sha2-ce-glue.c | 2 +- arch/arm64/crypto/sha256-glue.c | 2 +- arch/arm64/crypto/sha512-ce-glue.c | 2 +- arch/arm64/crypto/sha512-glue.c | 2 +- arch/mips/cavium-octeon/crypto/octeon-sha1.c | 2 +- .../mips/cavium-octeon/crypto/octeon-sha256.c | 2 +- .../mips/cavium-octeon/crypto/octeon-sha512.c | 2 +- arch/powerpc/crypto/sha1-spe-glue.c | 2 +- arch/powerpc/crypto/sha1.c | 2 +- arch/powerpc/crypto/sha256-spe-glue.c | 2 +- arch/s390/crypto/sha.h | 3 +- arch/s390/crypto/sha1_s390.c | 2 +- arch/s390/crypto/sha256_s390.c | 2 +- arch/s390/crypto/sha3_256_s390.c | 1 - arch/s390/crypto/sha3_512_s390.c | 1 - arch/s390/crypto/sha512_s390.c | 2 +- arch/s390/purgatory/purgatory.c | 2 +- arch/sparc/crypto/sha1_glue.c | 2 +- arch/sparc/crypto/sha256_glue.c | 2 +- arch/sparc/crypto/sha512_glue.c | 2 +- arch/x86/crypto/sha1_ssse3_glue.c | 2 +- arch/x86/crypto/sha256_ssse3_glue.c | 2 +- arch/x86/crypto/sha512_ssse3_glue.c | 2 +- arch/x86/purgatory/purgatory.c | 2 +- crypto/asymmetric_keys/asym_tpm.c | 2 +- crypto/sha1_generic.c | 2 +- crypto/sha256_generic.c | 2 +- crypto/sha512_generic.c | 2 +- drivers/char/random.c | 2 +- drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h | 2 +- .../crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 3 +- drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h | 3 +- .../crypto/allwinner/sun8i-ss/sun8i-ss-hash.c | 3 +- drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h | 3 +- drivers/crypto/amcc/crypto4xx_alg.c | 2 +- drivers/crypto/amcc/crypto4xx_core.c | 2 +- drivers/crypto/atmel-authenc.h | 3 +- drivers/crypto/atmel-sha.c | 3 +- drivers/crypto/axis/artpec6_crypto.c | 3 +- drivers/crypto/bcm/cipher.c | 3 +- drivers/crypto/bcm/cipher.h | 3 +- drivers/crypto/bcm/spu.h | 3 +- drivers/crypto/caam/compat.h | 3 +- drivers/crypto/cavium/nitrox/nitrox_aead.c | 1 - drivers/crypto/ccp/ccp-crypto-sha.c | 3 +- drivers/crypto/ccp/ccp-crypto.h | 3 +- drivers/crypto/ccree/cc_driver.h | 3 +- drivers/crypto/chelsio/chcr_algo.c | 3 +- drivers/crypto/hisilicon/sec2/sec_crypto.c | 3 +- drivers/crypto/img-hash.c | 3 +- drivers/crypto/inside-secure/safexcel.h | 3 +- .../crypto/inside-secure/safexcel_cipher.c | 3 +- drivers/crypto/inside-secure/safexcel_hash.c | 3 +- drivers/crypto/ixp4xx_crypto.c | 2 +- drivers/crypto/marvell/cesa/hash.c | 3 +- .../crypto/marvell/octeontx/otx_cptvf_algs.c | 3 +- drivers/crypto/mediatek/mtk-sha.c | 3 +- drivers/crypto/mxs-dcp.c | 3 +- drivers/crypto/n2_core.c | 3 +- drivers/crypto/nx/nx-sha256.c | 2 +- drivers/crypto/nx/nx-sha512.c | 2 +- drivers/crypto/nx/nx.c | 2 +- drivers/crypto/omap-sham.c | 3 +- drivers/crypto/padlock-sha.c | 3 +- drivers/crypto/picoxcell_crypto.c | 3 +- drivers/crypto/qat/qat_common/qat_algs.c | 3 +- drivers/crypto/qce/common.c | 3 +- drivers/crypto/qce/core.c | 1 - drivers/crypto/qce/sha.h | 3 +- drivers/crypto/rockchip/rk3288_crypto.h | 3 +- drivers/crypto/s5p-sss.c | 3 +- drivers/crypto/sa2ul.c | 3 +- drivers/crypto/sa2ul.h | 3 +- drivers/crypto/sahara.c | 3 +- drivers/crypto/stm32/stm32-hash.c | 3 +- drivers/crypto/talitos.c | 3 +- drivers/crypto/ux500/hash/hash_core.c | 3 +- drivers/firmware/efi/embedded-firmware.c | 2 +- .../inline_crypto/ch_ipsec/chcr_ipsec.c | 3 +- .../chelsio/inline_crypto/chtls/chtls.h | 3 +- drivers/nfc/s3fwrn5/firmware.c | 2 +- drivers/tee/tee_core.c | 2 +- fs/crypto/fname.c | 2 +- fs/crypto/hkdf.c | 2 +- fs/ubifs/auth.c | 1 - fs/verity/fsverity_private.h | 2 +- include/crypto/hash_info.h | 3 +- include/crypto/sha1.h | 46 +++++++++++++++++++ include/crypto/sha1_base.h | 2 +- include/crypto/{sha.h => sha2.h} | 41 ++--------------- include/crypto/sha256_base.h | 2 +- include/crypto/sha512_base.h | 2 +- include/linux/ccp.h | 3 +- include/linux/filter.h | 2 +- include/linux/purgatory.h | 2 +- kernel/crash_core.c | 2 +- kernel/kexec_core.c | 1 - kernel/kexec_file.c | 2 +- lib/crypto/sha256.c | 2 +- lib/digsig.c | 2 +- lib/sha1.c | 2 +- net/ipv6/seg6_hmac.c | 1 - net/mptcp/crypto.c | 2 +- net/mptcp/options.c | 2 +- net/mptcp/subflow.c | 2 +- security/integrity/integrity.h | 2 +- security/keys/encrypted-keys/encrypted.c | 2 +- security/keys/trusted-keys/trusted_tpm1.c | 2 +- sound/soc/codecs/cros_ec_codec.c | 2 +- 120 files changed, 205 insertions(+), 155 deletions(-) create mode 100644 include/crypto/sha1.h rename include/crypto/{sha.h => sha2.h} (77%) diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c index e79b1fb4b4dc..de9100c67b37 100644 --- a/arch/arm/crypto/sha1-ce-glue.c +++ b/arch/arm/crypto/sha1-ce-glue.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha1.h b/arch/arm/crypto/sha1.h index 758db3e9ff0a..b1b7e21da2c3 100644 --- a/arch/arm/crypto/sha1.h +++ b/arch/arm/crypto/sha1.h @@ -3,7 +3,7 @@ #define ASM_ARM_CRYPTO_SHA1_H #include -#include +#include extern int sha1_update_arm(struct shash_desc *desc, const u8 *data, unsigned int len); diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c index 4e954b3f7ecd..6c2b849e459d 100644 --- a/arch/arm/crypto/sha1_glue.c +++ b/arch/arm/crypto/sha1_glue.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c index 0071e5e4411a..cfe36ae0f3f5 100644 --- a/arch/arm/crypto/sha1_neon_glue.c +++ b/arch/arm/crypto/sha1_neon_glue.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha2-ce-glue.c b/arch/arm/crypto/sha2-ce-glue.c index 87f0b62386c6..c62ce89dd3e0 100644 --- a/arch/arm/crypto/sha2-ce-glue.c +++ b/arch/arm/crypto/sha2-ce-glue.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha256_glue.c b/arch/arm/crypto/sha256_glue.c index b8a4f79020cf..433ee4ddce6c 100644 --- a/arch/arm/crypto/sha256_glue.c +++ b/arch/arm/crypto/sha256_glue.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha256_neon_glue.c b/arch/arm/crypto/sha256_neon_glue.c index 79820b9e2541..701706262ef3 100644 --- a/arch/arm/crypto/sha256_neon_glue.c +++ b/arch/arm/crypto/sha256_neon_glue.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha512-glue.c b/arch/arm/crypto/sha512-glue.c index 8775aa42bbbe..0635a65aa488 100644 --- a/arch/arm/crypto/sha512-glue.c +++ b/arch/arm/crypto/sha512-glue.c @@ -6,7 +6,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/arm/crypto/sha512-neon-glue.c b/arch/arm/crypto/sha512-neon-glue.c index 96cb94403540..c879ad32db51 100644 --- a/arch/arm/crypto/sha512-neon-glue.c +++ b/arch/arm/crypto/sha512-neon-glue.c @@ -7,7 +7,7 @@ #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 395bbf64b2ab..34b8a89197be 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index c63b99211db3..c93121bcfdeb 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index 5e956d7582a5..31ba3da5e61b 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha256-glue.c b/arch/arm64/crypto/sha256-glue.c index 77bc6e72abae..9462f6088b3f 100644 --- a/arch/arm64/crypto/sha256-glue.c +++ b/arch/arm64/crypto/sha256-glue.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha512-ce-glue.c b/arch/arm64/crypto/sha512-ce-glue.c index dc890a719f54..faa83f6cf376 100644 --- a/arch/arm64/crypto/sha512-ce-glue.c +++ b/arch/arm64/crypto/sha512-ce-glue.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/arm64/crypto/sha512-glue.c b/arch/arm64/crypto/sha512-glue.c index 370ccb29602f..2acff1c7df5d 100644 --- a/arch/arm64/crypto/sha512-glue.c +++ b/arch/arm64/crypto/sha512-glue.c @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha1.c b/arch/mips/cavium-octeon/crypto/octeon-sha1.c index 75e79b47abfe..30f1d75208a5 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha1.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha1.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c index a682ce76716a..36cb92895d72 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha256.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -15,7 +15,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha512.c b/arch/mips/cavium-octeon/crypto/octeon-sha512.c index 50722a0cfb53..359f039820d8 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-sha512.c +++ b/arch/mips/cavium-octeon/crypto/octeon-sha512.c @@ -14,7 +14,7 @@ */ #include -#include +#include #include #include #include diff --git a/arch/powerpc/crypto/sha1-spe-glue.c b/arch/powerpc/crypto/sha1-spe-glue.c index cb57be4ada61..b1e577cbf00c 100644 --- a/arch/powerpc/crypto/sha1-spe-glue.c +++ b/arch/powerpc/crypto/sha1-spe-glue.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/powerpc/crypto/sha1.c b/arch/powerpc/crypto/sha1.c index b40dc50a6908..7a55d790cdb1 100644 --- a/arch/powerpc/crypto/sha1.c +++ b/arch/powerpc/crypto/sha1.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include void powerpc_sha_transform(u32 *state, const u8 *src); diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c index ceb0b6c980b3..88530ae0791f 100644 --- a/arch/powerpc/crypto/sha256-spe-glue.c +++ b/arch/powerpc/crypto/sha256-spe-glue.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/s390/crypto/sha.h b/arch/s390/crypto/sha.h index ada2f98c27b7..65ea12fc87a1 100644 --- a/arch/s390/crypto/sha.h +++ b/arch/s390/crypto/sha.h @@ -11,7 +11,8 @@ #define _CRYPTO_ARCH_S390_SHA_H #include -#include +#include +#include #include /* must be big enough for the largest SHA variant */ diff --git a/arch/s390/crypto/sha1_s390.c b/arch/s390/crypto/sha1_s390.c index 698b1e6d3c14..a3fabf310a38 100644 --- a/arch/s390/crypto/sha1_s390.c +++ b/arch/s390/crypto/sha1_s390.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include "sha.h" diff --git a/arch/s390/crypto/sha256_s390.c b/arch/s390/crypto/sha256_s390.c index b52c87e44939..24983f175676 100644 --- a/arch/s390/crypto/sha256_s390.c +++ b/arch/s390/crypto/sha256_s390.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include "sha.h" diff --git a/arch/s390/crypto/sha3_256_s390.c b/arch/s390/crypto/sha3_256_s390.c index 460cbbbaa44a..30ac49b635bf 100644 --- a/arch/s390/crypto/sha3_256_s390.c +++ b/arch/s390/crypto/sha3_256_s390.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include diff --git a/arch/s390/crypto/sha3_512_s390.c b/arch/s390/crypto/sha3_512_s390.c index 72cf460a53e5..e70d50f7620f 100644 --- a/arch/s390/crypto/sha3_512_s390.c +++ b/arch/s390/crypto/sha3_512_s390.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index ad29db085a18..29a6bd404c59 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -8,7 +8,7 @@ * Author(s): Jan Glauber (jang@de.ibm.com) */ #include -#include +#include #include #include #include diff --git a/arch/s390/purgatory/purgatory.c b/arch/s390/purgatory/purgatory.c index 0a423bcf6746..030efda05dbe 100644 --- a/arch/s390/purgatory/purgatory.c +++ b/arch/s390/purgatory/purgatory.c @@ -9,7 +9,7 @@ #include #include -#include +#include #include int verify_sha256_digest(void) diff --git a/arch/sparc/crypto/sha1_glue.c b/arch/sparc/crypto/sha1_glue.c index dc017782be52..86a654cce5ab 100644 --- a/arch/sparc/crypto/sha1_glue.c +++ b/arch/sparc/crypto/sha1_glue.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c index ca2547df9652..60ec524cf9ca 100644 --- a/arch/sparc/crypto/sha256_glue.c +++ b/arch/sparc/crypto/sha256_glue.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c index 3b2ca732ff7a..273ce21918c1 100644 --- a/arch/sparc/crypto/sha512_glue.c +++ b/arch/sparc/crypto/sha512_glue.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c index 18200135603f..44340a1139e0 100644 --- a/arch/x86/crypto/sha1_ssse3_glue.c +++ b/arch/x86/crypto/sha1_ssse3_glue.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c index dd06249229e1..3a5f6be7dbba 100644 --- a/arch/x86/crypto/sha256_ssse3_glue.c +++ b/arch/x86/crypto/sha256_ssse3_glue.c @@ -35,7 +35,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c index b0b05c93409e..30e70f4fe2f7 100644 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ b/arch/x86/crypto/sha512_ssse3_glue.c @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include #include diff --git a/arch/x86/purgatory/purgatory.c b/arch/x86/purgatory/purgatory.c index 7b37a412f829..f03b64d9cb51 100644 --- a/arch/x86/purgatory/purgatory.c +++ b/arch/x86/purgatory/purgatory.c @@ -9,7 +9,7 @@ */ #include -#include +#include #include #include "../boot/string.h" diff --git a/crypto/asymmetric_keys/asym_tpm.c b/crypto/asymmetric_keys/asym_tpm.c index 378b18b9bc34..511932aa94a6 100644 --- a/crypto/asymmetric_keys/asym_tpm.c +++ b/crypto/asymmetric_keys/asym_tpm.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c index 1d43472fecbd..325b57fe28dc 100644 --- a/crypto/sha1_generic.c +++ b/crypto/sha1_generic.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c index 88156e3e2a33..3b377197236e 100644 --- a/crypto/sha256_generic.c +++ b/crypto/sha256_generic.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index e34d09dd9971..c72d72ad828e 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/char/random.c b/drivers/char/random.c index 2a41b21623ae..5f3b8ac9d97b 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -336,7 +336,7 @@ #include #include #include -#include +#include #include #include diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h index 163962f9e284..5c291e4a6857 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss.h @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index fa2f1b4fad7b..4927a6c82d32 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -13,7 +13,8 @@ #include #include #include -#include +#include +#include #include #include "sun8i-ce.h" diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h index 558027516aed..cec781d5063c 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h @@ -16,7 +16,8 @@ #include #include #include -#include +#include +#include /* CE Registers */ #define CE_TDQ 0x00 diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index b6ab2054f217..11cbcbc83a7b 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -13,7 +13,8 @@ #include #include #include -#include +#include +#include #include #include "sun8i-ss.h" diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h index 1a66457f4a20..28188685b910 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include #define SS_START 1 diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 7729a637fb02..a3fa849b139a 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include "crypto4xx_reg_def.h" diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 2e3690f65786..8d1b918a0533 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/atmel-authenc.h b/drivers/crypto/atmel-authenc.h index c6530a1c8c20..45171e89a7d2 100644 --- a/drivers/crypto/atmel-authenc.h +++ b/drivers/crypto/atmel-authenc.h @@ -16,7 +16,8 @@ #include #include -#include +#include +#include #include "atmel-sha-regs.h" struct atmel_aes_dev; diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 0eb6f54e3b66..352d80cb5ae9 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -33,7 +33,8 @@ #include #include #include -#include +#include +#include #include #include #include "atmel-sha-regs.h" diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 809c3033ca74..9ad188cffd0d 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -28,7 +28,8 @@ #include #include #include -#include +#include +#include #include /* Max length of a line in all cache levels for Artpec SoCs. */ diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 50d169e61b41..30390a7324b2 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -26,11 +26,12 @@ #include #include #include -#include #include #include #include #include +#include +#include #include #include "util.h" diff --git a/drivers/crypto/bcm/cipher.h b/drivers/crypto/bcm/cipher.h index 035c8389cb3d..0ad5892b445d 100644 --- a/drivers/crypto/bcm/cipher.h +++ b/drivers/crypto/bcm/cipher.h @@ -16,7 +16,8 @@ #include #include #include -#include +#include +#include #include #include "spu.h" diff --git a/drivers/crypto/bcm/spu.h b/drivers/crypto/bcm/spu.h index dd132389bcaa..1c386a2d5506 100644 --- a/drivers/crypto/bcm/spu.h +++ b/drivers/crypto/bcm/spu.h @@ -17,7 +17,8 @@ #include #include -#include +#include +#include enum spu_cipher_alg { CIPHER_ALG_NONE = 0x0, diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h index c3c22a8de4c0..c4f79764172b 100644 --- a/drivers/crypto/caam/compat.h +++ b/drivers/crypto/caam/compat.h @@ -34,7 +34,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/cavium/nitrox/nitrox_aead.c b/drivers/crypto/cavium/nitrox/nitrox_aead.c index e5d8607ecb1d..c93c4e41d267 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_aead.c +++ b/drivers/crypto/cavium/nitrox/nitrox_aead.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c index 8fbfdb9e8cd3..74fa5360e722 100644 --- a/drivers/crypto/ccp/ccp-crypto-sha.c +++ b/drivers/crypto/ccp/ccp-crypto-sha.c @@ -17,7 +17,8 @@ #include #include #include -#include +#include +#include #include #include diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h index aed3d2192d01..e42450d07168 100644 --- a/drivers/crypto/ccp/ccp-crypto.h +++ b/drivers/crypto/ccp/ccp-crypto.h @@ -19,7 +19,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index af77b2020350..ed2b2f13a256 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -17,7 +17,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 13b908ea4873..f5a336634daa 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -53,7 +53,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 891e04914615..2eaa516b3231 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -7,7 +7,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index 91f555ccbb31..e813115d5432 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -19,7 +19,8 @@ #include #include -#include +#include +#include #define CR_RESET 0 #define CR_RESET_SET 1 diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 9045f2d7f4c6..ce1e611a163e 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -11,7 +11,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 9bcfb79a030f..d68ef16650d4 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -18,7 +18,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 56d5ccb5cc00..50fb6d90a2e0 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -8,7 +8,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index 276012e7c482..8b0f17fc09fb 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/marvell/cesa/hash.c b/drivers/crypto/marvell/cesa/hash.c index add7ea011c98..8cf9fd518d86 100644 --- a/drivers/crypto/marvell/cesa/hash.c +++ b/drivers/crypto/marvell/cesa/hash.c @@ -11,7 +11,8 @@ #include #include -#include +#include +#include #include #include diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c index 90bb31329d4b..ccbef01888d4 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_algs.c @@ -13,7 +13,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index 3d5d7d68b03b..f55aacdafbef 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -10,7 +10,8 @@ */ #include -#include +#include +#include #include "mtk-platform.h" #define SHA_ALIGN_MSK (sizeof(u32) - 1) diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index 909a7eb748e3..d6a7784d2988 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -17,7 +17,8 @@ #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index 3642bf83d809..3b0bf6fea491 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/nx/nx-sha256.c b/drivers/crypto/nx/nx-sha256.c index 02fb53453195..90d9a37a57f6 100644 --- a/drivers/crypto/nx/nx-sha256.c +++ b/drivers/crypto/nx/nx-sha256.c @@ -8,7 +8,7 @@ */ #include -#include +#include #include #include #include diff --git a/drivers/crypto/nx/nx-sha512.c b/drivers/crypto/nx/nx-sha512.c index 4c7a3e3eeebf..eb8627a0f317 100644 --- a/drivers/crypto/nx/nx-sha512.c +++ b/drivers/crypto/nx/nx-sha512.c @@ -8,7 +8,7 @@ */ #include -#include +#include #include #include diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index 40882d6d52c1..0d2dc5be7f19 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index a3b38d2c92e7..ae0d320d3c60 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -35,7 +35,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index a697a4a3f2d0..6865c7f1fc1a 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -9,7 +9,8 @@ #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index fb34bf92861d..84f9c16d984c 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -8,7 +8,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 0fab8bb8ca59..b3a68d986417 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -6,7 +6,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/qce/common.c b/drivers/crypto/qce/common.c index 5006e74c40cd..a73db2a5637f 100644 --- a/drivers/crypto/qce/common.c +++ b/drivers/crypto/qce/common.c @@ -7,7 +7,8 @@ #include #include #include -#include +#include +#include #include "cipher.h" #include "common.h" diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index ea616b7259ae..5e6717f9bbda 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -13,7 +13,6 @@ #include #include #include -#include #include "core.h" #include "cipher.h" diff --git a/drivers/crypto/qce/sha.h b/drivers/crypto/qce/sha.h index d63526e3804d..a22695361f16 100644 --- a/drivers/crypto/qce/sha.h +++ b/drivers/crypto/qce/sha.h @@ -7,7 +7,8 @@ #define _SHA_H_ #include -#include +#include +#include #include "common.h" #include "core.h" diff --git a/drivers/crypto/rockchip/rk3288_crypto.h b/drivers/crypto/rockchip/rk3288_crypto.h index 3db595570c9c..97278c2574ff 100644 --- a/drivers/crypto/rockchip/rk3288_crypto.h +++ b/drivers/crypto/rockchip/rk3288_crypto.h @@ -12,7 +12,8 @@ #include #include -#include +#include +#include #define _SBF(v, f) ((v) << (f)) diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 88a6c853ffd7..682c8a450a57 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -30,7 +30,8 @@ #include #include -#include +#include +#include #include #define _SBF(s, v) ((v) << (s)) diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index c357010a159e..f300b0a5958a 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -25,7 +25,8 @@ #include #include #include -#include +#include +#include #include "sa2ul.h" diff --git a/drivers/crypto/sa2ul.h b/drivers/crypto/sa2ul.h index bb40df3876e5..f597ddecde34 100644 --- a/drivers/crypto/sa2ul.h +++ b/drivers/crypto/sa2ul.h @@ -13,7 +13,8 @@ #define _K3_SA2UL_ #include -#include +#include +#include #define SA_ENGINE_ENABLE_CONTROL 0x1000 diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index d60679c79822..8b5be29cb4dc 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include #include #include diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index e3e25278a970..7ac0573ef663 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -25,7 +25,8 @@ #include #include #include -#include +#include +#include #include #define HASH_CR 0x00 diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index a713a35dc502..4fd85f31630a 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -31,7 +31,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c index 3d407eebb2ba..da284b0ea1b2 100644 --- a/drivers/crypto/ux500/hash/hash_core.c +++ b/drivers/crypto/ux500/hash/hash_core.c @@ -31,7 +31,8 @@ #include #include -#include +#include +#include #include #include diff --git a/drivers/firmware/efi/embedded-firmware.c b/drivers/firmware/efi/embedded-firmware.c index 21ae0c48232a..f5be8e22305b 100644 --- a/drivers/firmware/efi/embedded-firmware.c +++ b/drivers/firmware/efi/embedded-firmware.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include /* Exported for use by lib/test_firmware.c only */ LIST_HEAD(efi_embedded_fw_list); diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c index 072299b14b8d..47d9268a7e3c 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ipsec/chcr_ipsec.c @@ -51,7 +51,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h index 2d3dfdd2a716..65617752c630 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h @@ -9,7 +9,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/nfc/s3fwrn5/firmware.c b/drivers/nfc/s3fwrn5/firmware.c index ec930ee2c847..5d5ad8307211 100644 --- a/drivers/nfc/s3fwrn5/firmware.c +++ b/drivers/nfc/s3fwrn5/firmware.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include "s3fwrn5.h" #include "firmware.h" diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c index f53bf336c0a2..d70d4be91096 100644 --- a/drivers/tee/tee_core.c +++ b/drivers/tee/tee_core.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include "tee_private.h" #define TEE_NUM_DEVICES 32 diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c index 1fbe6c24d705..cf06ea3870eb 100644 --- a/fs/crypto/fname.c +++ b/fs/crypto/fname.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include "fscrypt_private.h" diff --git a/fs/crypto/hkdf.c b/fs/crypto/hkdf.c index 0cba7928446d..e0ec21055505 100644 --- a/fs/crypto/hkdf.c +++ b/fs/crypto/hkdf.c @@ -10,7 +10,7 @@ */ #include -#include +#include #include "fscrypt_private.h" diff --git a/fs/ubifs/auth.c b/fs/ubifs/auth.c index b93b3cd10bfd..0886d835f597 100644 --- a/fs/ubifs/auth.c +++ b/fs/ubifs/auth.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index e96d99d5145e..6a8f2e3cce6c 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -14,7 +14,7 @@ #define pr_fmt(fmt) "fs-verity: " fmt -#include +#include #include #include diff --git a/include/crypto/hash_info.h b/include/crypto/hash_info.h index eb9d2e368969..dd4f06785049 100644 --- a/include/crypto/hash_info.h +++ b/include/crypto/hash_info.h @@ -8,7 +8,8 @@ #ifndef _CRYPTO_HASH_INFO_H #define _CRYPTO_HASH_INFO_H -#include +#include +#include #include #include diff --git a/include/crypto/sha1.h b/include/crypto/sha1.h new file mode 100644 index 000000000000..044ecea60ac8 --- /dev/null +++ b/include/crypto/sha1.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common values for SHA-1 algorithms + */ + +#ifndef _CRYPTO_SHA1_H +#define _CRYPTO_SHA1_H + +#include + +#define SHA1_DIGEST_SIZE 20 +#define SHA1_BLOCK_SIZE 64 + +#define SHA1_H0 0x67452301UL +#define SHA1_H1 0xefcdab89UL +#define SHA1_H2 0x98badcfeUL +#define SHA1_H3 0x10325476UL +#define SHA1_H4 0xc3d2e1f0UL + +extern const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE]; + +struct sha1_state { + u32 state[SHA1_DIGEST_SIZE / 4]; + u64 count; + u8 buffer[SHA1_BLOCK_SIZE]; +}; + +struct shash_desc; + +extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, + unsigned int len); + +extern int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *hash); + +/* + * An implementation of SHA-1's compression function. Don't use in new code! + * You shouldn't be using SHA-1, and even if you *have* to use SHA-1, this isn't + * the correct way to hash something with SHA-1 (use crypto_shash instead). + */ +#define SHA1_DIGEST_WORDS (SHA1_DIGEST_SIZE / 4) +#define SHA1_WORKSPACE_WORDS 16 +void sha1_init(__u32 *buf); +void sha1_transform(__u32 *digest, const char *data, __u32 *W); + +#endif /* _CRYPTO_SHA1_H */ diff --git a/include/crypto/sha1_base.h b/include/crypto/sha1_base.h index a5d6033efef7..2e0e7c3827d1 100644 --- a/include/crypto/sha1_base.h +++ b/include/crypto/sha1_base.h @@ -9,7 +9,7 @@ #define _CRYPTO_SHA1_BASE_H #include -#include +#include #include #include #include diff --git a/include/crypto/sha.h b/include/crypto/sha2.h similarity index 77% rename from include/crypto/sha.h rename to include/crypto/sha2.h index 4ff3da816630..2838f529f31e 100644 --- a/include/crypto/sha.h +++ b/include/crypto/sha2.h @@ -1,16 +1,13 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Common values for SHA algorithms + * Common values for SHA-2 algorithms */ -#ifndef _CRYPTO_SHA_H -#define _CRYPTO_SHA_H +#ifndef _CRYPTO_SHA2_H +#define _CRYPTO_SHA2_H #include -#define SHA1_DIGEST_SIZE 20 -#define SHA1_BLOCK_SIZE 64 - #define SHA224_DIGEST_SIZE 28 #define SHA224_BLOCK_SIZE 64 @@ -23,12 +20,6 @@ #define SHA512_DIGEST_SIZE 64 #define SHA512_BLOCK_SIZE 128 -#define SHA1_H0 0x67452301UL -#define SHA1_H1 0xefcdab89UL -#define SHA1_H2 0x98badcfeUL -#define SHA1_H3 0x10325476UL -#define SHA1_H4 0xc3d2e1f0UL - #define SHA224_H0 0xc1059ed8UL #define SHA224_H1 0x367cd507UL #define SHA224_H2 0x3070dd17UL @@ -65,8 +56,6 @@ #define SHA512_H6 0x1f83d9abfb41bd6bULL #define SHA512_H7 0x5be0cd19137e2179ULL -extern const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE]; - extern const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE]; extern const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE]; @@ -75,12 +64,6 @@ extern const u8 sha384_zero_message_hash[SHA384_DIGEST_SIZE]; extern const u8 sha512_zero_message_hash[SHA512_DIGEST_SIZE]; -struct sha1_state { - u32 state[SHA1_DIGEST_SIZE / 4]; - u64 count; - u8 buffer[SHA1_BLOCK_SIZE]; -}; - struct sha256_state { u32 state[SHA256_DIGEST_SIZE / 4]; u64 count; @@ -95,12 +78,6 @@ struct sha512_state { struct shash_desc; -extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len); - -extern int crypto_sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *hash); - extern int crypto_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len); @@ -113,16 +90,6 @@ extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data, extern int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *hash); -/* - * An implementation of SHA-1's compression function. Don't use in new code! - * You shouldn't be using SHA-1, and even if you *have* to use SHA-1, this isn't - * the correct way to hash something with SHA-1 (use crypto_shash instead). - */ -#define SHA1_DIGEST_WORDS (SHA1_DIGEST_SIZE / 4) -#define SHA1_WORKSPACE_WORDS 16 -void sha1_init(__u32 *buf); -void sha1_transform(__u32 *digest, const char *data, __u32 *W); - /* * Stand-alone implementation of the SHA256 algorithm. It is designed to * have as little dependencies as possible so it can be used in the @@ -164,4 +131,4 @@ static inline void sha224_init(struct sha256_state *sctx) void sha224_update(struct sha256_state *sctx, const u8 *data, unsigned int len); void sha224_final(struct sha256_state *sctx, u8 *out); -#endif +#endif /* _CRYPTO_SHA2_H */ diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index 93f9fd21cc06..76173c613058 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -9,7 +9,7 @@ #define _CRYPTO_SHA256_BASE_H #include -#include +#include #include #include #include diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h index 93ab73baa38e..b370b3340b16 100644 --- a/include/crypto/sha512_base.h +++ b/include/crypto/sha512_base.h @@ -9,7 +9,7 @@ #define _CRYPTO_SHA512_BASE_H #include -#include +#include #include #include #include diff --git a/include/linux/ccp.h b/include/linux/ccp.h index a5dfbaf2470d..868924dec5a1 100644 --- a/include/linux/ccp.h +++ b/include/linux/ccp.h @@ -15,7 +15,8 @@ #include #include #include -#include +#include +#include struct ccp_device; struct ccp_cmd; diff --git a/include/linux/filter.h b/include/linux/filter.h index 72d62cbc1578..6c00140538b9 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include diff --git a/include/linux/purgatory.h b/include/linux/purgatory.h index b950e961cfa8..d7dc1559427f 100644 --- a/include/linux/purgatory.h +++ b/include/linux/purgatory.h @@ -3,7 +3,7 @@ #define _LINUX_PURGATORY_H #include -#include +#include #include struct kexec_sha_region { diff --git a/kernel/crash_core.c b/kernel/crash_core.c index 106e4500fd53..4fcfe0b70c4e 100644 --- a/kernel/crash_core.c +++ b/kernel/crash_core.c @@ -11,7 +11,7 @@ #include #include -#include +#include /* vmcoreinfo stuff */ unsigned char *vmcoreinfo_data; diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index 8798a8183974..4f8efc278aa7 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -42,7 +42,6 @@ #include #include -#include #include "kexec_internal.h" DEFINE_MUTEX(kexec_mutex); diff --git a/kernel/kexec_file.c b/kernel/kexec_file.c index e21f6b9234f7..b02086d70492 100644 --- a/kernel/kexec_file.c +++ b/kernel/kexec_file.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index cdef37c05972..72a4b0b1df28 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include static const u32 SHA256_K[] = { diff --git a/lib/digsig.c b/lib/digsig.c index e0627c3e53b2..04b5e55ed95f 100644 --- a/lib/digsig.c +++ b/lib/digsig.c @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/lib/sha1.c b/lib/sha1.c index 49257a915bb6..9bd1935a1472 100644 --- a/lib/sha1.c +++ b/lib/sha1.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include /* diff --git a/net/ipv6/seg6_hmac.c b/net/ipv6/seg6_hmac.c index 85dddfe3a2c6..687d95dce085 100644 --- a/net/ipv6/seg6_hmac.c +++ b/net/ipv6/seg6_hmac.c @@ -35,7 +35,6 @@ #include #include -#include #include #include #include diff --git a/net/mptcp/crypto.c b/net/mptcp/crypto.c index 05d398d3fde4..b472dc149856 100644 --- a/net/mptcp/crypto.c +++ b/net/mptcp/crypto.c @@ -21,7 +21,7 @@ */ #include -#include +#include #include #include "protocol.h" diff --git a/net/mptcp/options.c b/net/mptcp/options.c index a044dd43411d..90cd52df99a6 100644 --- a/net/mptcp/options.c +++ b/net/mptcp/options.c @@ -7,7 +7,7 @@ #define pr_fmt(fmt) "MPTCP: " fmt #include -#include +#include #include #include #include "protocol.h" diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index ac4a1fe3550b..b229ae914d76 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index 413c803c5208..547425c20e11 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include diff --git a/security/keys/encrypted-keys/encrypted.c b/security/keys/encrypted-keys/encrypted.c index 192e531c146f..87432b35d771 100644 --- a/security/keys/encrypted-keys/encrypted.c +++ b/security/keys/encrypted-keys/encrypted.c @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include "encrypted.h" diff --git a/security/keys/trusted-keys/trusted_tpm1.c b/security/keys/trusted-keys/trusted_tpm1.c index b9fe02e5f84f..74d82093cbaa 100644 --- a/security/keys/trusted-keys/trusted_tpm1.c +++ b/security/keys/trusted-keys/trusted_tpm1.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/sound/soc/codecs/cros_ec_codec.c b/sound/soc/codecs/cros_ec_codec.c index 28f039adfa13..58894bf47514 100644 --- a/sound/soc/codecs/cros_ec_codec.c +++ b/sound/soc/codecs/cros_ec_codec.c @@ -8,7 +8,7 @@ * EC for audio function. */ -#include +#include #include #include #include From 1201581c57925b8bc2cba8628b61add3d16d4615 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 13 Nov 2020 17:12:59 +1100 Subject: [PATCH 221/360] crypto: lib/curve25519 - Move selftest prototype into header file This patch moves the curve25519_selftest into curve25519.h so we don't get a warning from gcc complaining about a missing prototype. Reported-by: kernel test robot Signed-off-by: Herbert Xu --- include/crypto/curve25519.h | 2 ++ lib/crypto/curve25519.c | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/crypto/curve25519.h b/include/crypto/curve25519.h index 4e6dc840b159..ece6a9b5fafc 100644 --- a/include/crypto/curve25519.h +++ b/include/crypto/curve25519.h @@ -28,6 +28,8 @@ void curve25519_arch(u8 out[CURVE25519_KEY_SIZE], void curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE]); +bool curve25519_selftest(void); + static inline bool __must_check curve25519(u8 mypublic[CURVE25519_KEY_SIZE], const u8 secret[CURVE25519_KEY_SIZE], diff --git a/lib/crypto/curve25519.c b/lib/crypto/curve25519.c index 288a62cd29b2..fb29739e8c29 100644 --- a/lib/crypto/curve25519.c +++ b/lib/crypto/curve25519.c @@ -13,8 +13,6 @@ #include #include -bool curve25519_selftest(void); - static int __init mod_init(void) { if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && From 1dc440355e472a60a98cb4ec9aa5ec56267a96fc Mon Sep 17 00:00:00 2001 From: Yang Shen Date: Fri, 13 Nov 2020 17:32:35 +0800 Subject: [PATCH 222/360] crypto: hisilicon/zip - add a work_queue for zip irq The patch 'irqchip/gic-v3-its: Balance initial LPI affinity across CPUs' set the IRQ to an uncentain CPU. If an IRQ is bound to the CPU used by the thread which is sending request, the throughput will be just half. So allocate a 'work_queue' and set as 'WQ_UNBOUND' to do the back half work on some different CPUS. Signed-off-by: Yang Shen Reviewed-by: Zaibo Xu Reviewed-by: Zhou Wang Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/zip/zip_main.c | 26 ++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 3d1524b63ea8..4fb5a32bf830 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -747,6 +747,8 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) { + int ret; + qm->pdev = pdev; qm->ver = pdev->revision; qm->algs = "zlib\ngzip"; @@ -772,7 +774,25 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) qm->qp_num = HZIP_QUEUE_NUM_V1 - HZIP_PF_DEF_Q_NUM; } - return hisi_qm_init(qm); + qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM | + WQ_UNBOUND, num_online_cpus(), + pci_name(qm->pdev)); + if (!qm->wq) { + pci_err(qm->pdev, "fail to alloc workqueue\n"); + return -ENOMEM; + } + + ret = hisi_qm_init(qm); + if (ret) + destroy_workqueue(qm->wq); + + return ret; +} + +static void hisi_zip_qm_uninit(struct hisi_qm *qm) +{ + hisi_qm_uninit(qm); + destroy_workqueue(qm->wq); } static int hisi_zip_probe_init(struct hisi_zip *hisi_zip) @@ -854,7 +874,7 @@ err_dev_err_uninit: hisi_qm_dev_err_uninit(qm); err_qm_uninit: - hisi_qm_uninit(qm); + hisi_zip_qm_uninit(qm); return ret; } @@ -872,7 +892,7 @@ static void hisi_zip_remove(struct pci_dev *pdev) hisi_zip_debugfs_exit(qm); hisi_qm_stop(qm, QM_NORMAL); hisi_qm_dev_err_uninit(qm); - hisi_qm_uninit(qm); + hisi_zip_qm_uninit(qm); } static const struct pci_error_handlers hisi_zip_err_handler = { From ff8107200367f4abe0e5bce66a245e8d0f2d229e Mon Sep 17 00:00:00 2001 From: Zhang Qilong Date: Fri, 13 Nov 2020 21:17:28 +0800 Subject: [PATCH 223/360] crypto: omap-aes - Fix PM disable depth imbalance in omap_aes_probe The pm_runtime_enable will increase power disable depth. Thus a pairing decrement is needed on the error handling path to keep it balanced according to context. Fixes: f7b2b5dd6a62a ("crypto: omap-aes - add error check for pm_runtime_get_sync") Signed-off-by: Zhang Qilong Signed-off-by: Herbert Xu --- drivers/crypto/omap-aes.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 70ea5784a024..a45bdcf3026d 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -1138,7 +1138,7 @@ static int omap_aes_probe(struct platform_device *pdev) if (err < 0) { dev_err(dev, "%s: failed to get_sync(%d)\n", __func__, err); - goto err_res; + goto err_pm_disable; } omap_aes_dma_stop(dd); @@ -1247,6 +1247,7 @@ err_engine: omap_aes_dma_cleanup(dd); err_irq: tasklet_kill(&dd->done_task); +err_pm_disable: pm_runtime_disable(dev); err_res: dd = NULL; From e73916f20829be09932326ab0c9627b29df553e8 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Fri, 13 Nov 2020 16:46:41 +0000 Subject: [PATCH 224/360] crypto: qat - target fw images to specific AEs Introduce support for devices that require multiple firmware images. If a device requires more than a firmware image to operate, load the image to the appropriate Acceleration Engine (AE). Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- .../crypto/qat/qat_common/adf_accel_devices.h | 3 + .../crypto/qat/qat_common/adf_accel_engine.c | 58 +++++++++++++++++-- 2 files changed, 56 insertions(+), 5 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 5694422ec66c..1484f10dbfdf 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -175,6 +175,9 @@ struct adf_hw_device_data { void (*enable_ints)(struct adf_accel_dev *accel_dev); int (*enable_vf2pf_comms)(struct adf_accel_dev *accel_dev); void (*reset_device)(struct adf_accel_dev *accel_dev); + char *(*uof_get_name)(u32 obj_num); + u32 (*uof_get_num_objs)(void); + u32 (*uof_get_ae_mask)(u32 obj_num); struct adf_hw_csr_ops csr_ops; const char *fw_name; const char *fw_mmp_name; diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c index 08aaaf2b4659..ca4eae8cdd0b 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_engine.c +++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c @@ -7,12 +7,55 @@ #include "adf_common_drv.h" #include "icp_qat_uclo.h" +static int adf_ae_fw_load_images(struct adf_accel_dev *accel_dev, void *fw_addr, + u32 fw_size) +{ + struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct icp_qat_fw_loader_handle *loader; + char *obj_name; + u32 num_objs; + u32 ae_mask; + int i; + + loader = loader_data->fw_loader; + num_objs = hw_device->uof_get_num_objs(); + + for (i = 0; i < num_objs; i++) { + obj_name = hw_device->uof_get_name(i); + ae_mask = hw_device->uof_get_ae_mask(i); + + if (qat_uclo_set_cfg_ae_mask(loader, ae_mask)) { + dev_err(&GET_DEV(accel_dev), + "Invalid mask for UOF image\n"); + goto out_err; + } + if (qat_uclo_map_obj(loader, fw_addr, fw_size, obj_name)) { + dev_err(&GET_DEV(accel_dev), + "Failed to map UOF firmware\n"); + goto out_err; + } + if (qat_uclo_wr_all_uimage(loader)) { + dev_err(&GET_DEV(accel_dev), + "Failed to load UOF firmware\n"); + goto out_err; + } + qat_uclo_del_obj(loader); + } + + return 0; + +out_err: + adf_ae_fw_release(accel_dev); + return -EFAULT; +} + int adf_ae_fw_load(struct adf_accel_dev *accel_dev) { struct adf_fw_loader_data *loader_data = accel_dev->fw_loader; struct adf_hw_device_data *hw_device = accel_dev->hw_device; - void *uof_addr, *mmp_addr; - u32 uof_size, mmp_size; + void *fw_addr, *mmp_addr; + u32 fw_size, mmp_size; if (!hw_device->fw_name) return 0; @@ -30,15 +73,20 @@ int adf_ae_fw_load(struct adf_accel_dev *accel_dev) goto out_err; } - uof_size = loader_data->uof_fw->size; - uof_addr = (void *)loader_data->uof_fw->data; + fw_size = loader_data->uof_fw->size; + fw_addr = (void *)loader_data->uof_fw->data; mmp_size = loader_data->mmp_fw->size; mmp_addr = (void *)loader_data->mmp_fw->data; + if (qat_uclo_wr_mimage(loader_data->fw_loader, mmp_addr, mmp_size)) { dev_err(&GET_DEV(accel_dev), "Failed to load MMP\n"); goto out_err; } - if (qat_uclo_map_obj(loader_data->fw_loader, uof_addr, uof_size, NULL)) { + + if (hw_device->uof_get_num_objs) + return adf_ae_fw_load_images(accel_dev, fw_addr, fw_size); + + if (qat_uclo_map_obj(loader_data->fw_loader, fw_addr, fw_size, NULL)) { dev_err(&GET_DEV(accel_dev), "Failed to map FW\n"); goto out_err; } From 6644f49e7b71e13bcc5498d9fea8862c5ce96388 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Fri, 13 Nov 2020 16:46:42 +0000 Subject: [PATCH 225/360] crypto: qat - add hook to initialize vector routing table Add an hook to initialize the vector routing table with the default values before MSIx is enabled. The new function set_msix_rttable() is called only if present in the struct adf_hw_device_data of the device. This is to allow for QAT devices that do not support that functionality. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_accel_devices.h | 1 + drivers/crypto/qat/qat_common/adf_isr.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 1484f10dbfdf..26164d71f1d6 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -175,6 +175,7 @@ struct adf_hw_device_data { void (*enable_ints)(struct adf_accel_dev *accel_dev); int (*enable_vf2pf_comms)(struct adf_accel_dev *accel_dev); void (*reset_device)(struct adf_accel_dev *accel_dev); + void (*set_msix_rttable)(struct adf_accel_dev *accel_dev); char *(*uof_get_name)(u32 obj_num); u32 (*uof_get_num_objs)(void); u32 (*uof_get_ae_mask)(u32 obj_num); diff --git a/drivers/crypto/qat/qat_common/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index 5444f0ea0a1d..c45853463530 100644 --- a/drivers/crypto/qat/qat_common/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -21,6 +21,9 @@ static int adf_enable_msix(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; u32 msix_num_entries = 1; + if (hw_data->set_msix_rttable) + hw_data->set_msix_rttable(accel_dev); + /* If SR-IOV is disabled, add entries for each bank */ if (!accel_dev->pf.vf_info) { int i; From 8c8268166e83425243789c5781c92e7fa33b703b Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Fri, 13 Nov 2020 16:46:43 +0000 Subject: [PATCH 226/360] crypto: qat - add qat_4xxx driver Add support for QAT 4xxx devices. Signed-off-by: Giovanni Cabiddu Reviewed-by: Fiona Trahe Signed-off-by: Herbert Xu --- drivers/crypto/qat/Kconfig | 11 + drivers/crypto/qat/Makefile | 1 + drivers/crypto/qat/qat_4xxx/Makefile | 4 + .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 218 ++++++++++++ .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.h | 75 ++++ drivers/crypto/qat/qat_4xxx/adf_drv.c | 320 ++++++++++++++++++ drivers/crypto/qat/qat_common/Makefile | 1 + .../crypto/qat/qat_common/adf_accel_devices.h | 1 + .../crypto/qat/qat_common/adf_cfg_common.h | 3 +- .../crypto/qat/qat_common/adf_gen4_hw_data.c | 101 ++++++ .../crypto/qat/qat_common/adf_gen4_hw_data.h | 99 ++++++ 11 files changed, 833 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/qat/qat_4xxx/Makefile create mode 100644 drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c create mode 100644 drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h create mode 100644 drivers/crypto/qat/qat_4xxx/adf_drv.c create mode 100644 drivers/crypto/qat/qat_common/adf_gen4_hw_data.c create mode 100644 drivers/crypto/qat/qat_common/adf_gen4_hw_data.h diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig index 2006322345de..beb379b23dc3 100644 --- a/drivers/crypto/qat/Kconfig +++ b/drivers/crypto/qat/Kconfig @@ -46,6 +46,17 @@ config CRYPTO_DEV_QAT_C62X To compile this as a module, choose M here: the module will be called qat_c62x. +config CRYPTO_DEV_QAT_4XXX + tristate "Support for Intel(R) QAT_4XXX" + depends on X86 && PCI + select CRYPTO_DEV_QAT + help + Support for Intel(R) QuickAssist Technology QAT_4xxx + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_4xxx. + config CRYPTO_DEV_QAT_DH895xCCVF tristate "Support for Intel(R) DH895xCC Virtual Function" depends on X86 && PCI diff --git a/drivers/crypto/qat/Makefile b/drivers/crypto/qat/Makefile index 7dd15e751d02..258c8a626ce0 100644 --- a/drivers/crypto/qat/Makefile +++ b/drivers/crypto/qat/Makefile @@ -3,6 +3,7 @@ obj-$(CONFIG_CRYPTO_DEV_QAT) += qat_common/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx/ obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x/ +obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf/ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf/ obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf/ diff --git a/drivers/crypto/qat/qat_4xxx/Makefile b/drivers/crypto/qat/qat_4xxx/Makefile new file mode 100644 index 000000000000..ff9c8b5897ea --- /dev/null +++ b/drivers/crypto/qat/qat_4xxx/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +ccflags-y := -I $(srctree)/$(src)/../qat_common +obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx.o +qat_4xxx-objs := adf_drv.o adf_4xxx_hw_data.o diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c new file mode 100644 index 000000000000..e7a7c1e3da28 --- /dev/null +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ +#include +#include +#include +#include +#include "adf_4xxx_hw_data.h" + +struct adf_fw_config { + u32 ae_mask; + char *obj_name; +}; + +static struct adf_fw_config adf_4xxx_fw_config[] = { + {0xF0, ADF_4XXX_SYM_OBJ}, + {0xF, ADF_4XXX_ASYM_OBJ}, + {0x100, ADF_4XXX_ADMIN_OBJ}, +}; + +/* Worker thread to service arbiter mappings */ +static u32 thrd_to_arb_map[] = { + 0x5555555, 0x5555555, 0x5555555, 0x5555555, + 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, 0xAAAAAAA, + 0x0 +}; + +static struct adf_hw_device_class adf_4xxx_class = { + .name = ADF_4XXX_DEVICE_NAME, + .type = DEV_4XXX, + .instances = 0, +}; + +static u32 get_accel_mask(struct adf_hw_device_data *self) +{ + return ADF_4XXX_ACCELERATORS_MASK; +} + +static u32 get_ae_mask(struct adf_hw_device_data *self) +{ + u32 me_disable = self->fuses; + + return ~me_disable & ADF_4XXX_ACCELENGINES_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_4XXX_MAX_ACCELERATORS; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + if (!self || !self->ae_mask) + return 0; + + return hweight32(self->ae_mask); +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_4XXX_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_4XXX_ETR_BAR; +} + +static u32 get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_4XXX_SRAM_BAR; +} + +/* + * The vector routing table is used to select the MSI-X entry to use for each + * interrupt source. + * The first ADF_4XXX_ETR_MAX_BANKS entries correspond to ring interrupts. + * The final entry corresponds to VF2PF or error interrupts. + * This vector table could be used to configure one MSI-X entry to be shared + * between multiple interrupt sources. + * + * The default routing is set to have a one to one correspondence between the + * interrupt source and the MSI-X entry used. + */ +static void set_msix_default_rttable(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr; + int i; + + csr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; + for (i = 0; i <= ADF_4XXX_ETR_MAX_BANKS; i++) + ADF_CSR_WR(csr, ADF_4XXX_MSIX_RTTABLE_OFFSET(i), i); +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_1; +} + +static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, + u32 const **arb_map_config) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + unsigned long ae_mask = hw_device->ae_mask; + int i; + + for_each_clear_bit(i, &ae_mask, ADF_4XXX_MAX_ACCELENGINES) + thrd_to_arb_map[i] = 0; + + *arb_map_config = thrd_to_arb_map; +} + +static void get_arb_info(struct arb_info *arb_info) +{ + arb_info->arb_cfg = ADF_4XXX_ARB_CONFIG; + arb_info->arb_offset = ADF_4XXX_ARB_OFFSET; + arb_info->wt2sam_offset = ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET; +} + +static void get_admin_info(struct admin_info *admin_csrs_info) +{ + admin_csrs_info->mailbox_offset = ADF_4XXX_MAILBOX_BASE_OFFSET; + admin_csrs_info->admin_msg_ur = ADF_4XXX_ADMINMSGUR_OFFSET; + admin_csrs_info->admin_msg_lr = ADF_4XXX_ADMINMSGLR_OFFSET; +} + +static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) +{ + struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR]; + void __iomem *csr = misc_bar->virt_addr; + + /* Enable all in errsou3 except VFLR notification on host */ + ADF_CSR_WR(csr, ADF_4XXX_ERRMSK3, ADF_4XXX_VFLNOTIFY); +} + +static void adf_enable_ints(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr; + + addr = (&GET_BARS(accel_dev)[ADF_4XXX_PMISC_BAR])->virt_addr; + + /* Enable bundle interrupts */ + ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET, 0); + ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET, 0); + + /* Enable misc interrupts */ + ADF_CSR_WR(addr, ADF_4XXX_SMIAPF_MASK_OFFSET, 0); +} + +static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static u32 uof_get_num_objs(void) +{ + return ARRAY_SIZE(adf_4xxx_fw_config); +} + +static char *uof_get_name(u32 obj_num) +{ + return adf_4xxx_fw_config[obj_num].obj_name; +} + +static u32 uof_get_ae_mask(u32 obj_num) +{ + return adf_4xxx_fw_config[obj_num].ae_mask; +} + +void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &adf_4xxx_class; + hw_data->instance_id = adf_4xxx_class.instances++; + hw_data->num_banks = ADF_4XXX_ETR_MAX_BANKS; + hw_data->num_rings_per_bank = ADF_4XXX_NUM_RINGS_PER_BANK; + hw_data->num_accel = ADF_4XXX_MAX_ACCELERATORS; + hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES; + hw_data->num_logical_accel = 1; + hw_data->tx_rx_gap = ADF_4XXX_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_4XXX_TX_RINGS_MASK; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = adf_enable_error_correction; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_sram_bar_id = get_sram_bar_id; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_arb_info = get_arb_info; + hw_data->get_admin_info = get_admin_info; + hw_data->get_sku = get_sku; + hw_data->fw_name = ADF_4XXX_FW; + hw_data->fw_mmp_name = ADF_4XXX_MMP; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->disable_iov = adf_disable_sriov; + hw_data->send_admin_init = adf_send_admin_init; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = adf_enable_ints; + hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; + hw_data->reset_device = adf_reset_flr; + hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK; + hw_data->uof_get_num_objs = uof_get_num_objs; + hw_data->uof_get_name = uof_get_name; + hw_data->uof_get_ae_mask = uof_get_ae_mask; + hw_data->set_msix_rttable = set_msix_default_rttable; + + adf_gen4_init_hw_csr_ops(&hw_data->csr_ops); +} + +void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h new file mode 100644 index 000000000000..cdde0be886bf --- /dev/null +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2014 - 2020 Intel Corporation */ +#ifndef ADF_4XXX_HW_DATA_H_ +#define ADF_4XXX_HW_DATA_H_ + +#include + +/* PCIe configuration space */ +#define ADF_4XXX_SRAM_BAR 0 +#define ADF_4XXX_PMISC_BAR 1 +#define ADF_4XXX_ETR_BAR 2 +#define ADF_4XXX_RX_RINGS_OFFSET 1 +#define ADF_4XXX_TX_RINGS_MASK 0x1 +#define ADF_4XXX_MAX_ACCELERATORS 1 +#define ADF_4XXX_MAX_ACCELENGINES 9 +#define ADF_4XXX_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) + +/* Physical function fuses */ +#define ADF_4XXX_FUSECTL0_OFFSET (0x2C8) +#define ADF_4XXX_FUSECTL1_OFFSET (0x2CC) +#define ADF_4XXX_FUSECTL2_OFFSET (0x2D0) +#define ADF_4XXX_FUSECTL3_OFFSET (0x2D4) +#define ADF_4XXX_FUSECTL4_OFFSET (0x2D8) +#define ADF_4XXX_FUSECTL5_OFFSET (0x2DC) + +#define ADF_4XXX_ACCELERATORS_MASK (0x1) +#define ADF_4XXX_ACCELENGINES_MASK (0x1FF) +#define ADF_4XXX_ADMIN_AE_MASK (0x100) + +#define ADF_4XXX_ETR_MAX_BANKS 64 + +/* MSIX interrupt */ +#define ADF_4XXX_SMIAPF_RP_X0_MASK_OFFSET (0x41A040) +#define ADF_4XXX_SMIAPF_RP_X1_MASK_OFFSET (0x41A044) +#define ADF_4XXX_SMIAPF_MASK_OFFSET (0x41A084) +#define ADF_4XXX_MSIX_RTTABLE_OFFSET(i) (0x409000 + ((i) * 0x04)) + +/* Bank and ring configuration */ +#define ADF_4XXX_NUM_RINGS_PER_BANK 2 + +/* Error source registers */ +#define ADF_4XXX_ERRSOU0 (0x41A200) +#define ADF_4XXX_ERRSOU1 (0x41A204) +#define ADF_4XXX_ERRSOU2 (0x41A208) +#define ADF_4XXX_ERRSOU3 (0x41A20C) + +/* Error source mask registers */ +#define ADF_4XXX_ERRMSK0 (0x41A210) +#define ADF_4XXX_ERRMSK1 (0x41A214) +#define ADF_4XXX_ERRMSK2 (0x41A218) +#define ADF_4XXX_ERRMSK3 (0x41A21C) + +#define ADF_4XXX_VFLNOTIFY BIT(7) + +/* Arbiter configuration */ +#define ADF_4XXX_ARB_CONFIG (BIT(31) | BIT(6) | BIT(0)) +#define ADF_4XXX_ARB_OFFSET (0x0) +#define ADF_4XXX_ARB_WRK_2_SER_MAP_OFFSET (0x400) + +/* Admin Interface Reg Offset */ +#define ADF_4XXX_ADMINMSGUR_OFFSET (0x500574) +#define ADF_4XXX_ADMINMSGLR_OFFSET (0x500578) +#define ADF_4XXX_MAILBOX_BASE_OFFSET (0x600970) + +/* Firmware Binaries */ +#define ADF_4XXX_FW "qat_4xxx.bin" +#define ADF_4XXX_MMP "qat_4xxx_mmp.bin" +#define ADF_4XXX_SYM_OBJ "qat_4xxx_sym.bin" +#define ADF_4XXX_ASYM_OBJ "qat_4xxx_asym.bin" +#define ADF_4XXX_ADMIN_OBJ "qat_4xxx_admin.bin" + +void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data); + +#endif diff --git a/drivers/crypto/qat/qat_4xxx/adf_drv.c b/drivers/crypto/qat/qat_4xxx/adf_drv.c new file mode 100644 index 000000000000..de5a955f406a --- /dev/null +++ b/drivers/crypto/qat/qat_4xxx/adf_drv.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ +#include +#include +#include + +#include +#include +#include + +#include "adf_4xxx_hw_data.h" +#include "qat_crypto.h" +#include "adf_transport_access_macros.h" + +static const struct pci_device_id adf_pci_tbl[] = { + { PCI_VDEVICE(INTEL, ADF_4XXX_PCI_DEVICE_ID), }, + { } +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + if (accel_dev->hw_device) { + adf_clean_hw_data_4xxx(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_cfg_dev_remove(accel_dev); + debugfs_remove(accel_dev->debugfs_dir); + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev) +{ + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + int banks = GET_MAX_BANKS(accel_dev); + int cpus = num_online_cpus(); + unsigned long bank, val; + int instances; + int ret; + int i; + + if (adf_hw_dev_has_crypto(accel_dev)) + instances = min(cpus, banks / 2); + else + instances = 0; + + ret = adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC); + if (ret) + goto err; + + ret = adf_cfg_section_add(accel_dev, "Accelerator0"); + if (ret) + goto err; + + for (i = 0; i < instances; i++) { + val = i; + bank = i * 2; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &bank, ADF_DEC); + if (ret) + goto err; + + bank += 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_BANK_NUM, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &bank, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, + i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); + val = 128; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 512; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = 1; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, &val, ADF_DEC); + if (ret) + goto err; + + val = ADF_COALESCING_DEF_TIME; + snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); + ret = adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, &val, ADF_DEC); + if (ret) + goto err; + } + + val = i; + ret = adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, ADF_NUM_CY, + &val, ADF_DEC); + if (ret) + goto err; + + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + return 0; +err: + dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n"); + return ret; +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + char name[ADF_DEVICE_NAME_LENGTH]; + unsigned int i, bar_nr; + unsigned long bar_mask; + struct adf_bar *bar; + int ret; + + if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { + /* + * If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. + */ + dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); + return -EINVAL; + } + + accel_dev = devm_kzalloc(&pdev->dev, sizeof(*accel_dev), GFP_KERNEL); + if (!accel_dev) + return -ENOMEM; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* + * Add accel device to accel table + * This should be called before adf_cleanup_accel is called + */ + if (adf_devmgr_add_dev(accel_dev, NULL)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + return -EFAULT; + } + + accel_dev->owner = THIS_MODULE; + /* Allocate and initialise device hardware meta-data structure */ + hw_data = devm_kzalloc(&pdev->dev, sizeof(*hw_data), GFP_KERNEL); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + + accel_dev->hw_device = hw_data; + adf_init_hw_data_4xxx(accel_dev->hw_device); + + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_4XXX_FUSECTL4_OFFSET, &hw_data->fuses); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + /* If the device has no acceleration engines then ignore it */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + (~hw_data->ae_mask & 0x01)) { + dev_err(&pdev->dev, "No acceleration units found.\n"); + ret = -EFAULT; + goto out_err; + } + + /* Create dev top level debugfs entry */ + snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, + hw_data->dev_class->name, pci_name(pdev)); + + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* Enable PCI device */ + ret = pcim_enable_device(pdev); + if (ret) { + dev_err(&pdev->dev, "Can't enable PCI device.\n"); + goto out_err; + } + + /* Set DMA identifier */ + if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { + dev_err(&pdev->dev, "No usable DMA configuration.\n"); + ret = -EFAULT; + goto out_err; + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + } + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + } + + /* Find and map all the device's BARS */ + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_4XXX_BAR_MASK; + + ret = pcim_iomap_regions_request_all(pdev, bar_mask, pci_name(pdev)); + if (ret) { + dev_err(&pdev->dev, "Failed to map pci regions.\n"); + goto out_err; + } + + i = 0; + for_each_set_bit(bar_nr, &bar_mask, PCI_STD_NUM_BARS) { + bar = &accel_pci_dev->pci_bars[i++]; + bar->virt_addr = pcim_iomap_table(pdev)[bar_nr]; + } + + pci_set_master(pdev); + + if (adf_enable_aer(accel_dev)) { + dev_err(&pdev->dev, "Failed to enable aer.\n"); + ret = -EFAULT; + goto out_err; + } + + if (pci_save_state(pdev)) { + dev_err(&pdev->dev, "Failed to save pci state.\n"); + ret = -ENOMEM; + goto out_err_disable_aer; + } + + ret = adf_crypto_dev_config(accel_dev); + if (ret) + goto out_err_disable_aer; + + ret = adf_dev_init(accel_dev); + if (ret) + goto out_err_dev_shutdown; + + ret = adf_dev_start(accel_dev); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_stop(accel_dev); +out_err_dev_shutdown: + adf_dev_shutdown(accel_dev); +out_err_disable_aer: + adf_disable_aer(accel_dev); +out_err: + adf_cleanup_accel(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + adf_dev_stop(accel_dev); + adf_dev_shutdown(accel_dev); + adf_disable_aer(accel_dev); + adf_cleanup_accel(accel_dev); +} + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_4XXX_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .sriov_configure = adf_sriov_configure, +}; + +module_pci_driver(adf_driver); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_FIRMWARE(ADF_4XXX_FW); +MODULE_FIRMWARE(ADF_4XXX_MMP); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_VERSION(ADF_DRV_VERSION); +MODULE_SOFTDEP("pre: crypto-intel_qat"); diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 25d28516dcdd..9c57abdf56b7 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -11,6 +11,7 @@ intel_qat-objs := adf_cfg.o \ adf_admin.o \ adf_hw_arbiter.o \ adf_gen2_hw_data.o \ + adf_gen4_hw_data.o \ qat_crypto.o \ qat_algs.o \ qat_asym_algs.o \ diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 26164d71f1d6..c46a5805b294 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -15,6 +15,7 @@ #define ADF_C62XVF_DEVICE_NAME "c6xxvf" #define ADF_C3XXX_DEVICE_NAME "c3xxx" #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" +#define ADF_4XXX_DEVICE_NAME "4xxx" #define ADF_4XXX_PCI_DEVICE_ID 0x4940 #define ADF_4XXXIOV_PCI_DEVICE_ID 0x4941 #define ADF_ERRSOU3 (0x3A000 + 0x0C) diff --git a/drivers/crypto/qat/qat_common/adf_cfg_common.h b/drivers/crypto/qat/qat_common/adf_cfg_common.h index 1ef46ccfba47..4fabb70b1f18 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg_common.h +++ b/drivers/crypto/qat/qat_common/adf_cfg_common.h @@ -32,7 +32,8 @@ enum adf_device_type { DEV_C62X, DEV_C62XVF, DEV_C3XXX, - DEV_C3XXXVF + DEV_C3XXXVF, + DEV_4XXX, }; struct adf_dev_status_info { diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c new file mode 100644 index 000000000000..b72ff58e0bc7 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) +/* Copyright(c) 2020 Intel Corporation */ +#include "adf_accel_devices.h" +#include "adf_gen4_hw_data.h" + +static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size) +{ + return BUILD_RING_BASE_ADDR(addr, size); +} + +static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_HEAD(csr_base_addr, bank, ring); +} + +static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring) +{ + return READ_CSR_RING_TAIL(csr_base_addr, bank, ring); +} + +static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value); +} + +static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank) +{ + return READ_CSR_E_STAT(csr_base_addr, bank); +} + +static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, u32 ring, + u32 value) +{ + WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value); +} + +static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring, + dma_addr_t addr) +{ + WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr); +} + +static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG(csr_base_addr, bank, value); +} + +static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank) +{ + WRITE_CSR_INT_SRCSEL(csr_base_addr, bank); +} + +static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, u32 value) +{ + WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value); +} + +static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value); +} + +static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value); +} + +static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank, + u32 value) +{ + WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value); +} + +void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops) +{ + csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr; + csr_ops->read_csr_ring_head = read_csr_ring_head; + csr_ops->write_csr_ring_head = write_csr_ring_head; + csr_ops->read_csr_ring_tail = read_csr_ring_tail; + csr_ops->write_csr_ring_tail = write_csr_ring_tail; + csr_ops->read_csr_e_stat = read_csr_e_stat; + csr_ops->write_csr_ring_config = write_csr_ring_config; + csr_ops->write_csr_ring_base = write_csr_ring_base; + csr_ops->write_csr_int_flag = write_csr_int_flag; + csr_ops->write_csr_int_srcsel = write_csr_int_srcsel; + csr_ops->write_csr_int_col_en = write_csr_int_col_en; + csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl; + csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col; + csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_hw_csr_ops); diff --git a/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h new file mode 100644 index 000000000000..8ab62b2ac311 --- /dev/null +++ b/drivers/crypto/qat/qat_common/adf_gen4_hw_data.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */ +/* Copyright(c) 2020 Intel Corporation */ +#ifndef ADF_GEN4_HW_CSR_DATA_H_ +#define ADF_GEN4_HW_CSR_DATA_H_ + +#include "adf_accel_devices.h" + +/* Transport access */ +#define ADF_BANK_INT_SRC_SEL_MASK 0x44UL +#define ADF_RING_CSR_RING_CONFIG 0x1000 +#define ADF_RING_CSR_RING_LBASE 0x1040 +#define ADF_RING_CSR_RING_UBASE 0x1080 +#define ADF_RING_CSR_RING_HEAD 0x0C0 +#define ADF_RING_CSR_RING_TAIL 0x100 +#define ADF_RING_CSR_E_STAT 0x14C +#define ADF_RING_CSR_INT_FLAG 0x170 +#define ADF_RING_CSR_INT_SRCSEL 0x174 +#define ADF_RING_CSR_INT_COL_CTL 0x180 +#define ADF_RING_CSR_INT_FLAG_AND_COL 0x184 +#define ADF_RING_CSR_INT_COL_CTL_ENABLE 0x80000000 +#define ADF_RING_CSR_INT_COL_EN 0x17C +#define ADF_RING_CSR_ADDR_OFFSET 0x100000 +#define ADF_RING_BUNDLE_SIZE 0x2000 + +#define BUILD_RING_BASE_ADDR(addr, size) \ + ((((addr) >> 6) & (GENMASK_ULL(63, 0) << (size))) << 6) +#define READ_CSR_RING_HEAD(csr_base_addr, bank, ring) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2)) +#define READ_CSR_RING_TAIL(csr_base_addr, bank, ring) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2)) +#define READ_CSR_E_STAT(csr_base_addr, bank) \ + ADF_CSR_RD((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + ADF_RING_CSR_E_STAT) +#define WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_CONFIG + ((ring) << 2), value) +#define WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, value) \ +do { \ + void __iomem *_csr_base_addr = csr_base_addr; \ + u32 _bank = bank; \ + u32 _ring = ring; \ + dma_addr_t _value = value; \ + u32 l_base = 0, u_base = 0; \ + l_base = lower_32_bits(_value); \ + u_base = upper_32_bits(_value); \ + ADF_CSR_WR((_csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (_bank) + \ + ADF_RING_CSR_RING_LBASE + ((_ring) << 2), l_base); \ + ADF_CSR_WR((_csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (_bank) + \ + ADF_RING_CSR_RING_UBASE + ((_ring) << 2), u_base); \ +} while (0) + +#define WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_HEAD + ((ring) << 2), value) +#define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_TAIL + ((ring) << 2), value) +#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG, (value)) +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_SRCSEL, ADF_BANK_INT_SRC_SEL_MASK) +#define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_COL_EN, (value)) +#define WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_COL_CTL, \ + ADF_RING_CSR_INT_COL_CTL_ENABLE | (value)) +#define WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_INT_FLAG_AND_COL, (value)) + +/* Arbiter configuration */ +#define ADF_RING_CSR_RING_SRV_ARB_EN 0x19C + +#define WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value) \ + ADF_CSR_WR((csr_base_addr) + ADF_RING_CSR_ADDR_OFFSET, \ + ADF_RING_BUNDLE_SIZE * (bank) + \ + ADF_RING_CSR_RING_SRV_ARB_EN, (value)) + +void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops); + +#endif From 2e0e386af88429ec28fb9ab4531096a370578e97 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Sat, 14 Nov 2020 12:12:27 +0000 Subject: [PATCH 227/360] crypto: kconfig - fix a couple of spelling mistakes There are a couple of spelling mistakes in two crypto Kconfig files. Fix these. Signed-off-by: Colin Ian King Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 2 +- drivers/crypto/allwinner/Kconfig | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 37da0c070a88..129bbdc4993b 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -648,7 +648,7 @@ choice default CRYPTO_DEV_QCE_ENABLE_ALL depends on CRYPTO_DEV_QCE help - This option allows to choose whether to build support for all algorihtms + This option allows to choose whether to build support for all algorithms (default), hashes-only, or skciphers-only. The QCE engine does not appear to scale as well as the CPU to handle diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig index 0cdfe0e8cc66..180c8a9db819 100644 --- a/drivers/crypto/allwinner/Kconfig +++ b/drivers/crypto/allwinner/Kconfig @@ -43,7 +43,7 @@ config CRYPTO_DEV_SUN8I_CE depends on CRYPTO_DEV_ALLWINNER depends on PM help - Select y here to have support for the crypto Engine availlable on + Select y here to have support for the crypto Engine available on Allwinner SoC H2+, H3, H5, H6, R40 and A64. The Crypto Engine handle AES/3DES ciphers in ECB/CBC mode. From 732b764099f651a088fd931d7b8121b6aa84e62e Mon Sep 17 00:00:00 2001 From: Corentin Labbe Date: Sun, 15 Nov 2020 19:08:07 +0000 Subject: [PATCH 228/360] crypto: sun8i-ce - fix two error path's memory leak This patch fixes the following smatch warnings: drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c:412 sun8i_ce_hash_run() warn: possible memory leak of 'result' Note: "buf" is leaked as well. Furthermore, in case of ENOMEM, crypto_finalize_hash_request() was not called which was an error. Fixes: 56f6d5aee88d ("crypto: sun8i-ce - support hash algorithms") Reported-by: kernel test robot Reported-by: Dan Carpenter Signed-off-by: Corentin Labbe Signed-off-by: Herbert Xu --- .../crypto/allwinner/sun8i-ce/sun8i-ce-hash.c | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c index 4927a6c82d32..7ca4979190d0 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-hash.c @@ -263,13 +263,13 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) u32 common; u64 byte_count; __le32 *bf; - void *buf; + void *buf = NULL; int j, i, todo; int nbw = 0; u64 fill, min_fill; __be64 *bebits; __le64 *lebits; - void *result; + void *result = NULL; u64 bs; int digestsize; dma_addr_t addr_res, addr_pad; @@ -286,13 +286,17 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) /* the padding could be up to two block. */ buf = kzalloc(bs * 2, GFP_KERNEL | GFP_DMA); - if (!buf) - return -ENOMEM; + if (!buf) { + err = -ENOMEM; + goto theend; + } bf = (__le32 *)buf; result = kzalloc(digestsize, GFP_KERNEL | GFP_DMA); - if (!result) - return -ENOMEM; + if (!result) { + err = -ENOMEM; + goto theend; + } flow = rctx->flow; chan = &ce->chanlist[flow]; @@ -404,11 +408,11 @@ int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq) dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_TO_DEVICE); dma_unmap_single(ce->dev, addr_res, digestsize, DMA_FROM_DEVICE); - kfree(buf); memcpy(areq->result, result, algt->alg.hash.halg.digestsize); - kfree(result); theend: + kfree(buf); + kfree(result); crypto_finalize_hash_request(engine, breq, err); return 0; } From bab202ab87ba4da48018daf0f6810b22705a570d Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 28 Sep 2020 12:00:04 +0200 Subject: [PATCH 229/360] x86/mm: Declare 'start' variable where it is used It is not required to initialize the local variable start in memory_map_top_down(), as the variable will be initialized in any path before it is used. make clang-analyzer on x86_64 tinyconfig reports: arch/x86/mm/init.c:612:15: warning: Although the value stored to 'start' \ is used in the enclosing expression, the value is never actually read \ from 'start' [clang-analyzer-deadcode.DeadStores] Move the variable declaration into the loop, where it is used. No code changed: # arch/x86/mm/init.o: text data bss dec hex filename 7105 1424 26768 35297 89e1 init.o.before 7105 1424 26768 35297 89e1 init.o.after md5: a8d76c1bb5fce9cae251780a7ee7730f init.o.before.asm a8d76c1bb5fce9cae251780a7ee7730f init.o.after.asm [ bp: Massage. ] Signed-off-by: Lukas Bulwahn Signed-off-by: Borislav Petkov Acked-by: Dave Hansen Link: https://lkml.kernel.org/r/20200928100004.25674-1-lukas.bulwahn@gmail.com --- arch/x86/mm/init.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c index c7a47603537f..e26f5c5c6565 100644 --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -596,7 +596,7 @@ static unsigned long __init get_new_step_size(unsigned long step_size) static void __init memory_map_top_down(unsigned long map_start, unsigned long map_end) { - unsigned long real_end, start, last_start; + unsigned long real_end, last_start; unsigned long step_size; unsigned long addr; unsigned long mapped_ram_size = 0; @@ -609,7 +609,7 @@ static void __init memory_map_top_down(unsigned long map_start, step_size = PMD_SIZE; max_pfn_mapped = 0; /* will get exact value next */ min_pfn_mapped = real_end >> PAGE_SHIFT; - last_start = start = real_end; + last_start = real_end; /* * We start from the top (end of memory) and go to the bottom. @@ -618,6 +618,8 @@ static void __init memory_map_top_down(unsigned long map_start, * for page table. */ while (last_start > map_start) { + unsigned long start; + if (last_start > step_size) { start = round_down(last_start - 1, step_size); if (start < map_start) From 61b39ad9a7d26fe14a2f5f23e5e940e7f9664d41 Mon Sep 17 00:00:00 2001 From: Wang Qing Date: Mon, 9 Nov 2020 11:45:41 +0800 Subject: [PATCH 230/360] x86/head64: Remove duplicate include Remove duplicate header include. Signed-off-by: Wang Qing Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/1604893542-20961-1-git-send-email-wangqing@vivo.com --- arch/x86/kernel/head64.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index 05e117137b45..5e9beb77cafd 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include From 0cd9b7230cc57b0f9cfd13ef5c3830c7db1a68d4 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 11 Nov 2020 18:46:26 +0100 Subject: [PATCH 231/360] s390: add separate program check exit path System call and program check handler both use the system call exit path when returning to previous context. However the program check handler jumps right to the end of the system call exit path if the previous context is kernel context. This lead to the quite odd double disabling of interrupts in the system call exit path introduced with commit ce9dfafe29be ("s390: fix system call exit path"). To avoid that have a separate program check handler exit path if the previous context is kernel context. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/kernel/entry.S | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 5346545b9860..0a7811d993a7 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -430,8 +430,6 @@ ENTRY(system_call) TSTMSK __LC_CPU_FLAGS,(_CIF_WORK-_CIF_FPU) jnz .Lsysc_work BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP -.Lsysc_restore: - DISABLE_INTS TSTMSK __LC_CPU_FLAGS, _CIF_FPU jz .Lsysc_skip_fpu brasl %r14,load_fpu_regs @@ -709,10 +707,20 @@ ENTRY(pgm_check_handler) .Lpgm_return: LOCKDEP_SYS_EXIT tm __PT_PSW+1(%r11),0x01 # returning to user ? - jno .Lsysc_restore + jno .Lpgm_restore TSTMSK __PT_FLAGS(%r11),_PIF_SYSCALL jo .Lsysc_do_syscall j .Lsysc_tif +.Lpgm_restore: + DISABLE_INTS + TSTMSK __LC_CPU_FLAGS, _CIF_FPU + jz .Lpgm_skip_fpu + brasl %r14,load_fpu_regs +.Lpgm_skip_fpu: + mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) + stpt __LC_EXIT_TIMER + lmg %r0,%r15,__PT_R0(%r11) + b __LC_RETURN_LPSWE # # PER event in supervisor state, must be kprobes From 5ec11d0966406e4857a642539c5781fe72cc6e22 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 16 Nov 2020 11:18:00 +0100 Subject: [PATCH 232/360] s390/cio: fix kernel-doc markups in cio driver. Fix typo in the kernel-doc markups 1. ccw driver -> ccw_driver 2. ccw_device_id_is_equal() -> ccw_dev_id_is_equal Signed-off-by: Mauro Carvalho Chehab Reviewed-by: Cornelia Huck [vneethv@linux.ibm.com: slight modification in the changelog] Reviewed-by: Vineeth Vijayan Signed-off-by: Heiko Carstens --- arch/s390/include/asm/ccwdev.h | 2 +- arch/s390/include/asm/cio.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/include/asm/ccwdev.h b/arch/s390/include/asm/ccwdev.h index c0be5fe1ddba..069709b8e9e7 100644 --- a/arch/s390/include/asm/ccwdev.h +++ b/arch/s390/include/asm/ccwdev.h @@ -115,7 +115,7 @@ enum uc_todo { }; /** - * struct ccw driver - device driver for channel attached devices + * struct ccw_driver - device driver for channel attached devices * @ids: ids supported by this driver * @probe: function called on probe * @remove: function called on remove diff --git a/arch/s390/include/asm/cio.h b/arch/s390/include/asm/cio.h index 5c58756d6476..23dceb8d0453 100644 --- a/arch/s390/include/asm/cio.h +++ b/arch/s390/include/asm/cio.h @@ -329,7 +329,7 @@ struct ccw_dev_id { }; /** - * ccw_device_id_is_equal() - compare two ccw_dev_ids + * ccw_dev_id_is_equal() - compare two ccw_dev_ids * @dev_id1: a ccw_dev_id * @dev_id2: another ccw_dev_id * Returns: From 1e632eaa0f4b7f65a81301205ca122024991e1d3 Mon Sep 17 00:00:00 2001 From: Julian Wiedmann Date: Thu, 5 Nov 2020 09:34:34 +0200 Subject: [PATCH 233/360] s390/prng: let misc_register() add the prng sysfs attributes Instead of creating the sysfs attributes for the prng devices by hand, describe them in .groups and let the misdevice core handle it. This also ensures that the attributes are available when the KOBJ_ADD event is raised. Signed-off-by: Julian Wiedmann Reviewed-by: Harald Freudenberger Signed-off-by: Heiko Carstens --- arch/s390/crypto/prng.c | 53 +++++++++++++---------------------------- 1 file changed, 16 insertions(+), 37 deletions(-) diff --git a/arch/s390/crypto/prng.c b/arch/s390/crypto/prng.c index 5057773f82e9..b2f219ec379c 100644 --- a/arch/s390/crypto/prng.c +++ b/arch/s390/crypto/prng.c @@ -674,20 +674,6 @@ static const struct file_operations prng_tdes_fops = { .llseek = noop_llseek, }; -static struct miscdevice prng_sha512_dev = { - .name = "prandom", - .minor = MISC_DYNAMIC_MINOR, - .mode = 0644, - .fops = &prng_sha512_fops, -}; -static struct miscdevice prng_tdes_dev = { - .name = "prandom", - .minor = MISC_DYNAMIC_MINOR, - .mode = 0644, - .fops = &prng_tdes_fops, -}; - - /* chunksize attribute (ro) */ static ssize_t prng_chunksize_show(struct device *dev, struct device_attribute *attr, @@ -801,18 +787,30 @@ static struct attribute *prng_sha512_dev_attrs[] = { &dev_attr_strength.attr, NULL }; +ATTRIBUTE_GROUPS(prng_sha512_dev); + static struct attribute *prng_tdes_dev_attrs[] = { &dev_attr_chunksize.attr, &dev_attr_byte_counter.attr, &dev_attr_mode.attr, NULL }; +ATTRIBUTE_GROUPS(prng_tdes_dev); -static struct attribute_group prng_sha512_dev_attr_group = { - .attrs = prng_sha512_dev_attrs +static struct miscdevice prng_sha512_dev = { + .name = "prandom", + .minor = MISC_DYNAMIC_MINOR, + .mode = 0644, + .fops = &prng_sha512_fops, + .groups = prng_sha512_dev_groups, }; -static struct attribute_group prng_tdes_dev_attr_group = { - .attrs = prng_tdes_dev_attrs + +static struct miscdevice prng_tdes_dev = { + .name = "prandom", + .minor = MISC_DYNAMIC_MINOR, + .mode = 0644, + .fops = &prng_tdes_fops, + .groups = prng_tdes_dev_groups, }; @@ -867,13 +865,6 @@ static int __init prng_init(void) prng_sha512_deinstantiate(); goto out; } - ret = sysfs_create_group(&prng_sha512_dev.this_device->kobj, - &prng_sha512_dev_attr_group); - if (ret) { - misc_deregister(&prng_sha512_dev); - prng_sha512_deinstantiate(); - goto out; - } } else { @@ -898,14 +889,6 @@ static int __init prng_init(void) prng_tdes_deinstantiate(); goto out; } - ret = sysfs_create_group(&prng_tdes_dev.this_device->kobj, - &prng_tdes_dev_attr_group); - if (ret) { - misc_deregister(&prng_tdes_dev); - prng_tdes_deinstantiate(); - goto out; - } - } out: @@ -916,13 +899,9 @@ out: static void __exit prng_exit(void) { if (prng_mode == PRNG_MODE_SHA512) { - sysfs_remove_group(&prng_sha512_dev.this_device->kobj, - &prng_sha512_dev_attr_group); misc_deregister(&prng_sha512_dev); prng_sha512_deinstantiate(); } else { - sysfs_remove_group(&prng_tdes_dev.this_device->kobj, - &prng_tdes_dev_attr_group); misc_deregister(&prng_tdes_dev); prng_tdes_deinstantiate(); } From 73045a08cf5549cc7dee14463431fbeb2134dd67 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Mon, 19 Oct 2020 11:01:33 +0200 Subject: [PATCH 234/360] s390: unify identity mapping limits handling Currently we have to consider too many different values which in the end only affect identity mapping size. These are: 1. max_physmem_end - end of physical memory online or standby. Always <= end of the last online memory block (get_mem_detect_end()). 2. CONFIG_MAX_PHYSMEM_BITS - the maximum size of physical memory the kernel is able to support. 3. "mem=" kernel command line option which limits physical memory usage. 4. OLDMEM_BASE which is a kdump memory limit when the kernel is executed as crash kernel. 5. "hsa" size which is a memory limit when the kernel is executed during zfcp/nvme dump. Through out kernel startup and run we juggle all those values at once but that does not bring any amusement, only confusion and complexity. Unify all those values to a single one we should really care, that is our identity mapping size. Signed-off-by: Vasily Gorbik Reviewed-by: Alexander Gordeev Acked-by: Heiko Carstens Signed-off-by: Heiko Carstens --- arch/s390/boot/boot.h | 11 +++++--- arch/s390/boot/ipl_parm.c | 44 ++++++++++--------------------- arch/s390/boot/kaslr.c | 3 +-- arch/s390/boot/mem_detect.c | 13 ++++----- arch/s390/boot/startup.c | 48 +++++++++++++++++++++++++++++++--- arch/s390/include/asm/setup.h | 4 +-- arch/s390/kernel/setup.c | 37 ++++++++++++-------------- arch/s390/mm/dump_pagetables.c | 2 +- arch/s390/mm/kasan_init.c | 21 ++++++++++----- drivers/s390/char/sclp_cmd.c | 6 ++--- 10 files changed, 109 insertions(+), 80 deletions(-) diff --git a/arch/s390/boot/boot.h b/arch/s390/boot/boot.h index 4d4536299789..d5cf2e8c5eb4 100644 --- a/arch/s390/boot/boot.h +++ b/arch/s390/boot/boot.h @@ -2,20 +2,23 @@ #ifndef BOOT_BOOT_H #define BOOT_BOOT_H +#include + void startup_kernel(void); -void detect_memory(void); +unsigned long detect_memory(void); +bool is_ipl_block_dump(void); void store_ipl_parmblock(void); void setup_boot_command_line(void); void parse_boot_command_line(void); -void setup_memory_end(void); void verify_facilities(void); void print_missing_facilities(void); void print_pgm_check_info(void); unsigned long get_random_base(unsigned long safe_addr); -extern int kaslr_enabled; -extern int vmalloc_size_set; extern const char kernel_version[]; +extern unsigned long memory_limit; +extern int vmalloc_size_set; +extern int kaslr_enabled; unsigned long read_ipl_report(unsigned long safe_offset); diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c index 33f43a7d03f3..d372a45fe10e 100644 --- a/arch/s390/boot/ipl_parm.c +++ b/arch/s390/boot/ipl_parm.c @@ -17,10 +17,9 @@ int __bootdata_preserved(ipl_block_valid); unsigned int __bootdata_preserved(zlib_dfltcc_support) = ZLIB_DFLTCC_FULL; unsigned long __bootdata(vmalloc_size) = VMALLOC_DEFAULT_SIZE; -unsigned long __bootdata(memory_end); -int __bootdata(memory_end_set); int __bootdata(noexec_disabled); +unsigned long memory_limit; int vmalloc_size_set; int kaslr_enabled; @@ -58,6 +57,17 @@ void store_ipl_parmblock(void) ipl_block_valid = 1; } +bool is_ipl_block_dump(void) +{ + if (ipl_block.pb0_hdr.pbt == IPL_PBT_FCP && + ipl_block.fcp.opt == IPL_PB0_FCP_OPT_DUMP) + return true; + if (ipl_block.pb0_hdr.pbt == IPL_PBT_NVME && + ipl_block.nvme.opt == IPL_PB0_NVME_OPT_DUMP) + return true; + return false; +} + static size_t scpdata_length(const u8 *buf, size_t count) { while (count) { @@ -238,10 +248,8 @@ void parse_boot_command_line(void) while (*args) { args = next_arg(args, ¶m, &val); - if (!strcmp(param, "mem") && val) { - memory_end = round_down(memparse(val, NULL), PAGE_SIZE); - memory_end_set = 1; - } + if (!strcmp(param, "mem") && val) + memory_limit = round_down(memparse(val, NULL), PAGE_SIZE); if (!strcmp(param, "vmalloc") && val) { vmalloc_size = round_up(memparse(val, NULL), PAGE_SIZE); @@ -282,27 +290,3 @@ void parse_boot_command_line(void) #endif } } - -static inline bool is_ipl_block_dump(void) -{ - if (ipl_block.pb0_hdr.pbt == IPL_PBT_FCP && - ipl_block.fcp.opt == IPL_PB0_FCP_OPT_DUMP) - return true; - if (ipl_block.pb0_hdr.pbt == IPL_PBT_NVME && - ipl_block.nvme.opt == IPL_PB0_NVME_OPT_DUMP) - return true; - return false; -} - -void setup_memory_end(void) -{ -#ifdef CONFIG_CRASH_DUMP - if (OLDMEM_BASE) { - kaslr_enabled = 0; - } else if (ipl_block_valid && is_ipl_block_dump()) { - kaslr_enabled = 0; - if (!sclp_early_get_hsa_size(&memory_end) && memory_end) - memory_end_set = 1; - } -#endif -} diff --git a/arch/s390/boot/kaslr.c b/arch/s390/boot/kaslr.c index 987a9eaf228a..0dd48fbdbaa4 100644 --- a/arch/s390/boot/kaslr.c +++ b/arch/s390/boot/kaslr.c @@ -177,8 +177,7 @@ unsigned long get_random_base(unsigned long safe_addr) unsigned long kasan_needs; int i; - if (memory_end_set) - memory_limit = min(memory_limit, memory_end); + memory_limit = min(memory_limit, ident_map_size); /* * Avoid putting kernel in the end of physical memory diff --git a/arch/s390/boot/mem_detect.c b/arch/s390/boot/mem_detect.c index 62e7c13ce85c..40168e59abd3 100644 --- a/arch/s390/boot/mem_detect.c +++ b/arch/s390/boot/mem_detect.c @@ -8,7 +8,6 @@ #include "compressed/decompressor.h" #include "boot.h" -unsigned long __bootdata(max_physmem_end); struct mem_detect_info __bootdata(mem_detect); /* up to 256 storage elements, 1020 subincrements each */ @@ -149,27 +148,29 @@ static void search_mem_end(void) add_mem_detect_block(0, (offset + 1) << 20); } -void detect_memory(void) +unsigned long detect_memory(void) { + unsigned long max_physmem_end; + sclp_early_get_memsize(&max_physmem_end); if (!sclp_early_read_storage_info()) { mem_detect.info_source = MEM_DETECT_SCLP_STOR_INFO; - return; + return max_physmem_end; } if (!diag260()) { mem_detect.info_source = MEM_DETECT_DIAG260; - return; + return max_physmem_end; } if (max_physmem_end) { add_mem_detect_block(0, max_physmem_end); mem_detect.info_source = MEM_DETECT_SCLP_READ_INFO; - return; + return max_physmem_end; } search_mem_end(); mem_detect.info_source = MEM_DETECT_BIN_SEARCH; - max_physmem_end = get_mem_detect_end(); + return get_mem_detect_end(); } diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index eeda91fcf7cc..05f8eefa3dcf 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include #include #include @@ -14,6 +15,7 @@ extern char __boot_data_start[], __boot_data_end[]; extern char __boot_data_preserved_start[], __boot_data_preserved_end[]; unsigned long __bootdata_preserved(__kaslr_offset); +unsigned long __bootdata(ident_map_size); /* * Some code and data needs to stay below 2 GB, even when the kernel would be @@ -127,6 +129,46 @@ static void handle_relocs(unsigned long offset) } } +/* + * Merge information from several sources into a single ident_map_size value. + * "ident_map_size" represents the upper limit of physical memory we may ever + * reach. It might not be all online memory, but also include standby (offline) + * memory. "ident_map_size" could be lower then actual standby or even online + * memory present, due to limiting factors. We should never go above this limit. + * It is the size of our identity mapping. + * + * Consider the following factors: + * 1. max_physmem_end - end of physical memory online or standby. + * Always <= end of the last online memory block (get_mem_detect_end()). + * 2. CONFIG_MAX_PHYSMEM_BITS - the maximum size of physical memory the + * kernel is able to support. + * 3. "mem=" kernel command line option which limits physical memory usage. + * 4. OLDMEM_BASE which is a kdump memory limit when the kernel is executed as + * crash kernel. + * 5. "hsa" size which is a memory limit when the kernel is executed during + * zfcp/nvme dump. + */ +static void setup_ident_map_size(unsigned long max_physmem_end) +{ + unsigned long hsa_size; + + ident_map_size = max_physmem_end; + if (memory_limit) + ident_map_size = min(ident_map_size, memory_limit); + ident_map_size = min(ident_map_size, 1UL << MAX_PHYSMEM_BITS); + +#ifdef CONFIG_CRASH_DUMP + if (OLDMEM_BASE) { + kaslr_enabled = 0; + ident_map_size = min(ident_map_size, OLDMEM_SIZE); + } else if (ipl_block_valid && is_ipl_block_dump()) { + kaslr_enabled = 0; + if (!sclp_early_get_hsa_size(&hsa_size) && hsa_size) + ident_map_size = min(ident_map_size, hsa_size); + } +#endif +} + /* * This function clears the BSS section of the decompressed Linux kernel and NOT the decompressor's. */ @@ -145,8 +187,7 @@ static void setup_vmalloc_size(void) if (vmalloc_size_set) return; - size = (memory_end ?: max_physmem_end) >> 3; - size = round_up(size, _SEGMENT_SIZE); + size = round_up(ident_map_size / 8, _SEGMENT_SIZE); vmalloc_size = max(size, vmalloc_size); } @@ -165,8 +206,7 @@ void startup_kernel(void) sclp_early_read_info(); setup_boot_command_line(); parse_boot_command_line(); - setup_memory_end(); - detect_memory(); + setup_ident_map_size(detect_memory()); setup_vmalloc_size(); random_lma = __kaslr_offset = 0; diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index 1be6a064f317..3e388fa208d4 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -86,10 +86,8 @@ extern unsigned int zlib_dfltcc_support; #define ZLIB_DFLTCC_FULL_DEBUG 4 extern int noexec_disabled; -extern int memory_end_set; -extern unsigned long memory_end; +extern unsigned long ident_map_size; extern unsigned long vmalloc_size; -extern unsigned long max_physmem_end; /* The Write Back bit position in the physaddr is given by the SLPC PCI */ extern unsigned long mio_wb_bit_mask; diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 02bccee5ee85..756936785598 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -94,10 +94,8 @@ char elf_platform[ELF_PLATFORM_SIZE]; unsigned long int_hwcap = 0; int __bootdata(noexec_disabled); -int __bootdata(memory_end_set); -unsigned long __bootdata(memory_end); +unsigned long __bootdata(ident_map_size); unsigned long __bootdata(vmalloc_size); -unsigned long __bootdata(max_physmem_end); struct mem_detect_info __bootdata(mem_detect); struct exception_table_entry *__bootdata_preserved(__start_dma_ex_table); @@ -557,12 +555,12 @@ static void __init setup_resources(void) #endif } -static void __init setup_memory_end(void) +static void __init setup_ident_map_size(void) { unsigned long vmax, tmp; /* Choose kernel address space layout: 3 or 4 levels. */ - tmp = (memory_end ?: max_physmem_end) / PAGE_SIZE; + tmp = ident_map_size / PAGE_SIZE; tmp = tmp * (sizeof(struct page) + PAGE_SIZE); if (tmp + vmalloc_size + MODULES_LEN <= _REGION2_SIZE) vmax = _REGION2_SIZE; /* 3-level kernel page table */ @@ -589,22 +587,22 @@ static void __init setup_memory_end(void) tmp = min(tmp, 1UL << MAX_PHYSMEM_BITS); vmemmap = (struct page *) tmp; - /* Take care that memory_end is set and <= vmemmap */ - memory_end = min(memory_end ?: max_physmem_end, (unsigned long)vmemmap); + /* Take care that ident_map_size <= vmemmap */ + ident_map_size = min(ident_map_size, (unsigned long)vmemmap); #ifdef CONFIG_KASAN - memory_end = min(memory_end, KASAN_SHADOW_START); + ident_map_size = min(ident_map_size, KASAN_SHADOW_START); #endif - vmemmap_size = SECTION_ALIGN_UP(memory_end / PAGE_SIZE) * sizeof(struct page); + vmemmap_size = SECTION_ALIGN_UP(ident_map_size / PAGE_SIZE) * sizeof(struct page); #ifdef CONFIG_KASAN /* move vmemmap above kasan shadow only if stands in a way */ if (KASAN_SHADOW_END > (unsigned long)vmemmap && (unsigned long)vmemmap + vmemmap_size > KASAN_SHADOW_START) vmemmap = max(vmemmap, (struct page *)KASAN_SHADOW_END); #endif - max_pfn = max_low_pfn = PFN_DOWN(memory_end); - memblock_remove(memory_end, ULONG_MAX); + max_pfn = max_low_pfn = PFN_DOWN(ident_map_size); + memblock_remove(ident_map_size, ULONG_MAX); - pr_notice("The maximum memory size is %luMB\n", memory_end >> 20); + pr_notice("The maximum memory size is %luMB\n", ident_map_size >> 20); } #ifdef CONFIG_CRASH_DUMP @@ -634,12 +632,11 @@ static struct notifier_block kdump_mem_nb = { #endif /* - * Make sure that the area behind memory_end is protected + * Make sure that the area above identity mapping is protected */ -static void __init reserve_memory_end(void) +static void __init reserve_above_ident_map(void) { - if (memory_end_set) - memblock_reserve(memory_end, ULONG_MAX); + memblock_reserve(ident_map_size, ULONG_MAX); } /* @@ -676,7 +673,7 @@ static void __init reserve_crashkernel(void) phys_addr_t low, high; int rc; - rc = parse_crashkernel(boot_command_line, memory_end, &crash_size, + rc = parse_crashkernel(boot_command_line, ident_map_size, &crash_size, &crash_base); crash_base = ALIGN(crash_base, KEXEC_CRASH_MEM_ALIGN); @@ -1130,7 +1127,7 @@ void __init setup_arch(char **cmdline_p) setup_control_program_code(); /* Do some memory reservations *before* memory is added to memblock */ - reserve_memory_end(); + reserve_above_ident_map(); reserve_oldmem(); reserve_kernel(); reserve_initrd(); @@ -1145,9 +1142,9 @@ void __init setup_arch(char **cmdline_p) remove_oldmem(); setup_uv(); - setup_memory_end(); + setup_ident_map_size(); setup_memory(); - dma_contiguous_reserve(memory_end); + dma_contiguous_reserve(ident_map_size); vmcp_cma_reserve(); check_initrd(); diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index 8f9ff7e7187d..e40a30647d99 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -255,7 +255,7 @@ static int pt_dump_init(void) */ max_addr = (S390_lowcore.kernel_asce & _REGION_ENTRY_TYPE_MASK) >> 2; max_addr = 1UL << (max_addr * 11 + 31); - address_markers[IDENTITY_AFTER_END_NR].start_address = memory_end; + address_markers[IDENTITY_AFTER_END_NR].start_address = ident_map_size; address_markers[MODULES_NR].start_address = MODULES_VADDR; address_markers[MODULES_END_NR].start_address = MODULES_END; address_markers[VMEMMAP_NR].start_address = (unsigned long) vmemmap; diff --git a/arch/s390/mm/kasan_init.c b/arch/s390/mm/kasan_init.c index a9f561491110..db4d303aaaa9 100644 --- a/arch/s390/mm/kasan_init.c +++ b/arch/s390/mm/kasan_init.c @@ -289,12 +289,19 @@ void __init kasan_early_init(void) memsize = get_mem_detect_end(); if (!memsize) kasan_early_panic("cannot detect physical memory size\n"); - /* respect mem= cmdline parameter */ - if (memory_end_set && memsize > memory_end) - memsize = memory_end; - if (IS_ENABLED(CONFIG_CRASH_DUMP) && OLDMEM_BASE) - memsize = min(memsize, OLDMEM_SIZE); - memsize = min(memsize, KASAN_SHADOW_START); + /* + * Kasan currently supports standby memory but only if it follows + * online memory (default allocation), i.e. no memory holes. + * - memsize represents end of online memory + * - ident_map_size represents online + standby and memory limits + * accounted. + * Kasan maps "memsize" right away. + * [0, memsize] - as identity mapping + * [__sha(0), __sha(memsize)] - shadow memory for identity mapping + * The rest [memsize, ident_map_size] if memsize < ident_map_size + * could be mapped/unmapped dynamically later during memory hotplug. + */ + memsize = min(memsize, ident_map_size); BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE)); BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE)); @@ -377,7 +384,7 @@ void __init kasan_early_init(void) POPULATE_SHALLOW); } /* populate kasan shadow for untracked memory */ - kasan_early_pgtable_populate(__sha(max_physmem_end), __sha(untracked_mem_end), + kasan_early_pgtable_populate(__sha(ident_map_size), __sha(untracked_mem_end), POPULATE_ZERO_SHADOW); kasan_early_pgtable_populate(__sha(kasan_vmax), __sha(vmax_unlimited), POPULATE_ZERO_SHADOW); diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c index 7ebe89890a9b..d41bc144c183 100644 --- a/drivers/s390/char/sclp_cmd.c +++ b/drivers/s390/char/sclp_cmd.c @@ -401,10 +401,10 @@ static void __init add_memory_merged(u16 rn) goto skip_add; if (start + size > VMEM_MAX_PHYS) size = VMEM_MAX_PHYS - start; - if (memory_end_set && (start >= memory_end)) + if (start >= ident_map_size) goto skip_add; - if (memory_end_set && (start + size > memory_end)) - size = memory_end - start; + if (start + size > ident_map_size) + size = ident_map_size - start; block_size = memory_block_size_bytes(); align_to_block_size(&start, &size, block_size); if (!size) From c9343637d6b265127dfe37cd7e09eb6feac03032 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Thu, 12 Nov 2020 15:54:47 +0100 Subject: [PATCH 235/360] s390/ftrace: assume -mhotpatch or -mrecord-mcount always available Currently the kernel minimal compiler requirement is gcc 4.9 or clang 10.0.1. * gcc -mhotpatch option is supported since 4.8. * A combination of -pg -mrecord-mcount -mnop-mcount -mfentry flags is supported since gcc 9 and since clang 10. Drop support for old -pg function prologues. Which leaves binary compatible -mhotpatch / -mnop-mcount -mfentry prologues in a form: brcl 0,0 Which are also do not require initial nop optimization / conversion and presence of _mcount symbol. Signed-off-by: Vasily Gorbik Reviewed-by: Heiko Carstens Signed-off-by: Heiko Carstens --- arch/s390/include/asm/ftrace.h | 33 ++++++------------ arch/s390/kernel/ftrace.c | 63 ++++++++-------------------------- arch/s390/kernel/mcount.S | 8 ----- scripts/recordmcount.pl | 3 -- 4 files changed, 25 insertions(+), 82 deletions(-) diff --git a/arch/s390/include/asm/ftrace.h b/arch/s390/include/asm/ftrace.h index 68d362f8d6c1..695c61989f97 100644 --- a/arch/s390/include/asm/ftrace.h +++ b/arch/s390/include/asm/ftrace.h @@ -2,16 +2,9 @@ #ifndef _ASM_S390_FTRACE_H #define _ASM_S390_FTRACE_H -#define ARCH_SUPPORTS_FTRACE_OPS 1 - -#if defined(CC_USING_HOTPATCH) || defined(CC_USING_NOP_MCOUNT) -#define MCOUNT_INSN_SIZE 6 -#else -#define MCOUNT_INSN_SIZE 24 -#define MCOUNT_RETURN_FIXUP 18 -#endif - #define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR +#define ARCH_SUPPORTS_FTRACE_OPS 1 +#define MCOUNT_INSN_SIZE 6 #ifndef __ASSEMBLY__ @@ -22,7 +15,6 @@ #define ftrace_return_address(n) __builtin_return_address(n) #endif -void _mcount(void); void ftrace_caller(void); extern char ftrace_graph_caller_end; @@ -30,12 +22,20 @@ extern unsigned long ftrace_plt; struct dyn_arch_ftrace { }; -#define MCOUNT_ADDR ((unsigned long)_mcount) +#define MCOUNT_ADDR 0 #define FTRACE_ADDR ((unsigned long)ftrace_caller) #define KPROBE_ON_FTRACE_NOP 0 #define KPROBE_ON_FTRACE_CALL 1 +struct module; +struct dyn_ftrace; +/* + * Either -mhotpatch or -mnop-mcount is used - no explicit init is required + */ +static inline int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec) { return 0; } +#define ftrace_init_nop ftrace_init_nop + static inline unsigned long ftrace_call_adjust(unsigned long addr) { return addr; @@ -49,28 +49,17 @@ struct ftrace_insn { static inline void ftrace_generate_nop_insn(struct ftrace_insn *insn) { #ifdef CONFIG_FUNCTION_TRACER -#if defined(CC_USING_HOTPATCH) || defined(CC_USING_NOP_MCOUNT) /* brcl 0,0 */ insn->opc = 0xc004; insn->disp = 0; -#else - /* jg .+24 */ - insn->opc = 0xc0f4; - insn->disp = MCOUNT_INSN_SIZE / 2; -#endif #endif } static inline int is_ftrace_nop(struct ftrace_insn *insn) { #ifdef CONFIG_FUNCTION_TRACER -#if defined(CC_USING_HOTPATCH) || defined(CC_USING_NOP_MCOUNT) if (insn->disp == 0) return 1; -#else - if (insn->disp == MCOUNT_INSN_SIZE / 2) - return 1; -#endif #endif return 0; } diff --git a/arch/s390/kernel/ftrace.c b/arch/s390/kernel/ftrace.c index b388e87a08bf..ebc1284a618b 100644 --- a/arch/s390/kernel/ftrace.c +++ b/arch/s390/kernel/ftrace.c @@ -22,56 +22,26 @@ #include "entry.h" /* - * The mcount code looks like this: - * stg %r14,8(%r15) # offset 0 - * larl %r1,<&counter> # offset 6 - * brasl %r14,_mcount # offset 12 - * lg %r14,8(%r15) # offset 18 - * Total length is 24 bytes. Only the first instruction will be patched - * by ftrace_make_call / ftrace_make_nop. - * The enabled ftrace code block looks like this: + * To generate function prologue either gcc's hotpatch feature (since gcc 4.8) + * or a combination of -pg -mrecord-mcount -mnop-mcount -mfentry flags + * (since gcc 9 / clang 10) is used. + * In both cases the original and also the disabled function prologue contains + * only a single six byte instruction and looks like this: + * > brcl 0,0 # offset 0 + * To enable ftrace the code gets patched like above and afterwards looks + * like this: * > brasl %r0,ftrace_caller # offset 0 - * larl %r1,<&counter> # offset 6 - * brasl %r14,_mcount # offset 12 - * lg %r14,8(%r15) # offset 18 + * + * The instruction will be patched by ftrace_make_call / ftrace_make_nop. * The ftrace function gets called with a non-standard C function call ABI * where r0 contains the return address. It is also expected that the called * function only clobbers r0 and r1, but restores r2-r15. * For module code we can't directly jump to ftrace caller, but need a * trampoline (ftrace_plt), which clobbers also r1. - * The return point of the ftrace function has offset 24, so execution - * continues behind the mcount block. - * The disabled ftrace code block looks like this: - * > jg .+24 # offset 0 - * larl %r1,<&counter> # offset 6 - * brasl %r14,_mcount # offset 12 - * lg %r14,8(%r15) # offset 18 - * The jg instruction branches to offset 24 to skip as many instructions - * as possible. - * In case we use gcc's hotpatch feature the original and also the disabled - * function prologue contains only a single six byte instruction and looks - * like this: - * > brcl 0,0 # offset 0 - * To enable ftrace the code gets patched like above and afterwards looks - * like this: - * > brasl %r0,ftrace_caller # offset 0 */ unsigned long ftrace_plt; -static inline void ftrace_generate_orig_insn(struct ftrace_insn *insn) -{ -#if defined(CC_USING_HOTPATCH) || defined(CC_USING_NOP_MCOUNT) - /* brcl 0,0 */ - insn->opc = 0xc004; - insn->disp = 0; -#else - /* stg r14,8(r15) */ - insn->opc = 0xe3e0; - insn->disp = 0xf0080024; -#endif -} - int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { @@ -85,15 +55,10 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, if (copy_from_kernel_nofault(&old, (void *) rec->ip, sizeof(old))) return -EFAULT; - if (addr == MCOUNT_ADDR) { - /* Initial code replacement */ - ftrace_generate_orig_insn(&orig); - ftrace_generate_nop_insn(&new); - } else { - /* Replace ftrace call with a nop. */ - ftrace_generate_call_insn(&orig, rec->ip); - ftrace_generate_nop_insn(&new); - } + /* Replace ftrace call with a nop. */ + ftrace_generate_call_insn(&orig, rec->ip); + ftrace_generate_nop_insn(&new); + /* Verify that the to be replaced code matches what we expect. */ if (memcmp(&orig, &old, sizeof(old))) return -EINVAL; diff --git a/arch/s390/kernel/mcount.S b/arch/s390/kernel/mcount.S index 7458dcfd6464..faf64c2f90f5 100644 --- a/arch/s390/kernel/mcount.S +++ b/arch/s390/kernel/mcount.S @@ -33,11 +33,6 @@ ENDPROC(ftrace_stub) #define TRACED_FUNC_FRAME_SIZE STACK_FRAME_OVERHEAD #endif -ENTRY(_mcount) - BR_EX %r14 -ENDPROC(_mcount) -EXPORT_SYMBOL(_mcount) - ENTRY(ftrace_caller) .globl ftrace_regs_caller .set ftrace_regs_caller,ftrace_caller @@ -46,9 +41,6 @@ ENTRY(ftrace_caller) ipm %r14 # don't put any instructions sllg %r14,%r14,16 # clobbering CC before this point lgr %r1,%r15 -#if !(defined(CC_USING_HOTPATCH) || defined(CC_USING_NOP_MCOUNT)) - aghi %r0,MCOUNT_RETURN_FIXUP -#endif # allocate stack frame for ftrace_caller to contain traced function aghi %r15,-TRACED_FUNC_FRAME_SIZE stg %r1,__SF_BACKCHAIN(%r15) diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl index 3f77a5d695c1..56c801502b9a 100755 --- a/scripts/recordmcount.pl +++ b/scripts/recordmcount.pl @@ -254,9 +254,6 @@ if ($arch eq "x86_64") { if ($cc =~ /-DCC_USING_HOTPATCH/) { $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*c0 04 00 00 00 00\\s*brcl\\s*0,[0-9a-f]+ <([^\+]*)>\$"; $mcount_adjust = 0; - } else { - $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_390_(PC|PLT)32DBL\\s+_mcount\\+0x2\$"; - $mcount_adjust = -14; } $alignment = 8; $type = ".quad"; From 9a78c70a1ba03cb4a8fb96964c6ee77236dd487b Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Tue, 10 Nov 2020 17:05:26 +0100 Subject: [PATCH 236/360] s390/decompressor: add decompressor_printk The decompressor does not have any special debug means. Running the kernel under qemu with gdb is helpful but tedious exercise if done repeatedly. It is also not applicable to debugging under LPAR and z/VM. One special thing which stands out is a working sclp_early_printk, which could be used once the kernel switches to 64-bit addressing mode. But sclp_early_printk does not provide any string formating capabilities. Formatting and printing string without printk-alike function is a not fun. The lack of printk-alike function means people would save up on testing and introduce more bugs. So, finally, provide decompressor_printk function, which fits on one screen and trades features for simplicity. It only supports "%s", "%x" and "%lx" specifiers and zero padding for hex values. Reviewed-by: Alexander Egorenkov Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/boot/boot.h | 3 + arch/s390/boot/pgm_check_info.c | 143 +++++++++++++++----------------- 2 files changed, 68 insertions(+), 78 deletions(-) diff --git a/arch/s390/boot/boot.h b/arch/s390/boot/boot.h index d5cf2e8c5eb4..ca485bfa4e50 100644 --- a/arch/s390/boot/boot.h +++ b/arch/s390/boot/boot.h @@ -4,6 +4,8 @@ #include +#include + void startup_kernel(void); unsigned long detect_memory(void); bool is_ipl_block_dump(void); @@ -14,6 +16,7 @@ void verify_facilities(void); void print_missing_facilities(void); void print_pgm_check_info(void); unsigned long get_random_base(unsigned long safe_addr); +void __printf(1, 2) decompressor_printk(const char *fmt, ...); extern const char kernel_version[]; extern unsigned long memory_limit; diff --git a/arch/s390/boot/pgm_check_info.c b/arch/s390/boot/pgm_check_info.c index a3c9862bcede..829548c22444 100644 --- a/arch/s390/boot/pgm_check_info.c +++ b/arch/s390/boot/pgm_check_info.c @@ -1,99 +1,86 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include #include #include +#include #include "boot.h" const char hex_asc[] = "0123456789abcdef"; -#define add_val_as_hex(dst, val) \ - __add_val_as_hex(dst, (const unsigned char *)&val, sizeof(val)) - -static char *__add_val_as_hex(char *dst, const unsigned char *src, size_t count) +static char *as_hex(char *dst, unsigned long val, int pad) { - while (count--) - dst = hex_byte_pack(dst, *src++); - return dst; + char *p, *end = p = dst + max(pad, (int)__fls(val | 1) / 4 + 1); + + for (*p-- = 0; p >= dst; val >>= 4) + *p-- = hex_asc[val & 0x0f]; + return end; } -static char *add_str(char *dst, char *src) +void decompressor_printk(const char *fmt, ...) { - strcpy(dst, src); - return dst + strlen(dst); + char buf[1024] = { 0 }; + char *end = buf + sizeof(buf) - 1; /* make sure buf is 0 terminated */ + unsigned long pad; + char *p = buf; + va_list args; + + va_start(args, fmt); + for (; p < end && *fmt; fmt++) { + if (*fmt != '%') { + *p++ = *fmt; + continue; + } + pad = isdigit(*++fmt) ? simple_strtol(fmt, (char **)&fmt, 10) : 0; + switch (*fmt) { + case 's': + p = buf + strlcat(buf, va_arg(args, char *), sizeof(buf)); + break; + case 'l': + if (*++fmt != 'x' || end - p <= max(sizeof(long) * 2, pad)) + goto out; + p = as_hex(p, va_arg(args, unsigned long), pad); + break; + case 'x': + if (end - p <= max(sizeof(int) * 2, pad)) + goto out; + p = as_hex(p, va_arg(args, unsigned int), pad); + break; + default: + goto out; + } + } +out: + va_end(args); + sclp_early_printk(buf); } void print_pgm_check_info(void) { + unsigned long *gpregs = (unsigned long *)S390_lowcore.gpregs_save_area; struct psw_bits *psw = &psw_bits(S390_lowcore.psw_save_area); - unsigned short ilc = S390_lowcore.pgm_ilc >> 1; - char buf[256]; - int row, col; - char *p; - add_str(buf, "Linux version "); - strlcat(buf, kernel_version, sizeof(buf) - 1); - strlcat(buf, "\n", sizeof(buf)); - sclp_early_printk(buf); - - p = add_str(buf, "Kernel fault: interruption code "); - p = add_val_as_hex(buf + strlen(buf), S390_lowcore.pgm_code); - p = add_str(p, " ilc:"); - *p++ = hex_asc_lo(ilc); - add_str(p, "\n"); - sclp_early_printk(buf); - - if (kaslr_enabled) { - p = add_str(buf, "Kernel random base: "); - p = add_val_as_hex(p, __kaslr_offset); - add_str(p, "\n"); - sclp_early_printk(buf); - } - - p = add_str(buf, "PSW : "); - p = add_val_as_hex(p, S390_lowcore.psw_save_area.mask); - p = add_str(p, " "); - p = add_val_as_hex(p, S390_lowcore.psw_save_area.addr); - add_str(p, "\n"); - sclp_early_printk(buf); - - p = add_str(buf, " R:"); - *p++ = hex_asc_lo(psw->per); - p = add_str(p, " T:"); - *p++ = hex_asc_lo(psw->dat); - p = add_str(p, " IO:"); - *p++ = hex_asc_lo(psw->io); - p = add_str(p, " EX:"); - *p++ = hex_asc_lo(psw->ext); - p = add_str(p, " Key:"); - *p++ = hex_asc_lo(psw->key); - p = add_str(p, " M:"); - *p++ = hex_asc_lo(psw->mcheck); - p = add_str(p, " W:"); - *p++ = hex_asc_lo(psw->wait); - p = add_str(p, " P:"); - *p++ = hex_asc_lo(psw->pstate); - p = add_str(p, " AS:"); - *p++ = hex_asc_lo(psw->as); - p = add_str(p, " CC:"); - *p++ = hex_asc_lo(psw->cc); - p = add_str(p, " PM:"); - *p++ = hex_asc_lo(psw->pm); - p = add_str(p, " RI:"); - *p++ = hex_asc_lo(psw->ri); - p = add_str(p, " EA:"); - *p++ = hex_asc_lo(psw->eaba); - add_str(p, "\n"); - sclp_early_printk(buf); - - for (row = 0; row < 4; row++) { - p = add_str(buf, row == 0 ? "GPRS:" : " "); - for (col = 0; col < 4; col++) { - p = add_str(p, " "); - p = add_val_as_hex(p, S390_lowcore.gpregs_save_area[row * 4 + col]); - } - add_str(p, "\n"); - sclp_early_printk(buf); - } + decompressor_printk("Linux version %s\n", kernel_version); + decompressor_printk("Kernel fault: interruption code %04x ilc:%x\n", + S390_lowcore.pgm_code, S390_lowcore.pgm_ilc >> 1); + if (kaslr_enabled) + decompressor_printk("Kernel random base: %lx\n", __kaslr_offset); + decompressor_printk("PSW : %016lx %016lx\n", + S390_lowcore.psw_save_area.mask, + S390_lowcore.psw_save_area.addr); + decompressor_printk( + " R:%x T:%x IO:%x EX:%x Key:%x M:%x W:%x P:%x AS:%x CC:%x PM:%x RI:%x EA:%x\n", + psw->per, psw->dat, psw->io, psw->ext, psw->key, psw->mcheck, + psw->wait, psw->pstate, psw->as, psw->cc, psw->pm, psw->ri, + psw->eaba); + decompressor_printk("GPRS: %016lx %016lx %016lx %016lx\n", + gpregs[0], gpregs[1], gpregs[2], gpregs[3]); + decompressor_printk(" %016lx %016lx %016lx %016lx\n", + gpregs[4], gpregs[5], gpregs[6], gpregs[7]); + decompressor_printk(" %016lx %016lx %016lx %016lx\n", + gpregs[8], gpregs[9], gpregs[10], gpregs[11]); + decompressor_printk(" %016lx %016lx %016lx %016lx\n", + gpregs[12], gpregs[13], gpregs[14], gpregs[15]); } From ec55d1e1dbea990145644bd6838c061e8113cc4d Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Tue, 10 Nov 2020 17:05:29 +0100 Subject: [PATCH 237/360] s390/decompressor: correct some asm symbols annotations Use SYM_CODE_* annotations for asm functions, so that function lengths are recognized correctly. Also currently the most part of startup is marked as startup_kdump. Move misplaced startup_kdump where it belongs. $ nm -n -S arch/s390/boot/compressed/vmlinux Before: 0000000000010000 T startup 0000000000010010 T startup_kdump After: 0000000000010000 0000000000000014 T startup 0000000000010014 00000000000000b0 t startup_normal 0000000000010180 00000000000000b2 t startup_kdump Reviewed-by: Alexander Egorenkov Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/boot/head.S | 15 ++++++++------- arch/s390/boot/head_kdump.S | 8 ++++---- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/arch/s390/boot/head.S b/arch/s390/boot/head.S index bc34ad2c2e49..cf70917f98e8 100644 --- a/arch/s390/boot/head.S +++ b/arch/s390/boot/head.S @@ -279,8 +279,8 @@ iplstart: # or linload or SALIPL # .org 0x10000 -ENTRY(startup) - j .Lep_startup_normal +SYM_CODE_START(startup) + j startup_normal .org EP_OFFSET # # This is a list of s390 kernel entry points. At address 0x1000f the number of @@ -294,9 +294,9 @@ ENTRY(startup) # kdump startup-code at 0x10010, running in 64 bit absolute addressing mode # .org 0x10010 -ENTRY(startup_kdump) - j .Lep_startup_kdump -.Lep_startup_normal: + j startup_kdump +SYM_CODE_END(startup) +SYM_CODE_START_LOCAL(startup_normal) mvi __LC_AR_MODE_ID,1 # set esame flag slr %r0,%r0 # set cpuid to zero lhi %r1,2 # mode 2 = esame (dump) @@ -322,6 +322,7 @@ ENTRY(startup_kdump) l %r15,.Lstack-.LPG0(%r13) brasl %r14,verify_facilities brasl %r14,startup_kernel +SYM_CODE_END(startup_normal) .Lstack: .long 0x8000 + (1<<(PAGE_SHIFT+BOOT_STACK_ORDER)) - STACK_FRAME_OVERHEAD @@ -371,7 +372,7 @@ ENTRY(startup_kdump) # It simply saves general/control registers and psw in # the save area and does disabled wait with a faulty address. # -ENTRY(startup_pgm_check_handler) +SYM_CODE_START_LOCAL(startup_pgm_check_handler) stmg %r8,%r15,__LC_SAVE_AREA_SYNC la %r8,4095 stctg %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r8) @@ -390,9 +391,9 @@ ENTRY(startup_pgm_check_handler) la %r8,4095 lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r8) lpswe __LC_RETURN_PSW # disabled wait +SYM_CODE_END(startup_pgm_check_handler) .Ldump_info_stack: .long 0x5000 + PAGE_SIZE - STACK_FRAME_OVERHEAD -ENDPROC(startup_pgm_check_handler) # # params at 10400 (setup.h) diff --git a/arch/s390/boot/head_kdump.S b/arch/s390/boot/head_kdump.S index 174d6959bf5b..f015469e7db9 100644 --- a/arch/s390/boot/head_kdump.S +++ b/arch/s390/boot/head_kdump.S @@ -19,8 +19,7 @@ # Note: This code has to be position independent # -.align 2 -.Lep_startup_kdump: +SYM_CODE_START_LOCAL(startup_kdump) lhi %r1,2 # mode 2 = esame (dump) sigp %r1,%r0,SIGP_SET_ARCHITECTURE # Switch to esame mode sam64 # Switch to 64 bit addressing @@ -87,14 +86,15 @@ startup_kdump_relocated: basr %r13,0 0: lpswe .Lrestart_psw-0b(%r13) # Start new kernel... +SYM_CODE_END(startup_kdump) .align 8 .Lrestart_psw: .quad 0x0000000080000000,0x0000000000000000 + startup #else -.align 2 -.Lep_startup_kdump: +SYM_CODE_START_LOCAL(startup_kdump) larl %r13,startup_kdump_crash lpswe 0(%r13) +SYM_CODE_END(startup_kdump) .align 8 startup_kdump_crash: .quad 0x0002000080000000,0x0000000000000000 + startup_kdump_crash From 246218962e2175cd1dfa1e5467e9bffae5cc7b5e Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Wed, 11 Nov 2020 10:59:40 +0100 Subject: [PATCH 238/360] s390/decompressor: add symbols support Information printed by print_pgm_check_info() is crucial for debugging decompressor problems. Printing instruction addresses is better than nothing, but turns further debugging into tedious job of figuring out which function those addresses correspond to. This change adds simplistic symbols resolution support. And adds %pS format specifier support to decompressor_printk(). Decompressor symbols list is extracted and sorted with nm -n -S: ... 0000000000010000 0000000000000014 T startup 0000000000010014 00000000000000b0 t startup_normal 0000000000010180 00000000000000b2 t startup_kdump ... Then functions are filtered and contracted to a form: "10000 14 startup\0""10014 b0 startup_normal\0""10180 b2 startup_kdump\0" ... Which makes it trivial to find beginning of an entry and names are 0 terminated, so could be used as is. Symbols are binary-searched. To get symbols list with final addresses and then get it into the decompressor's image the same trick as for kallsyms is used. Decompressor's vmlinux is linked twice. Symbols are stored in .decompressor.syms section, current size is about 2kb. Reviewed-by: Alexander Egorenkov Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/boot/compressed/.gitignore | 1 + arch/s390/boot/compressed/Makefile | 24 +++++++-- arch/s390/boot/compressed/vmlinux.lds.S | 8 +++ arch/s390/boot/pgm_check_info.c | 69 ++++++++++++++++++++++++- 4 files changed, 97 insertions(+), 5 deletions(-) diff --git a/arch/s390/boot/compressed/.gitignore b/arch/s390/boot/compressed/.gitignore index 765a08f1bd77..01d93832cf4a 100644 --- a/arch/s390/boot/compressed/.gitignore +++ b/arch/s390/boot/compressed/.gitignore @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only vmlinux vmlinux.lds +vmlinux.syms diff --git a/arch/s390/boot/compressed/Makefile b/arch/s390/boot/compressed/Makefile index b235ed95a3d8..e0502222706a 100644 --- a/arch/s390/boot/compressed/Makefile +++ b/arch/s390/boot/compressed/Makefile @@ -10,21 +10,39 @@ GCOV_PROFILE := n UBSAN_SANITIZE := n KASAN_SANITIZE := n -obj-y := $(if $(CONFIG_KERNEL_UNCOMPRESSED),,decompressor.o) piggy.o info.o +obj-y := $(if $(CONFIG_KERNEL_UNCOMPRESSED),,decompressor.o) info.o +obj-all := $(obj-y) piggy.o syms.o targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 targets += vmlinux.bin.xz vmlinux.bin.lzma vmlinux.bin.lzo vmlinux.bin.lz4 -targets += info.bin $(obj-y) +targets += info.bin syms.bin vmlinux.syms $(obj-all) KBUILD_AFLAGS := $(KBUILD_AFLAGS_DECOMPRESSOR) KBUILD_CFLAGS := $(KBUILD_CFLAGS_DECOMPRESSOR) OBJCOPYFLAGS := OBJECTS := $(addprefix $(obj)/,$(obj-y)) +OBJECTS_ALL := $(addprefix $(obj)/,$(obj-all)) LDFLAGS_vmlinux := --oformat $(LD_BFD) -e startup -T -$(obj)/vmlinux: $(obj)/vmlinux.lds $(objtree)/arch/s390/boot/startup.a $(OBJECTS) FORCE +$(obj)/vmlinux: $(obj)/vmlinux.lds $(objtree)/arch/s390/boot/startup.a $(OBJECTS_ALL) FORCE $(call if_changed,ld) +LDFLAGS_vmlinux.syms := --oformat $(LD_BFD) -e startup -T +$(obj)/vmlinux.syms: $(obj)/vmlinux.lds $(objtree)/arch/s390/boot/startup.a $(OBJECTS) FORCE + $(call if_changed,ld) + +quiet_cmd_dumpsyms = DUMPSYMS $< +define cmd_dumpsyms + $(NM) -n -S --format=bsd "$<" | $(PERL) -ne '/(\w+)\s+(\w+)\s+[tT]\s+(\w+)/ and printf "%x %x %s\0",hex $$1,hex $$2,$$3' > "$@" +endef + +$(obj)/syms.bin: $(obj)/vmlinux.syms FORCE + $(call if_changed,dumpsyms) + +OBJCOPYFLAGS_syms.o := -I binary -O elf64-s390 -B s390:64-bit --rename-section .data=.decompressor.syms +$(obj)/syms.o: $(obj)/syms.bin FORCE + $(call if_changed,objcopy) + OBJCOPYFLAGS_info.bin := -O binary --only-section=.vmlinux.info --set-section-flags .vmlinux.info=load $(obj)/info.bin: vmlinux FORCE $(call if_changed,objcopy) diff --git a/arch/s390/boot/compressed/vmlinux.lds.S b/arch/s390/boot/compressed/vmlinux.lds.S index 9427e2cd0c15..152ac6f875c0 100644 --- a/arch/s390/boot/compressed/vmlinux.lds.S +++ b/arch/s390/boot/compressed/vmlinux.lds.S @@ -82,6 +82,14 @@ SECTIONS *(.vmlinux.info) } + .decompressor.syms : { + . += 1; /* make sure we have \0 before the first entry */ + . = ALIGN(2); + _decompressor_syms_start = .; + *(.decompressor.syms) + _decompressor_syms_end = .; + } + #ifdef CONFIG_KERNEL_UNCOMPRESSED . = 0x100000; #else diff --git a/arch/s390/boot/pgm_check_info.c b/arch/s390/boot/pgm_check_info.c index 829548c22444..8ba8136adb4d 100644 --- a/arch/s390/boot/pgm_check_info.c +++ b/arch/s390/boot/pgm_check_info.c @@ -19,6 +19,65 @@ static char *as_hex(char *dst, unsigned long val, int pad) return end; } +static char *symstart(char *p) +{ + while (*p) + p--; + return p + 1; +} + +extern char _decompressor_syms_start[], _decompressor_syms_end[]; +static noinline char *findsym(unsigned long ip, unsigned short *off, unsigned short *len) +{ + /* symbol entries are in a form "10000 c4 startup\0" */ + char *a = _decompressor_syms_start; + char *b = _decompressor_syms_end; + unsigned long start; + unsigned long size; + char *pivot; + char *endp; + + while (a < b) { + pivot = symstart(a + (b - a) / 2); + start = simple_strtoull(pivot, &endp, 16); + size = simple_strtoull(endp + 1, &endp, 16); + if (ip < start) { + b = pivot; + continue; + } + if (ip > start + size) { + a = pivot + strlen(pivot) + 1; + continue; + } + *off = ip - start; + *len = size; + return endp + 1; + } + return NULL; +} + +static noinline char *strsym(void *ip) +{ + static char buf[64]; + unsigned short off; + unsigned short len; + char *p; + + p = findsym((unsigned long)ip, &off, &len); + if (p) { + strncpy(buf, p, sizeof(buf)); + /* reserve 15 bytes for offset/len in symbol+0x1234/0x1234 */ + p = buf + strnlen(buf, sizeof(buf) - 15); + strcpy(p, "+0x"); + p = as_hex(p + 3, off, 0); + strcpy(p, "/0x"); + as_hex(p + 3, len, 0); + } else { + as_hex(buf, (unsigned long)ip, 16); + } + return buf; +} + void decompressor_printk(const char *fmt, ...) { char buf[1024] = { 0 }; @@ -38,6 +97,11 @@ void decompressor_printk(const char *fmt, ...) case 's': p = buf + strlcat(buf, va_arg(args, char *), sizeof(buf)); break; + case 'p': + if (*++fmt != 'S') + goto out; + p = buf + strlcat(buf, strsym(va_arg(args, void *)), sizeof(buf)); + break; case 'l': if (*++fmt != 'x' || end - p <= max(sizeof(long) * 2, pad)) goto out; @@ -67,9 +131,10 @@ void print_pgm_check_info(void) S390_lowcore.pgm_code, S390_lowcore.pgm_ilc >> 1); if (kaslr_enabled) decompressor_printk("Kernel random base: %lx\n", __kaslr_offset); - decompressor_printk("PSW : %016lx %016lx\n", + decompressor_printk("PSW : %016lx %016lx (%pS)\n", S390_lowcore.psw_save_area.mask, - S390_lowcore.psw_save_area.addr); + S390_lowcore.psw_save_area.addr, + (void *)S390_lowcore.psw_save_area.addr); decompressor_printk( " R:%x T:%x IO:%x EX:%x Key:%x M:%x W:%x P:%x AS:%x CC:%x PM:%x RI:%x EA:%x\n", psw->per, psw->dat, psw->io, psw->ext, psw->key, psw->mcheck, From 8977ab65b894d889c3919581bf545ed106a9a1a5 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Tue, 10 Nov 2020 17:05:35 +0100 Subject: [PATCH 239/360] s390/decompressor: add stacktrace support Decompressor works on a single statically allocated stack. Stacktrace implementation with -mbackchain just takes few lines of code. Linux version 5.10.0-rc3-22793-g0f84a355b776-dirty (gor@tuxmaker) #27 SMP PREEMPT Mon Nov 9 17:30:18 CET 2020 Kernel fault: interruption code 0005 ilc:2 PSW : 0000000180000000 0000000000012f92 (parse_boot_command_line+0x27a/0x46c) R:0 T:0 IO:0 EX:0 Key:0 M:0 W:0 P:0 AS:0 CC:0 PM:0 RI:0 EA:3 GPRS: 0000000000000000 00ffffffffffffff 0000000000000000 000000000001a62c 000000000000bf60 0000000000000000 00000000000003c0 0000000000000000 0000000000000080 000000000002322d 000000007f29ef20 0000000000efd018 000000000311c000 0000000000010070 0000000000012f82 000000000000bea8 Call Trace: (sp:000000000000bea8 [<000000000002016e>] 000000000002016e) sp:000000000000bf18 [<0000000000012408>] startup_kernel+0x88/0x2fc sp:000000000000bf60 [<00000000000100c4>] startup_normal+0xb0/0xb0 Reviewed-by: Alexander Egorenkov Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/Makefile | 2 +- arch/s390/boot/boot.h | 5 +++++ arch/s390/boot/head.S | 3 ++- arch/s390/boot/pgm_check_info.c | 23 +++++++++++++++++++++++ arch/s390/include/asm/thread_info.h | 2 +- 5 files changed, 32 insertions(+), 3 deletions(-) diff --git a/arch/s390/Makefile b/arch/s390/Makefile index ba94b03c8b2f..8db267d2a543 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -25,7 +25,7 @@ KBUILD_AFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -D__ASSEMBLY__ KBUILD_AFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO),$(aflags_dwarf)) KBUILD_CFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -O2 KBUILD_CFLAGS_DECOMPRESSOR += -DDISABLE_BRANCH_PROFILING -D__NO_FORTIFY -KBUILD_CFLAGS_DECOMPRESSOR += -fno-delete-null-pointer-checks -msoft-float +KBUILD_CFLAGS_DECOMPRESSOR += -fno-delete-null-pointer-checks -msoft-float -mbackchain KBUILD_CFLAGS_DECOMPRESSOR += -fno-asynchronous-unwind-tables KBUILD_CFLAGS_DECOMPRESSOR += -ffreestanding KBUILD_CFLAGS_DECOMPRESSOR += $(call cc-disable-warning, address-of-packed-member) diff --git a/arch/s390/boot/boot.h b/arch/s390/boot/boot.h index ca485bfa4e50..8b50967f5804 100644 --- a/arch/s390/boot/boot.h +++ b/arch/s390/boot/boot.h @@ -4,6 +4,10 @@ #include +#define BOOT_STACK_OFFSET 0x8000 + +#ifndef __ASSEMBLY__ + #include void startup_kernel(void); @@ -25,4 +29,5 @@ extern int kaslr_enabled; unsigned long read_ipl_report(unsigned long safe_offset); +#endif /* __ASSEMBLY__ */ #endif /* BOOT_BOOT_H */ diff --git a/arch/s390/boot/head.S b/arch/s390/boot/head.S index cf70917f98e8..dacb7813f982 100644 --- a/arch/s390/boot/head.S +++ b/arch/s390/boot/head.S @@ -28,6 +28,7 @@ #include #include #include +#include "boot.h" #define ARCH_OFFSET 4 @@ -325,7 +326,7 @@ SYM_CODE_START_LOCAL(startup_normal) SYM_CODE_END(startup_normal) .Lstack: - .long 0x8000 + (1<<(PAGE_SHIFT+BOOT_STACK_ORDER)) - STACK_FRAME_OVERHEAD + .long BOOT_STACK_OFFSET + BOOT_STACK_SIZE - STACK_FRAME_OVERHEAD .align 8 6: .long 0x7fffffff,0xffffffff .Lext_new_psw: diff --git a/arch/s390/boot/pgm_check_info.c b/arch/s390/boot/pgm_check_info.c index 8ba8136adb4d..d8bfc5673956 100644 --- a/arch/s390/boot/pgm_check_info.c +++ b/arch/s390/boot/pgm_check_info.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -121,6 +122,27 @@ out: sclp_early_printk(buf); } +static noinline void print_stacktrace(void) +{ + struct stack_info boot_stack = { STACK_TYPE_TASK, BOOT_STACK_OFFSET, + BOOT_STACK_OFFSET + BOOT_STACK_SIZE }; + unsigned long sp = S390_lowcore.gpregs_save_area[15]; + bool first = true; + + decompressor_printk("Call Trace:\n"); + while (!(sp & 0x7) && on_stack(&boot_stack, sp, sizeof(struct stack_frame))) { + struct stack_frame *sf = (struct stack_frame *)sp; + + decompressor_printk(first ? "(sp:%016lx [<%016lx>] %pS)\n" : + " sp:%016lx [<%016lx>] %pS\n", + sp, sf->gprs[8], (void *)sf->gprs[8]); + if (sf->back_chain <= sp) + break; + sp = sf->back_chain; + first = false; + } +} + void print_pgm_check_info(void) { unsigned long *gpregs = (unsigned long *)S390_lowcore.gpregs_save_area; @@ -148,4 +170,5 @@ void print_pgm_check_info(void) gpregs[8], gpregs[9], gpregs[10], gpregs[11]); decompressor_printk(" %016lx %016lx %016lx %016lx\n", gpregs[12], gpregs[13], gpregs[14], gpregs[15]); + print_stacktrace(); } diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h index 13a04fcf7762..ce788f3e534d 100644 --- a/arch/s390/include/asm/thread_info.h +++ b/arch/s390/include/asm/thread_info.h @@ -18,7 +18,7 @@ #else #define THREAD_SIZE_ORDER 2 #endif -#define BOOT_STACK_ORDER 2 +#define BOOT_STACK_SIZE (PAGE_SIZE << 2) #define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) #ifndef __ASSEMBLY__ From ba1a6be994e8444baf23c14d7be045811197ccb3 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Tue, 10 Nov 2020 23:30:23 +0100 Subject: [PATCH 240/360] s390/decompressor: print cmdline and BEAR on pgm_check Add kernel command line and last breaking event. The kernel command line is taken from early_command_line and printed only if kernel is not running as protected virtualization guest and if it has been already initialized from the "COMMAND_LINE". Linux version 5.10.0-rc3-22794-gecaa72788df0-dirty (gor@tuxmaker) #28 SMP PREEMPT Mon Nov 9 17:41:20 CET 2020 Kernel command line: audit_enable=0 audit=0 selinux=0 crashkernel=296M root=/dev/dasda1 dasd=ec5b memblock=debug die Kernel fault: interruption code 0005 ilc:2 PSW : 0000000180000000 0000000000012f92 (parse_boot_command_line+0x27a/0x46c) R:0 T:0 IO:0 EX:0 Key:0 M:0 W:0 P:0 AS:0 CC:0 PM:0 RI:0 EA:3 GPRS: 0000000000000000 00ffffffffffffff 0000000000000000 000000000001a65c 000000000000bf60 0000000000000000 00000000000003c0 0000000000000000 0000000000000080 000000000002322d 000000007f29ef20 0000000000efd018 000000000311c000 0000000000010070 0000000000012f82 000000000000bea8 Call Trace: (sp:000000000000bea8 [<000000000002016e>] 000000000002016e) sp:000000000000bf18 [<0000000000012408>] startup_kernel+0x88/0x2fc sp:000000000000bf60 [<00000000000100c4>] startup_normal+0xb0/0xb0 Last Breaking-Event-Address: [<00000000000135ba>] strcmp+0x22/0x24 Reviewed-by: Alexander Egorenkov Acked-by: Viktor Mihajlovski Signed-off-by: Vasily Gorbik Signed-off-by: Heiko Carstens --- arch/s390/boot/pgm_check_info.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/s390/boot/pgm_check_info.c b/arch/s390/boot/pgm_check_info.c index d8bfc5673956..3a46abed2549 100644 --- a/arch/s390/boot/pgm_check_info.c +++ b/arch/s390/boot/pgm_check_info.c @@ -3,9 +3,11 @@ #include #include #include +#include #include #include #include +#include #include #include "boot.h" @@ -149,6 +151,8 @@ void print_pgm_check_info(void) struct psw_bits *psw = &psw_bits(S390_lowcore.psw_save_area); decompressor_printk("Linux version %s\n", kernel_version); + if (!is_prot_virt_guest() && early_command_line[0]) + decompressor_printk("Kernel command line: %s\n", early_command_line); decompressor_printk("Kernel fault: interruption code %04x ilc:%x\n", S390_lowcore.pgm_code, S390_lowcore.pgm_ilc >> 1); if (kaslr_enabled) @@ -171,4 +175,7 @@ void print_pgm_check_info(void) decompressor_printk(" %016lx %016lx %016lx %016lx\n", gpregs[12], gpregs[13], gpregs[14], gpregs[15]); print_stacktrace(); + decompressor_printk("Last Breaking-Event-Address:\n"); + decompressor_printk(" [<%016lx>] %pS\n", (unsigned long)S390_lowcore.breaking_event_addr, + (void *)S390_lowcore.breaking_event_addr); } From 074ff04e279ab8a3f97fefd1b8a313b1e4f04e9b Mon Sep 17 00:00:00 2001 From: Julian Wiedmann Date: Wed, 11 Nov 2020 08:13:14 +0100 Subject: [PATCH 241/360] s390/stp: let subsys_system_register() sysfs attributes Instead of creating the sysfs attributes for the stp root_dev by hand, pass them to subsys_system_register() as parameter. This also ensures that the attributes are available when the KOBJ_ADD event is raised. Signed-off-by: Julian Wiedmann Signed-off-by: Heiko Carstens --- arch/s390/kernel/time.c | 44 +++++++++++++---------------------------- 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 0ac30ee2c633..c59cb44fbb7d 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -927,41 +927,25 @@ static ssize_t online_store(struct device *dev, */ static DEVICE_ATTR_RW(online); -static struct device_attribute *stp_attributes[] = { - &dev_attr_ctn_id, - &dev_attr_ctn_type, - &dev_attr_dst_offset, - &dev_attr_leap_seconds, - &dev_attr_online, - &dev_attr_leap_seconds_scheduled, - &dev_attr_stratum, - &dev_attr_time_offset, - &dev_attr_time_zone_offset, - &dev_attr_timing_mode, - &dev_attr_timing_state, +static struct attribute *stp_dev_attrs[] = { + &dev_attr_ctn_id.attr, + &dev_attr_ctn_type.attr, + &dev_attr_dst_offset.attr, + &dev_attr_leap_seconds.attr, + &dev_attr_online.attr, + &dev_attr_leap_seconds_scheduled.attr, + &dev_attr_stratum.attr, + &dev_attr_time_offset.attr, + &dev_attr_time_zone_offset.attr, + &dev_attr_timing_mode.attr, + &dev_attr_timing_state.attr, NULL }; +ATTRIBUTE_GROUPS(stp_dev); static int __init stp_init_sysfs(void) { - struct device_attribute **attr; - int rc; - - rc = subsys_system_register(&stp_subsys, NULL); - if (rc) - goto out; - for (attr = stp_attributes; *attr; attr++) { - rc = device_create_file(stp_subsys.dev_root, *attr); - if (rc) - goto out_unreg; - } - return 0; -out_unreg: - for (; attr >= stp_attributes; attr--) - device_remove_file(stp_subsys.dev_root, *attr); - bus_unregister(&stp_subsys); -out: - return rc; + return subsys_system_register(&stp_subsys, stp_dev_groups); } device_initcall(stp_init_sysfs); From af71657c153fc18d7bcd35a3d5240fbd1ac7ebbd Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Tue, 10 Nov 2020 10:36:21 +0100 Subject: [PATCH 242/360] s390/vmem: remove redundant check Reviewed-by: David Hildenbrand Signed-off-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/mm/vmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index b239f2ba93b0..56ab9bb770f3 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -223,7 +223,7 @@ static int __ref modify_pmd_table(pud_t *pud, unsigned long addr, if (!add) { if (pmd_none(*pmd)) continue; - if (pmd_large(*pmd) && !add) { + if (pmd_large(*pmd)) { if (IS_ALIGNED(addr, PMD_SIZE) && IS_ALIGNED(next, PMD_SIZE)) { if (!direct) From 12bb4c682354740e4aaf0103a9bf283df42adaa9 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Tue, 10 Nov 2020 10:36:23 +0100 Subject: [PATCH 243/360] s390/vmem: make variable and function names consistent Rename some variable and functions to better clarify what they are and what they do. Signed-off-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/mm/vmem.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 56ab9bb770f3..01f3a5f58e64 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -76,20 +76,20 @@ static void vmem_pte_free(unsigned long *table) /* * The unused vmemmap range, which was not yet memset(PAGE_UNUSED) ranges - * from unused_pmd_start to next PMD_SIZE boundary. + * from unused_sub_pmd_start to next PMD_SIZE boundary. */ -static unsigned long unused_pmd_start; +static unsigned long unused_sub_pmd_start; -static void vmemmap_flush_unused_pmd(void) +static void vmemmap_flush_unused_sub_pmd(void) { - if (!unused_pmd_start) + if (!unused_sub_pmd_start) return; - memset(__va(unused_pmd_start), PAGE_UNUSED, - ALIGN(unused_pmd_start, PMD_SIZE) - unused_pmd_start); - unused_pmd_start = 0; + memset(__va(unused_sub_pmd_start), PAGE_UNUSED, + ALIGN(unused_sub_pmd_start, PMD_SIZE) - unused_sub_pmd_start); + unused_sub_pmd_start = 0; } -static void __vmemmap_use_sub_pmd(unsigned long start, unsigned long end) +static void vmemmap_mark_sub_pmd_used(unsigned long start, unsigned long end) { /* * As we expect to add in the same granularity as we remove, it's @@ -106,24 +106,24 @@ static void vmemmap_use_sub_pmd(unsigned long start, unsigned long end) * We only optimize if the new used range directly follows the * previously unused range (esp., when populating consecutive sections). */ - if (unused_pmd_start == start) { - unused_pmd_start = end; - if (likely(IS_ALIGNED(unused_pmd_start, PMD_SIZE))) - unused_pmd_start = 0; + if (unused_sub_pmd_start == start) { + unused_sub_pmd_start = end; + if (likely(IS_ALIGNED(unused_sub_pmd_start, PMD_SIZE))) + unused_sub_pmd_start = 0; return; } - vmemmap_flush_unused_pmd(); - __vmemmap_use_sub_pmd(start, end); + vmemmap_flush_unused_sub_pmd(); + vmemmap_mark_sub_pmd_used(start, end); } static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end) { void *page = __va(ALIGN_DOWN(start, PMD_SIZE)); - vmemmap_flush_unused_pmd(); + vmemmap_flush_unused_sub_pmd(); /* Could be our memmap page is filled with PAGE_UNUSED already ... */ - __vmemmap_use_sub_pmd(start, end); + vmemmap_mark_sub_pmd_used(start, end); /* Mark the unused parts of the new memmap page PAGE_UNUSED. */ if (!IS_ALIGNED(start, PMD_SIZE)) @@ -134,7 +134,7 @@ static void vmemmap_use_new_sub_pmd(unsigned long start, unsigned long end) * unused range in the populated PMD. */ if (!IS_ALIGNED(end, PMD_SIZE)) - unused_pmd_start = end; + unused_sub_pmd_start = end; } /* Returns true if the PMD is completely unused and can be freed. */ @@ -142,7 +142,7 @@ static bool vmemmap_unuse_sub_pmd(unsigned long start, unsigned long end) { void *page = __va(ALIGN_DOWN(start, PMD_SIZE)); - vmemmap_flush_unused_pmd(); + vmemmap_flush_unused_sub_pmd(); memset(__va(start), PAGE_UNUSED, end - start); return !memchr_inv(page, PAGE_UNUSED, PMD_SIZE); } From 334ef6ed06fa1a54e35296b77b693bcf6d63ee9e Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 18 Nov 2020 21:32:33 +0100 Subject: [PATCH 244/360] init/Kconfig: make COMPILE_TEST depend on !S390 While allmodconfig and allyesconfig build for s390 there are also various bots running compile tests with randconfig, where PCI is disabled. This reveals that a lot of drivers should actually depend on HAS_IOMEM. Adding this to each device driver would be a never ending story, therefore just disable COMPILE_TEST for s390. The reasoning is more or less the same as described in commit bc083a64b6c0 ("init/Kconfig: make COMPILE_TEST depend on !UML"). Reported-by: kernel test robot Suggested-by: Arnd Bergmann Signed-off-by: Heiko Carstens --- init/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/Kconfig b/init/Kconfig index c9446911cf41..36e280846c5f 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -110,7 +110,7 @@ config INIT_ENV_ARG_LIMIT config COMPILE_TEST bool "Compile also drivers which will not load" - depends on !UML + depends on !UML && !S390 default n help Some drivers can be compiled on a different platform than they are From ab09b58e4bdfdbcec425e54ebeaf6e209a96318f Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Thu, 29 Oct 2020 17:02:58 +0100 Subject: [PATCH 245/360] x86/boot/compressed/64: Use TEST %reg,%reg instead of CMP $0,%reg Use TEST %reg,%reg which sets the zero flag in the same way as CMP $0,%reg, but the encoding uses one byte less. Signed-off-by: Uros Bizjak Signed-off-by: Borislav Petkov Reviewed-by: Andy Lutomirski Link: https://lkml.kernel.org/r/20201029160258.139216-1-ubizjak@gmail.com --- arch/x86/boot/compressed/head_64.S | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 017de6cc87dc..e94874f4bbc1 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -241,12 +241,12 @@ SYM_FUNC_START(startup_32) leal rva(startup_64)(%ebp), %eax #ifdef CONFIG_EFI_MIXED movl rva(efi32_boot_args)(%ebp), %edi - cmp $0, %edi + testl %edi, %edi jz 1f leal rva(efi64_stub_entry)(%ebp), %eax movl rva(efi32_boot_args+4)(%ebp), %esi movl rva(efi32_boot_args+8)(%ebp), %edx // saved bootparams pointer - cmpl $0, %edx + testl %edx, %edx jnz 1f /* * efi_pe_entry uses MS calling convention, which requires 32 bytes of @@ -592,7 +592,7 @@ SYM_CODE_START(trampoline_32bit_src) movl %eax, %cr0 /* Check what paging mode we want to be in after the trampoline */ - cmpl $0, %edx + testl %edx, %edx jz 1f /* We want 5-level paging: don't touch CR3 if it already points to 5-level page tables */ @@ -622,7 +622,7 @@ SYM_CODE_START(trampoline_32bit_src) /* Enable PAE and LA57 (if required) paging modes */ movl $X86_CR4_PAE, %eax - cmpl $0, %edx + testl %edx, %edx jz 1f orl $X86_CR4_LA57, %eax 1: From 4a24d80b8c3e9f89d6a6a7b89bd057c463b638d3 Mon Sep 17 00:00:00 2001 From: Smita Koralahalli Date: Thu, 19 Nov 2020 12:29:38 -0600 Subject: [PATCH 246/360] x86/mce, cper: Pass x86 CPER through the MCA handling chain The kernel uses ACPI Boot Error Record Table (BERT) to report fatal errors that occurred in a previous boot. The MCA errors in the BERT are reported using the x86 Processor Error Common Platform Error Record (CPER) format. Currently, the record prints out the raw MSR values and AMD relies on the raw record to provide MCA information. Extract the raw MSR values of MCA registers from the BERT and feed them into mce_log() to decode them properly. The implementation is SMCA-specific as the raw MCA register values are given in the register offset order of the SMCA address space. [ bp: Massage. ] [ Fix a build breakage in patch v1. ] Reported-by: kernel test robot Signed-off-by: Smita Koralahalli Signed-off-by: Borislav Petkov Reviewed-by: Punit Agrawal Acked-by: Ard Biesheuvel Link: https://lkml.kernel.org/r/20201119182938.151155-1-Smita.KoralahalliChannabasappa@amd.com --- arch/x86/include/asm/acpi.h | 11 ++++++ arch/x86/include/asm/mce.h | 6 ++++ arch/x86/kernel/acpi/apei.c | 5 +++ arch/x86/kernel/cpu/mce/apei.c | 61 +++++++++++++++++++++++++++++++++ drivers/firmware/efi/cper-x86.c | 11 ++++-- 5 files changed, 91 insertions(+), 3 deletions(-) diff --git a/arch/x86/include/asm/acpi.h b/arch/x86/include/asm/acpi.h index 6d2df1ee427b..65064d9f7fa6 100644 --- a/arch/x86/include/asm/acpi.h +++ b/arch/x86/include/asm/acpi.h @@ -159,6 +159,8 @@ static inline u64 x86_default_get_root_pointer(void) extern int x86_acpi_numa_init(void); #endif /* CONFIG_ACPI_NUMA */ +struct cper_ia_proc_ctx; + #ifdef CONFIG_ACPI_APEI static inline pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr) { @@ -177,6 +179,15 @@ static inline pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr) */ return PAGE_KERNEL_NOENC; } + +int arch_apei_report_x86_error(struct cper_ia_proc_ctx *ctx_info, + u64 lapic_id); +#else +static inline int arch_apei_report_x86_error(struct cper_ia_proc_ctx *ctx_info, + u64 lapic_id) +{ + return -EINVAL; +} #endif #define ACPI_TABLE_UPGRADE_MAX_PHYS (max_low_pfn_mapped << PAGE_SHIFT) diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h index fc25c88c7ff2..56cdeaac76a0 100644 --- a/arch/x86/include/asm/mce.h +++ b/arch/x86/include/asm/mce.h @@ -199,16 +199,22 @@ static inline void enable_copy_mc_fragile(void) } #endif +struct cper_ia_proc_ctx; + #ifdef CONFIG_X86_MCE int mcheck_init(void); void mcheck_cpu_init(struct cpuinfo_x86 *c); void mcheck_cpu_clear(struct cpuinfo_x86 *c); void mcheck_vendor_init_severity(void); +int apei_smca_report_x86_error(struct cper_ia_proc_ctx *ctx_info, + u64 lapic_id); #else static inline int mcheck_init(void) { return 0; } static inline void mcheck_cpu_init(struct cpuinfo_x86 *c) {} static inline void mcheck_cpu_clear(struct cpuinfo_x86 *c) {} static inline void mcheck_vendor_init_severity(void) {} +static inline int apei_smca_report_x86_error(struct cper_ia_proc_ctx *ctx_info, + u64 lapic_id) { return -EINVAL; } #endif #ifdef CONFIG_X86_ANCIENT_MCE diff --git a/arch/x86/kernel/acpi/apei.c b/arch/x86/kernel/acpi/apei.c index c22fb55abcfd..0916f00a992e 100644 --- a/arch/x86/kernel/acpi/apei.c +++ b/arch/x86/kernel/acpi/apei.c @@ -43,3 +43,8 @@ void arch_apei_report_mem_error(int sev, struct cper_sec_mem_err *mem_err) apei_mce_report_mem_error(sev, mem_err); #endif } + +int arch_apei_report_x86_error(struct cper_ia_proc_ctx *ctx_info, u64 lapic_id) +{ + return apei_smca_report_x86_error(ctx_info, lapic_id); +} diff --git a/arch/x86/kernel/cpu/mce/apei.c b/arch/x86/kernel/cpu/mce/apei.c index af8d37962586..b58b85380ddb 100644 --- a/arch/x86/kernel/cpu/mce/apei.c +++ b/arch/x86/kernel/cpu/mce/apei.c @@ -51,6 +51,67 @@ void apei_mce_report_mem_error(int severity, struct cper_sec_mem_err *mem_err) } EXPORT_SYMBOL_GPL(apei_mce_report_mem_error); +int apei_smca_report_x86_error(struct cper_ia_proc_ctx *ctx_info, u64 lapic_id) +{ + const u64 *i_mce = ((const u64 *) (ctx_info + 1)); + unsigned int cpu; + struct mce m; + + if (!boot_cpu_has(X86_FEATURE_SMCA)) + return -EINVAL; + + /* + * The starting address of the register array extracted from BERT must + * match with the first expected register in the register layout of + * SMCA address space. This address corresponds to banks's MCA_STATUS + * register. + * + * Match any MCi_STATUS register by turning off bank numbers. + */ + if ((ctx_info->msr_addr & MSR_AMD64_SMCA_MC0_STATUS) != + MSR_AMD64_SMCA_MC0_STATUS) + return -EINVAL; + + /* + * The register array size must be large enough to include all the + * SMCA registers which need to be extracted. + * + * The number of registers in the register array is determined by + * Register Array Size/8 as defined in UEFI spec v2.8, sec N.2.4.2.2. + * The register layout is fixed and currently the raw data in the + * register array includes 6 SMCA registers which the kernel can + * extract. + */ + if (ctx_info->reg_arr_size < 48) + return -EINVAL; + + mce_setup(&m); + + m.extcpu = -1; + m.socketid = -1; + + for_each_possible_cpu(cpu) { + if (cpu_data(cpu).initial_apicid == lapic_id) { + m.extcpu = cpu; + m.socketid = cpu_data(m.extcpu).phys_proc_id; + break; + } + } + + m.apicid = lapic_id; + m.bank = (ctx_info->msr_addr >> 4) & 0xFF; + m.status = *i_mce; + m.addr = *(i_mce + 1); + m.misc = *(i_mce + 2); + /* Skipping MCA_CONFIG */ + m.ipid = *(i_mce + 4); + m.synd = *(i_mce + 5); + + mce_log(&m); + + return 0; +} + #define CPER_CREATOR_MCE \ GUID_INIT(0x75a574e3, 0x5052, 0x4b29, 0x8a, 0x8e, 0xbe, 0x2c, \ 0x64, 0x90, 0xb8, 0x9d) diff --git a/drivers/firmware/efi/cper-x86.c b/drivers/firmware/efi/cper-x86.c index 2531de49f56c..438ed9eff6d0 100644 --- a/drivers/firmware/efi/cper-x86.c +++ b/drivers/firmware/efi/cper-x86.c @@ -2,6 +2,7 @@ // Copyright (C) 2018, Advanced Micro Devices, Inc. #include +#include /* * We don't need a "CPER_IA" prefix since these are all locally defined. @@ -347,9 +348,13 @@ void cper_print_proc_ia(const char *pfx, const struct cper_sec_proc_ia *proc) ctx_info->mm_reg_addr); } - printk("%sRegister Array:\n", newpfx); - print_hex_dump(newpfx, "", DUMP_PREFIX_OFFSET, 16, groupsize, - (ctx_info + 1), ctx_info->reg_arr_size, 0); + if (ctx_info->reg_ctx_type != CTX_TYPE_MSR || + arch_apei_report_x86_error(ctx_info, proc->lapic_id)) { + printk("%sRegister Array:\n", newpfx); + print_hex_dump(newpfx, "", DUMP_PREFIX_OFFSET, 16, + groupsize, (ctx_info + 1), + ctx_info->reg_arr_size, 0); + } ctx_info = (struct cper_ia_proc_ctx *)((long)ctx_info + size); } From 266994e3ac78955f2f7f4ac3fe588002c7bd679c Mon Sep 17 00:00:00 2001 From: Finn Thain Date: Fri, 20 Nov 2020 15:39:56 +1100 Subject: [PATCH 247/360] m68k: mac: Refactor iop_preinit() and iop_init() The idea behind iop_preinit() was to put the SCC IOP into bypass mode. However, that remains unimplemented and implementing it would be difficult. Let the comments and code reflect this. Even if iop_preinit() worked as described in the comments, it gets called immediately before iop_init() so it might as well part of iop_init(). Cc: Joshua Thompson Tested-by: Stan Johnson Signed-off-by: Finn Thain Link: https://lore.kernel.org/r/0a7b09f5e5f48e270b82041c19e8f20f54c69216.1605847196.git.fthain@telegraphics.com.au Signed-off-by: Geert Uytterhoeven --- arch/m68k/mac/config.c | 8 ------- arch/m68k/mac/iop.c | 54 ++++++++++++++---------------------------- 2 files changed, 18 insertions(+), 44 deletions(-) diff --git a/arch/m68k/mac/config.c b/arch/m68k/mac/config.c index eff442e932cf..36bbefe60117 100644 --- a/arch/m68k/mac/config.c +++ b/arch/m68k/mac/config.c @@ -55,7 +55,6 @@ struct mac_booter_data mac_bi_data; static unsigned long mac_orig_videoaddr; extern int mac_hwclk(int, struct rtc_time *); -extern void iop_preinit(void); extern void iop_init(void); extern void via_init(void); extern void via_init_clock(irq_handler_t func); @@ -834,13 +833,6 @@ static void __init mac_identify(void) break; } - /* - * We need to pre-init the IOPs, if any. Otherwise - * the serial console won't work if the user had - * the serial ports set to "Faster" mode in MacOS. - */ - iop_preinit(); - pr_info("Detected Macintosh model: %d\n", model); /* diff --git a/arch/m68k/mac/iop.c b/arch/m68k/mac/iop.c index c669a7644301..de156a027f5b 100644 --- a/arch/m68k/mac/iop.c +++ b/arch/m68k/mac/iop.c @@ -47,6 +47,10 @@ * * TODO: * + * o The SCC IOP has to be placed in bypass mode before the serial console + * gets initialized. iop_init() would be one place to do that. Or the + * bootloader could do that. For now, the Serial Switch control panel + * is needed for that -- contrary to the changelog above. * o Something should be periodically checking iop_alive() to make sure the * IOP hasn't died. * o Some of the IOP manager routines need better error checking and @@ -224,40 +228,6 @@ static struct iop_msg *iop_get_unused_msg(void) return NULL; } -/* - * This is called by the startup code before anything else. Its purpose - * is to find and initialize the IOPs early in the boot sequence, so that - * the serial IOP can be placed into bypass mode _before_ we try to - * initialize the serial console. - */ - -void __init iop_preinit(void) -{ - if (macintosh_config->scc_type == MAC_SCC_IOP) { - if (macintosh_config->ident == MAC_MODEL_IIFX) { - iop_base[IOP_NUM_SCC] = (struct mac_iop *) SCC_IOP_BASE_IIFX; - } else { - iop_base[IOP_NUM_SCC] = (struct mac_iop *) SCC_IOP_BASE_QUADRA; - } - iop_scc_present = 1; - } else { - iop_base[IOP_NUM_SCC] = NULL; - iop_scc_present = 0; - } - if (macintosh_config->adb_type == MAC_ADB_IOP) { - if (macintosh_config->ident == MAC_MODEL_IIFX) { - iop_base[IOP_NUM_ISM] = (struct mac_iop *) ISM_IOP_BASE_IIFX; - } else { - iop_base[IOP_NUM_ISM] = (struct mac_iop *) ISM_IOP_BASE_QUADRA; - } - iop_stop(iop_base[IOP_NUM_ISM]); - iop_ism_present = 1; - } else { - iop_base[IOP_NUM_ISM] = NULL; - iop_ism_present = 0; - } -} - /* * Initialize the IOPs, if present. */ @@ -266,11 +236,23 @@ void __init iop_init(void) { int i; - if (iop_scc_present) { + if (macintosh_config->scc_type == MAC_SCC_IOP) { + if (macintosh_config->ident == MAC_MODEL_IIFX) + iop_base[IOP_NUM_SCC] = (struct mac_iop *)SCC_IOP_BASE_IIFX; + else + iop_base[IOP_NUM_SCC] = (struct mac_iop *)SCC_IOP_BASE_QUADRA; + iop_scc_present = 1; pr_debug("SCC IOP detected at %p\n", iop_base[IOP_NUM_SCC]); } - if (iop_ism_present) { + if (macintosh_config->adb_type == MAC_ADB_IOP) { + if (macintosh_config->ident == MAC_MODEL_IIFX) + iop_base[IOP_NUM_ISM] = (struct mac_iop *)ISM_IOP_BASE_IIFX; + else + iop_base[IOP_NUM_ISM] = (struct mac_iop *)ISM_IOP_BASE_QUADRA; + iop_ism_present = 1; pr_debug("ISM IOP detected at %p\n", iop_base[IOP_NUM_ISM]); + + iop_stop(iop_base[IOP_NUM_ISM]); iop_start(iop_base[IOP_NUM_ISM]); iop_alive(iop_base[IOP_NUM_ISM]); /* clears the alive flag */ } From 471037e2c9bbc96048a024b68bc9873b588fbe96 Mon Sep 17 00:00:00 2001 From: Finn Thain Date: Fri, 20 Nov 2020 15:39:56 +1100 Subject: [PATCH 248/360] m68k: mac: Remove dead code Cc: Joshua Thompson Signed-off-by: Finn Thain Link: https://lore.kernel.org/r/effef6339c919a4ef2e81a47e4383f712cdd7626.1605847196.git.fthain@telegraphics.com.au Signed-off-by: Geert Uytterhoeven --- arch/m68k/mac/via.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/arch/m68k/mac/via.c b/arch/m68k/mac/via.c index ac77d73af19a..2217ca688b64 100644 --- a/arch/m68k/mac/via.c +++ b/arch/m68k/mac/via.c @@ -304,21 +304,6 @@ void via_l2_flush(int writeback) local_irq_restore(flags); } -/* - * Return the status of the L2 cache on a IIci - */ - -int via_get_cache_disable(void) -{ - /* Safeguard against being called accidentally */ - if (!via2) { - printk(KERN_ERR "via_get_cache_disable called on a non-VIA machine!\n"); - return 1; - } - - return (int) via2[gBufB] & VIA2B_vCDis; -} - /* * Initialize VIA2 for Nubus access */ From 0c450b8e7882e42c3f65ab434c48cc110640de92 Mon Sep 17 00:00:00 2001 From: Finn Thain Date: Fri, 20 Nov 2020 15:39:56 +1100 Subject: [PATCH 249/360] m68k: mac: Remove redundant VIA register writes There's no need to write the same value to the timer latch and timer counter registers. Values written to the counter registers get stored in the latches anyway. The write to vT1CH copies the latch values to the counter. Cc: Joshua Thompson Tested-by: Stan Johnson Signed-off-by: Finn Thain Link: https://lore.kernel.org/r/c6b1d9620af3e8f89dd0157a41fa4147294b251d.1605847196.git.fthain@telegraphics.com.au Signed-off-by: Geert Uytterhoeven --- arch/m68k/mac/via.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/arch/m68k/mac/via.c b/arch/m68k/mac/via.c index 2217ca688b64..9f2b32f22f16 100644 --- a/arch/m68k/mac/via.c +++ b/arch/m68k/mac/via.c @@ -169,8 +169,6 @@ void __init via_init(void) via1[vIER] = 0x7F; via1[vIFR] = 0x7F; - via1[vT1LL] = 0; - via1[vT1LH] = 0; via1[vT1CL] = 0; via1[vT1CH] = 0; via1[vT2CL] = 0; @@ -225,8 +223,6 @@ void __init via_init(void) via2[gIER] = 0x7F; via2[gIFR] = 0x7F | rbv_clear; if (!rbv_present) { - via2[vT1LL] = 0; - via2[vT1LH] = 0; via2[vT1CL] = 0; via2[vT1CH] = 0; via2[vT2CL] = 0; @@ -604,8 +600,6 @@ void __init via_init_clock(irq_handler_t timer_routine) return; } - via1[vT1LL] = VIA_TC_LOW; - via1[vT1LH] = VIA_TC_HIGH; via1[vT1CL] = VIA_TC_LOW; via1[vT1CH] = VIA_TC_HIGH; via1[vACR] |= 0x40; From 07ce9b7ab0bff8f7e2b51db43e69d36f018c8118 Mon Sep 17 00:00:00 2001 From: Finn Thain Date: Fri, 20 Nov 2020 15:39:56 +1100 Subject: [PATCH 250/360] m68k: mac: Update Kconfig help There is still some missing hardware support that affects all models, such as sound chip and localtalk support. However, many models are well supported, including the Quadra 800 emulated by QEMU. Missing hardware support is mostly documented at the web site, so add the URL. Cc: Joshua Thompson Signed-off-by: Finn Thain Link: https://lore.kernel.org/r/bb327f05f8fb61eeb332cc2ba4e8335570976474.1605847196.git.fthain@telegraphics.com.au Signed-off-by: Geert Uytterhoeven --- arch/m68k/Kconfig.machine | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/arch/m68k/Kconfig.machine b/arch/m68k/Kconfig.machine index 17e8c3a292d7..054ff6392329 100644 --- a/arch/m68k/Kconfig.machine +++ b/arch/m68k/Kconfig.machine @@ -30,11 +30,9 @@ config MAC select HAVE_ARCH_NVRAM_OPS help This option enables support for the Apple Macintosh series of - computers (yes, there is experimental support now, at least for part - of the series). - - Say N unless you're willing to code the remaining necessary support. - ;) + computers. If you plan to use this kernel on a Mac, say Y here and + browse the documentation available at ; + otherwise say N. config APOLLO bool "Apollo support" From 549aeec256caa80e03607c88f1030b58fb87aaf3 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Fri, 20 Nov 2020 14:07:54 +0800 Subject: [PATCH 251/360] m68k: Drop redundant NOTES in link script Commit eaf937075c9a ("vmlinux.lds.h: Move NOTES into RO_DATA") after should remove redundant NOTES. Signed-off-by: Youling Tang Link: https://lore.kernel.org/r/1605852474-23446-1-git-send-email-tangyouling@loongson.cn Signed-off-by: Geert Uytterhoeven --- arch/m68k/kernel/vmlinux-nommu.lds | 1 - arch/m68k/kernel/vmlinux-std.lds | 1 - arch/m68k/kernel/vmlinux-sun3.lds | 1 - 3 files changed, 3 deletions(-) diff --git a/arch/m68k/kernel/vmlinux-nommu.lds b/arch/m68k/kernel/vmlinux-nommu.lds index 7b975420c3d9..247e19f0d0ea 100644 --- a/arch/m68k/kernel/vmlinux-nommu.lds +++ b/arch/m68k/kernel/vmlinux-nommu.lds @@ -65,7 +65,6 @@ SECTIONS { _edata = .; EXCEPTION_TABLE(16) - NOTES . = ALIGN(PAGE_SIZE); __init_begin = .; diff --git a/arch/m68k/kernel/vmlinux-std.lds b/arch/m68k/kernel/vmlinux-std.lds index 4d33da4e7106..15113468d326 100644 --- a/arch/m68k/kernel/vmlinux-std.lds +++ b/arch/m68k/kernel/vmlinux-std.lds @@ -49,7 +49,6 @@ SECTIONS *(.m68k_fixup) __stop_fixup = .; } - NOTES .init_end : { /* This ALIGN be in a section so that _end is at the end of the load segment. */ diff --git a/arch/m68k/kernel/vmlinux-sun3.lds b/arch/m68k/kernel/vmlinux-sun3.lds index 87d9f4d08f65..90ff8e54995a 100644 --- a/arch/m68k/kernel/vmlinux-sun3.lds +++ b/arch/m68k/kernel/vmlinux-sun3.lds @@ -33,7 +33,6 @@ SECTIONS RW_DATA(16, PAGE_SIZE, THREAD_SIZE) :data /* End of data goes *here* so that freeing init code works properly. */ _edata = .; - NOTES /* will be freed after init */ . = ALIGN(PAGE_SIZE); /* Init code and data */ From 428ec5f9dde73fa93d1bb2dc27db1db167dd9530 Mon Sep 17 00:00:00 2001 From: Youling Tang Date: Fri, 20 Nov 2020 14:08:14 +0800 Subject: [PATCH 252/360] m68k: Add a missing ELF_DETAILS in link script Commit c604abc3f6e3 ("vmlinux.lds.h: Split ELF_DETAILS from STABS_DEBUG") after should add a missing ELF_DETAILS, at the same time, the .comment section has been included in the ELF_DETAILS. Signed-off-by: Youling Tang Link: https://lore.kernel.org/r/1605852494-23515-1-git-send-email-tangyouling@loongson.cn Signed-off-by: Geert Uytterhoeven --- arch/m68k/kernel/vmlinux-nommu.lds | 2 +- arch/m68k/kernel/vmlinux-std.lds | 2 +- arch/m68k/kernel/vmlinux-sun3.lds | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/m68k/kernel/vmlinux-nommu.lds b/arch/m68k/kernel/vmlinux-nommu.lds index 247e19f0d0ea..396e126a4258 100644 --- a/arch/m68k/kernel/vmlinux-nommu.lds +++ b/arch/m68k/kernel/vmlinux-nommu.lds @@ -86,7 +86,7 @@ SECTIONS { _end = .; STABS_DEBUG - .comment 0 : { *(.comment) } + ELF_DETAILS /* Sections to be discarded */ DISCARDS diff --git a/arch/m68k/kernel/vmlinux-std.lds b/arch/m68k/kernel/vmlinux-std.lds index 15113468d326..ed1d9eda3190 100644 --- a/arch/m68k/kernel/vmlinux-std.lds +++ b/arch/m68k/kernel/vmlinux-std.lds @@ -59,7 +59,7 @@ SECTIONS _end = . ; STABS_DEBUG - .comment 0 : { *(.comment) } + ELF_DETAILS /* Sections to be discarded */ DISCARDS diff --git a/arch/m68k/kernel/vmlinux-sun3.lds b/arch/m68k/kernel/vmlinux-sun3.lds index 90ff8e54995a..4a52f44f2ef0 100644 --- a/arch/m68k/kernel/vmlinux-sun3.lds +++ b/arch/m68k/kernel/vmlinux-sun3.lds @@ -52,6 +52,7 @@ __init_begin = .; _end = . ; STABS_DEBUG + ELF_DETAILS /* Sections to be discarded */ DISCARDS From a7b5458ce73b235be027cf2658c39b19b7e58cf2 Mon Sep 17 00:00:00 2001 From: Finn Thain Date: Sun, 22 Nov 2020 10:28:17 +1100 Subject: [PATCH 253/360] m68k: Fix WARNING splat in pmac_zilog driver Don't add platform resources that won't be used. This avoids a recently-added warning from the driver core, that can show up on a multi-platform kernel when !MACH_IS_MAC. ------------[ cut here ]------------ WARNING: CPU: 0 PID: 0 at drivers/base/platform.c:224 platform_get_irq_optional+0x8e/0xce 0 is an invalid IRQ number Modules linked in: CPU: 0 PID: 0 Comm: swapper Not tainted 5.9.0-multi #1 Stack from 004b3f04: 004b3f04 00462c2f 00462c2f 004b3f20 0002e128 004754db 004b6ad4 004b3f4c 0002e19c 004754f7 000000e0 00285ba0 00000009 00000000 004b3f44 ffffffff 004754db 004b3f64 004b3f74 00285ba0 004754f7 000000e0 00000009 004754db 004fdf0c 005269e2 004fdf0c 00000000 004b3f88 00285cae 004b6964 00000000 004fdf0c 004b3fac 0051cc68 004b6964 00000000 004b6964 00000200 00000000 0051cc3e 0023c18a 004b3fc0 0051cd8a 004fdf0c 00000002 0052b43c 004b3fc8 Call Trace: [<0002e128>] __warn+0xa6/0xd6 [<0002e19c>] warn_slowpath_fmt+0x44/0x76 [<00285ba0>] platform_get_irq_optional+0x8e/0xce [<00285ba0>] platform_get_irq_optional+0x8e/0xce [<00285cae>] platform_get_irq+0x12/0x4c [<0051cc68>] pmz_init_port+0x2a/0xa6 [<0051cc3e>] pmz_init_port+0x0/0xa6 [<0023c18a>] strlen+0x0/0x22 [<0051cd8a>] pmz_probe+0x34/0x88 [<0051cde6>] pmz_console_init+0x8/0x28 [<00511776>] console_init+0x1e/0x28 [<0005a3bc>] printk+0x0/0x16 [<0050a8a6>] start_kernel+0x368/0x4ce [<005094f8>] _sinittext+0x4f8/0xc48 random: get_random_bytes called from print_oops_end_marker+0x56/0x80 with crng_init=0 ---[ end trace 392d8e82eed68d6c ]--- Commit a85a6c86c25b ("driver core: platform: Clarify that IRQ 0 is invalid"), which introduced the WARNING, suggests that testing for irq == 0 is undesirable. Instead of that comparison, just test for resource existence. Cc: Michael Ellerman Cc: Benjamin Herrenschmidt Cc: Paul Mackerras Cc: Joshua Thompson Cc: Greg Kroah-Hartman Cc: Jiri Slaby Cc: stable@vger.kernel.org # v5.8+ Reported-by: Laurent Vivier Signed-off-by: Finn Thain Link: https://lore.kernel.org/r/0c0fe1e4f11ccec202d4df09ea7d9d98155d101a.1606001297.git.fthain@telegraphics.com.au Signed-off-by: Geert Uytterhoeven --- arch/m68k/mac/config.c | 17 +++++++++-------- drivers/tty/serial/pmac_zilog.c | 14 +++++++++----- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/arch/m68k/mac/config.c b/arch/m68k/mac/config.c index 36bbefe60117..e239d2ddccec 100644 --- a/arch/m68k/mac/config.c +++ b/arch/m68k/mac/config.c @@ -775,16 +775,12 @@ static struct resource scc_b_rsrcs[] = { struct platform_device scc_a_pdev = { .name = "scc", .id = 0, - .num_resources = ARRAY_SIZE(scc_a_rsrcs), - .resource = scc_a_rsrcs, }; EXPORT_SYMBOL(scc_a_pdev); struct platform_device scc_b_pdev = { .name = "scc", .id = 1, - .num_resources = ARRAY_SIZE(scc_b_rsrcs), - .resource = scc_b_rsrcs, }; EXPORT_SYMBOL(scc_b_pdev); @@ -811,10 +807,15 @@ static void __init mac_identify(void) /* Set up serial port resources for the console initcall. */ - scc_a_rsrcs[0].start = (resource_size_t) mac_bi_data.sccbase + 2; - scc_a_rsrcs[0].end = scc_a_rsrcs[0].start; - scc_b_rsrcs[0].start = (resource_size_t) mac_bi_data.sccbase; - scc_b_rsrcs[0].end = scc_b_rsrcs[0].start; + scc_a_rsrcs[0].start = (resource_size_t)mac_bi_data.sccbase + 2; + scc_a_rsrcs[0].end = scc_a_rsrcs[0].start; + scc_a_pdev.num_resources = ARRAY_SIZE(scc_a_rsrcs); + scc_a_pdev.resource = scc_a_rsrcs; + + scc_b_rsrcs[0].start = (resource_size_t)mac_bi_data.sccbase; + scc_b_rsrcs[0].end = scc_b_rsrcs[0].start; + scc_b_pdev.num_resources = ARRAY_SIZE(scc_b_rsrcs); + scc_b_pdev.resource = scc_b_rsrcs; switch (macintosh_config->scc_type) { case MAC_SCC_PSC: diff --git a/drivers/tty/serial/pmac_zilog.c b/drivers/tty/serial/pmac_zilog.c index 063484b22523..d6aef8a1f0a4 100644 --- a/drivers/tty/serial/pmac_zilog.c +++ b/drivers/tty/serial/pmac_zilog.c @@ -1693,22 +1693,26 @@ static int __init pmz_probe(void) #else +/* On PCI PowerMacs, pmz_probe() does an explicit search of the OpenFirmware + * tree to obtain the device_nodes needed to start the console before the + * macio driver. On Macs without OpenFirmware, global platform_devices take + * the place of those device_nodes. + */ extern struct platform_device scc_a_pdev, scc_b_pdev; static int __init pmz_init_port(struct uart_pmac_port *uap) { - struct resource *r_ports; - int irq; + struct resource *r_ports, *r_irq; r_ports = platform_get_resource(uap->pdev, IORESOURCE_MEM, 0); - irq = platform_get_irq(uap->pdev, 0); - if (!r_ports || irq <= 0) + r_irq = platform_get_resource(uap->pdev, IORESOURCE_IRQ, 0); + if (!r_ports || !r_irq) return -ENODEV; uap->port.mapbase = r_ports->start; uap->port.membase = (unsigned char __iomem *) r_ports->start; uap->port.iotype = UPIO_MEM; - uap->port.irq = irq; + uap->port.irq = r_irq->start; uap->port.uartclk = ZS_CLOCK; uap->port.fifosize = 1; uap->port.ops = &pmz_pops; From 87d5986345219a7e4f204726d9085ea87f3e22d0 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 16 Nov 2020 08:06:40 +0100 Subject: [PATCH 254/360] s390/mm: remove set_fs / rework address space handling Remove set_fs support from s390. With doing this rework address space handling and simplify it. As a result address spaces are now setup like this: CPU running in | %cr1 ASCE | %cr7 ASCE | %cr13 ASCE ----------------------------|-----------|-----------|----------- user space | user | user | kernel kernel, normal execution | kernel | user | kernel kernel, kvm guest execution | gmap | user | kernel To achieve this the getcpu vdso syscall is removed in order to avoid secondary address mode and a separate vdso address space in for user space. The getcpu vdso syscall will be implemented differently with a subsequent patch. The kernel accesses user space always via secondary address space. This happens in different ways: - with mvcos in home space mode and directly read/write to secondary address space - with mvcs/mvcp in primary space mode and copy from primary space to secondary space or vice versa - with e.g. cs in secondary space mode and access secondary space Switching translation modes happens with sacf before and after instructions which access user space, like before. Lazy handling of control register reloading is removed in the hope to make everything simpler, but at the cost of making kernel entry and exit a bit slower. That is: on kernel entry the primary asce is always changed to contain the kernel asce, and on kernel exit the primary asce is changed again so it contains the user asce. In kernel mode there is only one exception to the primary asce: when kvm guests are executed the primary asce contains the gmap asce (which describes the guest address space). The primary asce is reset to kernel asce whenever kvm guest execution is interrupted, so that this doesn't has to be taken into account for any user space accesses. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/Kconfig | 1 - arch/s390/include/asm/futex.h | 6 -- arch/s390/include/asm/lowcore.h | 4 +- arch/s390/include/asm/mmu_context.h | 25 ++------ arch/s390/include/asm/processor.h | 7 --- arch/s390/include/asm/ptrace.h | 1 + arch/s390/include/asm/uaccess.h | 22 +------ arch/s390/include/asm/vdso.h | 25 -------- arch/s390/kernel/asm-offsets.c | 7 +-- arch/s390/kernel/entry.S | 75 ++++++++--------------- arch/s390/kernel/entry.h | 1 - arch/s390/kernel/process.c | 14 ----- arch/s390/kernel/smp.c | 10 +--- arch/s390/kernel/vdso.c | 57 ------------------ arch/s390/kernel/vdso64/Makefile | 2 +- arch/s390/kernel/vdso64/getcpu.S | 31 ---------- arch/s390/kernel/vdso64/vdso64.lds.S | 1 - arch/s390/lib/uaccess.c | 89 ++-------------------------- arch/s390/mm/fault.c | 29 +++------ arch/s390/mm/pgalloc.c | 13 +--- arch/s390/pci/pci_mmio.c | 6 -- 21 files changed, 51 insertions(+), 375 deletions(-) delete mode 100644 arch/s390/kernel/vdso64/getcpu.S diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 28bdb11e9db2..64c72b1ecc77 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -190,7 +190,6 @@ config S390 select PCI_DOMAINS if PCI select PCI_MSI if PCI select PCI_MSI_ARCH_FALLBACKS if PCI_MSI - select SET_FS select SPARSE_IRQ select SYSCTL_EXCEPTION_TRACE select THREAD_INFO_IN_TASK diff --git a/arch/s390/include/asm/futex.h b/arch/s390/include/asm/futex.h index 26f9144562c9..c22debfcebf1 100644 --- a/arch/s390/include/asm/futex.h +++ b/arch/s390/include/asm/futex.h @@ -26,9 +26,7 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) { int oldval = 0, newval, ret; - mm_segment_t old_fs; - old_fs = enable_sacf_uaccess(); switch (op) { case FUTEX_OP_SET: __futex_atomic_op("lr %2,%5\n", @@ -53,7 +51,6 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, default: ret = -ENOSYS; } - disable_sacf_uaccess(old_fs); if (!ret) *oval = oldval; @@ -64,10 +61,8 @@ static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newval) { - mm_segment_t old_fs; int ret; - old_fs = enable_sacf_uaccess(); asm volatile( " sacf 256\n" "0: cs %1,%4,0(%5)\n" @@ -77,7 +72,6 @@ static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, : "=d" (ret), "+d" (oldval), "=m" (*uaddr) : "0" (-EFAULT), "d" (newval), "a" (uaddr), "m" (*uaddr) : "cc", "memory"); - disable_sacf_uaccess(old_fs); *uval = oldval; return ret; } diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 612ed3c6d581..69ce9191eaf1 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -116,7 +116,7 @@ struct lowcore { /* Address space pointer. */ __u64 kernel_asce; /* 0x0380 */ __u64 user_asce; /* 0x0388 */ - __u64 vdso_asce; /* 0x0390 */ + __u8 pad_0x0390[0x0398-0x0390]; /* 0x0390 */ /* * The lpp and current_pid fields form a @@ -134,7 +134,7 @@ struct lowcore { __u32 spinlock_index; /* 0x03b0 */ __u32 fpu_flags; /* 0x03b4 */ __u64 percpu_offset; /* 0x03b8 */ - __u64 vdso_per_cpu_data; /* 0x03c0 */ + __u8 pad_0x03c0[0x03c8-0x03c0]; /* 0x03c0 */ __u64 machine_flags; /* 0x03c8 */ __u64 gmap; /* 0x03d0 */ __u8 pad_0x03d8[0x0400-0x03d8]; /* 0x03d8 */ diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index cec19ae164eb..51def960a3dd 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -71,16 +71,6 @@ static inline int init_new_context(struct task_struct *tsk, #define destroy_context(mm) do { } while (0) -static inline void set_user_asce(struct mm_struct *mm) -{ - S390_lowcore.user_asce = mm->context.asce; - __ctl_load(S390_lowcore.user_asce, 1, 1); - clear_cpu_flag(CIF_ASCE_PRIMARY); -} - -mm_segment_t enable_sacf_uaccess(void); -void disable_sacf_uaccess(mm_segment_t old_fs); - static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk) { @@ -88,15 +78,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, S390_lowcore.user_asce = next->context.asce; cpumask_set_cpu(cpu, &next->context.cpu_attach_mask); - /* Clear previous user-ASCE from CR1 and CR7 */ - if (!test_cpu_flag(CIF_ASCE_PRIMARY)) { - __ctl_load(S390_lowcore.kernel_asce, 1, 1); - set_cpu_flag(CIF_ASCE_PRIMARY); - } - if (test_cpu_flag(CIF_ASCE_SECONDARY)) { - __ctl_load(S390_lowcore.vdso_asce, 7, 7); - clear_cpu_flag(CIF_ASCE_SECONDARY); - } + /* Clear previous user-ASCE from CR7 */ + __ctl_load(S390_lowcore.kernel_asce, 7, 7); if (prev != next) cpumask_clear_cpu(cpu, &prev->context.cpu_attach_mask); } @@ -115,7 +98,7 @@ static inline void finish_arch_post_lock_switch(void) __tlb_flush_mm_lazy(mm); preempt_enable(); } - set_fs(current->thread.mm_segment); + __ctl_load(S390_lowcore.user_asce, 7, 7); } #define enter_lazy_tlb(mm,tsk) do { } while (0) @@ -126,7 +109,7 @@ static inline void activate_mm(struct mm_struct *prev, { switch_mm(prev, next, current); cpumask_set_cpu(smp_processor_id(), mm_cpumask(next)); - set_user_asce(next); + __ctl_load(S390_lowcore.user_asce, 7, 7); } #endif /* __S390_MMU_CONTEXT_H */ diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 2043c562ec55..6b7269f51f83 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -14,8 +14,6 @@ #include -#define CIF_ASCE_PRIMARY 0 /* primary asce needs fixup / uaccess */ -#define CIF_ASCE_SECONDARY 1 /* secondary asce needs fixup / uaccess */ #define CIF_NOHZ_DELAY 2 /* delay HZ disable for a tick */ #define CIF_FPU 3 /* restore FPU registers */ #define CIF_IGNORE_IRQ 4 /* ignore interrupt (for udelay) */ @@ -23,8 +21,6 @@ #define CIF_MCCK_GUEST 6 /* machine check happening in guest */ #define CIF_DEDICATED_CPU 7 /* this CPU is dedicated */ -#define _CIF_ASCE_PRIMARY BIT(CIF_ASCE_PRIMARY) -#define _CIF_ASCE_SECONDARY BIT(CIF_ASCE_SECONDARY) #define _CIF_NOHZ_DELAY BIT(CIF_NOHZ_DELAY) #define _CIF_FPU BIT(CIF_FPU) #define _CIF_IGNORE_IRQ BIT(CIF_IGNORE_IRQ) @@ -102,8 +98,6 @@ extern void __bpon(void); #define HAVE_ARCH_PICK_MMAP_LAYOUT -typedef unsigned int mm_segment_t; - /* * Thread structure */ @@ -116,7 +110,6 @@ struct thread_struct { unsigned long hardirq_timer; /* task cputime in hardirq context */ unsigned long softirq_timer; /* task cputime in softirq context */ unsigned long sys_call_table; /* system call table address */ - mm_segment_t mm_segment; unsigned long gmap_addr; /* address of last gmap fault. */ unsigned int gmap_write_flag; /* gmap fault write indication */ unsigned int gmap_int_code; /* int code of last gmap fault */ diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index 16b3e4396312..73ca7f7cac33 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -87,6 +87,7 @@ struct pt_regs unsigned int int_parm; unsigned long int_parm_long; unsigned long flags; + unsigned long cr1; }; /* diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index c868e7ee49b3..e59fd96a1561 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -18,24 +18,6 @@ #include #include -/* - * The fs value determines whether argument validity checking should be - * performed or not. If get_fs() == USER_DS, checking is performed, with - * get_fs() == KERNEL_DS, checking is bypassed. - * - * For historical reasons, these macros are grossly misnamed. - */ - -#define KERNEL_DS (0) -#define KERNEL_DS_SACF (1) -#define USER_DS (2) -#define USER_DS_SACF (3) - -#define get_fs() (current->thread.mm_segment) -#define uaccess_kernel() ((get_fs() & 2) == KERNEL_DS) - -void set_fs(mm_segment_t fs); - static inline int __range_ok(unsigned long addr, unsigned long size) { return 1; @@ -88,7 +70,7 @@ int __get_user_bad(void) __attribute__((noreturn)); static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned long size) { - unsigned long spec = 0x010000UL; + unsigned long spec = 0x810000UL; int rc; switch (size) { @@ -121,7 +103,7 @@ static __always_inline int __put_user_fn(void *x, void __user *ptr, unsigned lon static __always_inline int __get_user_fn(void *x, const void __user *ptr, unsigned long size) { - unsigned long spec = 0x01UL; + unsigned long spec = 0x81UL; int rc; switch (size) { diff --git a/arch/s390/include/asm/vdso.h b/arch/s390/include/asm/vdso.h index 29b44a930e71..9b299c05abf1 100644 --- a/arch/s390/include/asm/vdso.h +++ b/arch/s390/include/asm/vdso.h @@ -12,32 +12,7 @@ #ifndef __ASSEMBLY__ -/* - * Note about the vdso_data and vdso_per_cpu_data structures: - * - * NEVER USE THEM IN USERSPACE CODE DIRECTLY. The layout of the - * structure is supposed to be known only to the function in the vdso - * itself and may change without notice. - */ - -struct vdso_per_cpu_data { - /* - * Note: node_id and cpu_nr must be at adjacent memory locations. - * VDSO userspace must read both values with a single instruction. - */ - union { - __u64 getcpu_val; - struct { - __u32 node_id; - __u32 cpu_nr; - }; - }; -}; - extern struct vdso_data *vdso_data; -int vdso_alloc_per_cpu(struct lowcore *lowcore); -void vdso_free_per_cpu(struct lowcore *lowcore); - #endif /* __ASSEMBLY__ */ #endif /* __S390_VDSO_H__ */ diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index 483051e10db3..79724d861dc9 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -48,6 +47,7 @@ int main(void) OFFSET(__PT_INT_PARM, pt_regs, int_parm); OFFSET(__PT_INT_PARM_LONG, pt_regs, int_parm_long); OFFSET(__PT_FLAGS, pt_regs, flags); + OFFSET(__PT_CR1, pt_regs, cr1); DEFINE(__PT_SIZE, sizeof(struct pt_regs)); BLANK(); /* stack_frame offsets */ @@ -59,8 +59,6 @@ int main(void) OFFSET(__SF_SIE_REASON, stack_frame, empty1[3]); OFFSET(__SF_SIE_FLAGS, stack_frame, empty1[4]); BLANK(); - OFFSET(__VDSO_GETCPU_VAL, vdso_per_cpu_data, getcpu_val); - BLANK(); /* idle data offsets */ OFFSET(__CLOCK_IDLE_ENTER, s390_idle_data, clock_idle_enter); OFFSET(__CLOCK_IDLE_EXIT, s390_idle_data, clock_idle_exit); @@ -138,12 +136,11 @@ int main(void) OFFSET(__LC_RESTART_FN, lowcore, restart_fn); OFFSET(__LC_RESTART_DATA, lowcore, restart_data); OFFSET(__LC_RESTART_SOURCE, lowcore, restart_source); + OFFSET(__LC_KERNEL_ASCE, lowcore, kernel_asce); OFFSET(__LC_USER_ASCE, lowcore, user_asce); - OFFSET(__LC_VDSO_ASCE, lowcore, vdso_asce); OFFSET(__LC_LPP, lowcore, lpp); OFFSET(__LC_CURRENT_PID, lowcore, current_pid); OFFSET(__LC_PERCPU_OFFSET, lowcore, percpu_offset); - OFFSET(__LC_VDSO_PER_CPU, lowcore, vdso_per_cpu_data); OFFSET(__LC_MACHINE_FLAGS, lowcore, machine_flags); OFFSET(__LC_PREEMPT_COUNT, lowcore, preempt_count); OFFSET(__LC_GMAP, lowcore, gmap); diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index b454654ce5b5..d43ef46dc683 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -55,7 +55,7 @@ _TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ _TIF_UPROBE | _TIF_GUARDED_STORAGE | _TIF_PATCH_PENDING) _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ _TIF_SYSCALL_TRACEPOINT) -_CIF_WORK = (_CIF_ASCE_PRIMARY | _CIF_ASCE_SECONDARY | _CIF_FPU) +_CIF_WORK = (_CIF_FPU) _PIF_WORK = (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART) _LPP_OFFSET = __LC_LPP @@ -327,7 +327,7 @@ ENTRY(sie64a) BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) .Lsie_skip: ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE - lctlg %c1,%c1,__LC_USER_ASCE # load primary asce + lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce .Lsie_done: # some program checks are suppressing. C code (e.g. do_protection_exception) # will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There @@ -380,6 +380,7 @@ ENTRY(system_call) lg %r12,__LC_CURRENT lghi %r14,_PIF_SYSCALL .Lsysc_per: + lctlg %c1,%c1,__LC_KERNEL_ASCE lghi %r13,__TASK_thread lg %r15,__LC_KERNEL_STACK la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs @@ -427,8 +428,7 @@ ENTRY(system_call) jnz .Lsysc_work TSTMSK __TI_flags(%r12),_TIF_WORK jnz .Lsysc_work # check for work - TSTMSK __LC_CPU_FLAGS,(_CIF_WORK-_CIF_FPU) - jnz .Lsysc_work + lctlg %c1,%c1,__LC_USER_ASCE BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP TSTMSK __LC_CPU_FLAGS, _CIF_FPU jz .Lsysc_skip_fpu @@ -467,8 +467,6 @@ ENTRY(system_call) jo .Lsysc_sigpending TSTMSK __TI_flags(%r12),_TIF_NOTIFY_RESUME jo .Lsysc_notify_resume - TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY) - jnz .Lsysc_asce j .Lsysc_return # @@ -478,26 +476,6 @@ ENTRY(system_call) larl %r14,.Lsysc_return jg schedule -# -# _CIF_ASCE_PRIMARY and/or _CIF_ASCE_SECONDARY set, load user space asce -# -.Lsysc_asce: - ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_SECONDARY - lctlg %c7,%c7,__LC_VDSO_ASCE # load secondary asce - TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_PRIMARY - jz .Lsysc_return -#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES - tm __LC_STFLE_FAC_LIST+3,0x10 # has MVCOS ? - jnz .Lsysc_set_fs_fixup - ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY - lctlg %c1,%c1,__LC_USER_ASCE # load primary asce - j .Lsysc_return -.Lsysc_set_fs_fixup: -#endif - larl %r14,.Lsysc_return - jg set_fs_fixup - - # # _TIF_SIGPENDING is set, call do_signal # @@ -634,8 +612,11 @@ ENTRY(pgm_check_handler) 0: lg %r12,__LC_CURRENT lghi %r11,0 lmg %r8,%r9,__LC_PGM_OLD_PSW - tmhh %r8,0x0001 # test problem state bit - jnz 3f # -> fault in user space + tmhh %r8,0x0001 # coming from user space? + jno .Lpgm_skip_asce + lctlg %c1,%c1,__LC_KERNEL_ASCE + j 3f +.Lpgm_skip_asce: #if IS_ENABLED(CONFIG_KVM) # cleanup critical section for program checks in sie64a lgr %r14,%r9 @@ -646,7 +627,7 @@ ENTRY(pgm_check_handler) jhe 1f lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE - lctlg %c1,%c1,__LC_USER_ASCE # load primary asce + lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce larl %r9,sie_exit # skip forward to sie_exit lghi %r11,_PIF_GUEST_FAULT #endif @@ -767,6 +748,10 @@ ENTRY(io_int_handler) xgr %r10,%r10 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC stmg %r8,%r9,__PT_PSW(%r11) + tm __PT_PSW+1(%r11),0x01 # coming from user space? + jno .Lio_skip_asce + lctlg %c1,%c1,__LC_KERNEL_ASCE +.Lio_skip_asce: mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) TSTMSK __LC_CPU_FLAGS,_CIF_IGNORE_IRQ @@ -808,6 +793,7 @@ ENTRY(io_int_handler) mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) tm __PT_PSW+1(%r11),0x01 # returning to user ? jno .Lio_exit_kernel + lctlg %c1,%c1,__LC_USER_ASCE BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP stpt __LC_EXIT_TIMER .Lio_exit_kernel: @@ -873,29 +859,8 @@ ENTRY(io_int_handler) jo .Lio_guarded_storage TSTMSK __LC_CPU_FLAGS,_CIF_FPU jo .Lio_vxrs - TSTMSK __LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY) - jnz .Lio_asce j .Lio_return -# -# _CIF_ASCE_PRIMARY and/or CIF_ASCE_SECONDARY set, load user space asce -# -.Lio_asce: - ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_SECONDARY - lctlg %c7,%c7,__LC_VDSO_ASCE # load secondary asce - TSTMSK __LC_CPU_FLAGS,_CIF_ASCE_PRIMARY - jz .Lio_return -#ifndef CONFIG_HAVE_MARCH_Z10_FEATURES - tm __LC_STFLE_FAC_LIST+3,0x10 # has MVCOS ? - jnz .Lio_set_fs_fixup - ni __LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY - lctlg %c1,%c1,__LC_USER_ASCE # load primary asce - j .Lio_return -.Lio_set_fs_fixup: -#endif - larl %r14,.Lio_return - jg set_fs_fixup - # # CIF_FPU is set, restore floating-point controls and floating-point registers. # @@ -977,6 +942,10 @@ ENTRY(ext_int_handler) xgr %r10,%r10 mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC stmg %r8,%r9,__PT_PSW(%r11) + tm __PT_PSW+1(%r11),0x01 # coming from user space? + jno .Lext_skip_asce + lctlg %c1,%c1,__LC_KERNEL_ASCE +.Lext_skip_asce: lghi %r1,__LC_EXT_PARAMS2 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS @@ -1206,6 +1175,9 @@ ENTRY(mcck_int_handler) xgr %r10,%r10 mvc __PT_R8(64,%r11),0(%r14) stmg %r8,%r9,__PT_PSW(%r11) + la %r14,4095 + mvc __PT_CR1(8,%r11),__LC_CREGS_SAVE_AREA-4095+8(%r14) + lctlg %c1,%c1,__LC_KERNEL_ASCE xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) lgr %r2,%r11 # pass pointer to pt_regs @@ -1221,6 +1193,7 @@ ENTRY(mcck_int_handler) brasl %r14,s390_handle_mcck TRACE_IRQS_ON .Lmcck_return: + lctlg %c1,%c1,__PT_CR1(%r11) lmg %r0,%r10,__PT_R0(%r11) mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? @@ -1297,7 +1270,7 @@ ENDPROC(stack_overflow) 1: BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST) lg %r9,__SF_SIE_CONTROL(%r15) # get control block pointer ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE - lctlg %c1,%c1,__LC_USER_ASCE # load primary asce + lctlg %c1,%c1,__LC_KERNEL_ASCE larl %r9,sie_exit # skip forward to sie_exit BR_EX %r14,%r11 diff --git a/arch/s390/kernel/entry.h b/arch/s390/kernel/entry.h index d2ca3fe51f8e..a16c33b32ab0 100644 --- a/arch/s390/kernel/entry.h +++ b/arch/s390/kernel/entry.h @@ -83,7 +83,6 @@ long sys_s390_sthyi(unsigned long function_code, void __user *buffer, u64 __user DECLARE_PER_CPU(u64, mt_cycles[8]); void gs_load_bc_cb(struct pt_regs *regs); -void set_fs_fixup(void); unsigned long stack_alloc(void); void stack_free(unsigned long stack); diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index ec801d3bbb37..bc3ca54edfb4 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -94,7 +94,6 @@ int copy_thread(unsigned long clone_flags, unsigned long new_stackp, /* Save access registers to new thread structure. */ save_access_regs(&p->thread.acrs[0]); /* start new process with ar4 pointing to the correct address space */ - p->thread.mm_segment = get_fs(); /* Don't copy debug registers */ memset(&p->thread.per_user, 0, sizeof(p->thread.per_user)); memset(&p->thread.per_event, 0, sizeof(p->thread.per_event)); @@ -208,16 +207,3 @@ unsigned long arch_randomize_brk(struct mm_struct *mm) ret = PAGE_ALIGN(mm->brk + brk_rnd()); return (ret > mm->brk) ? ret : mm->brk; } - -void set_fs_fixup(void) -{ - struct pt_regs *regs = current_pt_regs(); - static bool warned; - - set_fs(USER_DS); - if (warned) - return; - WARN(1, "Unbalanced set_fs - int code: 0x%x\n", regs->int_code); - show_registers(regs); - warned = true; -} diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 390d97daa2b3..cac96f240dd8 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -47,7 +47,6 @@ #include #include #include -#include #include #include #include @@ -217,14 +216,10 @@ static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) lc->return_mcck_lpswe = gen_lpswe(__LC_RETURN_MCCK_PSW); if (nmi_alloc_per_cpu(lc)) goto out_async; - if (vdso_alloc_per_cpu(lc)) - goto out_mcesa; lowcore_ptr[cpu] = lc; pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, (u32)(unsigned long) lc); return 0; -out_mcesa: - nmi_free_per_cpu(lc); out_async: stack_free(async_stack); out: @@ -245,7 +240,6 @@ static void pcpu_free_lowcore(struct pcpu *pcpu) pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, 0); lowcore_ptr[pcpu - pcpu_devices] = NULL; - vdso_free_per_cpu(pcpu->lowcore); nmi_free_per_cpu(pcpu->lowcore); stack_free(async_stack); if (pcpu == &pcpu_devices[0]) @@ -271,7 +265,7 @@ static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu) lc->steal_timer = lc->avg_steal_timer = 0; __ctl_store(lc->cregs_save_area, 0, 15); lc->cregs_save_area[1] = lc->kernel_asce; - lc->cregs_save_area[7] = lc->vdso_asce; + lc->cregs_save_area[7] = lc->user_asce; save_access_regs((unsigned int *) lc->access_regs_save_area); memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list, sizeof(lc->stfle_fac_list)); @@ -859,8 +853,6 @@ static void smp_init_secondary(void) S390_lowcore.last_update_clock = get_tod_clock(); restore_access_regs(S390_lowcore.access_regs_save_area); - set_cpu_flag(CIF_ASCE_PRIMARY); - set_cpu_flag(CIF_ASCE_SECONDARY); cpu_init(); rcu_cpu_starting(cpu); preempt_disable(); diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index f9da5b149141..53c983f1ea51 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -99,61 +99,6 @@ static union { u8 page[PAGE_SIZE]; } vdso_data_store __page_aligned_data; struct vdso_data *vdso_data = (struct vdso_data *)&vdso_data_store.data; -/* - * Allocate/free per cpu vdso data. - */ -#define SEGMENT_ORDER 2 - -int vdso_alloc_per_cpu(struct lowcore *lowcore) -{ - unsigned long segment_table, page_table, page_frame; - struct vdso_per_cpu_data *vd; - - segment_table = __get_free_pages(GFP_KERNEL, SEGMENT_ORDER); - page_table = get_zeroed_page(GFP_KERNEL); - page_frame = get_zeroed_page(GFP_KERNEL); - if (!segment_table || !page_table || !page_frame) - goto out; - arch_set_page_dat(virt_to_page(segment_table), SEGMENT_ORDER); - arch_set_page_dat(virt_to_page(page_table), 0); - - /* Initialize per-cpu vdso data page */ - vd = (struct vdso_per_cpu_data *) page_frame; - vd->cpu_nr = lowcore->cpu_nr; - vd->node_id = cpu_to_node(vd->cpu_nr); - - /* Set up page table for the vdso address space */ - memset64((u64 *)segment_table, _SEGMENT_ENTRY_EMPTY, _CRST_ENTRIES); - memset64((u64 *)page_table, _PAGE_INVALID, PTRS_PER_PTE); - - *(unsigned long *) segment_table = _SEGMENT_ENTRY + page_table; - *(unsigned long *) page_table = _PAGE_PROTECT + page_frame; - - lowcore->vdso_asce = segment_table + - _ASCE_TABLE_LENGTH + _ASCE_USER_BITS + _ASCE_TYPE_SEGMENT; - lowcore->vdso_per_cpu_data = page_frame; - - return 0; - -out: - free_page(page_frame); - free_page(page_table); - free_pages(segment_table, SEGMENT_ORDER); - return -ENOMEM; -} - -void vdso_free_per_cpu(struct lowcore *lowcore) -{ - unsigned long segment_table, page_table, page_frame; - - segment_table = lowcore->vdso_asce & PAGE_MASK; - page_table = *(unsigned long *) segment_table; - page_frame = *(unsigned long *) page_table; - - free_page(page_frame); - free_page(page_table); - free_pages(segment_table, SEGMENT_ORDER); -} /* * This is called from binfmt_elf, we create the special vma for the @@ -240,8 +185,6 @@ static int __init vdso_init(void) } vdso64_pagelist[vdso64_pages - 1] = virt_to_page(vdso_data); vdso64_pagelist[vdso64_pages] = NULL; - if (vdso_alloc_per_cpu(&S390_lowcore)) - BUG(); get_page(virt_to_page(vdso_data)); diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile index 13cc5a3f9abf..b0cf58ae82fe 100644 --- a/arch/s390/kernel/vdso64/Makefile +++ b/arch/s390/kernel/vdso64/Makefile @@ -6,7 +6,7 @@ ARCH_REL_TYPE_ABS := R_390_COPY|R_390_GLOB_DAT|R_390_JMP_SLOT|R_390_RELATIVE ARCH_REL_TYPE_ABS += R_390_GOT|R_390_PLT include $(srctree)/lib/vdso/Makefile -obj-vdso64 = vdso_user_wrapper.o note.o getcpu.o +obj-vdso64 = vdso_user_wrapper.o note.o obj-cvdso64 = vdso64_generic.o CFLAGS_REMOVE_vdso64_generic.o = -pg $(CC_FLAGS_FTRACE) $(CC_FLAGS_EXPOLINE) diff --git a/arch/s390/kernel/vdso64/getcpu.S b/arch/s390/kernel/vdso64/getcpu.S deleted file mode 100644 index 3c04f7328500..000000000000 --- a/arch/s390/kernel/vdso64/getcpu.S +++ /dev/null @@ -1,31 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Userland implementation of getcpu() for 64 bits processes in a - * s390 kernel for use in the vDSO - * - * Copyright IBM Corp. 2016 - * Author(s): Martin Schwidefsky - */ -#include -#include -#include - - .text - .align 4 - .globl __kernel_getcpu - .type __kernel_getcpu,@function -__kernel_getcpu: - CFI_STARTPROC - sacf 256 - lm %r4,%r5,__VDSO_GETCPU_VAL(%r0) - sacf 0 - ltgr %r2,%r2 - jz 2f - st %r5,0(%r2) -2: ltgr %r3,%r3 - jz 3f - st %r4,0(%r3) -3: lghi %r2,0 - br %r14 - CFI_ENDPROC - .size __kernel_getcpu,.-__kernel_getcpu diff --git a/arch/s390/kernel/vdso64/vdso64.lds.S b/arch/s390/kernel/vdso64/vdso64.lds.S index 7ddb116b5e2e..b59006584a9d 100644 --- a/arch/s390/kernel/vdso64/vdso64.lds.S +++ b/arch/s390/kernel/vdso64/vdso64.lds.S @@ -135,7 +135,6 @@ VERSION __kernel_gettimeofday; __kernel_clock_gettime; __kernel_clock_getres; - __kernel_getcpu; local: *; }; diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 0267405ab7c6..0ffbe1fad72a 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -40,71 +40,10 @@ static inline int copy_with_mvcos(void) } #endif -void set_fs(mm_segment_t fs) -{ - current->thread.mm_segment = fs; - if (fs == USER_DS) { - __ctl_load(S390_lowcore.user_asce, 1, 1); - clear_cpu_flag(CIF_ASCE_PRIMARY); - } else { - __ctl_load(S390_lowcore.kernel_asce, 1, 1); - set_cpu_flag(CIF_ASCE_PRIMARY); - } - if (fs & 1) { - if (fs == USER_DS_SACF) - __ctl_load(S390_lowcore.user_asce, 7, 7); - else - __ctl_load(S390_lowcore.kernel_asce, 7, 7); - set_cpu_flag(CIF_ASCE_SECONDARY); - } -} -EXPORT_SYMBOL(set_fs); - -mm_segment_t enable_sacf_uaccess(void) -{ - mm_segment_t old_fs; - unsigned long asce, cr; - unsigned long flags; - - old_fs = current->thread.mm_segment; - if (old_fs & 1) - return old_fs; - /* protect against a concurrent page table upgrade */ - local_irq_save(flags); - current->thread.mm_segment |= 1; - asce = S390_lowcore.kernel_asce; - if (likely(old_fs == USER_DS)) { - __ctl_store(cr, 1, 1); - if (cr != S390_lowcore.kernel_asce) { - __ctl_load(S390_lowcore.kernel_asce, 1, 1); - set_cpu_flag(CIF_ASCE_PRIMARY); - } - asce = S390_lowcore.user_asce; - } - __ctl_store(cr, 7, 7); - if (cr != asce) { - __ctl_load(asce, 7, 7); - set_cpu_flag(CIF_ASCE_SECONDARY); - } - local_irq_restore(flags); - return old_fs; -} -EXPORT_SYMBOL(enable_sacf_uaccess); - -void disable_sacf_uaccess(mm_segment_t old_fs) -{ - current->thread.mm_segment = old_fs; - if (old_fs == USER_DS && test_facility(27)) { - __ctl_load(S390_lowcore.user_asce, 1, 1); - clear_cpu_flag(CIF_ASCE_PRIMARY); - } -} -EXPORT_SYMBOL(disable_sacf_uaccess); - static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr, unsigned long size) { - register unsigned long reg0 asm("0") = 0x01UL; + register unsigned long reg0 asm("0") = 0x81UL; unsigned long tmp1, tmp2; tmp1 = -4096UL; @@ -135,9 +74,7 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr, unsigned long size) { unsigned long tmp1, tmp2; - mm_segment_t old_fs; - old_fs = enable_sacf_uaccess(); tmp1 = -256UL; asm volatile( " sacf 0\n" @@ -164,7 +101,6 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr, EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b) : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) : : "cc", "memory"); - disable_sacf_uaccess(old_fs); return size; } @@ -179,7 +115,7 @@ EXPORT_SYMBOL(raw_copy_from_user); static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x, unsigned long size) { - register unsigned long reg0 asm("0") = 0x010000UL; + register unsigned long reg0 asm("0") = 0x810000UL; unsigned long tmp1, tmp2; tmp1 = -4096UL; @@ -210,9 +146,7 @@ static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x, unsigned long size) { unsigned long tmp1, tmp2; - mm_segment_t old_fs; - old_fs = enable_sacf_uaccess(); tmp1 = -256UL; asm volatile( " sacf 0\n" @@ -239,7 +173,6 @@ static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x, EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b) : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) : : "cc", "memory"); - disable_sacf_uaccess(old_fs); return size; } @@ -254,7 +187,7 @@ EXPORT_SYMBOL(raw_copy_to_user); static inline unsigned long copy_in_user_mvcos(void __user *to, const void __user *from, unsigned long size) { - register unsigned long reg0 asm("0") = 0x010001UL; + register unsigned long reg0 asm("0") = 0x810081UL; unsigned long tmp1, tmp2; tmp1 = -4096UL; @@ -277,10 +210,8 @@ static inline unsigned long copy_in_user_mvcos(void __user *to, const void __use static inline unsigned long copy_in_user_mvc(void __user *to, const void __user *from, unsigned long size) { - mm_segment_t old_fs; unsigned long tmp1; - old_fs = enable_sacf_uaccess(); asm volatile( " sacf 256\n" " aghi %0,-1\n" @@ -304,7 +235,6 @@ static inline unsigned long copy_in_user_mvc(void __user *to, const void __user EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b) : "+a" (size), "+a" (to), "+a" (from), "=a" (tmp1) : : "cc", "memory"); - disable_sacf_uaccess(old_fs); return size; } @@ -318,7 +248,7 @@ EXPORT_SYMBOL(raw_copy_in_user); static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size) { - register unsigned long reg0 asm("0") = 0x010000UL; + register unsigned long reg0 asm("0") = 0x810000UL; unsigned long tmp1, tmp2; tmp1 = -4096UL; @@ -346,10 +276,8 @@ static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size static inline unsigned long clear_user_xc(void __user *to, unsigned long size) { - mm_segment_t old_fs; unsigned long tmp1, tmp2; - old_fs = enable_sacf_uaccess(); asm volatile( " sacf 256\n" " aghi %0,-1\n" @@ -378,7 +306,6 @@ static inline unsigned long clear_user_xc(void __user *to, unsigned long size) EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b) : "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2) : : "cc", "memory"); - disable_sacf_uaccess(old_fs); return size; } @@ -414,15 +341,9 @@ static inline unsigned long strnlen_user_srst(const char __user *src, unsigned long __strnlen_user(const char __user *src, unsigned long size) { - mm_segment_t old_fs; - unsigned long len; - if (unlikely(!size)) return 0; - old_fs = enable_sacf_uaccess(); - len = strnlen_user_srst(src, size); - disable_sacf_uaccess(old_fs); - return len; + return strnlen_user_srst(src, size); } EXPORT_SYMBOL(__strnlen_user); diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 996884dcc9fd..b8210103de14 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -53,7 +53,6 @@ enum fault_type { KERNEL_FAULT, USER_FAULT, - VDSO_FAULT, GMAP_FAULT, }; @@ -77,22 +76,16 @@ static enum fault_type get_fault_type(struct pt_regs *regs) trans_exc_code = regs->int_parm_long & 3; if (likely(trans_exc_code == 0)) { /* primary space exception */ - if (IS_ENABLED(CONFIG_PGSTE) && - test_pt_regs_flag(regs, PIF_GUEST_FAULT)) - return GMAP_FAULT; - if (current->thread.mm_segment == USER_DS) + if (user_mode(regs)) return USER_FAULT; + if (!IS_ENABLED(CONFIG_PGSTE)) + return KERNEL_FAULT; + if (test_pt_regs_flag(regs, PIF_GUEST_FAULT)) + return GMAP_FAULT; return KERNEL_FAULT; } - if (trans_exc_code == 2) { - /* secondary space exception */ - if (current->thread.mm_segment & 1) { - if (current->thread.mm_segment == USER_DS_SACF) - return USER_FAULT; - return KERNEL_FAULT; - } - return VDSO_FAULT; - } + if (trans_exc_code == 2) + return USER_FAULT; if (trans_exc_code == 1) { /* access register mode, not used in the kernel */ return USER_FAULT; @@ -188,10 +181,6 @@ static void dump_fault_info(struct pt_regs *regs) asce = S390_lowcore.user_asce; pr_cont("user "); break; - case VDSO_FAULT: - asce = S390_lowcore.vdso_asce; - pr_cont("vdso "); - break; case GMAP_FAULT: asce = ((struct gmap *) S390_lowcore.gmap)->asce; pr_cont("gmap "); @@ -414,9 +403,6 @@ static inline vm_fault_t do_exception(struct pt_regs *regs, int access) switch (type) { case KERNEL_FAULT: goto out; - case VDSO_FAULT: - fault = VM_FAULT_BADMAP; - goto out; case USER_FAULT: case GMAP_FAULT: if (faulthandler_disabled() || !mm) @@ -834,7 +820,6 @@ void do_secure_storage_access(struct pt_regs *regs) if (rc) BUG(); break; - case VDSO_FAULT: case GMAP_FAULT: default: do_fault_error(regs, VM_READ | VM_WRITE, VM_FAULT_BADMAP); diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c index 11d2c8395e2a..4e87c819ddea 100644 --- a/arch/s390/mm/pgalloc.c +++ b/arch/s390/mm/pgalloc.c @@ -70,19 +70,10 @@ static void __crst_table_upgrade(void *arg) { struct mm_struct *mm = arg; - /* we must change all active ASCEs to avoid the creation of new TLBs */ + /* change all active ASCEs to avoid the creation of new TLBs */ if (current->active_mm == mm) { S390_lowcore.user_asce = mm->context.asce; - if (current->thread.mm_segment == USER_DS) { - __ctl_load(S390_lowcore.user_asce, 1, 1); - /* Mark user-ASCE present in CR1 */ - clear_cpu_flag(CIF_ASCE_PRIMARY); - } - if (current->thread.mm_segment == USER_DS_SACF) { - __ctl_load(S390_lowcore.user_asce, 7, 7); - /* enable_sacf_uaccess does all or nothing */ - WARN_ON(!test_cpu_flag(CIF_ASCE_SECONDARY)); - } + __ctl_load(S390_lowcore.user_asce, 7, 7); } __tlb_flush_local(); } diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index 1a6adbc68ee8..de3bdbed8881 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -93,12 +93,10 @@ static inline int __memcpy_toio_inuser(void __iomem *dst, { int size, rc = 0; u8 status = 0; - mm_segment_t old_fs; if (!src) return -EINVAL; - old_fs = enable_sacf_uaccess(); while (n > 0) { size = zpci_get_max_write_size((u64 __force) dst, (u64 __force) src, n, @@ -113,7 +111,6 @@ static inline int __memcpy_toio_inuser(void __iomem *dst, dst += size; n -= size; } - disable_sacf_uaccess(old_fs); if (rc) zpci_err_mmio(rc, status, (__force u64) dst); return rc; @@ -246,9 +243,7 @@ static inline int __memcpy_fromio_inuser(void __user *dst, { int size, rc = 0; u8 status; - mm_segment_t old_fs; - old_fs = enable_sacf_uaccess(); while (n > 0) { size = zpci_get_max_write_size((u64 __force) src, (u64 __force) dst, n, @@ -260,7 +255,6 @@ static inline int __memcpy_fromio_inuser(void __user *dst, dst += size; n -= size; } - disable_sacf_uaccess(old_fs); if (rc) zpci_err_mmio(rc, status, (__force u64) dst); return rc; From 0290c9e328e04052e317171953feb18177a34aed Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 16 Nov 2020 08:06:41 +0100 Subject: [PATCH 255/360] s390/mm: use invalid asce instead of kernel asce Create a region 3 page table which contains only invalid entries, and use that via "s390_invalid_asce" instead of the kernel ASCE whenever there is either - no user address space available, e.g. during early startup - as an intermediate ASCE when address spaces are switched This makes sure that user space accesses in such situations are guaranteed to fail. Reviewed-by: Sven Schnelle Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/include/asm/mmu_context.h | 2 +- arch/s390/include/asm/pgtable.h | 1 + arch/s390/kernel/smp.c | 2 +- arch/s390/kernel/vmlinux.lds.S | 3 ++- arch/s390/mm/init.c | 10 ++++++++-- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index 51def960a3dd..87a84fc59fc3 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -79,7 +79,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, S390_lowcore.user_asce = next->context.asce; cpumask_set_cpu(cpu, &next->context.cpu_attach_mask); /* Clear previous user-ASCE from CR7 */ - __ctl_load(S390_lowcore.kernel_asce, 7, 7); + __ctl_load(s390_invalid_asce, 7, 7); if (prev != next) cpumask_clear_cpu(cpu, &prev->context.cpu_attach_mask); } diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index a8edd96b2103..794746a32806 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -23,6 +23,7 @@ extern pgd_t swapper_pg_dir[]; extern void paging_init(void); +extern unsigned long s390_invalid_asce; enum { PG_DIRECT_MAP_4K = 0, diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index cac96f240dd8..7f7d81f19292 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -259,7 +259,7 @@ static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu) lc->spinlock_index = 0; lc->percpu_offset = __per_cpu_offset[cpu]; lc->kernel_asce = S390_lowcore.kernel_asce; - lc->user_asce = S390_lowcore.kernel_asce; + lc->user_asce = s390_invalid_asce; lc->machine_flags = S390_lowcore.machine_flags; lc->user_timer = lc->system_timer = lc->steal_timer = lc->avg_steal_timer = 0; diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 177ccfbda40a..4c0e19145cc6 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -10,7 +10,8 @@ * Put .bss..swapper_pg_dir as the first thing in .bss. This will * make sure it has 16k alignment. */ -#define BSS_FIRST_SECTIONS *(.bss..swapper_pg_dir) +#define BSS_FIRST_SECTIONS *(.bss..swapper_pg_dir) \ + *(.bss..invalid_pg_dir) /* Handle ro_after_init data on our own. */ #define RO_AFTER_INIT_DATA diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 69e6e2a5072e..73a163065b95 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -49,6 +49,9 @@ #include pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir"); +static pgd_t invalid_pg_dir[PTRS_PER_PGD] __section(".bss..invalid_pg_dir"); + +unsigned long s390_invalid_asce; unsigned long empty_zero_page, zero_page_mask; EXPORT_SYMBOL(empty_zero_page); @@ -92,6 +95,9 @@ void __init paging_init(void) unsigned long pgd_type, asce_bits; psw_t psw; + s390_invalid_asce = (unsigned long)invalid_pg_dir; + s390_invalid_asce |= _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; + crst_table_init((unsigned long *)invalid_pg_dir, _REGION3_ENTRY_EMPTY); init_mm.pgd = swapper_pg_dir; if (VMALLOC_END > _REGION2_SIZE) { asce_bits = _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH; @@ -102,14 +108,14 @@ void __init paging_init(void) } init_mm.context.asce = (__pa(init_mm.pgd) & PAGE_MASK) | asce_bits; S390_lowcore.kernel_asce = init_mm.context.asce; - S390_lowcore.user_asce = S390_lowcore.kernel_asce; + S390_lowcore.user_asce = s390_invalid_asce; crst_table_init((unsigned long *) init_mm.pgd, pgd_type); vmem_map_init(); kasan_copy_shadow_mapping(); /* enable virtual mapping in kernel mode */ __ctl_load(S390_lowcore.kernel_asce, 1, 1); - __ctl_load(S390_lowcore.kernel_asce, 7, 7); + __ctl_load(S390_lowcore.user_asce, 7, 7); __ctl_load(S390_lowcore.kernel_asce, 13, 13); psw.mask = __extract_psw(); psw_bits(psw).dat = 1; From 062e527956d05fae02f143c0d5ff9e8525c6799f Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 16 Nov 2020 08:06:41 +0100 Subject: [PATCH 256/360] s390/mm: add debug user asce support Verify on exit to user space that always - the primary ASCE (cr1) is set to kernel ASCE - the secondary ASCE (cr7) is set to user ASCE If this is not the case: panic since something went terribly wrong. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/Kconfig.debug | 8 ++++++++ arch/s390/configs/debug_defconfig | 1 + arch/s390/include/asm/uaccess.h | 2 ++ arch/s390/kernel/entry.S | 8 ++++++++ arch/s390/lib/uaccess.c | 16 ++++++++++++++++ 5 files changed, 35 insertions(+) diff --git a/arch/s390/Kconfig.debug b/arch/s390/Kconfig.debug index ab48b694ade8..6bfaceebbbc0 100644 --- a/arch/s390/Kconfig.debug +++ b/arch/s390/Kconfig.debug @@ -5,3 +5,11 @@ config TRACE_IRQFLAGS_SUPPORT config EARLY_PRINTK def_bool y + +config DEBUG_USER_ASCE + bool "Debug User ASCE" + help + Check on exit to user space that address space control + elements are setup correctly. + + If unsure, say N. diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index fe6f529ac82c..c52113a238b1 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -826,6 +826,7 @@ CONFIG_FTRACE_SYSCALLS=y CONFIG_BLK_DEV_IO_TRACE=y CONFIG_BPF_KPROBE_OVERRIDE=y CONFIG_HIST_TRIGGERS=y +CONFIG_DEBUG_USER_ASCE=y CONFIG_NOTIFIER_ERROR_INJECTION=m CONFIG_NETDEV_NOTIFIER_ERROR_INJECT=m CONFIG_FAULT_INJECTION=y diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index e59fd96a1561..c6707885e7c2 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -18,6 +18,8 @@ #include #include +void debug_user_asce(void); + static inline int __range_ok(unsigned long addr, unsigned long size) { return 1; diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index d43ef46dc683..377f75616693 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -90,6 +90,12 @@ _LPP_OFFSET = __LC_LPP #endif .endm + .macro DEBUG_USER_ASCE +#ifdef CONFIG_DEBUG_USER_ASCE + brasl %r14,debug_user_asce +#endif + .endm + .macro CHECK_VMAP_STACK savearea,oklabel #ifdef CONFIG_VMAP_STACK lgr %r14,%r15 @@ -428,6 +434,7 @@ ENTRY(system_call) jnz .Lsysc_work TSTMSK __TI_flags(%r12),_TIF_WORK jnz .Lsysc_work # check for work + DEBUG_USER_ASCE lctlg %c1,%c1,__LC_USER_ASCE BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP TSTMSK __LC_CPU_FLAGS, _CIF_FPU @@ -793,6 +800,7 @@ ENTRY(io_int_handler) mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) tm __PT_PSW+1(%r11),0x01 # returning to user ? jno .Lio_exit_kernel + DEBUG_USER_ASCE lctlg %c1,%c1,__LC_USER_ASCE BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP stpt __LC_EXIT_TIMER diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 0ffbe1fad72a..e8f642446fed 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -16,6 +16,22 @@ #include #include +#ifdef CONFIG_DEBUG_USER_ASCE +void debug_user_asce(void) +{ + unsigned long cr1, cr7; + + __ctl_store(cr1, 1, 1); + __ctl_store(cr7, 7, 7); + if (cr1 == S390_lowcore.kernel_asce && cr7 == S390_lowcore.user_asce) + return; + panic("incorrect ASCE on kernel exit\n" + "cr1: %016lx cr7: %016lx\n" + "kernel: %016llx user: %016llx\n", + cr1, cr7, S390_lowcore.kernel_asce, S390_lowcore.user_asce); +} +#endif /*CONFIG_DEBUG_USER_ASCE */ + #ifndef CONFIG_HAVE_MARCH_Z10_FEATURES static DEFINE_STATIC_KEY_FALSE(have_mvcos); From 80f06306240e0ad1c75116111be11950474dfda7 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 16 Nov 2020 08:06:41 +0100 Subject: [PATCH 257/360] s390/vdso: reimplement getcpu vdso syscall Implement the previously removed getcpu vdso syscall by using the TOD programmable field to pass the cpu number to user space. Reviewed-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/include/asm/timex.h | 7 +++++++ arch/s390/include/asm/vdso.h | 2 ++ arch/s390/kernel/smp.c | 2 ++ arch/s390/kernel/vdso.c | 7 +++++++ arch/s390/kernel/vdso64/Makefile | 3 ++- arch/s390/kernel/vdso64/getcpu.c | 20 ++++++++++++++++++++ arch/s390/kernel/vdso64/vdso64.lds.S | 2 +- arch/s390/kernel/vdso64/vdso_user_wrapper.S | 1 + 8 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 arch/s390/kernel/vdso64/getcpu.c diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h index 289aaff4d365..c8e244ecdfde 100644 --- a/arch/s390/include/asm/timex.h +++ b/arch/s390/include/asm/timex.h @@ -49,6 +49,13 @@ static inline void set_clock_comparator(__u64 time) asm volatile("sckc %0" : : "Q" (time)); } +static inline void set_tod_programmable_field(u16 val) +{ + register unsigned long reg0 asm("0") = val; + + asm volatile("sckpf" : : "d" (reg0)); +} + void clock_comparator_work(void); void __init time_early_init(void); diff --git a/arch/s390/include/asm/vdso.h b/arch/s390/include/asm/vdso.h index 9b299c05abf1..f65590889054 100644 --- a/arch/s390/include/asm/vdso.h +++ b/arch/s390/include/asm/vdso.h @@ -14,5 +14,7 @@ extern struct vdso_data *vdso_data; +void vdso_getcpu_init(void); + #endif /* __ASSEMBLY__ */ #endif /* __S390_VDSO_H__ */ diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 7f7d81f19292..647226e50c80 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -54,6 +54,7 @@ #include #include #include +#include #include "entry.h" enum { @@ -858,6 +859,7 @@ static void smp_init_secondary(void) preempt_disable(); init_cpu_timer(); vtime_init(); + vdso_getcpu_init(); pfault_init(); notify_cpu_starting(cpu); if (topology_cpu_dedicated(cpu)) diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index 53c983f1ea51..aef2edff9959 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -29,6 +29,7 @@ #include #include #include +#include extern char vdso64_start, vdso64_end; static void *vdso64_kbase = &vdso64_start; @@ -100,6 +101,11 @@ static union { } vdso_data_store __page_aligned_data; struct vdso_data *vdso_data = (struct vdso_data *)&vdso_data_store.data; +void vdso_getcpu_init(void) +{ + set_tod_programmable_field(smp_processor_id()); +} + /* * This is called from binfmt_elf, we create the special vma for the * vDSO and insert it into the mm struct tree @@ -170,6 +176,7 @@ static int __init vdso_init(void) { int i; + vdso_getcpu_init(); /* Calculate the size of the 64 bit vDSO */ vdso64_pages = ((&vdso64_end - &vdso64_start + PAGE_SIZE - 1) >> PAGE_SHIFT) + 1; diff --git a/arch/s390/kernel/vdso64/Makefile b/arch/s390/kernel/vdso64/Makefile index b0cf58ae82fe..a6e0fb6b91d6 100644 --- a/arch/s390/kernel/vdso64/Makefile +++ b/arch/s390/kernel/vdso64/Makefile @@ -7,7 +7,8 @@ ARCH_REL_TYPE_ABS += R_390_GOT|R_390_PLT include $(srctree)/lib/vdso/Makefile obj-vdso64 = vdso_user_wrapper.o note.o -obj-cvdso64 = vdso64_generic.o +obj-cvdso64 = vdso64_generic.o getcpu.o +CFLAGS_REMOVE_getcpu.o = -pg $(CC_FLAGS_FTRACE) $(CC_FLAGS_EXPOLINE) CFLAGS_REMOVE_vdso64_generic.o = -pg $(CC_FLAGS_FTRACE) $(CC_FLAGS_EXPOLINE) # Build rules diff --git a/arch/s390/kernel/vdso64/getcpu.c b/arch/s390/kernel/vdso64/getcpu.c new file mode 100644 index 000000000000..a5da7a9eb43d --- /dev/null +++ b/arch/s390/kernel/vdso64/getcpu.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright IBM Corp. 2020 */ + +#include +#include +#include + +int __s390_vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) +{ + __u16 todval[8]; + + /* CPU number is stored in the programmable field of the TOD clock */ + get_tod_clock_ext((char *)todval); + if (cpu) + *cpu = todval[7]; + /* NUMA node is always zero */ + if (node) + *node = 0; + return 0; +} diff --git a/arch/s390/kernel/vdso64/vdso64.lds.S b/arch/s390/kernel/vdso64/vdso64.lds.S index b59006584a9d..7bde3909290f 100644 --- a/arch/s390/kernel/vdso64/vdso64.lds.S +++ b/arch/s390/kernel/vdso64/vdso64.lds.S @@ -135,7 +135,7 @@ VERSION __kernel_gettimeofday; __kernel_clock_gettime; __kernel_clock_getres; - + __kernel_getcpu; local: *; }; } diff --git a/arch/s390/kernel/vdso64/vdso_user_wrapper.S b/arch/s390/kernel/vdso64/vdso_user_wrapper.S index a775d7e52872..f773505c7e63 100644 --- a/arch/s390/kernel/vdso64/vdso_user_wrapper.S +++ b/arch/s390/kernel/vdso64/vdso_user_wrapper.S @@ -36,3 +36,4 @@ __kernel_\func: vdso_func gettimeofday vdso_func clock_getres vdso_func clock_gettime +vdso_func getcpu From 77429eebd9b1af516bf1b6898e63b098ed748374 Mon Sep 17 00:00:00 2001 From: kernel test robot Date: Mon, 23 Nov 2020 11:18:50 +0800 Subject: [PATCH 258/360] EDAC/igen6: ecclog_llist can be static Fixes: 10590a9d4f23 ("EDAC/igen6: Add EDAC driver for Intel client SoCs using IBECC") Reported-by: kernel test robot Signed-off-by: kernel test robot Link: https://lore.kernel.org/r/20201123031850.GA20416@aef56166e5fc Signed-off-by: Tony Luck --- drivers/edac/igen6_edac.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/edac/igen6_edac.c b/drivers/edac/igen6_edac.c index 6c0039e1171f..6be9986fc6bd 100644 --- a/drivers/edac/igen6_edac.c +++ b/drivers/edac/igen6_edac.c @@ -164,7 +164,7 @@ struct ecclog_node { * to EDAC core in a worker. */ #define ECCLOG_POOL_SIZE PAGE_SIZE -LLIST_HEAD(ecclog_llist); +static LLIST_HEAD(ecclog_llist); static struct gen_pool *ecclog_pool; static char ecclog_buf[ECCLOG_POOL_SIZE]; static struct irq_work ecclog_irq_work; From bde493349025ca0559e2fff88592935af3b8df19 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 13 Nov 2020 13:19:18 -0800 Subject: [PATCH 259/360] fs-verity: move structs needed for file signing to UAPI header Although it isn't used directly by the ioctls, "struct fsverity_descriptor" is required by userspace programs that need to compute fs-verity file digests in a standalone way. Therefore it's also needed to sign files in a standalone way. Similarly, "struct fsverity_formatted_digest" (previously called "struct fsverity_signed_digest" which was misleading) is also needed to sign files if the built-in signature verification is being used. Therefore, move these structs to the UAPI header. While doing this, try to make it clear that the signature-related fields in fsverity_descriptor aren't used in the file digest computation. Acked-by: Luca Boccassi Link: https://lore.kernel.org/r/20201113211918.71883-5-ebiggers@kernel.org Signed-off-by: Eric Biggers --- Documentation/filesystems/fsverity.rst | 6 +--- fs/verity/fsverity_private.h | 37 ------------------- include/uapi/linux/fsverity.h | 49 ++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 42 deletions(-) diff --git a/Documentation/filesystems/fsverity.rst b/Documentation/filesystems/fsverity.rst index 2eee558b7f5f..e0204a23e997 100644 --- a/Documentation/filesystems/fsverity.rst +++ b/Documentation/filesystems/fsverity.rst @@ -334,17 +334,13 @@ root hash as well as other fields such as the file size:: __u8 hash_algorithm; /* Merkle tree hash algorithm */ __u8 log_blocksize; /* log2 of size of data and tree blocks */ __u8 salt_size; /* size of salt in bytes; 0 if none */ - __le32 sig_size; /* must be 0 */ + __le32 __reserved_0x04; /* must be 0 */ __le64 data_size; /* size of file the Merkle tree is built over */ __u8 root_hash[64]; /* Merkle tree root hash */ __u8 salt[32]; /* salt prepended to each hashed block */ __u8 __reserved[144]; /* must be 0's */ }; -Note that the ``sig_size`` field must be set to 0 for the purpose of -computing the file measurement, even if a signature was provided (or -will be provided) to `FS_IOC_ENABLE_VERITY`_. - Built-in signature verification =============================== diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index 21e9930d65fb..96f7b332f54f 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -77,49 +77,12 @@ struct fsverity_info { const struct inode *inode; }; -/* - * Merkle tree properties. The fs-verity file digest is the hash of this - * structure excluding the signature and with the sig_size field set to 0. - */ -struct fsverity_descriptor { - __u8 version; /* must be 1 */ - __u8 hash_algorithm; /* Merkle tree hash algorithm */ - __u8 log_blocksize; /* log2 of size of data and tree blocks */ - __u8 salt_size; /* size of salt in bytes; 0 if none */ - __le32 sig_size; /* size of signature in bytes; 0 if none */ - __le64 data_size; /* size of file the Merkle tree is built over */ - __u8 root_hash[64]; /* Merkle tree root hash */ - __u8 salt[32]; /* salt prepended to each hashed block */ - __u8 __reserved[144]; /* must be 0's */ - __u8 signature[]; /* optional PKCS#7 signature */ -}; - /* Arbitrary limit to bound the kmalloc() size. Can be changed. */ #define FS_VERITY_MAX_DESCRIPTOR_SIZE 16384 #define FS_VERITY_MAX_SIGNATURE_SIZE (FS_VERITY_MAX_DESCRIPTOR_SIZE - \ sizeof(struct fsverity_descriptor)) -/* - * Format in which fs-verity file digests are signed in built-in signatures. - * This is the same as 'struct fsverity_digest', except here some magic bytes - * are prepended to provide some context about what is being signed in case the - * same key is used for non-fsverity purposes, and here the fields have fixed - * endianness. - * - * This struct is specific to the built-in signature verification support, which - * is optional. fs-verity users may also verify signatures in userspace, in - * which case userspace is responsible for deciding on what bytes are signed. - * This struct may still be used, but it doesn't have to be. For example, - * userspace could instead use a string like "sha256:$digest_as_hex_string". - */ -struct fsverity_formatted_digest { - char magic[8]; /* must be "FSVerity" */ - __le16 digest_algorithm; - __le16 digest_size; - __u8 digest[]; -}; - /* hash_algs.c */ extern struct fsverity_hash_alg fsverity_hash_algs[]; diff --git a/include/uapi/linux/fsverity.h b/include/uapi/linux/fsverity.h index da0daf6c193b..33f44156f8ea 100644 --- a/include/uapi/linux/fsverity.h +++ b/include/uapi/linux/fsverity.h @@ -34,6 +34,55 @@ struct fsverity_digest { __u8 digest[]; }; +/* + * Struct containing a file's Merkle tree properties. The fs-verity file digest + * is the hash of this struct. A userspace program needs this struct only if it + * needs to compute fs-verity file digests itself, e.g. in order to sign files. + * It isn't needed just to enable fs-verity on a file. + * + * Note: when computing the file digest, 'sig_size' and 'signature' must be left + * zero and empty, respectively. These fields are present only because some + * filesystems reuse this struct as part of their on-disk format. + */ +struct fsverity_descriptor { + __u8 version; /* must be 1 */ + __u8 hash_algorithm; /* Merkle tree hash algorithm */ + __u8 log_blocksize; /* log2 of size of data and tree blocks */ + __u8 salt_size; /* size of salt in bytes; 0 if none */ +#ifdef __KERNEL__ + __le32 sig_size; +#else + __le32 __reserved_0x04; /* must be 0 */ +#endif + __le64 data_size; /* size of file the Merkle tree is built over */ + __u8 root_hash[64]; /* Merkle tree root hash */ + __u8 salt[32]; /* salt prepended to each hashed block */ + __u8 __reserved[144]; /* must be 0's */ +#ifdef __KERNEL__ + __u8 signature[]; +#endif +}; + +/* + * Format in which fs-verity file digests are signed in built-in signatures. + * This is the same as 'struct fsverity_digest', except here some magic bytes + * are prepended to provide some context about what is being signed in case the + * same key is used for non-fsverity purposes, and here the fields have fixed + * endianness. + * + * This struct is specific to the built-in signature verification support, which + * is optional. fs-verity users may also verify signatures in userspace, in + * which case userspace is responsible for deciding on what bytes are signed. + * This struct may still be used, but it doesn't have to be. For example, + * userspace could instead use a string like "sha256:$digest_as_hex_string". + */ +struct fsverity_formatted_digest { + char magic[8]; /* must be "FSVerity" */ + __le16 digest_algorithm; + __le16 digest_size; + __u8 digest[]; +}; + #define FS_IOC_ENABLE_VERITY _IOW('f', 133, struct fsverity_enable_arg) #define FS_IOC_MEASURE_VERITY _IOWR('f', 134, struct fsverity_digest) From afe76eca862ccde2a0c30105fc97a46a0b59339b Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Mon, 23 Nov 2020 11:11:17 +0100 Subject: [PATCH 260/360] x86/sgx: Fix sgx_ioc_enclave_provision() kernel-doc comment Fix ./arch/x86/kernel/cpu/sgx/ioctl.c:666: warning: Function parameter or member \ 'encl' not described in 'sgx_ioc_enclave_provision' ./arch/x86/kernel/cpu/sgx/ioctl.c:666: warning: Excess function parameter \ 'enclave' description in 'sgx_ioc_enclave_provision' Reported-by: Stephen Rothwell Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201123181922.0c009406@canb.auug.org.au --- arch/x86/kernel/cpu/sgx/ioctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/sgx/ioctl.c b/arch/x86/kernel/cpu/sgx/ioctl.c index 30aefc93a31d..c206aee80a04 100644 --- a/arch/x86/kernel/cpu/sgx/ioctl.c +++ b/arch/x86/kernel/cpu/sgx/ioctl.c @@ -652,7 +652,7 @@ out: /** * sgx_ioc_enclave_provision() - handler for %SGX_IOC_ENCLAVE_PROVISION - * @enclave: an enclave pointer + * @encl: an enclave pointer * @arg: userspace pointer to a struct sgx_enclave_provision instance * * Allow ATTRIBUTE.PROVISION_KEY for an enclave by providing a file handle to From 716572b0003ef67a4889bd7d85baf5099c5a0248 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Mon, 2 Nov 2020 11:51:10 -0800 Subject: [PATCH 261/360] selftests/x86/fsgsbase: Fix GS == 1, 2, and 3 tests Setting GS to 1, 2, or 3 causes a nonsensical part of the IRET microcode to change GS back to zero on a return from kernel mode to user mode. The result is that these tests fail randomly depending on when interrupts happen. Detect when this happens and let the test pass. Signed-off-by: Andy Lutomirski Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/7567fd44a1d60a9424f25b19a998f12149993b0d.1604346596.git.luto@kernel.org --- tools/testing/selftests/x86/fsgsbase.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/x86/fsgsbase.c b/tools/testing/selftests/x86/fsgsbase.c index 7161cfc2e60b..8c780cce941d 100644 --- a/tools/testing/selftests/x86/fsgsbase.c +++ b/tools/testing/selftests/x86/fsgsbase.c @@ -392,8 +392,8 @@ static void set_gs_and_switch_to(unsigned long local, local = read_base(GS); /* - * Signal delivery seems to mess up weird selectors. Put it - * back. + * Signal delivery is quite likely to change a selector + * of 1, 2, or 3 back to 0 due to IRET being defective. */ asm volatile ("mov %0, %%gs" : : "rm" (force_sel)); } else { @@ -411,6 +411,14 @@ static void set_gs_and_switch_to(unsigned long local, if (base == local && sel_pre_sched == sel_post_sched) { printf("[OK]\tGS/BASE remained 0x%hx/0x%lx\n", sel_pre_sched, local); + } else if (base == local && sel_pre_sched >= 1 && sel_pre_sched <= 3 && + sel_post_sched == 0) { + /* + * IRET is misdesigned and will squash selectors 1, 2, or 3 + * to zero. Don't fail the test just because this happened. + */ + printf("[OK]\tGS/BASE changed from 0x%hx/0x%lx to 0x%hx/0x%lx because IRET is defective\n", + sel_pre_sched, local, sel_post_sched, base); } else { nerrs++; printf("[FAIL]\tGS/BASE changed from 0x%hx/0x%lx to 0x%hx/0x%lx\n", From aeaaf005da1de075929e56562dced4a58238efc4 Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Mon, 2 Nov 2020 11:51:11 -0800 Subject: [PATCH 262/360] selftests/x86: Add missing .note.GNU-stack sections Several of the x86 selftests end up with executable stacks because the asm was missing the annotation that says that they are modern and don't need executable stacks. Add the annotations. Signed-off-by: Andy Lutomirski Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/6f043c03e9e0e4557e1e975a63b07a4d18965a68.1604346596.git.luto@kernel.org --- tools/testing/selftests/x86/raw_syscall_helper_32.S | 2 ++ tools/testing/selftests/x86/thunks.S | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tools/testing/selftests/x86/raw_syscall_helper_32.S b/tools/testing/selftests/x86/raw_syscall_helper_32.S index 94410fa2b5ed..a10d36afdca0 100644 --- a/tools/testing/selftests/x86/raw_syscall_helper_32.S +++ b/tools/testing/selftests/x86/raw_syscall_helper_32.S @@ -45,3 +45,5 @@ int80_and_ret: .type int80_and_ret, @function .size int80_and_ret, .-int80_and_ret + +.section .note.GNU-stack,"",%progbits diff --git a/tools/testing/selftests/x86/thunks.S b/tools/testing/selftests/x86/thunks.S index 1bb5d62c16a4..a2d47d8344d4 100644 --- a/tools/testing/selftests/x86/thunks.S +++ b/tools/testing/selftests/x86/thunks.S @@ -57,3 +57,5 @@ call32_from_64: ret .size call32_from_64, .-call32_from_64 + +.section .note.GNU-stack,"",%progbits From 8f061abbf543355d77fac5c23521b6b452da6310 Mon Sep 17 00:00:00 2001 From: Justin Ernst Date: Wed, 25 Nov 2020 11:54:40 -0600 Subject: [PATCH 263/360] x86/platform/uv: Remove existing /sys/firmware/sgi_uv/ interface Remove existing interface at /sys/firmware/sgi_uv/, created by arch/x86/platform/uv/uv_sysfs.c This interface includes: /sys/firmware/sgi_uv/coherence_id /sys/firmware/sgi_uv/partition_id Both coherence_id and partition_id will be re-introduced via a new uv_sysfs driver. Signed-off-by: Justin Ernst Signed-off-by: Borislav Petkov Reviewed-by: Steve Wahl Acked-by: Hans de Goede Link: https://lkml.kernel.org/r/20201125175444.279074-2-justin.ernst@hpe.com --- arch/x86/platform/uv/Makefile | 2 +- arch/x86/platform/uv/uv_sysfs.c | 63 --------------------------------- 2 files changed, 1 insertion(+), 64 deletions(-) delete mode 100644 arch/x86/platform/uv/uv_sysfs.c diff --git a/arch/x86/platform/uv/Makefile b/arch/x86/platform/uv/Makefile index 224ff0504890..1441dda8edf7 100644 --- a/arch/x86/platform/uv/Makefile +++ b/arch/x86/platform/uv/Makefile @@ -1,2 +1,2 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_X86_UV) += bios_uv.o uv_irq.o uv_sysfs.o uv_time.o uv_nmi.o +obj-$(CONFIG_X86_UV) += bios_uv.o uv_irq.o uv_time.o uv_nmi.o diff --git a/arch/x86/platform/uv/uv_sysfs.c b/arch/x86/platform/uv/uv_sysfs.c deleted file mode 100644 index 266773e2fb37..000000000000 --- a/arch/x86/platform/uv/uv_sysfs.c +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * This file supports the /sys/firmware/sgi_uv interfaces for SGI UV. - * - * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. - * Copyright (c) Russ Anderson - */ - -#include -#include -#include - -struct kobject *sgi_uv_kobj; - -static ssize_t partition_id_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - return snprintf(buf, PAGE_SIZE, "%ld\n", sn_partition_id); -} - -static ssize_t coherence_id_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - return snprintf(buf, PAGE_SIZE, "%ld\n", sn_coherency_id); -} - -static struct kobj_attribute partition_id_attr = - __ATTR(partition_id, S_IRUGO, partition_id_show, NULL); - -static struct kobj_attribute coherence_id_attr = - __ATTR(coherence_id, S_IRUGO, coherence_id_show, NULL); - - -static int __init sgi_uv_sysfs_init(void) -{ - unsigned long ret; - - if (!is_uv_system()) - return -ENODEV; - - if (!sgi_uv_kobj) - sgi_uv_kobj = kobject_create_and_add("sgi_uv", firmware_kobj); - if (!sgi_uv_kobj) { - printk(KERN_WARNING "kobject_create_and_add sgi_uv failed\n"); - return -EINVAL; - } - - ret = sysfs_create_file(sgi_uv_kobj, &partition_id_attr.attr); - if (ret) { - printk(KERN_WARNING "sysfs_create_file partition_id failed\n"); - return ret; - } - - ret = sysfs_create_file(sgi_uv_kobj, &coherence_id_attr.attr); - if (ret) { - printk(KERN_WARNING "sysfs_create_file coherence_id failed\n"); - return ret; - } - - return 0; -} - -device_initcall(sgi_uv_sysfs_init); From 9a3c425cfdfee169622f1cb1a974b2f287e5560c Mon Sep 17 00:00:00 2001 From: Justin Ernst Date: Wed, 25 Nov 2020 11:54:41 -0600 Subject: [PATCH 264/360] x86/platform/uv: Add and export uv_bios_* functions Add additional uv_bios_call() variant functions to expose information needed by the new uv_sysfs driver. This includes the addition of several new data types defined by UV BIOS and used in the new functions. Signed-off-by: Justin Ernst Signed-off-by: Borislav Petkov Reviewed-by: Steve Wahl Acked-by: Hans de Goede Link: https://lkml.kernel.org/r/20201125175444.279074-3-justin.ernst@hpe.com --- arch/x86/include/asm/uv/bios.h | 49 +++++++++++++++ arch/x86/include/asm/uv/uv_geo.h | 103 +++++++++++++++++++++++++++++++ arch/x86/platform/uv/bios_uv.c | 55 +++++++++++++++++ 3 files changed, 207 insertions(+) create mode 100644 arch/x86/include/asm/uv/uv_geo.h diff --git a/arch/x86/include/asm/uv/bios.h b/arch/x86/include/asm/uv/bios.h index 08b3d810dfba..01ba080887b3 100644 --- a/arch/x86/include/asm/uv/bios.h +++ b/arch/x86/include/asm/uv/bios.h @@ -28,6 +28,20 @@ enum uv_bios_cmd { UV_BIOS_SET_LEGACY_VGA_TARGET }; +#define UV_BIOS_EXTRA 0x10000 +#define UV_BIOS_GET_PCI_TOPOLOGY 0x10001 +#define UV_BIOS_GET_GEOINFO 0x10003 + +#define UV_BIOS_EXTRA_OP_MEM_COPYIN 0x1000 +#define UV_BIOS_EXTRA_OP_MEM_COPYOUT 0x2000 +#define UV_BIOS_EXTRA_OP_MASK 0x0fff +#define UV_BIOS_EXTRA_GET_HEAPSIZE 1 +#define UV_BIOS_EXTRA_INSTALL_HEAP 2 +#define UV_BIOS_EXTRA_MASTER_NASID 3 +#define UV_BIOS_EXTRA_OBJECT_COUNT (10|UV_BIOS_EXTRA_OP_MEM_COPYOUT) +#define UV_BIOS_EXTRA_ENUM_OBJECTS (12|UV_BIOS_EXTRA_OP_MEM_COPYOUT) +#define UV_BIOS_EXTRA_ENUM_PORTS (13|UV_BIOS_EXTRA_OP_MEM_COPYOUT) + /* * Status values returned from a BIOS call. */ @@ -109,6 +123,32 @@ struct uv_systab { } entry[1]; /* additional entries follow */ }; extern struct uv_systab *uv_systab; + +#define UV_BIOS_MAXSTRING 128 +struct uv_bios_hub_info { + unsigned int id; + union { + struct { + unsigned long long this_part:1; + unsigned long long is_shared:1; + unsigned long long is_disabled:1; + } fields; + struct { + unsigned long long flags; + unsigned long long reserved; + } b; + } f; + char name[UV_BIOS_MAXSTRING]; + char location[UV_BIOS_MAXSTRING]; + unsigned int ports; +}; + +struct uv_bios_port_info { + unsigned int port; + unsigned int conn_id; + unsigned int conn_port; +}; + /* (... end of definitions from UV BIOS ...) */ enum { @@ -142,6 +182,15 @@ extern s64 uv_bios_change_memprotect(u64, u64, enum uv_memprotect); extern s64 uv_bios_reserved_page_pa(u64, u64 *, u64 *, u64 *); extern int uv_bios_set_legacy_vga_target(bool decode, int domain, int bus); +extern s64 uv_bios_get_master_nasid(u64 sz, u64 *nasid); +extern s64 uv_bios_get_heapsize(u64 nasid, u64 sz, u64 *heap_sz); +extern s64 uv_bios_install_heap(u64 nasid, u64 sz, u64 *heap); +extern s64 uv_bios_obj_count(u64 nasid, u64 sz, u64 *objcnt); +extern s64 uv_bios_enum_objs(u64 nasid, u64 sz, u64 *objbuf); +extern s64 uv_bios_enum_ports(u64 nasid, u64 obj_id, u64 sz, u64 *portbuf); +extern s64 uv_bios_get_geoinfo(u64 nasid, u64 sz, u64 *geo); +extern s64 uv_bios_get_pci_topology(u64 sz, u64 *buf); + extern int uv_bios_init(void); extern unsigned long get_uv_systab_phys(bool msg); diff --git a/arch/x86/include/asm/uv/uv_geo.h b/arch/x86/include/asm/uv/uv_geo.h new file mode 100644 index 000000000000..f241451035fb --- /dev/null +++ b/arch/x86/include/asm/uv/uv_geo.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2020 Hewlett Packard Enterprise Development LP. All rights reserved. + */ + +#ifndef _ASM_UV_GEO_H +#define _ASM_UV_GEO_H + +/* Type declaractions */ + +/* Size of a geoid_s structure (must be before decl. of geoid_u) */ +#define GEOID_SIZE 8 + +/* Fields common to all substructures */ +struct geo_common_s { + unsigned char type; /* What type of h/w is named by this geoid_s */ + unsigned char blade; + unsigned char slot; /* slot is IRU */ + unsigned char upos; + unsigned char rack; +}; + +/* Additional fields for particular types of hardware */ +struct geo_node_s { + struct geo_common_s common; /* No additional fields needed */ +}; + +struct geo_rtr_s { + struct geo_common_s common; /* No additional fields needed */ +}; + +struct geo_iocntl_s { + struct geo_common_s common; /* No additional fields needed */ +}; + +struct geo_pcicard_s { + struct geo_iocntl_s common; + char bus; /* Bus/widget number */ + char slot; /* PCI slot number */ +}; + +/* Subcomponents of a node */ +struct geo_cpu_s { + struct geo_node_s node; + unsigned char socket:4, /* Which CPU on the node */ + thread:4; + unsigned char core; +}; + +struct geo_mem_s { + struct geo_node_s node; + char membus; /* The memory bus on the node */ + char memslot; /* The memory slot on the bus */ +}; + +union geoid_u { + struct geo_common_s common; + struct geo_node_s node; + struct geo_iocntl_s iocntl; + struct geo_pcicard_s pcicard; + struct geo_rtr_s rtr; + struct geo_cpu_s cpu; + struct geo_mem_s mem; + char padsize[GEOID_SIZE]; +}; + +/* Defined constants */ + +#define GEO_MAX_LEN 48 + +#define GEO_TYPE_INVALID 0 +#define GEO_TYPE_MODULE 1 +#define GEO_TYPE_NODE 2 +#define GEO_TYPE_RTR 3 +#define GEO_TYPE_IOCNTL 4 +#define GEO_TYPE_IOCARD 5 +#define GEO_TYPE_CPU 6 +#define GEO_TYPE_MEM 7 +#define GEO_TYPE_MAX (GEO_TYPE_MEM+1) + +static inline int geo_rack(union geoid_u g) +{ + return (g.common.type == GEO_TYPE_INVALID) ? + -1 : g.common.rack; +} + +static inline int geo_slot(union geoid_u g) +{ + return (g.common.type == GEO_TYPE_INVALID) ? + -1 : g.common.upos; +} + +static inline int geo_blade(union geoid_u g) +{ + return (g.common.type == GEO_TYPE_INVALID) ? + -1 : g.common.blade * 2 + g.common.slot; +} + +#endif /* _ASM_UV_GEO_H */ diff --git a/arch/x86/platform/uv/bios_uv.c b/arch/x86/platform/uv/bios_uv.c index 54511eaccf4d..bf31af3d32d6 100644 --- a/arch/x86/platform/uv/bios_uv.c +++ b/arch/x86/platform/uv/bios_uv.c @@ -72,6 +72,7 @@ static s64 uv_bios_call_irqsave(enum uv_bios_cmd which, u64 a1, u64 a2, u64 a3, long sn_partition_id; EXPORT_SYMBOL_GPL(sn_partition_id); long sn_coherency_id; +EXPORT_SYMBOL_GPL(sn_coherency_id); long sn_region_size; EXPORT_SYMBOL_GPL(sn_region_size); long system_serial_number; @@ -171,6 +172,60 @@ int uv_bios_set_legacy_vga_target(bool decode, int domain, int bus) (u64)decode, (u64)domain, (u64)bus, 0, 0); } +extern s64 uv_bios_get_master_nasid(u64 size, u64 *master_nasid) +{ + return uv_bios_call(UV_BIOS_EXTRA, 0, UV_BIOS_EXTRA_MASTER_NASID, 0, + size, (u64)master_nasid); +} +EXPORT_SYMBOL_GPL(uv_bios_get_master_nasid); + +extern s64 uv_bios_get_heapsize(u64 nasid, u64 size, u64 *heap_size) +{ + return uv_bios_call(UV_BIOS_EXTRA, nasid, UV_BIOS_EXTRA_GET_HEAPSIZE, + 0, size, (u64)heap_size); +} +EXPORT_SYMBOL_GPL(uv_bios_get_heapsize); + +extern s64 uv_bios_install_heap(u64 nasid, u64 heap_size, u64 *bios_heap) +{ + return uv_bios_call(UV_BIOS_EXTRA, nasid, UV_BIOS_EXTRA_INSTALL_HEAP, + 0, heap_size, (u64)bios_heap); +} +EXPORT_SYMBOL_GPL(uv_bios_install_heap); + +extern s64 uv_bios_obj_count(u64 nasid, u64 size, u64 *objcnt) +{ + return uv_bios_call(UV_BIOS_EXTRA, nasid, UV_BIOS_EXTRA_OBJECT_COUNT, + 0, size, (u64)objcnt); +} +EXPORT_SYMBOL_GPL(uv_bios_obj_count); + +extern s64 uv_bios_enum_objs(u64 nasid, u64 size, u64 *objbuf) +{ + return uv_bios_call(UV_BIOS_EXTRA, nasid, UV_BIOS_EXTRA_ENUM_OBJECTS, + 0, size, (u64)objbuf); +} +EXPORT_SYMBOL_GPL(uv_bios_enum_objs); + +extern s64 uv_bios_enum_ports(u64 nasid, u64 obj_id, u64 size, u64 *portbuf) +{ + return uv_bios_call(UV_BIOS_EXTRA, nasid, UV_BIOS_EXTRA_ENUM_PORTS, + obj_id, size, (u64)portbuf); +} +EXPORT_SYMBOL_GPL(uv_bios_enum_ports); + +extern s64 uv_bios_get_geoinfo(u64 nasid, u64 size, u64 *buf) +{ + return uv_bios_call(UV_BIOS_GET_GEOINFO, nasid, (u64)buf, size, 0, 0); +} +EXPORT_SYMBOL_GPL(uv_bios_get_geoinfo); + +extern s64 uv_bios_get_pci_topology(u64 size, u64 *buf) +{ + return uv_bios_call(UV_BIOS_GET_PCI_TOPOLOGY, (u64)buf, size, 0, 0, 0); +} +EXPORT_SYMBOL_GPL(uv_bios_get_pci_topology); + unsigned long get_uv_systab_phys(bool msg) { if ((uv_systab_phys == EFI_INVALID_TABLE_ADDR) || From 8539d3f06710a9e91b9968fa736549d7c6b44206 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 27 Oct 2020 14:45:32 -0700 Subject: [PATCH 265/360] x86/asm: Drop unused RDPID macro Drop the GAS-compatible RDPID macro. RDPID is unsafe in the kernel because KVM loads guest's TSC_AUX on VM-entry and may not restore the host's value until the CPU returns to userspace. See 6a3ea3e68b8a ("x86/entry/64: Do not use RDPID in paranoid entry to accomodate KVM") for details. It can always be resurrected from git history, if needed. [ bp: Massage commit message. ] Signed-off-by: Sean Christopherson Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201027214532.1792-1-sean.j.christopherson@intel.com --- arch/x86/include/asm/inst.h | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/arch/x86/include/asm/inst.h b/arch/x86/include/asm/inst.h index bd7f02480ca1..438ccd4f3cc4 100644 --- a/arch/x86/include/asm/inst.h +++ b/arch/x86/include/asm/inst.h @@ -143,21 +143,6 @@ .macro MODRM mod opd1 opd2 .byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3) .endm - -.macro RDPID opd - REG_TYPE rdpid_opd_type \opd - .if rdpid_opd_type == REG_TYPE_R64 - R64_NUM rdpid_opd \opd - .else - R32_NUM rdpid_opd \opd - .endif - .byte 0xf3 - .if rdpid_opd > 7 - PFX_REX rdpid_opd 0 - .endif - .byte 0x0f, 0xc7 - MODRM 0xc0 rdpid_opd 0x7 -.endm #endif #endif From 4fc2cf1f2daf8303000efb7c9dc0307ea638a8f3 Mon Sep 17 00:00:00 2001 From: Justin Ernst Date: Wed, 25 Nov 2020 11:54:42 -0600 Subject: [PATCH 266/360] x86/platform/uv: Add new uv_sysfs platform driver Add the uv_sysfs driver to construct a read-only sysfs interface at /sys/firmware/sgi_uv/ to expose information gathered from UV BIOS. This information includes: * UV Hub descriptions, including physical location * Cabling layout between hubs on the fabric * PCI topology, including physical location of PCI cards Together, the information provides a robust physical description of a UV system, useful for correlating to performance data or performing remote support. Signed-off-by: Justin Ernst Signed-off-by: Borislav Petkov Reviewed-by: Steve Wahl Acked-by: Hans de Goede Link: https://lkml.kernel.org/r/20201125175444.279074-4-justin.ernst@hpe.com --- drivers/platform/x86/Kconfig | 11 + drivers/platform/x86/Makefile | 3 + drivers/platform/x86/uv_sysfs.c | 862 ++++++++++++++++++++++++++++++++ 3 files changed, 876 insertions(+) create mode 100644 drivers/platform/x86/uv_sysfs.c diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig index 0d91d136bc3b..ba34153571b8 100644 --- a/drivers/platform/x86/Kconfig +++ b/drivers/platform/x86/Kconfig @@ -78,6 +78,17 @@ config HUAWEI_WMI To compile this driver as a module, choose M here: the module will be called huawei-wmi. +config UV_SYSFS + tristate "Sysfs structure for UV systems" + depends on X86_UV + depends on SYSFS + help + This driver supports a sysfs tree describing information about + UV systems at /sys/firmware/sgi_uv/. + + To compile this driver as a module, choose M here: the module will + be called uv_sysfs. + config INTEL_WMI_SBL_FW_UPDATE tristate "Intel WMI Slim Bootloader firmware update signaling driver" depends on ACPI_WMI diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile index 5f823f7eff45..a34875d833dd 100644 --- a/drivers/platform/x86/Makefile +++ b/drivers/platform/x86/Makefile @@ -62,6 +62,9 @@ obj-$(CONFIG_HP_WIRELESS) += hp-wireless.o obj-$(CONFIG_HP_WMI) += hp-wmi.o obj-$(CONFIG_TC1100_WMI) += tc1100-wmi.o +# Hewlett Packard Enterprise +obj-$(CONFIG_UV_SYSFS) += uv_sysfs.o + # IBM Thinkpad and Lenovo obj-$(CONFIG_IBM_RTL) += ibm_rtl.o obj-$(CONFIG_IDEAPAD_LAPTOP) += ideapad-laptop.o diff --git a/drivers/platform/x86/uv_sysfs.c b/drivers/platform/x86/uv_sysfs.c new file mode 100644 index 000000000000..54c342579f1c --- /dev/null +++ b/drivers/platform/x86/uv_sysfs.c @@ -0,0 +1,862 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * This file supports the /sys/firmware/sgi_uv topology tree on HPE UV. + * + * Copyright (c) 2020 Hewlett Packard Enterprise. All Rights Reserved. + * Copyright (c) Justin Ernst + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define INVALID_CNODE -1 + +struct kobject *sgi_uv_kobj; +struct kset *uv_pcibus_kset; +struct kset *uv_hubs_kset; +static struct uv_bios_hub_info *hub_buf; +static struct uv_bios_port_info **port_buf; +static struct uv_hub **uv_hubs; +static struct uv_pci_top_obj **uv_pci_objs; +static int num_pci_lines; +static int num_cnodes; +static int *prev_obj_to_cnode; +static int uv_bios_obj_cnt; +static signed short uv_master_nasid = -1; +static void *uv_biosheap; + +static const char *uv_type_string(void) +{ + if (is_uv5_hub()) + return "9.0"; + else if (is_uv4a_hub()) + return "7.1"; + else if (is_uv4_hub()) + return "7.0"; + else if (is_uv3_hub()) + return "5.0"; + else if (is_uv2_hub()) + return "3.0"; + else + return "unknown"; +} + +static int ordinal_to_nasid(int ordinal) +{ + if (ordinal < num_cnodes && ordinal >= 0) + return UV_PNODE_TO_NASID(uv_blade_to_pnode(ordinal)); + else + return -1; +} + +static union geoid_u cnode_to_geoid(int cnode) +{ + union geoid_u geoid; + + uv_bios_get_geoinfo(ordinal_to_nasid(cnode), (u64)sizeof(union geoid_u), (u64 *)&geoid); + return geoid; +} + +static int location_to_bpos(char *location, int *rack, int *slot, int *blade) +{ + char type, r, b, h; + int idb, idh; + + if (sscanf(location, "%c%03d%c%02d%c%2d%c%d", + &r, rack, &type, slot, &b, &idb, &h, &idh) != 8) + return -1; + *blade = idb * 2 + idh; + + return 0; +} + +static int cache_obj_to_cnode(struct uv_bios_hub_info *obj) +{ + int cnode; + union geoid_u geoid; + int obj_rack, obj_slot, obj_blade; + int rack, slot, blade; + + if (!obj->f.fields.this_part && !obj->f.fields.is_shared) + return 0; + + if (location_to_bpos(obj->location, &obj_rack, &obj_slot, &obj_blade)) + return -1; + + for (cnode = 0; cnode < num_cnodes; cnode++) { + geoid = cnode_to_geoid(cnode); + rack = geo_rack(geoid); + slot = geo_slot(geoid); + blade = geo_blade(geoid); + if (obj_rack == rack && obj_slot == slot && obj_blade == blade) + prev_obj_to_cnode[obj->id] = cnode; + } + + return 0; +} + +static int get_obj_to_cnode(int obj_id) +{ + return prev_obj_to_cnode[obj_id]; +} + +struct uv_hub { + struct kobject kobj; + struct uv_bios_hub_info *hub_info; + struct uv_port **ports; +}; + +#define to_uv_hub(kobj_ptr) container_of(kobj_ptr, struct uv_hub, kobj) + +static ssize_t hub_name_show(struct uv_bios_hub_info *hub_info, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%s\n", hub_info->name); +} + +static ssize_t hub_location_show(struct uv_bios_hub_info *hub_info, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%s\n", hub_info->location); +} + +static ssize_t hub_partition_show(struct uv_bios_hub_info *hub_info, char *buf) +{ + return sprintf(buf, "%d\n", hub_info->f.fields.this_part); +} + +static ssize_t hub_shared_show(struct uv_bios_hub_info *hub_info, char *buf) +{ + return sprintf(buf, "%d\n", hub_info->f.fields.is_shared); +} +static ssize_t hub_nasid_show(struct uv_bios_hub_info *hub_info, char *buf) +{ + int cnode = get_obj_to_cnode(hub_info->id); + + return sprintf(buf, "%d\n", ordinal_to_nasid(cnode)); +} +static ssize_t hub_cnode_show(struct uv_bios_hub_info *hub_info, char *buf) +{ + return sprintf(buf, "%d\n", get_obj_to_cnode(hub_info->id)); +} + +struct hub_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct uv_bios_hub_info *hub_info, char *buf); + ssize_t (*store)(struct uv_bios_hub_info *hub_info, const char *buf, size_t sz); +}; + +static struct hub_sysfs_entry name_attribute = + __ATTR(name, 0444, hub_name_show, NULL); +static struct hub_sysfs_entry location_attribute = + __ATTR(location, 0444, hub_location_show, NULL); +static struct hub_sysfs_entry partition_attribute = + __ATTR(this_partition, 0444, hub_partition_show, NULL); +static struct hub_sysfs_entry shared_attribute = + __ATTR(shared, 0444, hub_shared_show, NULL); +static struct hub_sysfs_entry nasid_attribute = + __ATTR(nasid, 0444, hub_nasid_show, NULL); +static struct hub_sysfs_entry cnode_attribute = + __ATTR(cnode, 0444, hub_cnode_show, NULL); + +static struct attribute *uv_hub_attrs[] = { + &name_attribute.attr, + &location_attribute.attr, + &partition_attribute.attr, + &shared_attribute.attr, + &nasid_attribute.attr, + &cnode_attribute.attr, + NULL, +}; + +static void hub_release(struct kobject *kobj) +{ + struct uv_hub *hub = to_uv_hub(kobj); + + kfree(hub); +} + +static ssize_t hub_type_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct uv_hub *hub = to_uv_hub(kobj); + struct uv_bios_hub_info *bios_hub_info = hub->hub_info; + struct hub_sysfs_entry *entry; + + entry = container_of(attr, struct hub_sysfs_entry, attr); + + if (!entry->show) + return -EIO; + + return entry->show(bios_hub_info, buf); +} + +static const struct sysfs_ops hub_sysfs_ops = { + .show = hub_type_show, +}; + +static struct kobj_type hub_attr_type = { + .release = hub_release, + .sysfs_ops = &hub_sysfs_ops, + .default_attrs = uv_hub_attrs, +}; + +static int uv_hubs_init(void) +{ + s64 biosr; + u64 sz; + int i, ret; + + prev_obj_to_cnode = kmalloc_array(uv_bios_obj_cnt, sizeof(*prev_obj_to_cnode), + GFP_KERNEL); + if (!prev_obj_to_cnode) + return -ENOMEM; + + for (i = 0; i < uv_bios_obj_cnt; i++) + prev_obj_to_cnode[i] = INVALID_CNODE; + + uv_hubs_kset = kset_create_and_add("hubs", NULL, sgi_uv_kobj); + if (!uv_hubs_kset) { + ret = -ENOMEM; + goto err_hubs_kset; + } + sz = uv_bios_obj_cnt * sizeof(*hub_buf); + hub_buf = kzalloc(sz, GFP_KERNEL); + if (!hub_buf) { + ret = -ENOMEM; + goto err_hub_buf; + } + + biosr = uv_bios_enum_objs((u64)uv_master_nasid, sz, (u64 *)hub_buf); + if (biosr) { + ret = -EINVAL; + goto err_enum_objs; + } + + uv_hubs = kcalloc(uv_bios_obj_cnt, sizeof(*uv_hubs), GFP_KERNEL); + if (!uv_hubs) { + ret = -ENOMEM; + goto err_enum_objs; + } + + for (i = 0; i < uv_bios_obj_cnt; i++) { + uv_hubs[i] = kzalloc(sizeof(*uv_hubs[i]), GFP_KERNEL); + if (!uv_hubs[i]) { + i--; + goto err_hubs; + } + + uv_hubs[i]->hub_info = &hub_buf[i]; + cache_obj_to_cnode(uv_hubs[i]->hub_info); + + uv_hubs[i]->kobj.kset = uv_hubs_kset; + + ret = kobject_init_and_add(&uv_hubs[i]->kobj, &hub_attr_type, + NULL, "hub_%u", hub_buf[i].id); + if (ret) + goto err_hubs; + kobject_uevent(&uv_hubs[i]->kobj, KOBJ_ADD); + } + return 0; + +err_hubs: + for (; i >= 0; i--) + kobject_put(&uv_hubs[i]->kobj); + kfree(uv_hubs); +err_enum_objs: + kfree(hub_buf); +err_hub_buf: + kset_unregister(uv_hubs_kset); +err_hubs_kset: + kfree(prev_obj_to_cnode); + return ret; + +} + +static void uv_hubs_exit(void) +{ + int i; + + for (i = 0; i < uv_bios_obj_cnt; i++) + kobject_put(&uv_hubs[i]->kobj); + + kfree(uv_hubs); + kfree(hub_buf); + kset_unregister(uv_hubs_kset); + kfree(prev_obj_to_cnode); +} + +struct uv_port { + struct kobject kobj; + struct uv_bios_port_info *port_info; +}; + +#define to_uv_port(kobj_ptr) container_of(kobj_ptr, struct uv_port, kobj) + +static ssize_t uv_port_conn_hub_show(struct uv_bios_port_info *port, char *buf) +{ + return sprintf(buf, "%d\n", port->conn_id); +} + +static ssize_t uv_port_conn_port_show(struct uv_bios_port_info *port, char *buf) +{ + return sprintf(buf, "%d\n", port->conn_port); +} + +struct uv_port_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct uv_bios_port_info *port_info, char *buf); + ssize_t (*store)(struct uv_bios_port_info *port_info, const char *buf, size_t size); +}; + +static struct uv_port_sysfs_entry uv_port_conn_hub_attribute = + __ATTR(conn_hub, 0444, uv_port_conn_hub_show, NULL); +static struct uv_port_sysfs_entry uv_port_conn_port_attribute = + __ATTR(conn_port, 0444, uv_port_conn_port_show, NULL); + +static struct attribute *uv_port_attrs[] = { + &uv_port_conn_hub_attribute.attr, + &uv_port_conn_port_attribute.attr, + NULL, +}; + +static void uv_port_release(struct kobject *kobj) +{ + struct uv_port *port = to_uv_port(kobj); + + kfree(port); +} + +static ssize_t uv_port_type_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct uv_port *port = to_uv_port(kobj); + struct uv_bios_port_info *port_info = port->port_info; + struct uv_port_sysfs_entry *entry; + + entry = container_of(attr, struct uv_port_sysfs_entry, attr); + + if (!entry->show) + return -EIO; + + return entry->show(port_info, buf); +} + +static const struct sysfs_ops uv_port_sysfs_ops = { + .show = uv_port_type_show, +}; + +static struct kobj_type uv_port_attr_type = { + .release = uv_port_release, + .sysfs_ops = &uv_port_sysfs_ops, + .default_attrs = uv_port_attrs, +}; + +static int uv_ports_init(void) +{ + s64 biosr; + int j = 0, k = 0, ret, sz; + + port_buf = kcalloc(uv_bios_obj_cnt, sizeof(*port_buf), GFP_KERNEL); + if (!port_buf) + return -ENOMEM; + + for (j = 0; j < uv_bios_obj_cnt; j++) { + sz = hub_buf[j].ports * sizeof(*port_buf[j]); + port_buf[j] = kzalloc(sz, GFP_KERNEL); + if (!port_buf[j]) { + ret = -ENOMEM; + j--; + goto err_port_info; + } + biosr = uv_bios_enum_ports((u64)uv_master_nasid, (u64)hub_buf[j].id, sz, + (u64 *)port_buf[j]); + if (biosr) { + ret = -EINVAL; + goto err_port_info; + } + } + for (j = 0; j < uv_bios_obj_cnt; j++) { + uv_hubs[j]->ports = kcalloc(hub_buf[j].ports, + sizeof(*uv_hubs[j]->ports), GFP_KERNEL); + if (!uv_hubs[j]->ports) { + ret = -ENOMEM; + j--; + goto err_ports; + } + } + for (j = 0; j < uv_bios_obj_cnt; j++) { + for (k = 0; k < hub_buf[j].ports; k++) { + uv_hubs[j]->ports[k] = kzalloc(sizeof(*uv_hubs[j]->ports[k]), GFP_KERNEL); + if (!uv_hubs[j]->ports[k]) { + ret = -ENOMEM; + k--; + goto err_kobj_ports; + } + uv_hubs[j]->ports[k]->port_info = &port_buf[j][k]; + ret = kobject_init_and_add(&uv_hubs[j]->ports[k]->kobj, &uv_port_attr_type, + &uv_hubs[j]->kobj, "port_%d", port_buf[j][k].port); + if (ret) + goto err_kobj_ports; + kobject_uevent(&uv_hubs[j]->ports[k]->kobj, KOBJ_ADD); + } + } + return 0; + +err_kobj_ports: + for (; j >= 0; j--) { + for (; k >= 0; k--) + kobject_put(&uv_hubs[j]->ports[k]->kobj); + if (j > 0) + k = hub_buf[j-1].ports - 1; + } + j = uv_bios_obj_cnt - 1; +err_ports: + for (; j >= 0; j--) + kfree(uv_hubs[j]->ports); + j = uv_bios_obj_cnt - 1; +err_port_info: + for (; j >= 0; j--) + kfree(port_buf[j]); + kfree(port_buf); + return ret; +} + +static void uv_ports_exit(void) +{ + int j, k; + + for (j = 0; j < uv_bios_obj_cnt; j++) { + for (k = hub_buf[j].ports - 1; k >= 0; k--) + kobject_put(&uv_hubs[j]->ports[k]->kobj); + } + for (j = 0; j < uv_bios_obj_cnt; j++) { + kfree(uv_hubs[j]->ports); + kfree(port_buf[j]); + } + kfree(port_buf); +} + +struct uv_pci_top_obj { + struct kobject kobj; + char *type; + char *location; + int iio_stack; + char *ppb_addr; + int slot; +}; + +#define to_uv_pci_top_obj(kobj_ptr) container_of(kobj_ptr, struct uv_pci_top_obj, kobj) + +static ssize_t uv_pci_type_show(struct uv_pci_top_obj *top_obj, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%s\n", top_obj->type); +} + +static ssize_t uv_pci_location_show(struct uv_pci_top_obj *top_obj, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%s\n", top_obj->location); +} + +static ssize_t uv_pci_iio_stack_show(struct uv_pci_top_obj *top_obj, char *buf) +{ + return sprintf(buf, "%d\n", top_obj->iio_stack); +} + +static ssize_t uv_pci_ppb_addr_show(struct uv_pci_top_obj *top_obj, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%s\n", top_obj->ppb_addr); +} + +static ssize_t uv_pci_slot_show(struct uv_pci_top_obj *top_obj, char *buf) +{ + return sprintf(buf, "%d\n", top_obj->slot); +} + +struct uv_pci_top_sysfs_entry { + struct attribute attr; + ssize_t (*show)(struct uv_pci_top_obj *top_obj, char *buf); + ssize_t (*store)(struct uv_pci_top_obj *top_obj, const char *buf, size_t size); +}; + +static struct uv_pci_top_sysfs_entry uv_pci_type_attribute = + __ATTR(type, 0444, uv_pci_type_show, NULL); +static struct uv_pci_top_sysfs_entry uv_pci_location_attribute = + __ATTR(location, 0444, uv_pci_location_show, NULL); +static struct uv_pci_top_sysfs_entry uv_pci_iio_stack_attribute = + __ATTR(iio_stack, 0444, uv_pci_iio_stack_show, NULL); +static struct uv_pci_top_sysfs_entry uv_pci_ppb_addr_attribute = + __ATTR(ppb_addr, 0444, uv_pci_ppb_addr_show, NULL); +static struct uv_pci_top_sysfs_entry uv_pci_slot_attribute = + __ATTR(slot, 0444, uv_pci_slot_show, NULL); + +static void uv_pci_top_release(struct kobject *kobj) +{ + struct uv_pci_top_obj *top_obj = to_uv_pci_top_obj(kobj); + + kfree(top_obj->type); + kfree(top_obj->location); + kfree(top_obj->ppb_addr); + kfree(top_obj); +} + +static ssize_t pci_top_type_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct uv_pci_top_obj *top_obj = to_uv_pci_top_obj(kobj); + struct uv_pci_top_sysfs_entry *entry; + + entry = container_of(attr, struct uv_pci_top_sysfs_entry, attr); + + if (!entry->show) + return -EIO; + + return entry->show(top_obj, buf); +} + +static const struct sysfs_ops uv_pci_top_sysfs_ops = { + .show = pci_top_type_show, +}; + +static struct kobj_type uv_pci_top_attr_type = { + .release = uv_pci_top_release, + .sysfs_ops = &uv_pci_top_sysfs_ops, +}; + +static int init_pci_top_obj(struct uv_pci_top_obj *top_obj, char *line) +{ + char *start; + char type[11], location[14], ppb_addr[15]; + int str_cnt, ret; + unsigned int tmp_match[2]; + + // Minimum line length + if (strlen(line) < 36) + return -EINVAL; + + //Line must match format "pcibus %4x:%2x" to be valid + str_cnt = sscanf(line, "pcibus %4x:%2x", &tmp_match[0], &tmp_match[1]); + if (str_cnt < 2) + return -EINVAL; + + /* Connect pcibus to segment:bus number with '_' + * to concatenate name tokens. + * pcibus 0000:00 ... -> pcibus_0000:00 ... + */ + line[6] = '_'; + + /* Null terminate after the concatencated name tokens + * to produce kobj name string. + */ + line[14] = '\0'; + + // Use start to index after name tokens string for remainder of line info. + start = &line[15]; + + top_obj->iio_stack = -1; + top_obj->slot = -1; + + /* r001i01b00h0 BASE IO (IIO Stack 0) + * r001i01b00h1 PCIe IO (IIO Stack 1) + * r001i01b03h1 PCIe SLOT + * r001i01b00h0 NODE IO + * r001i01b00h0 Riser + * (IIO Stack #) may not be present. + */ + if (start[0] == 'r') { + str_cnt = sscanf(start, "%13s %10[^(] %*s %*s %d)", + location, type, &top_obj->iio_stack); + if (str_cnt < 2) + return -EINVAL; + top_obj->type = kstrdup(type, GFP_KERNEL); + if (!top_obj->type) + return -ENOMEM; + top_obj->location = kstrdup(location, GFP_KERNEL); + if (!top_obj->location) { + kfree(top_obj->type); + return -ENOMEM; + } + } + /* PPB at 0000:80:00.00 (slot 3) + * (slot #) may not be present. + */ + else if (start[0] == 'P') { + str_cnt = sscanf(start, "%10s %*s %14s %*s %d)", + type, ppb_addr, &top_obj->slot); + if (str_cnt < 2) + return -EINVAL; + top_obj->type = kstrdup(type, GFP_KERNEL); + if (!top_obj->type) + return -ENOMEM; + top_obj->ppb_addr = kstrdup(ppb_addr, GFP_KERNEL); + if (!top_obj->ppb_addr) { + kfree(top_obj->type); + return -ENOMEM; + } + } else + return -EINVAL; + + top_obj->kobj.kset = uv_pcibus_kset; + + ret = kobject_init_and_add(&top_obj->kobj, &uv_pci_top_attr_type, NULL, "%s", line); + if (ret) + goto err_add_sysfs; + + if (top_obj->type) { + ret = sysfs_create_file(&top_obj->kobj, &uv_pci_type_attribute.attr); + if (ret) + goto err_add_sysfs; + } + if (top_obj->location) { + ret = sysfs_create_file(&top_obj->kobj, &uv_pci_location_attribute.attr); + if (ret) + goto err_add_sysfs; + } + if (top_obj->iio_stack >= 0) { + ret = sysfs_create_file(&top_obj->kobj, &uv_pci_iio_stack_attribute.attr); + if (ret) + goto err_add_sysfs; + } + if (top_obj->ppb_addr) { + ret = sysfs_create_file(&top_obj->kobj, &uv_pci_ppb_addr_attribute.attr); + if (ret) + goto err_add_sysfs; + } + if (top_obj->slot >= 0) { + ret = sysfs_create_file(&top_obj->kobj, &uv_pci_slot_attribute.attr); + if (ret) + goto err_add_sysfs; + } + + kobject_uevent(&top_obj->kobj, KOBJ_ADD); + return 0; + +err_add_sysfs: + kobject_put(&top_obj->kobj); + return ret; +} + +static int pci_topology_init(void) +{ + char *pci_top_str, *start, *found, *count; + size_t sz; + s64 biosr; + int l = 0, k = 0; + int len, ret; + + uv_pcibus_kset = kset_create_and_add("pcibuses", NULL, sgi_uv_kobj); + if (!uv_pcibus_kset) + return -ENOMEM; + + for (sz = PAGE_SIZE; sz < 16 * PAGE_SIZE; sz += PAGE_SIZE) { + pci_top_str = kmalloc(sz, GFP_KERNEL); + if (!pci_top_str) { + ret = -ENOMEM; + goto err_pci_top_str; + } + biosr = uv_bios_get_pci_topology((u64)sz, (u64 *)pci_top_str); + if (biosr == BIOS_STATUS_SUCCESS) { + len = strnlen(pci_top_str, sz); + for (count = pci_top_str; count < pci_top_str + len; count++) { + if (*count == '\n') + l++; + } + num_pci_lines = l; + + uv_pci_objs = kcalloc(num_pci_lines, + sizeof(*uv_pci_objs), GFP_KERNEL); + if (!uv_pci_objs) { + kfree(pci_top_str); + ret = -ENOMEM; + goto err_pci_top_str; + } + start = pci_top_str; + while ((found = strsep(&start, "\n")) != NULL) { + uv_pci_objs[k] = kzalloc(sizeof(*uv_pci_objs[k]), GFP_KERNEL); + if (!uv_pci_objs[k]) { + ret = -ENOMEM; + goto err_pci_obj; + } + ret = init_pci_top_obj(uv_pci_objs[k], found); + if (ret) + goto err_pci_obj; + k++; + if (k == num_pci_lines) + break; + } + } + kfree(pci_top_str); + if (biosr == BIOS_STATUS_SUCCESS || biosr == BIOS_STATUS_UNIMPLEMENTED) + break; + } + + return 0; +err_pci_obj: + k--; + for (; k >= 0; k--) + kobject_put(&uv_pci_objs[k]->kobj); + kfree(uv_pci_objs); + kfree(pci_top_str); +err_pci_top_str: + kset_unregister(uv_pcibus_kset); + return ret; +} + +static void pci_topology_exit(void) +{ + int k; + + for (k = 0; k < num_pci_lines; k++) + kobject_put(&uv_pci_objs[k]->kobj); + kset_unregister(uv_pcibus_kset); + kfree(uv_pci_objs); +} + +static ssize_t partition_id_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%ld\n", sn_partition_id); +} + +static ssize_t coherence_id_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%ld\n", sn_coherency_id); +} + +static ssize_t uv_type_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "%s\n", uv_type_string()); +} + +static struct kobj_attribute partition_id_attr = + __ATTR(partition_id, 0444, partition_id_show, NULL); +static struct kobj_attribute coherence_id_attr = + __ATTR(coherence_id, 0444, coherence_id_show, NULL); +static struct kobj_attribute uv_type_attr = + __ATTR(uv_type, 0444, uv_type_show, NULL); + +static struct attribute *base_attrs[] = { + &partition_id_attr.attr, + &coherence_id_attr.attr, + &uv_type_attr.attr, + NULL, +}; + +static struct attribute_group base_attr_group = { + .attrs = base_attrs +}; + +static int initial_bios_setup(void) +{ + u64 v; + s64 biosr; + + biosr = uv_bios_get_master_nasid((u64)sizeof(uv_master_nasid), (u64 *)&uv_master_nasid); + if (biosr) + return -EINVAL; + + biosr = uv_bios_get_heapsize((u64)uv_master_nasid, (u64)sizeof(u64), &v); + if (biosr) + return -EINVAL; + + uv_biosheap = vmalloc(v); + if (!uv_biosheap) + return -ENOMEM; + + biosr = uv_bios_install_heap((u64)uv_master_nasid, v, (u64 *)uv_biosheap); + if (biosr) { + vfree(uv_biosheap); + return -EINVAL; + } + + biosr = uv_bios_obj_count((u64)uv_master_nasid, sizeof(u64), &v); + if (biosr) { + vfree(uv_biosheap); + return -EINVAL; + } + uv_bios_obj_cnt = (int)v; + + return 0; +} + +static int __init uv_sysfs_init(void) +{ + int ret = 0; + + if (!is_uv_system()) + return -ENODEV; + + num_cnodes = uv_num_possible_blades(); + + if (!sgi_uv_kobj) + sgi_uv_kobj = kobject_create_and_add("sgi_uv", firmware_kobj); + if (!sgi_uv_kobj) { + pr_warn("kobject_create_and_add sgi_uv failed\n"); + return -EINVAL; + } + ret = sysfs_create_group(sgi_uv_kobj, &base_attr_group); + if (ret) { + pr_warn("sysfs_create_group base_attr_group failed\n"); + goto err_create_group; + } + + ret = initial_bios_setup(); + if (ret) + goto err_bios_setup; + + ret = uv_hubs_init(); + if (ret) + goto err_hubs_init; + + ret = uv_ports_init(); + if (ret) + goto err_ports_init; + + ret = pci_topology_init(); + if (ret) + goto err_pci_init; + + return 0; + +err_pci_init: + uv_ports_exit(); +err_ports_init: + uv_hubs_exit(); +err_hubs_init: + vfree(uv_biosheap); +err_bios_setup: + sysfs_remove_group(sgi_uv_kobj, &base_attr_group); +err_create_group: + kobject_put(sgi_uv_kobj); + return ret; +} + +static void __exit uv_sysfs_exit(void) +{ + if (!is_uv_system()) + return; + + pci_topology_exit(); + uv_ports_exit(); + uv_hubs_exit(); + vfree(uv_biosheap); + sysfs_remove_group(sgi_uv_kobj, &base_attr_group); + kobject_put(sgi_uv_kobj); +} + +#ifndef MODULE +device_initcall(uv_sysfs_init); +#else +module_init(uv_sysfs_init); +#endif +module_exit(uv_sysfs_exit); + +MODULE_AUTHOR("Hewlett Packard Enterprise"); +MODULE_LICENSE("GPL"); From 02685906d3afa4f7c72d86cf99242e3b08078865 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 17 Nov 2020 14:32:11 +0100 Subject: [PATCH 267/360] crypto: aegis128 - wipe plaintext and tag if decryption fails The AEGIS spec mentions explicitly that the security guarantees hold only if the resulting plaintext and tag of a failed decryption are withheld. So ensure that we abide by this. While at it, drop the unused struct aead_request *req parameter from crypto_aegis128_process_crypt(). Reviewed-by: Ondrej Mosnacek Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/aegis128-core.c | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/crypto/aegis128-core.c b/crypto/aegis128-core.c index 44fb4956f0dd..3a71235892f5 100644 --- a/crypto/aegis128-core.c +++ b/crypto/aegis128-core.c @@ -154,6 +154,12 @@ static void crypto_aegis128_ad(struct aegis_state *state, } } +static void crypto_aegis128_wipe_chunk(struct aegis_state *state, u8 *dst, + const u8 *src, unsigned int size) +{ + memzero_explicit(dst, size); +} + static void crypto_aegis128_encrypt_chunk(struct aegis_state *state, u8 *dst, const u8 *src, unsigned int size) { @@ -324,7 +330,6 @@ static void crypto_aegis128_process_ad(struct aegis_state *state, static __always_inline int crypto_aegis128_process_crypt(struct aegis_state *state, - struct aead_request *req, struct skcipher_walk *walk, void (*crypt)(struct aegis_state *state, u8 *dst, const u8 *src, @@ -403,14 +408,14 @@ static int crypto_aegis128_encrypt(struct aead_request *req) if (aegis128_do_simd()) { crypto_aegis128_init_simd(&state, &ctx->key, req->iv); crypto_aegis128_process_ad(&state, req->src, req->assoclen); - crypto_aegis128_process_crypt(&state, req, &walk, + crypto_aegis128_process_crypt(&state, &walk, crypto_aegis128_encrypt_chunk_simd); crypto_aegis128_final_simd(&state, &tag, req->assoclen, cryptlen); } else { crypto_aegis128_init(&state, &ctx->key, req->iv); crypto_aegis128_process_ad(&state, req->src, req->assoclen); - crypto_aegis128_process_crypt(&state, req, &walk, + crypto_aegis128_process_crypt(&state, &walk, crypto_aegis128_encrypt_chunk); crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen); } @@ -438,19 +443,34 @@ static int crypto_aegis128_decrypt(struct aead_request *req) if (aegis128_do_simd()) { crypto_aegis128_init_simd(&state, &ctx->key, req->iv); crypto_aegis128_process_ad(&state, req->src, req->assoclen); - crypto_aegis128_process_crypt(&state, req, &walk, + crypto_aegis128_process_crypt(&state, &walk, crypto_aegis128_decrypt_chunk_simd); crypto_aegis128_final_simd(&state, &tag, req->assoclen, cryptlen); } else { crypto_aegis128_init(&state, &ctx->key, req->iv); crypto_aegis128_process_ad(&state, req->src, req->assoclen); - crypto_aegis128_process_crypt(&state, req, &walk, + crypto_aegis128_process_crypt(&state, &walk, crypto_aegis128_decrypt_chunk); crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen); } - return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0; + if (unlikely(crypto_memneq(tag.bytes, zeros, authsize))) { + /* + * From Chapter 4. 'Security Analysis' of the AEGIS spec [0] + * + * "3. If verification fails, the decrypted plaintext and the + * wrong authentication tag should not be given as output." + * + * [0] https://competitions.cr.yp.to/round3/aegisv11.pdf + */ + skcipher_walk_aead_decrypt(&walk, req, false); + crypto_aegis128_process_crypt(NULL, &walk, + crypto_aegis128_wipe_chunk); + memzero_explicit(&tag, sizeof(tag)); + return -EBADMSG; + } + return 0; } static struct aead_alg crypto_aegis128_alg = { From ad00d41b47e6c86f4da61b9812b81cd4cd74be64 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 17 Nov 2020 14:32:12 +0100 Subject: [PATCH 268/360] crypto: aegis128/neon - optimize tail block handling Avoid copying the tail block via a stack buffer if the total size exceeds a single AEGIS block. In this case, we can use overlapping loads and stores and NEON permutation instructions instead, which leads to a modest performance improvement on some cores (< 5%), and is slightly cleaner. Note that we still need to use a stack buffer if the entire input is smaller than 16 bytes, given that we cannot use 16 byte NEON loads and stores safely in this case. Signed-off-by: Ard Biesheuvel Reviewed-by: Ondrej Mosnacek Signed-off-by: Herbert Xu --- crypto/aegis128-neon-inner.c | 89 ++++++++++++++++++++++++++++++------ 1 file changed, 75 insertions(+), 14 deletions(-) diff --git a/crypto/aegis128-neon-inner.c b/crypto/aegis128-neon-inner.c index 2a660ac1bc3a..cd1b3ad1d1f3 100644 --- a/crypto/aegis128-neon-inner.c +++ b/crypto/aegis128-neon-inner.c @@ -20,7 +20,6 @@ extern int aegis128_have_aes_insn; void *memcpy(void *dest, const void *src, size_t n); -void *memset(void *s, int c, size_t n); struct aegis128_state { uint8x16_t v[5]; @@ -173,10 +172,46 @@ void crypto_aegis128_update_neon(void *state, const void *msg) aegis128_save_state_neon(st, state); } +#ifdef CONFIG_ARM +/* + * AArch32 does not provide these intrinsics natively because it does not + * implement the underlying instructions. AArch32 only provides 64-bit + * wide vtbl.8/vtbx.8 instruction, so use those instead. + */ +static uint8x16_t vqtbl1q_u8(uint8x16_t a, uint8x16_t b) +{ + union { + uint8x16_t val; + uint8x8x2_t pair; + } __a = { a }; + + return vcombine_u8(vtbl2_u8(__a.pair, vget_low_u8(b)), + vtbl2_u8(__a.pair, vget_high_u8(b))); +} + +static uint8x16_t vqtbx1q_u8(uint8x16_t v, uint8x16_t a, uint8x16_t b) +{ + union { + uint8x16_t val; + uint8x8x2_t pair; + } __a = { a }; + + return vcombine_u8(vtbx2_u8(vget_low_u8(v), __a.pair, vget_low_u8(b)), + vtbx2_u8(vget_high_u8(v), __a.pair, vget_high_u8(b))); +} +#endif + +static const uint8_t permute[] __aligned(64) = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, +}; + void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src, unsigned int size) { struct aegis128_state st = aegis128_load_state_neon(state); + const int short_input = size < AEGIS_BLOCK_SIZE; uint8x16_t msg; preload_sbox(); @@ -186,7 +221,8 @@ void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src, msg = vld1q_u8(src); st = aegis128_update_neon(st, msg); - vst1q_u8(dst, msg ^ s); + msg ^= s; + vst1q_u8(dst, msg); size -= AEGIS_BLOCK_SIZE; src += AEGIS_BLOCK_SIZE; @@ -195,13 +231,26 @@ void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src, if (size > 0) { uint8x16_t s = st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4]; - uint8_t buf[AEGIS_BLOCK_SIZE] = {}; + uint8_t buf[AEGIS_BLOCK_SIZE]; + const void *in = src; + void *out = dst; + uint8x16_t m; - memcpy(buf, src, size); - msg = vld1q_u8(buf); - st = aegis128_update_neon(st, msg); - vst1q_u8(buf, msg ^ s); - memcpy(dst, buf, size); + if (__builtin_expect(short_input, 0)) + in = out = memcpy(buf + AEGIS_BLOCK_SIZE - size, src, size); + + m = vqtbl1q_u8(vld1q_u8(in + size - AEGIS_BLOCK_SIZE), + vld1q_u8(permute + 32 - size)); + + st = aegis128_update_neon(st, m); + + vst1q_u8(out + size - AEGIS_BLOCK_SIZE, + vqtbl1q_u8(m ^ s, vld1q_u8(permute + size))); + + if (__builtin_expect(short_input, 0)) + memcpy(dst, out, size); + else + vst1q_u8(out - AEGIS_BLOCK_SIZE, msg); } aegis128_save_state_neon(st, state); @@ -211,6 +260,7 @@ void crypto_aegis128_decrypt_chunk_neon(void *state, void *dst, const void *src, unsigned int size) { struct aegis128_state st = aegis128_load_state_neon(state); + const int short_input = size < AEGIS_BLOCK_SIZE; uint8x16_t msg; preload_sbox(); @@ -228,14 +278,25 @@ void crypto_aegis128_decrypt_chunk_neon(void *state, void *dst, const void *src, if (size > 0) { uint8x16_t s = st.v[1] ^ (st.v[2] & st.v[3]) ^ st.v[4]; uint8_t buf[AEGIS_BLOCK_SIZE]; + const void *in = src; + void *out = dst; + uint8x16_t m; - vst1q_u8(buf, s); - memcpy(buf, src, size); - msg = vld1q_u8(buf) ^ s; - vst1q_u8(buf, msg); - memcpy(dst, buf, size); + if (__builtin_expect(short_input, 0)) + in = out = memcpy(buf + AEGIS_BLOCK_SIZE - size, src, size); - st = aegis128_update_neon(st, msg); + m = s ^ vqtbx1q_u8(s, vld1q_u8(in + size - AEGIS_BLOCK_SIZE), + vld1q_u8(permute + 32 - size)); + + st = aegis128_update_neon(st, m); + + vst1q_u8(out + size - AEGIS_BLOCK_SIZE, + vqtbl1q_u8(m, vld1q_u8(permute + size))); + + if (__builtin_expect(short_input, 0)) + memcpy(dst, out, size); + else + vst1q_u8(out - AEGIS_BLOCK_SIZE, msg); } aegis128_save_state_neon(st, state); From 97b70180b7f97224762b63f211305a8052d07960 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 17 Nov 2020 14:32:13 +0100 Subject: [PATCH 269/360] crypto: aegis128/neon - move final tag check to SIMD domain Instead of calculating the tag and returning it to the caller on decryption, use a SIMD compare and min across vector to perform the comparison. This is slightly more efficient, and removes the need on the caller's part to wipe the tag from memory if the decryption failed. While at it, switch to unsigned int when passing cryptlen and assoclen - we don't support input sizes where it matters anyway. Signed-off-by: Ard Biesheuvel Reviewed-by: Ondrej Mosnacek Signed-off-by: Herbert Xu --- crypto/aegis128-core.c | 21 +++++++++++++++------ crypto/aegis128-neon-inner.c | 33 +++++++++++++++++++++++++++------ crypto/aegis128-neon.c | 21 +++++++++++++++------ 3 files changed, 57 insertions(+), 18 deletions(-) diff --git a/crypto/aegis128-core.c b/crypto/aegis128-core.c index 3a71235892f5..859c7b905618 100644 --- a/crypto/aegis128-core.c +++ b/crypto/aegis128-core.c @@ -67,9 +67,11 @@ void crypto_aegis128_encrypt_chunk_simd(struct aegis_state *state, u8 *dst, const u8 *src, unsigned int size); void crypto_aegis128_decrypt_chunk_simd(struct aegis_state *state, u8 *dst, const u8 *src, unsigned int size); -void crypto_aegis128_final_simd(struct aegis_state *state, - union aegis_block *tag_xor, - u64 assoclen, u64 cryptlen); +int crypto_aegis128_final_simd(struct aegis_state *state, + union aegis_block *tag_xor, + unsigned int assoclen, + unsigned int cryptlen, + unsigned int authsize); static void crypto_aegis128_update(struct aegis_state *state) { @@ -411,7 +413,7 @@ static int crypto_aegis128_encrypt(struct aead_request *req) crypto_aegis128_process_crypt(&state, &walk, crypto_aegis128_encrypt_chunk_simd); crypto_aegis128_final_simd(&state, &tag, req->assoclen, - cryptlen); + cryptlen, 0); } else { crypto_aegis128_init(&state, &ctx->key, req->iv); crypto_aegis128_process_ad(&state, req->src, req->assoclen); @@ -445,8 +447,15 @@ static int crypto_aegis128_decrypt(struct aead_request *req) crypto_aegis128_process_ad(&state, req->src, req->assoclen); crypto_aegis128_process_crypt(&state, &walk, crypto_aegis128_decrypt_chunk_simd); - crypto_aegis128_final_simd(&state, &tag, req->assoclen, - cryptlen); + if (unlikely(crypto_aegis128_final_simd(&state, &tag, + req->assoclen, + cryptlen, authsize))) { + skcipher_walk_aead_decrypt(&walk, req, false); + crypto_aegis128_process_crypt(NULL, req, &walk, + crypto_aegis128_wipe_chunk); + return -EBADMSG; + } + return 0; } else { crypto_aegis128_init(&state, &ctx->key, req->iv); crypto_aegis128_process_ad(&state, req->src, req->assoclen); diff --git a/crypto/aegis128-neon-inner.c b/crypto/aegis128-neon-inner.c index cd1b3ad1d1f3..7de485907d81 100644 --- a/crypto/aegis128-neon-inner.c +++ b/crypto/aegis128-neon-inner.c @@ -199,6 +199,17 @@ static uint8x16_t vqtbx1q_u8(uint8x16_t v, uint8x16_t a, uint8x16_t b) return vcombine_u8(vtbx2_u8(vget_low_u8(v), __a.pair, vget_low_u8(b)), vtbx2_u8(vget_high_u8(v), __a.pair, vget_high_u8(b))); } + +static int8_t vminvq_s8(int8x16_t v) +{ + int8x8_t s = vpmin_s8(vget_low_s8(v), vget_high_s8(v)); + + s = vpmin_s8(s, s); + s = vpmin_s8(s, s); + s = vpmin_s8(s, s); + + return vget_lane_s8(s, 0); +} #endif static const uint8_t permute[] __aligned(64) = { @@ -302,8 +313,10 @@ void crypto_aegis128_decrypt_chunk_neon(void *state, void *dst, const void *src, aegis128_save_state_neon(st, state); } -void crypto_aegis128_final_neon(void *state, void *tag_xor, uint64_t assoclen, - uint64_t cryptlen) +int crypto_aegis128_final_neon(void *state, void *tag_xor, + unsigned int assoclen, + unsigned int cryptlen, + unsigned int authsize) { struct aegis128_state st = aegis128_load_state_neon(state); uint8x16_t v; @@ -311,13 +324,21 @@ void crypto_aegis128_final_neon(void *state, void *tag_xor, uint64_t assoclen, preload_sbox(); - v = st.v[3] ^ (uint8x16_t)vcombine_u64(vmov_n_u64(8 * assoclen), - vmov_n_u64(8 * cryptlen)); + v = st.v[3] ^ (uint8x16_t)vcombine_u64(vmov_n_u64(8ULL * assoclen), + vmov_n_u64(8ULL * cryptlen)); for (i = 0; i < 7; i++) st = aegis128_update_neon(st, v); - v = vld1q_u8(tag_xor); - v ^= st.v[0] ^ st.v[1] ^ st.v[2] ^ st.v[3] ^ st.v[4]; + v = st.v[0] ^ st.v[1] ^ st.v[2] ^ st.v[3] ^ st.v[4]; + + if (authsize > 0) { + v = vqtbl1q_u8(~vceqq_u8(v, vld1q_u8(tag_xor)), + vld1q_u8(permute + authsize)); + + return vminvq_s8((int8x16_t)v); + } + vst1q_u8(tag_xor, v); + return 0; } diff --git a/crypto/aegis128-neon.c b/crypto/aegis128-neon.c index 8271b1fa0fbc..94d591a002a4 100644 --- a/crypto/aegis128-neon.c +++ b/crypto/aegis128-neon.c @@ -14,8 +14,10 @@ void crypto_aegis128_encrypt_chunk_neon(void *state, void *dst, const void *src, unsigned int size); void crypto_aegis128_decrypt_chunk_neon(void *state, void *dst, const void *src, unsigned int size); -void crypto_aegis128_final_neon(void *state, void *tag_xor, uint64_t assoclen, - uint64_t cryptlen); +int crypto_aegis128_final_neon(void *state, void *tag_xor, + unsigned int assoclen, + unsigned int cryptlen, + unsigned int authsize); int aegis128_have_aes_insn __ro_after_init; @@ -60,11 +62,18 @@ void crypto_aegis128_decrypt_chunk_simd(union aegis_block *state, u8 *dst, kernel_neon_end(); } -void crypto_aegis128_final_simd(union aegis_block *state, - union aegis_block *tag_xor, - u64 assoclen, u64 cryptlen) +int crypto_aegis128_final_simd(union aegis_block *state, + union aegis_block *tag_xor, + unsigned int assoclen, + unsigned int cryptlen, + unsigned int authsize) { + int ret; + kernel_neon_begin(); - crypto_aegis128_final_neon(state, tag_xor, assoclen, cryptlen); + ret = crypto_aegis128_final_neon(state, tag_xor, assoclen, cryptlen, + authsize); kernel_neon_end(); + + return ret; } From ac50aec41a9f9590b1d48bd4daa2251f0025052a Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 17 Nov 2020 14:32:14 +0100 Subject: [PATCH 270/360] crypto: aegis128 - expose SIMD code path as separate driver Wiring the SIMD code into the generic driver has the unfortunate side effect that the tcrypt testing code cannot distinguish them, and will therefore not use the latter to fuzz test the former, as it does for other algorithms. So let's refactor the code a bit so we can register two implementations: aegis128-generic and aegis128-simd. Signed-off-by: Ard Biesheuvel Reviewed-by: Ondrej Mosnacek Signed-off-by: Herbert Xu --- crypto/aegis128-core.c | 222 ++++++++++++++++++++++++++--------------- 1 file changed, 144 insertions(+), 78 deletions(-) diff --git a/crypto/aegis128-core.c b/crypto/aegis128-core.c index 859c7b905618..2b05f79475d3 100644 --- a/crypto/aegis128-core.c +++ b/crypto/aegis128-core.c @@ -86,9 +86,10 @@ static void crypto_aegis128_update(struct aegis_state *state) } static void crypto_aegis128_update_a(struct aegis_state *state, - const union aegis_block *msg) + const union aegis_block *msg, + bool do_simd) { - if (aegis128_do_simd()) { + if (do_simd) { crypto_aegis128_update_simd(state, msg); return; } @@ -97,9 +98,10 @@ static void crypto_aegis128_update_a(struct aegis_state *state, crypto_aegis_block_xor(&state->blocks[0], msg); } -static void crypto_aegis128_update_u(struct aegis_state *state, const void *msg) +static void crypto_aegis128_update_u(struct aegis_state *state, const void *msg, + bool do_simd) { - if (aegis128_do_simd()) { + if (do_simd) { crypto_aegis128_update_simd(state, msg); return; } @@ -128,27 +130,28 @@ static void crypto_aegis128_init(struct aegis_state *state, crypto_aegis_block_xor(&state->blocks[4], &crypto_aegis_const[1]); for (i = 0; i < 5; i++) { - crypto_aegis128_update_a(state, key); - crypto_aegis128_update_a(state, &key_iv); + crypto_aegis128_update_a(state, key, false); + crypto_aegis128_update_a(state, &key_iv, false); } } static void crypto_aegis128_ad(struct aegis_state *state, - const u8 *src, unsigned int size) + const u8 *src, unsigned int size, + bool do_simd) { if (AEGIS_ALIGNED(src)) { const union aegis_block *src_blk = (const union aegis_block *)src; while (size >= AEGIS_BLOCK_SIZE) { - crypto_aegis128_update_a(state, src_blk); + crypto_aegis128_update_a(state, src_blk, do_simd); size -= AEGIS_BLOCK_SIZE; src_blk++; } } else { while (size >= AEGIS_BLOCK_SIZE) { - crypto_aegis128_update_u(state, src); + crypto_aegis128_update_u(state, src, do_simd); size -= AEGIS_BLOCK_SIZE; src += AEGIS_BLOCK_SIZE; @@ -180,7 +183,7 @@ static void crypto_aegis128_encrypt_chunk(struct aegis_state *state, u8 *dst, crypto_aegis_block_xor(&tmp, &state->blocks[1]); crypto_aegis_block_xor(&tmp, src_blk); - crypto_aegis128_update_a(state, src_blk); + crypto_aegis128_update_a(state, src_blk, false); *dst_blk = tmp; @@ -196,7 +199,7 @@ static void crypto_aegis128_encrypt_chunk(struct aegis_state *state, u8 *dst, crypto_aegis_block_xor(&tmp, &state->blocks[1]); crypto_xor(tmp.bytes, src, AEGIS_BLOCK_SIZE); - crypto_aegis128_update_u(state, src); + crypto_aegis128_update_u(state, src, false); memcpy(dst, tmp.bytes, AEGIS_BLOCK_SIZE); @@ -215,7 +218,7 @@ static void crypto_aegis128_encrypt_chunk(struct aegis_state *state, u8 *dst, crypto_aegis_block_xor(&tmp, &state->blocks[4]); crypto_aegis_block_xor(&tmp, &state->blocks[1]); - crypto_aegis128_update_a(state, &msg); + crypto_aegis128_update_a(state, &msg, false); crypto_aegis_block_xor(&msg, &tmp); @@ -241,7 +244,7 @@ static void crypto_aegis128_decrypt_chunk(struct aegis_state *state, u8 *dst, crypto_aegis_block_xor(&tmp, &state->blocks[1]); crypto_aegis_block_xor(&tmp, src_blk); - crypto_aegis128_update_a(state, &tmp); + crypto_aegis128_update_a(state, &tmp, false); *dst_blk = tmp; @@ -257,7 +260,7 @@ static void crypto_aegis128_decrypt_chunk(struct aegis_state *state, u8 *dst, crypto_aegis_block_xor(&tmp, &state->blocks[1]); crypto_xor(tmp.bytes, src, AEGIS_BLOCK_SIZE); - crypto_aegis128_update_a(state, &tmp); + crypto_aegis128_update_a(state, &tmp, false); memcpy(dst, tmp.bytes, AEGIS_BLOCK_SIZE); @@ -279,7 +282,7 @@ static void crypto_aegis128_decrypt_chunk(struct aegis_state *state, u8 *dst, memset(msg.bytes + size, 0, AEGIS_BLOCK_SIZE - size); - crypto_aegis128_update_a(state, &msg); + crypto_aegis128_update_a(state, &msg, false); memcpy(dst, msg.bytes, size); } @@ -287,7 +290,8 @@ static void crypto_aegis128_decrypt_chunk(struct aegis_state *state, u8 *dst, static void crypto_aegis128_process_ad(struct aegis_state *state, struct scatterlist *sg_src, - unsigned int assoclen) + unsigned int assoclen, + bool do_simd) { struct scatter_walk walk; union aegis_block buf; @@ -304,13 +308,13 @@ static void crypto_aegis128_process_ad(struct aegis_state *state, if (pos > 0) { unsigned int fill = AEGIS_BLOCK_SIZE - pos; memcpy(buf.bytes + pos, src, fill); - crypto_aegis128_update_a(state, &buf); + crypto_aegis128_update_a(state, &buf, do_simd); pos = 0; left -= fill; src += fill; } - crypto_aegis128_ad(state, src, left); + crypto_aegis128_ad(state, src, left, do_simd); src += left & ~(AEGIS_BLOCK_SIZE - 1); left &= AEGIS_BLOCK_SIZE - 1; } @@ -326,7 +330,7 @@ static void crypto_aegis128_process_ad(struct aegis_state *state, if (pos > 0) { memset(buf.bytes + pos, 0, AEGIS_BLOCK_SIZE - pos); - crypto_aegis128_update_a(state, &buf); + crypto_aegis128_update_a(state, &buf, do_simd); } } @@ -368,7 +372,7 @@ static void crypto_aegis128_final(struct aegis_state *state, crypto_aegis_block_xor(&tmp, &state->blocks[3]); for (i = 0; i < 7; i++) - crypto_aegis128_update_a(state, &tmp); + crypto_aegis128_update_a(state, &tmp, false); for (i = 0; i < AEGIS128_STATE_BLOCKS; i++) crypto_aegis_block_xor(tag_xor, &state->blocks[i]); @@ -396,7 +400,7 @@ static int crypto_aegis128_setauthsize(struct crypto_aead *tfm, return 0; } -static int crypto_aegis128_encrypt(struct aead_request *req) +static int crypto_aegis128_encrypt_generic(struct aead_request *req) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); union aegis_block tag = {}; @@ -407,27 +411,18 @@ static int crypto_aegis128_encrypt(struct aead_request *req) struct aegis_state state; skcipher_walk_aead_encrypt(&walk, req, false); - if (aegis128_do_simd()) { - crypto_aegis128_init_simd(&state, &ctx->key, req->iv); - crypto_aegis128_process_ad(&state, req->src, req->assoclen); - crypto_aegis128_process_crypt(&state, &walk, - crypto_aegis128_encrypt_chunk_simd); - crypto_aegis128_final_simd(&state, &tag, req->assoclen, - cryptlen, 0); - } else { - crypto_aegis128_init(&state, &ctx->key, req->iv); - crypto_aegis128_process_ad(&state, req->src, req->assoclen); - crypto_aegis128_process_crypt(&state, &walk, - crypto_aegis128_encrypt_chunk); - crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen); - } + crypto_aegis128_init(&state, &ctx->key, req->iv); + crypto_aegis128_process_ad(&state, req->src, req->assoclen, false); + crypto_aegis128_process_crypt(&state, &walk, + crypto_aegis128_encrypt_chunk); + crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen); scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen, authsize, 1); return 0; } -static int crypto_aegis128_decrypt(struct aead_request *req) +static int crypto_aegis128_decrypt_generic(struct aead_request *req) { static const u8 zeros[AEGIS128_MAX_AUTH_SIZE] = {}; struct crypto_aead *tfm = crypto_aead_reqtfm(req); @@ -442,27 +437,11 @@ static int crypto_aegis128_decrypt(struct aead_request *req) authsize, 0); skcipher_walk_aead_decrypt(&walk, req, false); - if (aegis128_do_simd()) { - crypto_aegis128_init_simd(&state, &ctx->key, req->iv); - crypto_aegis128_process_ad(&state, req->src, req->assoclen); - crypto_aegis128_process_crypt(&state, &walk, - crypto_aegis128_decrypt_chunk_simd); - if (unlikely(crypto_aegis128_final_simd(&state, &tag, - req->assoclen, - cryptlen, authsize))) { - skcipher_walk_aead_decrypt(&walk, req, false); - crypto_aegis128_process_crypt(NULL, req, &walk, - crypto_aegis128_wipe_chunk); - return -EBADMSG; - } - return 0; - } else { - crypto_aegis128_init(&state, &ctx->key, req->iv); - crypto_aegis128_process_ad(&state, req->src, req->assoclen); - crypto_aegis128_process_crypt(&state, &walk, - crypto_aegis128_decrypt_chunk); - crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen); - } + crypto_aegis128_init(&state, &ctx->key, req->iv); + crypto_aegis128_process_ad(&state, req->src, req->assoclen, false); + crypto_aegis128_process_crypt(&state, &walk, + crypto_aegis128_decrypt_chunk); + crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen); if (unlikely(crypto_memneq(tag.bytes, zeros, authsize))) { /* @@ -482,42 +461,128 @@ static int crypto_aegis128_decrypt(struct aead_request *req) return 0; } -static struct aead_alg crypto_aegis128_alg = { - .setkey = crypto_aegis128_setkey, - .setauthsize = crypto_aegis128_setauthsize, - .encrypt = crypto_aegis128_encrypt, - .decrypt = crypto_aegis128_decrypt, +static int crypto_aegis128_encrypt_simd(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + union aegis_block tag = {}; + unsigned int authsize = crypto_aead_authsize(tfm); + struct aegis_ctx *ctx = crypto_aead_ctx(tfm); + unsigned int cryptlen = req->cryptlen; + struct skcipher_walk walk; + struct aegis_state state; - .ivsize = AEGIS128_NONCE_SIZE, - .maxauthsize = AEGIS128_MAX_AUTH_SIZE, - .chunksize = AEGIS_BLOCK_SIZE, + if (!aegis128_do_simd()) + return crypto_aegis128_encrypt_generic(req); - .base = { - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct aegis_ctx), - .cra_alignmask = 0, + skcipher_walk_aead_encrypt(&walk, req, false); + crypto_aegis128_init_simd(&state, &ctx->key, req->iv); + crypto_aegis128_process_ad(&state, req->src, req->assoclen, true); + crypto_aegis128_process_crypt(&state, &walk, + crypto_aegis128_encrypt_chunk_simd); + crypto_aegis128_final_simd(&state, &tag, req->assoclen, cryptlen, 0); - .cra_priority = 100, + scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen, + authsize, 1); + return 0; +} - .cra_name = "aegis128", - .cra_driver_name = "aegis128-generic", +static int crypto_aegis128_decrypt_simd(struct aead_request *req) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + union aegis_block tag; + unsigned int authsize = crypto_aead_authsize(tfm); + unsigned int cryptlen = req->cryptlen - authsize; + struct aegis_ctx *ctx = crypto_aead_ctx(tfm); + struct skcipher_walk walk; + struct aegis_state state; - .cra_module = THIS_MODULE, + if (!aegis128_do_simd()) + return crypto_aegis128_decrypt_generic(req); + + scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen, + authsize, 0); + + skcipher_walk_aead_decrypt(&walk, req, false); + crypto_aegis128_init_simd(&state, &ctx->key, req->iv); + crypto_aegis128_process_ad(&state, req->src, req->assoclen, true); + crypto_aegis128_process_crypt(&state, &walk, + crypto_aegis128_decrypt_chunk_simd); + + if (unlikely(crypto_aegis128_final_simd(&state, &tag, req->assoclen, + cryptlen, authsize))) { + skcipher_walk_aead_decrypt(&walk, req, false); + crypto_aegis128_process_crypt(NULL, &walk, + crypto_aegis128_wipe_chunk); + return -EBADMSG; } + return 0; +} + +static struct aead_alg crypto_aegis128_alg_generic = { + .setkey = crypto_aegis128_setkey, + .setauthsize = crypto_aegis128_setauthsize, + .encrypt = crypto_aegis128_encrypt_generic, + .decrypt = crypto_aegis128_decrypt_generic, + + .ivsize = AEGIS128_NONCE_SIZE, + .maxauthsize = AEGIS128_MAX_AUTH_SIZE, + .chunksize = AEGIS_BLOCK_SIZE, + + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct aegis_ctx), + .base.cra_alignmask = 0, + .base.cra_priority = 100, + .base.cra_name = "aegis128", + .base.cra_driver_name = "aegis128-generic", + .base.cra_module = THIS_MODULE, +}; + +static struct aead_alg crypto_aegis128_alg_simd = { + .setkey = crypto_aegis128_setkey, + .setauthsize = crypto_aegis128_setauthsize, + .encrypt = crypto_aegis128_encrypt_simd, + .decrypt = crypto_aegis128_decrypt_simd, + + .ivsize = AEGIS128_NONCE_SIZE, + .maxauthsize = AEGIS128_MAX_AUTH_SIZE, + .chunksize = AEGIS_BLOCK_SIZE, + + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct aegis_ctx), + .base.cra_alignmask = 0, + .base.cra_priority = 200, + .base.cra_name = "aegis128", + .base.cra_driver_name = "aegis128-simd", + .base.cra_module = THIS_MODULE, }; static int __init crypto_aegis128_module_init(void) { - if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && - crypto_aegis128_have_simd()) - static_branch_enable(&have_simd); + int ret; - return crypto_register_aead(&crypto_aegis128_alg); + ret = crypto_register_aead(&crypto_aegis128_alg_generic); + if (ret) + return ret; + + if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && + crypto_aegis128_have_simd()) { + ret = crypto_register_aead(&crypto_aegis128_alg_simd); + if (ret) { + crypto_unregister_aead(&crypto_aegis128_alg_generic); + return ret; + } + static_branch_enable(&have_simd); + } + return 0; } static void __exit crypto_aegis128_module_exit(void) { - crypto_unregister_aead(&crypto_aegis128_alg); + if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && + crypto_aegis128_have_simd()) + crypto_unregister_aead(&crypto_aegis128_alg_simd); + + crypto_unregister_aead(&crypto_aegis128_alg_generic); } subsys_initcall(crypto_aegis128_module_init); @@ -528,3 +593,4 @@ MODULE_AUTHOR("Ondrej Mosnacek "); MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm"); MODULE_ALIAS_CRYPTO("aegis128"); MODULE_ALIAS_CRYPTO("aegis128-generic"); +MODULE_ALIAS_CRYPTO("aegis128-simd"); From 8cbc3448214a9d4534f8381ec23ef5add0ae8d91 Mon Sep 17 00:00:00 2001 From: Thara Gopinath Date: Thu, 19 Nov 2020 10:52:30 -0500 Subject: [PATCH 271/360] crypto: qce - Enable support for crypto engine on sdm845 Add support Qualcomm Crypto Engine accelerated encryption and authentication algorithms on sdm845. Reviewed-by: Bjorn Andersson Signed-off-by: Thara Gopinath Signed-off-by: Herbert Xu --- drivers/crypto/qce/core.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index 5e6717f9bbda..80b75085c265 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -159,7 +159,21 @@ static int qce_check_version(struct qce_device *qce) return -ENODEV; qce->burst_size = QCE_BAM_BURST_SIZE; - qce->pipe_pair_id = 1; + + /* + * Rx and tx pipes are treated as a pair inside CE. + * Pipe pair number depends on the actual BAM dma pipe + * that is used for transfers. The BAM dma pipes are passed + * from the device tree and used to derive the pipe pair + * id in the CE driver as follows. + * BAM dma pipes(rx, tx) CE pipe pair id + * 0,1 0 + * 2,3 1 + * 4,5 2 + * 6,7 3 + * ... + */ + qce->pipe_pair_id = qce->dma.rxchan->chan_id >> 1; dev_dbg(qce->dev, "Crypto device found, version %d.%d.%d\n", major, minor, step); @@ -260,6 +274,7 @@ static int qce_crypto_remove(struct platform_device *pdev) static const struct of_device_id qce_crypto_of_match[] = { { .compatible = "qcom,crypto-v5.1", }, + { .compatible = "qcom,crypto-v5.4", }, {} }; MODULE_DEVICE_TABLE(of, qce_crypto_of_match); From 1148a9654b5a69611d33e14719251c6ec20f5f2c Mon Sep 17 00:00:00 2001 From: Thara Gopinath Date: Thu, 19 Nov 2020 10:52:31 -0500 Subject: [PATCH 272/360] crypto: qce - Fix SHA result buffer corruption issues Partial hash was being copied into the final result buffer without the entire message block processed. Depending on how the end user processes this result buffer, errors vary from result buffer corruption to result buffer poisoing. Fix this issue by ensuring that only the final hash value is copied into the result buffer. Reviewed-by: Bjorn Andersson Signed-off-by: Thara Gopinath Signed-off-by: Herbert Xu --- drivers/crypto/qce/sha.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index 87be96a0b0bb..61c418c12345 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -48,7 +48,7 @@ static void qce_ahash_done(void *data) dma_unmap_sg(qce->dev, &rctx->result_sg, 1, DMA_FROM_DEVICE); memcpy(rctx->digest, result->auth_iv, digestsize); - if (req->result) + if (req->result && rctx->last_blk) memcpy(req->result, result->auth_iv, digestsize); rctx->byte_count[0] = cpu_to_be32(result->auth_byte_count[0]); From da6d57948fa23f6d3aa7405b9f1c1de96f7e450e Mon Sep 17 00:00:00 2001 From: kernel test robot Date: Fri, 13 Nov 2020 18:14:00 +0100 Subject: [PATCH 273/360] crypto: qat - fix excluded_middle.cocci warnings Condition !A || A && B is equivalent to !A || B. Generated by: scripts/coccinelle/misc/excluded_middle.cocci Fixes: b76f0ea01312 ("coccinelle: misc: add excluded_middle.cocci script") CC: Denis Efremov Reported-by: kernel test robot Signed-off-by: kernel test robot Signed-off-by: Julia Lawall Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/adf_dev_mgr.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c index 29dc2e3d38ff..4c752eed10fe 100644 --- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c +++ b/drivers/crypto/qat/qat_common/adf_dev_mgr.c @@ -151,8 +151,8 @@ int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, mutex_lock(&table_lock); atomic_set(&accel_dev->ref_count, 0); - /* PF on host or VF on guest */ - if (!accel_dev->is_vf || (accel_dev->is_vf && !pf)) { + /* PF on host or VF on guest - optimized to remove redundant is_vf */ + if (!accel_dev->is_vf || !pf) { struct vf_id_map *map; list_for_each(itr, &accel_table) { @@ -248,7 +248,8 @@ void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, struct adf_accel_dev *pf) { mutex_lock(&table_lock); - if (!accel_dev->is_vf || (accel_dev->is_vf && !pf)) { + /* PF on host or VF on guest - optimized to remove redundant is_vf */ + if (!accel_dev->is_vf || !pf) { id_map[accel_dev->accel_id] = 0; num_devices--; } else if (accel_dev->is_vf && pf) { From 7c2f5537ca196ddd00639f66f0d58c9f8d265b27 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 20 Nov 2020 17:01:31 +1100 Subject: [PATCH 274/360] crypto: mips/octeon - Fix sparse endianness warnings This patch fixes a number of endianness warnings in the mips/octeon code. Signed-off-by: Herbert Xu --- arch/mips/cavium-octeon/crypto/octeon-crypto.h | 2 +- arch/mips/cavium-octeon/crypto/octeon-md5.c | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.h b/arch/mips/cavium-octeon/crypto/octeon-crypto.h index 7315cc307397..cb68f9e284bb 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-crypto.h +++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.h @@ -41,7 +41,7 @@ do { \ */ #define read_octeon_64bit_hash_dword(index) \ ({ \ - u64 __value; \ + __be64 __value; \ \ __asm__ __volatile__ ( \ "dmfc2 %[rt],0x0048+" STR(index) \ diff --git a/arch/mips/cavium-octeon/crypto/octeon-md5.c b/arch/mips/cavium-octeon/crypto/octeon-md5.c index 8c8ea139653e..5ee4ade99b99 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-md5.c +++ b/arch/mips/cavium-octeon/crypto/octeon-md5.c @@ -68,10 +68,11 @@ static int octeon_md5_init(struct shash_desc *desc) { struct md5_state *mctx = shash_desc_ctx(desc); - mctx->hash[0] = cpu_to_le32(MD5_H0); - mctx->hash[1] = cpu_to_le32(MD5_H1); - mctx->hash[2] = cpu_to_le32(MD5_H2); - mctx->hash[3] = cpu_to_le32(MD5_H3); + mctx->hash[0] = MD5_H0; + mctx->hash[1] = MD5_H1; + mctx->hash[2] = MD5_H2; + mctx->hash[3] = MD5_H3; + cpu_to_le32_array(mctx->hash, 4); mctx->byte_count = 0; return 0; @@ -139,8 +140,9 @@ static int octeon_md5_final(struct shash_desc *desc, u8 *out) } memset(p, 0, padding); - mctx->block[14] = cpu_to_le32(mctx->byte_count << 3); - mctx->block[15] = cpu_to_le32(mctx->byte_count >> 29); + mctx->block[14] = mctx->byte_count << 3; + mctx->block[15] = mctx->byte_count >> 29; + cpu_to_le32_array(mctx->block + 14, 2); octeon_md5_transform(mctx->block); octeon_md5_read_hash(mctx); From e547655238f70a7b238e14f18d968428ff6b6b93 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 20 Nov 2020 17:28:38 +1100 Subject: [PATCH 275/360] crypto: powerpc/sha256-spe - Fix sparse endianness warning This patch fixes a sparse endianness warning in sha256-spe. Signed-off-by: Herbert Xu --- arch/powerpc/crypto/sha256-spe-glue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c index 88530ae0791f..a6e650a97d8f 100644 --- a/arch/powerpc/crypto/sha256-spe-glue.c +++ b/arch/powerpc/crypto/sha256-spe-glue.c @@ -177,7 +177,7 @@ static int ppc_spe_sha256_final(struct shash_desc *desc, u8 *out) static int ppc_spe_sha224_final(struct shash_desc *desc, u8 *out) { - u32 D[SHA256_DIGEST_SIZE >> 2]; + __be32 D[SHA256_DIGEST_SIZE >> 2]; __be32 *dst = (__be32 *)out; ppc_spe_sha256_final(desc, (u8 *)D); From 6cf1a144d3f5752400cad99d9142fba668cc4ddb Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 20 Nov 2020 17:41:26 +1100 Subject: [PATCH 276/360] crypto: sparc - Fix sparse endianness warnings This patch fixes a coulpe of sparse endianness warnings. Signed-off-by: Herbert Xu --- arch/sparc/crypto/crc32c_glue.c | 2 +- arch/sparc/crypto/md5_glue.c | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/arch/sparc/crypto/crc32c_glue.c b/arch/sparc/crypto/crc32c_glue.c index 4e9323229e71..82efb7f81c28 100644 --- a/arch/sparc/crypto/crc32c_glue.c +++ b/arch/sparc/crypto/crc32c_glue.c @@ -35,7 +35,7 @@ static int crc32c_sparc64_setkey(struct crypto_shash *hash, const u8 *key, if (keylen != sizeof(u32)) return -EINVAL; - *(__le32 *)mctx = le32_to_cpup((__le32 *)key); + *mctx = le32_to_cpup((__le32 *)key); return 0; } diff --git a/arch/sparc/crypto/md5_glue.c b/arch/sparc/crypto/md5_glue.c index 111283fe837e..511db98d590a 100644 --- a/arch/sparc/crypto/md5_glue.c +++ b/arch/sparc/crypto/md5_glue.c @@ -33,10 +33,11 @@ static int md5_sparc64_init(struct shash_desc *desc) { struct md5_state *mctx = shash_desc_ctx(desc); - mctx->hash[0] = cpu_to_le32(MD5_H0); - mctx->hash[1] = cpu_to_le32(MD5_H1); - mctx->hash[2] = cpu_to_le32(MD5_H2); - mctx->hash[3] = cpu_to_le32(MD5_H3); + mctx->hash[0] = MD5_H0; + mctx->hash[1] = MD5_H1; + mctx->hash[2] = MD5_H2; + mctx->hash[3] = MD5_H3; + le32_to_cpu_array(mctx->hash, 4); mctx->byte_count = 0; return 0; From 4e0b858b1a8f165016d124c7c1e2baed88f6250f Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Fri, 20 Nov 2020 17:02:31 +0800 Subject: [PATCH 277/360] hwrng: hisi - remove HiSilicon TRNG driver Driver of HiSilicon true random number generator(TRNG) is removed from 'drivers/char/hw_random'. Both 'Kunpeng 920' and 'Kunpeng 930' chips have TRNG, however, PRNG is only supported by 'Kunpeng 930'. So, this driver is moved to 'drivers/crypto/hisilicon/trng/' in the next to enable the two's TRNG better. Signed-off-by: Weili Qian Reviewed-by: Zaibo Xu Signed-off-by: Herbert Xu --- drivers/char/hw_random/Kconfig | 13 ---- drivers/char/hw_random/Makefile | 1 - drivers/char/hw_random/hisi-trng-v2.c | 99 --------------------------- 3 files changed, 113 deletions(-) delete mode 100644 drivers/char/hw_random/hisi-trng-v2.c diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index e92c4d9469d8..eb0d51a78b44 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -348,19 +348,6 @@ config HW_RANDOM_HISI If unsure, say Y. -config HW_RANDOM_HISI_V2 - tristate "HiSilicon True Random Number Generator V2 support" - depends on HW_RANDOM && ARM64 && ACPI - default HW_RANDOM - help - This driver provides kernel-side support for the True Random Number - Generator V2 hardware found on HiSilicon Hi1620 SoC. - - To compile this driver as a module, choose M here: the - module will be called hisi-trng-v2. - - If unsure, say Y. - config HW_RANDOM_ST tristate "ST Microelectronics HW Random Number Generator support" depends on HW_RANDOM && ARCH_STI diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index 5da344509a4d..8933fada74f2 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -30,7 +30,6 @@ obj-$(CONFIG_HW_RANDOM_NOMADIK) += nomadik-rng.o obj-$(CONFIG_HW_RANDOM_PSERIES) += pseries-rng.o obj-$(CONFIG_HW_RANDOM_POWERNV) += powernv-rng.o obj-$(CONFIG_HW_RANDOM_HISI) += hisi-rng.o -obj-$(CONFIG_HW_RANDOM_HISI_V2) += hisi-trng-v2.o obj-$(CONFIG_HW_RANDOM_BCM2835) += bcm2835-rng.o obj-$(CONFIG_HW_RANDOM_IPROC_RNG200) += iproc-rng200.o obj-$(CONFIG_HW_RANDOM_ST) += st-rng.o diff --git a/drivers/char/hw_random/hisi-trng-v2.c b/drivers/char/hw_random/hisi-trng-v2.c deleted file mode 100644 index 6a65b8232ce0..000000000000 --- a/drivers/char/hw_random/hisi-trng-v2.c +++ /dev/null @@ -1,99 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2019 HiSilicon Limited. */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define HISI_TRNG_REG 0x00F0 -#define HISI_TRNG_BYTES 4 -#define HISI_TRNG_QUALITY 512 -#define SLEEP_US 10 -#define TIMEOUT_US 10000 - -struct hisi_trng { - void __iomem *base; - struct hwrng rng; -}; - -static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) -{ - struct hisi_trng *trng; - int currsize = 0; - u32 val = 0; - u32 ret; - - trng = container_of(rng, struct hisi_trng, rng); - - do { - ret = readl_poll_timeout(trng->base + HISI_TRNG_REG, val, - val, SLEEP_US, TIMEOUT_US); - if (ret) - return currsize; - - if (max - currsize >= HISI_TRNG_BYTES) { - memcpy(buf + currsize, &val, HISI_TRNG_BYTES); - currsize += HISI_TRNG_BYTES; - if (currsize == max) - return currsize; - continue; - } - - /* copy remaining bytes */ - memcpy(buf + currsize, &val, max - currsize); - currsize = max; - } while (currsize < max); - - return currsize; -} - -static int hisi_trng_probe(struct platform_device *pdev) -{ - struct hisi_trng *trng; - int ret; - - trng = devm_kzalloc(&pdev->dev, sizeof(*trng), GFP_KERNEL); - if (!trng) - return -ENOMEM; - - trng->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(trng->base)) - return PTR_ERR(trng->base); - - trng->rng.name = pdev->name; - trng->rng.read = hisi_trng_read; - trng->rng.quality = HISI_TRNG_QUALITY; - - ret = devm_hwrng_register(&pdev->dev, &trng->rng); - if (ret) - dev_err(&pdev->dev, "failed to register hwrng!\n"); - - return ret; -} - -static const struct acpi_device_id hisi_trng_acpi_match[] = { - { "HISI02B3", 0 }, - { } -}; -MODULE_DEVICE_TABLE(acpi, hisi_trng_acpi_match); - -static struct platform_driver hisi_trng_driver = { - .probe = hisi_trng_probe, - .driver = { - .name = "hisi-trng-v2", - .acpi_match_table = ACPI_PTR(hisi_trng_acpi_match), - }, -}; - -module_platform_driver(hisi_trng_driver); - -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Weili Qian "); -MODULE_AUTHOR("Zaibo Xu "); -MODULE_DESCRIPTION("HiSilicon true random number generator V2 driver"); From 56c6da16c3631f953fb20d8b7ddccdf493377ad4 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Fri, 20 Nov 2020 17:02:32 +0800 Subject: [PATCH 278/360] crypto: hisilicon/trng - add HiSilicon TRNG driver support Move existing char/hw_random/hisi-trng-v2.c to crypto/hisilicon/trng.c. Signed-off-by: Weili Qian Reviewed-by: Zaibo Xu Signed-off-by: Herbert Xu --- arch/arm64/configs/defconfig | 1 + drivers/crypto/hisilicon/Kconfig | 7 ++ drivers/crypto/hisilicon/Makefile | 1 + drivers/crypto/hisilicon/trng/Makefile | 2 + drivers/crypto/hisilicon/trng/trng.c | 99 ++++++++++++++++++++++++++ 5 files changed, 110 insertions(+) create mode 100644 drivers/crypto/hisilicon/trng/Makefile create mode 100644 drivers/crypto/hisilicon/trng/trng.c diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index 17a2df6a263e..646c3b41c60a 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1081,6 +1081,7 @@ CONFIG_CRYPTO_DEV_CCREE=m CONFIG_CRYPTO_DEV_HISI_SEC2=m CONFIG_CRYPTO_DEV_HISI_ZIP=m CONFIG_CRYPTO_DEV_HISI_HPRE=m +CONFIG_CRYPTO_DEV_HISI_TRNG=m CONFIG_CMA_SIZE_MBYTES=32 CONFIG_PRINTK_TIME=y CONFIG_DEBUG_INFO=y diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 9c3b3ca815e6..2cd3298920fa 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -71,3 +71,10 @@ config CRYPTO_DEV_HISI_HPRE help Support for HiSilicon HPRE(High Performance RSA Engine) accelerator, which can accelerate RSA and DH algorithms. + +config CRYPTO_DEV_HISI_TRNG + tristate "Support for HISI TRNG Driver" + depends on ARM64 && ACPI + select HW_RANDOM + help + Support for HiSilicon TRNG Driver. diff --git a/drivers/crypto/hisilicon/Makefile b/drivers/crypto/hisilicon/Makefile index 7f5f74c72baa..1e89269a2e4b 100644 --- a/drivers/crypto/hisilicon/Makefile +++ b/drivers/crypto/hisilicon/Makefile @@ -5,3 +5,4 @@ obj-$(CONFIG_CRYPTO_DEV_HISI_SEC2) += sec2/ obj-$(CONFIG_CRYPTO_DEV_HISI_QM) += hisi_qm.o hisi_qm-objs = qm.o sgl.o obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += zip/ +obj-$(CONFIG_CRYPTO_DEV_HISI_TRNG) += trng/ diff --git a/drivers/crypto/hisilicon/trng/Makefile b/drivers/crypto/hisilicon/trng/Makefile new file mode 100644 index 000000000000..d909079f351c --- /dev/null +++ b/drivers/crypto/hisilicon/trng/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_HISI_TRNG) += hisi-trng-v2.o +hisi-trng-v2-objs = trng.o diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c new file mode 100644 index 000000000000..6a65b8232ce0 --- /dev/null +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 HiSilicon Limited. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define HISI_TRNG_REG 0x00F0 +#define HISI_TRNG_BYTES 4 +#define HISI_TRNG_QUALITY 512 +#define SLEEP_US 10 +#define TIMEOUT_US 10000 + +struct hisi_trng { + void __iomem *base; + struct hwrng rng; +}; + +static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) +{ + struct hisi_trng *trng; + int currsize = 0; + u32 val = 0; + u32 ret; + + trng = container_of(rng, struct hisi_trng, rng); + + do { + ret = readl_poll_timeout(trng->base + HISI_TRNG_REG, val, + val, SLEEP_US, TIMEOUT_US); + if (ret) + return currsize; + + if (max - currsize >= HISI_TRNG_BYTES) { + memcpy(buf + currsize, &val, HISI_TRNG_BYTES); + currsize += HISI_TRNG_BYTES; + if (currsize == max) + return currsize; + continue; + } + + /* copy remaining bytes */ + memcpy(buf + currsize, &val, max - currsize); + currsize = max; + } while (currsize < max); + + return currsize; +} + +static int hisi_trng_probe(struct platform_device *pdev) +{ + struct hisi_trng *trng; + int ret; + + trng = devm_kzalloc(&pdev->dev, sizeof(*trng), GFP_KERNEL); + if (!trng) + return -ENOMEM; + + trng->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(trng->base)) + return PTR_ERR(trng->base); + + trng->rng.name = pdev->name; + trng->rng.read = hisi_trng_read; + trng->rng.quality = HISI_TRNG_QUALITY; + + ret = devm_hwrng_register(&pdev->dev, &trng->rng); + if (ret) + dev_err(&pdev->dev, "failed to register hwrng!\n"); + + return ret; +} + +static const struct acpi_device_id hisi_trng_acpi_match[] = { + { "HISI02B3", 0 }, + { } +}; +MODULE_DEVICE_TABLE(acpi, hisi_trng_acpi_match); + +static struct platform_driver hisi_trng_driver = { + .probe = hisi_trng_probe, + .driver = { + .name = "hisi-trng-v2", + .acpi_match_table = ACPI_PTR(hisi_trng_acpi_match), + }, +}; + +module_platform_driver(hisi_trng_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Weili Qian "); +MODULE_AUTHOR("Zaibo Xu "); +MODULE_DESCRIPTION("HiSilicon true random number generator V2 driver"); From e4d9d10ef4be0366316b3114593e4becf5b98a49 Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Fri, 20 Nov 2020 17:02:33 +0800 Subject: [PATCH 279/360] crypto: hisilicon/trng - add support for PRNG This patch adds support for pseudo random number generator(PRNG) in Crypto subsystem. Signed-off-by: Weili Qian Reviewed-by: Zaibo Xu Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/Kconfig | 1 + drivers/crypto/hisilicon/trng/trng.c | 241 ++++++++++++++++++++++++++- 2 files changed, 239 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 2cd3298920fa..843192666dc3 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -76,5 +76,6 @@ config CRYPTO_DEV_HISI_TRNG tristate "Support for HISI TRNG Driver" depends on ARM64 && ACPI select HW_RANDOM + select CRYPTO_RNG help Support for HiSilicon TRNG Driver. diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c index 6a65b8232ce0..a5033cf09e2b 100644 --- a/drivers/crypto/hisilicon/trng/trng.c +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -2,26 +2,181 @@ /* Copyright (c) 2019 HiSilicon Limited. */ #include +#include #include #include #include #include #include +#include #include +#include #include #include +#include #define HISI_TRNG_REG 0x00F0 #define HISI_TRNG_BYTES 4 #define HISI_TRNG_QUALITY 512 #define SLEEP_US 10 #define TIMEOUT_US 10000 +#define SW_DRBG_NUM_SHIFT 2 +#define SW_DRBG_KEY_BASE 0x082C +#define SW_DRBG_SEED(n) (SW_DRBG_KEY_BASE - ((n) << SW_DRBG_NUM_SHIFT)) +#define SW_DRBG_SEED_REGS_NUM 12 +#define SW_DRBG_SEED_SIZE 48 +#define SW_DRBG_BLOCKS 0x0830 +#define SW_DRBG_INIT 0x0834 +#define SW_DRBG_GEN 0x083c +#define SW_DRBG_STATUS 0x0840 +#define SW_DRBG_BLOCKS_NUM 4095 +#define SW_DRBG_DATA_BASE 0x0850 +#define SW_DRBG_DATA_NUM 4 +#define SW_DRBG_DATA(n) (SW_DRBG_DATA_BASE - ((n) << SW_DRBG_NUM_SHIFT)) +#define SW_DRBG_BYTES 16 +#define SW_DRBG_ENABLE_SHIFT 12 +#define SEED_SHIFT_24 24 +#define SEED_SHIFT_16 16 +#define SEED_SHIFT_8 8 + +struct hisi_trng_list { + struct mutex lock; + struct list_head list; + bool is_init; +}; struct hisi_trng { void __iomem *base; + struct hisi_trng_list *trng_list; + struct list_head list; struct hwrng rng; + bool is_used; + struct mutex mutex; }; +struct hisi_trng_ctx { + struct hisi_trng *trng; +}; + +static atomic_t trng_active_devs; +static struct hisi_trng_list trng_devices; + +static void hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) +{ + u32 val, seed_reg, i; + + for (i = 0; i < SW_DRBG_SEED_SIZE; + i += SW_DRBG_SEED_SIZE / SW_DRBG_SEED_REGS_NUM) { + val = seed[i] << SEED_SHIFT_24; + val |= seed[i + 1UL] << SEED_SHIFT_16; + val |= seed[i + 2UL] << SEED_SHIFT_8; + val |= seed[i + 3UL]; + + seed_reg = (i >> SW_DRBG_NUM_SHIFT) % SW_DRBG_SEED_REGS_NUM; + writel(val, trng->base + SW_DRBG_SEED(seed_reg)); + } +} + +static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, + unsigned int slen) +{ + struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); + struct hisi_trng *trng = ctx->trng; + u32 val = 0; + int ret = 0; + + if (slen < SW_DRBG_SEED_SIZE) { + pr_err("slen(%u) is not matched with trng(%d)\n", slen, + SW_DRBG_SEED_SIZE); + return -EINVAL; + } + + writel(0x0, trng->base + SW_DRBG_BLOCKS); + hisi_trng_set_seed(trng, seed); + + writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), + trng->base + SW_DRBG_BLOCKS); + writel(0x1, trng->base + SW_DRBG_INIT); + + ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, + val, val & BIT(0), SLEEP_US, TIMEOUT_US); + if (ret) + pr_err("fail to init trng(%d)\n", ret); + + return ret; +} + +static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, + unsigned int slen, u8 *dstn, unsigned int dlen) +{ + struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); + struct hisi_trng *trng = ctx->trng; + u32 data[SW_DRBG_DATA_NUM]; + u32 currsize = 0; + u32 val = 0; + int ret; + u32 i; + + if (dlen > SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES || dlen == 0) { + pr_err("dlen(%d) exceeds limit(%d)!\n", dlen, + SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES); + return -EINVAL; + } + + do { + ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, + val, val & BIT(1), SLEEP_US, TIMEOUT_US); + if (ret) { + pr_err("fail to generate random number(%d)!\n", ret); + break; + } + + for (i = 0; i < SW_DRBG_DATA_NUM; i++) + data[i] = readl(trng->base + SW_DRBG_DATA(i)); + + if (dlen - currsize >= SW_DRBG_BYTES) { + memcpy(dstn + currsize, data, SW_DRBG_BYTES); + currsize += SW_DRBG_BYTES; + } else { + memcpy(dstn + currsize, data, dlen - currsize); + currsize = dlen; + } + + writel(0x1, trng->base + SW_DRBG_GEN); + } while (currsize < dlen); + + return ret; +} + +static int hisi_trng_init(struct crypto_tfm *tfm) +{ + struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); + struct hisi_trng *trng; + int ret = -EBUSY; + + mutex_lock(&trng_devices.lock); + list_for_each_entry(trng, &trng_devices.list, list) { + if (!trng->is_used) { + trng->is_used = true; + ctx->trng = trng; + ret = 0; + break; + } + } + mutex_unlock(&trng_devices.lock); + + return ret; +} + +static void hisi_trng_exit(struct crypto_tfm *tfm) +{ + struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); + + mutex_lock(&trng_devices.lock); + ctx->trng->is_used = false; + mutex_unlock(&trng_devices.lock); +} + static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) { struct hisi_trng *trng; @@ -53,6 +208,42 @@ static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) return currsize; } +static struct rng_alg hisi_trng_alg = { + .generate = hisi_trng_generate, + .seed = hisi_trng_seed, + .seedsize = SW_DRBG_SEED_SIZE, + .base = { + .cra_name = "stdrng", + .cra_driver_name = "hisi_stdrng", + .cra_priority = 300, + .cra_ctxsize = sizeof(struct hisi_trng_ctx), + .cra_module = THIS_MODULE, + .cra_init = hisi_trng_init, + .cra_exit = hisi_trng_exit, + }, +}; + +static void hisi_trng_add_to_list(struct hisi_trng *trng) +{ + mutex_lock(&trng_devices.lock); + list_add_tail(&trng->list, &trng_devices.list); + mutex_unlock(&trng_devices.lock); +} + +static int hisi_trng_del_from_list(struct hisi_trng *trng) +{ + int ret = -EBUSY; + + mutex_lock(&trng_devices.lock); + if (!trng->is_used) { + list_del(&trng->list); + ret = 0; + } + mutex_unlock(&trng_devices.lock); + + return ret; +} + static int hisi_trng_probe(struct platform_device *pdev) { struct hisi_trng *trng; @@ -62,19 +253,62 @@ static int hisi_trng_probe(struct platform_device *pdev) if (!trng) return -ENOMEM; + platform_set_drvdata(pdev, trng); + trng->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(trng->base)) return PTR_ERR(trng->base); + trng->is_used = false; + if (!trng_devices.is_init) { + INIT_LIST_HEAD(&trng_devices.list); + mutex_init(&trng_devices.lock); + trng_devices.is_init = true; + } + + hisi_trng_add_to_list(trng); + if (atomic_add_return(1, &trng_active_devs) == 1) { + ret = crypto_register_rng(&hisi_trng_alg); + if (ret) { + dev_err(&pdev->dev, + "failed to register crypto(%d)\n", ret); + atomic_sub_return(1, &trng_active_devs); + goto err_remove_from_list; + } + } + trng->rng.name = pdev->name; trng->rng.read = hisi_trng_read; trng->rng.quality = HISI_TRNG_QUALITY; - ret = devm_hwrng_register(&pdev->dev, &trng->rng); - if (ret) - dev_err(&pdev->dev, "failed to register hwrng!\n"); + if (ret) { + dev_err(&pdev->dev, "failed to register hwrng: %d!\n", ret); + goto err_crypto_unregister; + } return ret; + +err_crypto_unregister: + if (atomic_sub_return(1, &trng_active_devs) == 0) + crypto_unregister_rng(&hisi_trng_alg); + +err_remove_from_list: + hisi_trng_del_from_list(trng); + return ret; +} + +static int hisi_trng_remove(struct platform_device *pdev) +{ + struct hisi_trng *trng = platform_get_drvdata(pdev); + + /* Wait until the task is finished */ + while (hisi_trng_del_from_list(trng)) + ; + + if (atomic_sub_return(1, &trng_active_devs) == 0) + crypto_unregister_rng(&hisi_trng_alg); + + return 0; } static const struct acpi_device_id hisi_trng_acpi_match[] = { @@ -85,6 +319,7 @@ MODULE_DEVICE_TABLE(acpi, hisi_trng_acpi_match); static struct platform_driver hisi_trng_driver = { .probe = hisi_trng_probe, + .remove = hisi_trng_remove, .driver = { .name = "hisi-trng-v2", .acpi_match_table = ACPI_PTR(hisi_trng_acpi_match), From e8a3dae6910157180c88c8b0e0dd06aee849b7fc Mon Sep 17 00:00:00 2001 From: Weili Qian Date: Fri, 20 Nov 2020 17:02:34 +0800 Subject: [PATCH 280/360] MAINTAINERS: Move HiSilicon TRNG V2 driver Move HiSilicon TRNG V2 driver into 'drivers/crypto/hisilicon/trng' with some updating on 'MAINTAINERS'. Signed-off-by: Weili Qian Reviewed-by: Zaibo Xu Signed-off-by: Herbert Xu --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index e73636b75f29..a835a74d4b29 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7999,7 +7999,7 @@ F: drivers/staging/hikey9xx/ HISILICON TRUE RANDOM NUMBER GENERATOR V2 SUPPORT M: Zaibo Xu S: Maintained -F: drivers/char/hw_random/hisi-trng-v2.c +F: drivers/crypto/hisilicon/trng/trng.c HISILICON V3XX SPI NOR FLASH Controller Driver M: John Garry From 08a7e33c083b60c1ddd330df22fb56038e4a40ad Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 20 Nov 2020 12:04:31 +0100 Subject: [PATCH 281/360] crypto: tcrypt - don't initialize at subsys_initcall time Commit c4741b2305979 ("crypto: run initcalls for generic implementations earlier") converted tcrypt.ko's module_init() to subsys_initcall(), but this was unintentional: tcrypt.ko currently cannot be built into the core kernel, and so the subsys_initcall() gets converted into module_init() under the hood. Given that tcrypt.ko does not implement a generic version of a crypto algorithm that has to be available early during boot, there is no point in running the tcrypt init code earlier than implied by module_init(). However, for crypto development purposes, we will lift the restriction that tcrypt.ko must be built as a module, and when builtin, it makes sense for tcrypt.ko (which does its work inside the module init function) to run as late as possible. So let's switch to late_initcall() instead. Signed-off-by: Ard Biesheuvel Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index eea0f453cfb6..fc1f3e516694 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -3066,7 +3066,7 @@ err_free_tv: */ static void __exit tcrypt_mod_fini(void) { } -subsys_initcall(tcrypt_mod_init); +late_initcall(tcrypt_mod_init); module_exit(tcrypt_mod_fini); module_param(alg, charp, 0); From 00ea27f11c4f96ffc9ebc147b5ea045babb02ce3 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 20 Nov 2020 12:04:32 +0100 Subject: [PATCH 282/360] crypto: tcrypt - permit tcrypt.ko to be builtin When working on crypto algorithms, being able to run tcrypt quickly without booting an entire Linux installation can be very useful. For instance, QEMU/kvm can be used to boot a kernel from the command line, and having tcrypt.ko builtin would allow tcrypt to be executed to run benchmarks, or to run tests for algorithms that need to be instantiated from templates, without the need to make it past the point where the rootfs is mounted. So let's relax the requirement that tcrypt can only be built as a module when CONFIG_EXPERT is enabled. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 37de7d006858..a367fcfeb5d4 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -201,7 +201,7 @@ config CRYPTO_AUTHENC config CRYPTO_TEST tristate "Testing module" - depends on m + depends on m || EXPERT select CRYPTO_MANAGER help Quick & dirty crypto test module. From ad6d66bcac77e5145eb67449f8354ed0f936258c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 20 Nov 2020 12:04:33 +0100 Subject: [PATCH 283/360] crypto: tcrypt - include 1420 byte blocks in aead and skcipher benchmarks WireGuard and IPsec both typically operate on input blocks that are ~1420 bytes in size, given the default Ethernet MTU of 1500 bytes and the overhead of the VPN metadata. Many aead and sckipher implementations are optimized for power-of-2 block sizes, and whether they perform well when operating on 1420 byte blocks cannot be easily extrapolated from the performance on power-of-2 block size. So let's add 1420 bytes explicitly, and round it up to the next blocksize multiple of the algo in question if it does not support 1420 byte blocks. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 81 +++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index fc1f3e516694..a647bb298fbc 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -77,8 +77,8 @@ static const char *check[] = { NULL }; -static u32 block_sizes[] = { 16, 64, 256, 1024, 1472, 8192, 0 }; -static u32 aead_sizes[] = { 16, 64, 256, 512, 1024, 2048, 4096, 8192, 0 }; +static const int block_sizes[] = { 16, 64, 256, 1024, 1420, 4096, 0 }; +static const int aead_sizes[] = { 16, 64, 256, 512, 1024, 1420, 4096, 8192, 0 }; #define XBUFSIZE 8 #define MAX_IVLEN 32 @@ -256,10 +256,10 @@ static void test_mb_aead_speed(const char *algo, int enc, int secs, struct test_mb_aead_data *data; struct crypto_aead *tfm; unsigned int i, j, iv_len; + const int *b_size; const char *key; const char *e; void *assoc; - u32 *b_size; char *iv; int ret; @@ -337,15 +337,17 @@ static void test_mb_aead_speed(const char *algo, int enc, int secs, do { b_size = aead_sizes; do { - if (*b_size + authsize > XBUFSIZE * PAGE_SIZE) { + int bs = round_up(*b_size, crypto_aead_blocksize(tfm)); + + if (bs + authsize > XBUFSIZE * PAGE_SIZE) { pr_err("template (%u) too big for buffer (%lu)\n", - authsize + *b_size, + authsize + bs, XBUFSIZE * PAGE_SIZE); goto out; } pr_info("test %u (%d bit key, %d byte blocks): ", i, - *keysize * 8, *b_size); + *keysize * 8, bs); /* Set up tfm global state, i.e. the key */ @@ -380,11 +382,11 @@ static void test_mb_aead_speed(const char *algo, int enc, int secs, memset(assoc, 0xff, aad_size); sg_init_aead(cur->sg, cur->xbuf, - *b_size + (enc ? 0 : authsize), + bs + (enc ? 0 : authsize), assoc, aad_size); sg_init_aead(cur->sgout, cur->xoutbuf, - *b_size + (enc ? authsize : 0), + bs + (enc ? authsize : 0), assoc, aad_size); aead_request_set_ad(cur->req, aad_size); @@ -394,7 +396,7 @@ static void test_mb_aead_speed(const char *algo, int enc, int secs, aead_request_set_crypt(cur->req, cur->sgout, cur->sg, - *b_size, iv); + bs, iv); ret = crypto_aead_encrypt(cur->req); ret = do_one_aead_op(cur->req, ret); @@ -406,18 +408,18 @@ static void test_mb_aead_speed(const char *algo, int enc, int secs, } aead_request_set_crypt(cur->req, cur->sg, - cur->sgout, *b_size + + cur->sgout, bs + (enc ? 0 : authsize), iv); } if (secs) { - ret = test_mb_aead_jiffies(data, enc, *b_size, + ret = test_mb_aead_jiffies(data, enc, bs, secs, num_mb); cond_resched(); } else { - ret = test_mb_aead_cycles(data, enc, *b_size, + ret = test_mb_aead_cycles(data, enc, bs, num_mb); } @@ -534,7 +536,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, char *xbuf[XBUFSIZE]; char *xoutbuf[XBUFSIZE]; char *axbuf[XBUFSIZE]; - unsigned int *b_size; + const int *b_size; unsigned int iv_len; struct crypto_wait wait; @@ -590,12 +592,14 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, do { b_size = aead_sizes; do { + u32 bs = round_up(*b_size, crypto_aead_blocksize(tfm)); + assoc = axbuf[0]; memset(assoc, 0xff, aad_size); - if ((*keysize + *b_size) > TVMEMSIZE * PAGE_SIZE) { + if ((*keysize + bs) > TVMEMSIZE * PAGE_SIZE) { pr_err("template (%u) too big for tvmem (%lu)\n", - *keysize + *b_size, + *keysize + bs, TVMEMSIZE * PAGE_SIZE); goto out; } @@ -616,7 +620,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, crypto_aead_clear_flags(tfm, ~0); printk(KERN_INFO "test %u (%d bit key, %d byte blocks): ", - i, *keysize * 8, *b_size); + i, *keysize * 8, bs); memset(tvmem[0], 0xff, PAGE_SIZE); @@ -627,11 +631,11 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, goto out; } - sg_init_aead(sg, xbuf, *b_size + (enc ? 0 : authsize), + sg_init_aead(sg, xbuf, bs + (enc ? 0 : authsize), assoc, aad_size); sg_init_aead(sgout, xoutbuf, - *b_size + (enc ? authsize : 0), assoc, + bs + (enc ? authsize : 0), assoc, aad_size); aead_request_set_ad(req, aad_size); @@ -644,7 +648,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, * reversed (input <-> output) to calculate it */ aead_request_set_crypt(req, sgout, sg, - *b_size, iv); + bs, iv); ret = do_one_aead_op(req, crypto_aead_encrypt(req)); @@ -656,15 +660,15 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, } aead_request_set_crypt(req, sg, sgout, - *b_size + (enc ? 0 : authsize), + bs + (enc ? 0 : authsize), iv); if (secs) { - ret = test_aead_jiffies(req, enc, *b_size, + ret = test_aead_jiffies(req, enc, bs, secs); cond_resched(); } else { - ret = test_aead_cycles(req, enc, *b_size); + ret = test_aead_cycles(req, enc, bs); } if (ret) { @@ -1253,9 +1257,9 @@ static void test_mb_skcipher_speed(const char *algo, int enc, int secs, struct test_mb_skcipher_data *data; struct crypto_skcipher *tfm; unsigned int i, j, iv_len; + const int *b_size; const char *key; const char *e; - u32 *b_size; char iv[128]; int ret; @@ -1316,14 +1320,16 @@ static void test_mb_skcipher_speed(const char *algo, int enc, int secs, do { b_size = block_sizes; do { - if (*b_size > XBUFSIZE * PAGE_SIZE) { + u32 bs = round_up(*b_size, crypto_skcipher_blocksize(tfm)); + + if (bs > XBUFSIZE * PAGE_SIZE) { pr_err("template (%u) too big for buffer (%lu)\n", *b_size, XBUFSIZE * PAGE_SIZE); goto out; } pr_info("test %u (%d bit key, %d byte blocks): ", i, - *keysize * 8, *b_size); + *keysize * 8, bs); /* Set up tfm global state, i.e. the key */ @@ -1353,7 +1359,7 @@ static void test_mb_skcipher_speed(const char *algo, int enc, int secs, for (j = 0; j < num_mb; ++j) { struct test_mb_skcipher_data *cur = &data[j]; - unsigned int k = *b_size; + unsigned int k = bs; unsigned int pages = DIV_ROUND_UP(k, PAGE_SIZE); unsigned int p = 0; @@ -1377,12 +1383,12 @@ static void test_mb_skcipher_speed(const char *algo, int enc, int secs, if (secs) { ret = test_mb_acipher_jiffies(data, enc, - *b_size, secs, + bs, secs, num_mb); cond_resched(); } else { ret = test_mb_acipher_cycles(data, enc, - *b_size, num_mb); + bs, num_mb); } if (ret) { @@ -1497,8 +1503,8 @@ static void test_skcipher_speed(const char *algo, int enc, unsigned int secs, char iv[128]; struct skcipher_request *req; struct crypto_skcipher *tfm; + const int *b_size; const char *e; - u32 *b_size; if (enc == ENCRYPT) e = "encryption"; @@ -1533,17 +1539,18 @@ static void test_skcipher_speed(const char *algo, int enc, unsigned int secs, b_size = block_sizes; do { + u32 bs = round_up(*b_size, crypto_skcipher_blocksize(tfm)); struct scatterlist sg[TVMEMSIZE]; - if ((*keysize + *b_size) > TVMEMSIZE * PAGE_SIZE) { + if ((*keysize + bs) > TVMEMSIZE * PAGE_SIZE) { pr_err("template (%u) too big for " - "tvmem (%lu)\n", *keysize + *b_size, + "tvmem (%lu)\n", *keysize + bs, TVMEMSIZE * PAGE_SIZE); goto out_free_req; } pr_info("test %u (%d bit key, %d byte blocks): ", i, - *keysize * 8, *b_size); + *keysize * 8, bs); memset(tvmem[0], 0xff, PAGE_SIZE); @@ -1565,7 +1572,7 @@ static void test_skcipher_speed(const char *algo, int enc, unsigned int secs, goto out_free_req; } - k = *keysize + *b_size; + k = *keysize + bs; sg_init_table(sg, DIV_ROUND_UP(k, PAGE_SIZE)); if (k > PAGE_SIZE) { @@ -1582,22 +1589,22 @@ static void test_skcipher_speed(const char *algo, int enc, unsigned int secs, sg_set_buf(sg + j, tvmem[j], k); memset(tvmem[j], 0xff, k); } else { - sg_set_buf(sg, tvmem[0] + *keysize, *b_size); + sg_set_buf(sg, tvmem[0] + *keysize, bs); } iv_len = crypto_skcipher_ivsize(tfm); if (iv_len) memset(&iv, 0xff, iv_len); - skcipher_request_set_crypt(req, sg, sg, *b_size, iv); + skcipher_request_set_crypt(req, sg, sg, bs, iv); if (secs) { ret = test_acipher_jiffies(req, enc, - *b_size, secs); + bs, secs); cond_resched(); } else { ret = test_acipher_cycles(req, enc, - *b_size); + bs); } if (ret) { From abc6146aba40ea3b8996773e6ad0e60fd70f7a3f Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Fri, 20 Nov 2020 12:34:56 -0600 Subject: [PATCH 284/360] crypto: ccree - Fix fall-through warnings for Clang In preparation to enable -Wimplicit-fallthrough for Clang, fix multiple warnings by explicitly adding multiple break statements instead of letting the code fall through to the next case. Link: https://github.com/KSPP/linux/issues/115 Signed-off-by: Gustavo A. R. Silva Acked-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- drivers/crypto/ccree/cc_cipher.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c index dafa6577a845..cdfee501fbd9 100644 --- a/drivers/crypto/ccree/cc_cipher.c +++ b/drivers/crypto/ccree/cc_cipher.c @@ -97,6 +97,7 @@ static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) case S_DIN_to_SM4: if (size == SM4_KEY_SIZE) return 0; + break; default: break; } @@ -139,9 +140,11 @@ static int validate_data_size(struct cc_cipher_ctx *ctx_p, case DRV_CIPHER_CBC: if (IS_ALIGNED(size, SM4_BLOCK_SIZE)) return 0; + break; default: break; } + break; default: break; } From d4f9afb23378f50e40cd3bc8aee35679bfd1d27b Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 21 Nov 2020 08:31:31 +0100 Subject: [PATCH 285/360] crypto: cavium/zip - Use dma_set_mask_and_coherent to simplify code 'pci_set_dma_mask()' + 'pci_set_consistent_dma_mask()' can be replaced by an equivalent 'dma_set_mask_and_coherent()' which is much less verbose. Signed-off-by: Christophe JAILLET Signed-off-by: Herbert Xu --- drivers/crypto/cavium/zip/zip_main.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/cavium/zip/zip_main.c b/drivers/crypto/cavium/zip/zip_main.c index d35216e2f6cd..812b4ac9afd6 100644 --- a/drivers/crypto/cavium/zip/zip_main.c +++ b/drivers/crypto/cavium/zip/zip_main.c @@ -263,15 +263,9 @@ static int zip_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto err_disable_device; } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto err_release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto err_release_regions; } From 7f6c383b9505f7f9ec75831b154be97e7df49cac Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 21 Nov 2020 08:49:16 +0100 Subject: [PATCH 286/360] crypto: marvell/octeontx - Use dma_set_mask_and_coherent to simplify code 'pci_set_dma_mask()' + 'pci_set_consistent_dma_mask()' can be replaced by an equivalent 'dma_set_mask_and_coherent()' which is much less verbose. Signed-off-by: Christophe JAILLET Signed-off-by: Herbert Xu --- drivers/crypto/marvell/octeontx/otx_cptpf_main.c | 10 ++-------- drivers/crypto/marvell/octeontx/otx_cptvf_main.c | 10 ++-------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_main.c b/drivers/crypto/marvell/octeontx/otx_cptpf_main.c index 34bb3063eb70..14a42559f81d 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptpf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptpf_main.c @@ -212,15 +212,9 @@ static int otx_cpt_probe(struct pci_dev *pdev, goto err_disable_device; } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto err_release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto err_release_regions; } diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c index 228fe8e47e0e..c076d0b3ad5f 100644 --- a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c @@ -804,15 +804,9 @@ static int otx_cptvf_probe(struct pci_dev *pdev, dev_err(dev, "PCI request regions failed 0x%x\n", err); goto disable_device; } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto release_regions; } From aeb4d8c0f855304c158195ebae3917e244e63e57 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 21 Nov 2020 08:56:47 +0100 Subject: [PATCH 287/360] crypto: cavium - Use dma_set_mask_and_coherent to simplify code 'pci_set_dma_mask()' + 'pci_set_consistent_dma_mask()' can be replaced by an equivalent 'dma_set_mask_and_coherent()' which is much less verbose. Signed-off-by: Christophe JAILLET Signed-off-by: Herbert Xu --- drivers/crypto/cavium/cpt/cptpf_main.c | 10 ++-------- drivers/crypto/cavium/cpt/cptvf_main.c | 10 ++-------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c index 781949027451..24d63bdc5dd2 100644 --- a/drivers/crypto/cavium/cpt/cptpf_main.c +++ b/drivers/crypto/cavium/cpt/cptpf_main.c @@ -569,15 +569,9 @@ static int cpt_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto cpt_err_disable_device; } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto cpt_err_release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto cpt_err_release_regions; } diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c index a15245992cf9..f016448e43bb 100644 --- a/drivers/crypto/cavium/cpt/cptvf_main.c +++ b/drivers/crypto/cavium/cpt/cptvf_main.c @@ -687,15 +687,9 @@ static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } /* Mark as VF driver */ cptvf->flags |= CPT_FLAG_VF_DRIVER; - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(48)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); if (err) { - dev_err(dev, "Unable to get usable DMA configuration\n"); - goto cptvf_err_release_regions; - } - - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(48)); - if (err) { - dev_err(dev, "Unable to get 48-bit DMA for consistent allocations\n"); + dev_err(dev, "Unable to get usable 48-bit DMA configuration\n"); goto cptvf_err_release_regions; } From 05c2a705917b77e0915cca3551583583f4eafcf8 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Sun, 22 Nov 2020 09:51:53 +0200 Subject: [PATCH 288/360] crypto: ccree - rework cache parameters handling Rework the setting of DMA cache parameters, program more appropriate values and explicitly set sharability domain. Signed-off-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- drivers/crypto/ccree/cc_driver.c | 75 +++++++++++++++++++++++++------- drivers/crypto/ccree/cc_driver.h | 6 +-- drivers/crypto/ccree/cc_pm.c | 2 +- 3 files changed, 63 insertions(+), 20 deletions(-) diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c index 6f519d3e896c..d0e59e942568 100644 --- a/drivers/crypto/ccree/cc_driver.c +++ b/drivers/crypto/ccree/cc_driver.c @@ -100,6 +100,57 @@ static const struct of_device_id arm_ccree_dev_of_match[] = { }; MODULE_DEVICE_TABLE(of, arm_ccree_dev_of_match); +static void init_cc_cache_params(struct cc_drvdata *drvdata) +{ + struct device *dev = drvdata_to_dev(drvdata); + u32 cache_params, ace_const, val, mask; + + /* compute CC_AXIM_CACHE_PARAMS */ + cache_params = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS)); + dev_dbg(dev, "Cache params previous: 0x%08X\n", cache_params); + + /* non cached or write-back, write allocate */ + val = drvdata->coherent ? 0xb : 0x2; + + mask = CC_GENMASK(CC_AXIM_CACHE_PARAMS_AWCACHE); + cache_params &= ~mask; + cache_params |= FIELD_PREP(mask, val); + + mask = CC_GENMASK(CC_AXIM_CACHE_PARAMS_AWCACHE_LAST); + cache_params &= ~mask; + cache_params |= FIELD_PREP(mask, val); + + mask = CC_GENMASK(CC_AXIM_CACHE_PARAMS_ARCACHE); + cache_params &= ~mask; + cache_params |= FIELD_PREP(mask, val); + + drvdata->cache_params = cache_params; + + dev_dbg(dev, "Cache params current: 0x%08X\n", cache_params); + + if (drvdata->hw_rev <= CC_HW_REV_710) + return; + + /* compute CC_AXIM_ACE_CONST */ + ace_const = cc_ioread(drvdata, CC_REG(AXIM_ACE_CONST)); + dev_dbg(dev, "ACE-const previous: 0x%08X\n", ace_const); + + /* system or outer-sharable */ + val = drvdata->coherent ? 0x2 : 0x3; + + mask = CC_GENMASK(CC_AXIM_ACE_CONST_ARDOMAIN); + ace_const &= ~mask; + ace_const |= FIELD_PREP(mask, val); + + mask = CC_GENMASK(CC_AXIM_ACE_CONST_AWDOMAIN); + ace_const &= ~mask; + ace_const |= FIELD_PREP(mask, val); + + dev_dbg(dev, "ACE-const current: 0x%08X\n", ace_const); + + drvdata->ace_const = ace_const; +} + static u32 cc_read_idr(struct cc_drvdata *drvdata, const u32 *idr_offsets) { int i; @@ -218,9 +269,9 @@ bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata) return false; } -int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe) +int init_cc_regs(struct cc_drvdata *drvdata) { - unsigned int val, cache_params; + unsigned int val; struct device *dev = drvdata_to_dev(drvdata); /* Unmask all AXI interrupt sources AXI_CFG1 register */ @@ -245,19 +296,9 @@ int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe) cc_iowrite(drvdata, CC_REG(HOST_IMR), ~val); - cache_params = (drvdata->coherent ? CC_COHERENT_CACHE_PARAMS : 0x0); - - val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS)); - - if (is_probe) - dev_dbg(dev, "Cache params previous: 0x%08X\n", val); - - cc_iowrite(drvdata, CC_REG(AXIM_CACHE_PARAMS), cache_params); - val = cc_ioread(drvdata, CC_REG(AXIM_CACHE_PARAMS)); - - if (is_probe) - dev_dbg(dev, "Cache params current: 0x%08X (expect: 0x%08X)\n", - val, cache_params); + cc_iowrite(drvdata, CC_REG(AXIM_CACHE_PARAMS), drvdata->cache_params); + if (drvdata->hw_rev >= CC_HW_REV_712) + cc_iowrite(drvdata, CC_REG(AXIM_ACE_CONST), drvdata->ace_const); return 0; } @@ -445,7 +486,9 @@ static int init_cc_resources(struct platform_device *plat_dev) } dev_dbg(dev, "Registered to IRQ: %d\n", irq); - rc = init_cc_regs(new_drvdata, true); + init_cc_cache_params(new_drvdata); + + rc = init_cc_regs(new_drvdata); if (rc) { dev_err(dev, "init_cc_regs failed\n"); goto post_pm_err; diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index ed2b2f13a256..5f1d4602eb8f 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -50,8 +50,6 @@ enum cc_std_body { CC_STD_ALL = 0x3 }; -#define CC_COHERENT_CACHE_PARAMS 0xEEE - #define CC_PINS_FULL 0x0 #define CC_PINS_SLIM 0x9F @@ -156,6 +154,8 @@ struct cc_drvdata { int std_bodies; bool sec_disabled; u32 comp_mask; + u32 cache_params; + u32 ace_const; }; struct cc_crypto_alg { @@ -206,7 +206,7 @@ static inline void dump_byte_array(const char *name, const u8 *the_array, } bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata); -int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe); +int init_cc_regs(struct cc_drvdata *drvdata); void fini_cc_regs(struct cc_drvdata *drvdata); unsigned int cc_get_default_hash_len(struct cc_drvdata *drvdata); diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c index 3c65bf070c90..d5421b0c6831 100644 --- a/drivers/crypto/ccree/cc_pm.c +++ b/drivers/crypto/ccree/cc_pm.c @@ -45,7 +45,7 @@ static int cc_pm_resume(struct device *dev) } cc_iowrite(drvdata, CC_REG(HOST_POWER_DOWN_EN), POWER_DOWN_DISABLE); - rc = init_cc_regs(drvdata, false); + rc = init_cc_regs(drvdata); if (rc) { dev_err(dev, "init_cc_regs (%x)\n", rc); return rc; From 706657b1febf446a9ba37dc51b89f46604f57ee9 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Sun, 22 Nov 2020 15:57:21 +0100 Subject: [PATCH 289/360] EDAC/amd64: Fix PCI component registration In order to setup its PCI component, the driver needs any node private instance in order to get a reference to the PCI device and hand that into edac_pci_create_generic_ctl(). For convenience, it uses the 0th memory controller descriptor under the assumption that if any, the 0th will be always present. However, this assumption goes wrong when the 0th node doesn't have memory and the driver doesn't initialize an instance for it: EDAC amd64: F17h detected (node 0). ... EDAC amd64: Node 0: No DIMMs detected. But looking up node instances is not really needed - all one needs is the pointer to the proper device which gets discovered during instance init. So stash that pointer into a variable and use it when setting up the EDAC PCI component. Clear that variable when the driver needs to unwind due to some instances failing init to avoid any registration imbalance. Cc: Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201122150815.13808-1-bp@alien8.de --- drivers/edac/amd64_edac.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 3bac76efd3f6..15b0d39d2845 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -18,6 +18,9 @@ static struct amd64_family_type *fam_type; /* Per-node stuff */ static struct ecc_settings **ecc_stngs; +/* Device for the PCI component */ +static struct device *pci_ctl_dev; + /* * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching- @@ -2675,6 +2678,9 @@ reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2) return -ENODEV; } + if (!pci_ctl_dev) + pci_ctl_dev = &pvt->F0->dev; + edac_dbg(1, "F0: %s\n", pci_name(pvt->F0)); edac_dbg(1, "F3: %s\n", pci_name(pvt->F3)); edac_dbg(1, "F6: %s\n", pci_name(pvt->F6)); @@ -2699,6 +2705,9 @@ reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2) return -ENODEV; } + if (!pci_ctl_dev) + pci_ctl_dev = &pvt->F2->dev; + edac_dbg(1, "F1: %s\n", pci_name(pvt->F1)); edac_dbg(1, "F2: %s\n", pci_name(pvt->F2)); edac_dbg(1, "F3: %s\n", pci_name(pvt->F3)); @@ -3615,21 +3624,10 @@ static void remove_one_instance(unsigned int nid) static void setup_pci_device(void) { - struct mem_ctl_info *mci; - struct amd64_pvt *pvt; - if (pci_ctl) return; - mci = edac_mc_find(0); - if (!mci) - return; - - pvt = mci->pvt_info; - if (pvt->umc) - pci_ctl = edac_pci_create_generic_ctl(&pvt->F0->dev, EDAC_MOD_STR); - else - pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR); + pci_ctl = edac_pci_create_generic_ctl(pci_ctl_dev, EDAC_MOD_STR); if (!pci_ctl) { pr_warn("%s(): Unable to create PCI control\n", __func__); pr_warn("%s(): PCI error report via EDAC not set\n", __func__); @@ -3708,6 +3706,8 @@ static int __init amd64_edac_init(void) return 0; err_pci: + pci_ctl_dev = NULL; + msrs_free(msrs); msrs = NULL; @@ -3737,6 +3737,8 @@ static void __exit amd64_edac_exit(void) kfree(ecc_stngs); ecc_stngs = NULL; + pci_ctl_dev = NULL; + msrs_free(msrs); msrs = NULL; } From 638920a66a17c8e1f4415cbab0d49dc4a344c2a7 Mon Sep 17 00:00:00 2001 From: Alex Shi Date: Fri, 13 Nov 2020 16:58:14 +0800 Subject: [PATCH 290/360] x86/PCI: Make a kernel-doc comment a normal one The comment is using kernel-doc markup but that comment isn't a kernel-doc comment so make it a normal one to avoid: arch/x86/pci/i386.c:373: warning: Function parameter or member \ 'pcibios_assign_resources' not described in 'fs_initcall' [ bp: Massage and fixup comment while at it. ] Signed-off-by: Alex Shi Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/1605257895-5536-5-git-send-email-alex.shi@linux.alibaba.com --- arch/x86/pci/i386.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/pci/i386.c b/arch/x86/pci/i386.c index fa855bbaebaf..f2f4a5d50b27 100644 --- a/arch/x86/pci/i386.c +++ b/arch/x86/pci/i386.c @@ -366,9 +366,9 @@ static int __init pcibios_assign_resources(void) return 0; } -/** - * called in fs_initcall (one below subsys_initcall), - * give a chance for motherboard reserve resources +/* + * This is an fs_initcall (one below subsys_initcall) in order to reserve + * resources properly. */ fs_initcall(pcibios_assign_resources); From 10e59217479d733ff50b6e8c6316ecffe8b857cb Mon Sep 17 00:00:00 2001 From: Niklas Schnelle Date: Thu, 19 Nov 2020 11:38:46 +0100 Subject: [PATCH 291/360] s390/Kconfig: default PCI_NR_FUNCTIONS to 512 With the addition of more complete SR-IOV support, we are recommending to raise this limit for distributions to 512, as the previous default of 128 can easily be hit with just the VFs of a single PCI physical function. With at least one distribution now shipping with this, supporting only one fourth as many PCI functions on a default upstream build may lead to confusion and reduced testing of the higher limit so increase the default to 512. Acked-by: Christian Borntraeger Signed-off-by: Niklas Schnelle Signed-off-by: Heiko Carstens --- arch/s390/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 64c72b1ecc77..a60cc523d810 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -712,7 +712,7 @@ if PCI config PCI_NR_FUNCTIONS int "Maximum number of PCI functions (1-4096)" range 1 4096 - default "128" + default "512" help This allows you to specify the maximum number of PCI functions which this kernel will support. From 1ab3001b6efb78e0293f3f02307720f6094db1ae Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 24 Nov 2020 14:02:35 +0100 Subject: [PATCH 292/360] s390/vdso: add missing prototypes for vdso functions clang W=1 warns about missing prototypes: >> arch/s390/kernel/vdso64/getcpu.c:8:5: warning: no previous prototype for function '__s390_vdso_getcpu' [-Wmissing-prototypes] int __s390_vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) ^ Add a local header file in order to get rid of this warnings. Reported-by: kernel test robot Signed-off-by: Heiko Carstens --- arch/s390/kernel/vdso64/getcpu.c | 1 + arch/s390/kernel/vdso64/vdso.h | 14 ++++++++++++++ arch/s390/kernel/vdso64/vdso64_generic.c | 1 + 3 files changed, 16 insertions(+) create mode 100644 arch/s390/kernel/vdso64/vdso.h diff --git a/arch/s390/kernel/vdso64/getcpu.c b/arch/s390/kernel/vdso64/getcpu.c index a5da7a9eb43d..5b2bc7494d5b 100644 --- a/arch/s390/kernel/vdso64/getcpu.c +++ b/arch/s390/kernel/vdso64/getcpu.c @@ -4,6 +4,7 @@ #include #include #include +#include "vdso.h" int __s390_vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused) { diff --git a/arch/s390/kernel/vdso64/vdso.h b/arch/s390/kernel/vdso64/vdso.h new file mode 100644 index 000000000000..34c7a2312f9d --- /dev/null +++ b/arch/s390/kernel/vdso64/vdso.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ARCH_S390_KERNEL_VDSO64_VDSO_H +#define __ARCH_S390_KERNEL_VDSO64_VDSO_H + +#include + +struct getcpu_cache; + +int __s390_vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused); +int __s390_vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz); +int __s390_vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts); +int __s390_vdso_clock_getres(clockid_t clock, struct __kernel_timespec *ts); + +#endif /* __ARCH_S390_KERNEL_VDSO64_VDSO_H */ diff --git a/arch/s390/kernel/vdso64/vdso64_generic.c b/arch/s390/kernel/vdso64/vdso64_generic.c index a8cef7e4d137..a9aa75643c08 100644 --- a/arch/s390/kernel/vdso64/vdso64_generic.c +++ b/arch/s390/kernel/vdso64/vdso64_generic.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #include "../../../../lib/vdso/gettimeofday.c" +#include "vdso.h" int __s390_vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) From 8cb4c20f32f542f78a57e1c6464721526d046acc Mon Sep 17 00:00:00 2001 From: Julian Wiedmann Date: Mon, 9 Nov 2020 14:41:07 +0100 Subject: [PATCH 293/360] s390/ap: let bus_register() add the AP bus sysfs attributes Instead of creating the sysfs attributes for the AP bus by hand, describe them in .bus_groups and let the driver core handle it. Signed-off-by: Julian Wiedmann Signed-off-my: Harald Freudenberger Signed-off-by: Heiko Carstens --- drivers/s390/crypto/ap_bus.c | 53 ++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 13bd6b27f00e..2758d05a802d 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -711,12 +711,6 @@ static int __ap_queue_devices_with_id_unregister(struct device *dev, void *data) return 0; } -static struct bus_type ap_bus_type = { - .name = "ap", - .match = &ap_bus_match, - .uevent = &ap_uevent, -}; - static int __ap_revise_reserved(struct device *dev, void *dummy) { int rc, card, queue, devres, drvres; @@ -1301,23 +1295,31 @@ static ssize_t bindings_show(struct bus_type *bus, char *buf) static BUS_ATTR_RO(bindings); -static struct bus_attribute *const ap_bus_attrs[] = { - &bus_attr_ap_domain, - &bus_attr_ap_control_domain_mask, - &bus_attr_ap_usage_domain_mask, - &bus_attr_ap_adapter_mask, - &bus_attr_config_time, - &bus_attr_poll_thread, - &bus_attr_ap_interrupts, - &bus_attr_poll_timeout, - &bus_attr_ap_max_domain_id, - &bus_attr_ap_max_adapter_id, - &bus_attr_apmask, - &bus_attr_aqmask, - &bus_attr_scans, - &bus_attr_bindings, +static struct attribute *ap_bus_attrs[] = { + &bus_attr_ap_domain.attr, + &bus_attr_ap_control_domain_mask.attr, + &bus_attr_ap_usage_domain_mask.attr, + &bus_attr_ap_adapter_mask.attr, + &bus_attr_config_time.attr, + &bus_attr_poll_thread.attr, + &bus_attr_ap_interrupts.attr, + &bus_attr_poll_timeout.attr, + &bus_attr_ap_max_domain_id.attr, + &bus_attr_ap_max_adapter_id.attr, + &bus_attr_apmask.attr, + &bus_attr_aqmask.attr, + &bus_attr_scans.attr, + &bus_attr_bindings.attr, NULL, }; +ATTRIBUTE_GROUPS(ap_bus); + +static struct bus_type ap_bus_type = { + .name = "ap", + .bus_groups = ap_bus_groups, + .match = &ap_bus_match, + .uevent = &ap_uevent, +}; /** * ap_select_domain(): Select an AP domain if possible and we haven't @@ -1798,7 +1800,7 @@ static void __init ap_perms_init(void) */ static int __init ap_module_init(void) { - int rc, i; + int rc; rc = ap_debug_init(); if (rc) @@ -1837,11 +1839,6 @@ static int __init ap_module_init(void) rc = bus_register(&ap_bus_type); if (rc) goto out; - for (i = 0; ap_bus_attrs[i]; i++) { - rc = bus_create_file(&ap_bus_type, ap_bus_attrs[i]); - if (rc) - goto out_bus; - } /* Create /sys/devices/ap. */ ap_root_device = root_device_register("ap"); @@ -1877,8 +1874,6 @@ out_work: hrtimer_cancel(&ap_poll_timer); root_device_unregister(ap_root_device); out_bus: - while (i--) - bus_remove_file(&ap_bus_type, ap_bus_attrs[i]); bus_unregister(&ap_bus_type); out: if (ap_using_interrupts()) From c159376490eef39f0f2cb1ce5dd38a6d41c859b4 Mon Sep 17 00:00:00 2001 From: Justin Ernst Date: Wed, 25 Nov 2020 11:54:43 -0600 Subject: [PATCH 294/360] x86/platform/uv: Update ABI documentation of /sys/firmware/sgi_uv/ Update the ABI documentation to describe the sysfs interface provided by the new uv_sysfs platform driver. [ bp: Merge in kernel-doc warning fixes, see second Link: below. ] Signed-off-by: Justin Ernst Signed-off-by: Borislav Petkov Reviewed-by: Steve Wahl Acked-by: Hans de Goede Link: https://lkml.kernel.org/r/20201125175444.279074-5-justin.ernst@hpe.com Link: https://lkml.kernel.org/r/20201130214304.369348-1-justin.ernst@hpe.com --- .../ABI/testing/sysfs-firmware-sgi_uv | 144 ++++++++++++++++-- 1 file changed, 130 insertions(+), 14 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-firmware-sgi_uv b/Documentation/ABI/testing/sysfs-firmware-sgi_uv index 66800baab096..351b1f41b6bf 100644 --- a/Documentation/ABI/testing/sysfs-firmware-sgi_uv +++ b/Documentation/ABI/testing/sysfs-firmware-sgi_uv @@ -1,27 +1,143 @@ What: /sys/firmware/sgi_uv/ -Date: August 2008 -Contact: Russ Anderson +Date: September 2020 +Contact: Justin Ernst Description: The /sys/firmware/sgi_uv directory contains information - about the SGI UV platform. + about the UV platform. - Under that directory are a number of files:: + Under that directory are a number of read-only attributes:: partition_id coherence_id + uv_type The partition_id entry contains the partition id. - SGI UV systems can be partitioned into multiple physical + UV systems can be partitioned into multiple physical machines, which each partition running a unique copy - of the operating system. Each partition will have a unique - partition id. To display the partition id, use the command:: - - cat /sys/firmware/sgi_uv/partition_id + of the operating system. Each partition will have a unique + partition id. The coherence_id entry contains the coherence id. - A partitioned SGI UV system can have one or more coherence - domain. The coherence id indicates which coherence domain - this partition is in. To display the coherence id, use the - command:: + A partitioned UV system can have one or more coherence + domains. The coherence id indicates which coherence domain + this partition is in. - cat /sys/firmware/sgi_uv/coherence_id + The uv_type entry contains the hub revision number. + This value can be used to identify the UV system version:: + + "3.0" = UV2 + "5.0" = UV3 + "7.0" = UV4 + "7.1" = UV4a + "9.0" = UV5 + + The /sys/firmware/sgi_uv directory also contains two directories:: + + hubs/ + pcibuses/ + + The hubs directory contains a number of hub objects, each representing + a UV Hub visible to the BIOS. Each hub object's name is appended by a + unique ordinal value (ex. /sys/firmware/sgi_uv/hubs/hub_5) + + Each hub object directory contains a number of read-only attributes:: + + cnode + location + name + nasid + shared + this_partition + + The cnode entry contains the cnode number of the corresponding hub. + If a cnode value is not applicable, the value returned will be -1. + + The location entry contains the location string of the corresponding hub. + This value is used to physically identify a hub within a system. + + The name entry contains the name of the corresponding hub. This name can + be two variants:: + + "UVHub x.x" = A 'node' ASIC, connecting a CPU to the interconnect + fabric. The 'x.x' value represents the ASIC revision. + (ex. 'UVHub 5.0') + + "NLxRouter" = A 'router ASIC, only connecting other ASICs to + the interconnect fabric. The 'x' value representing + the fabric technology version. (ex. 'NL8Router') + + The nasid entry contains the nasid number of the corresponding hub. + If a nasid value is not applicable, the value returned will be -1. + + The shared entry contains a boolean value describing whether the + corresponding hub is shared between system partitions. + + The this_partition entry contains a boolean value describing whether + the corresponding hub is local to the current partition. + + Each hub object directory also contains a number of port objects, + each representing a fabric port on the corresponding hub. + A port object's name is appended by a unique ordinal value + (ex. /sys/firmware/sgi_uv/hubs/hub_5/port_3) + + Each port object directory contains a number of read-only attributes:: + + conn_hub + conn_port + + The conn_hub entry contains a value representing the unique + oridinal value of the hub on the other end of the fabric + cable plugged into the port. If the port is disconnected, + the value returned will be -1. + + The conn_port entry contains a value representing the unique + oridinal value of the port on the other end of the fabric cable + plugged into the port. If the port is disconnected, the value + returned will be -1. + + Ex: + A value of '3' is read from: + /sys/firmware/sgi_uv/hubs/hub_5/port_3/conn_hub + + and a value of '6' is read from: + /sys/firmware/sgi_uv/hubs/hub_5/port_3/conn_port + + representing that this port is connected to: + /sys/firmware/sgi_uv/hubs/hub_3/port_6 + + The pcibuses directory contains a number of PCI bus objects. + Each PCI bus object's name is appended by its PCI bus address. + (ex. pcibus_0003:80) + + Each pcibus object has a number of possible read-only attributes:: + + type + location + slot + ppb_addr + iio_stack + + The type entry contains a value describing the type of IO at + the corresponding PCI bus address. Known possible values + across all UV versions are:: + + BASE IO + PCIe IO + PCIe SLOT + NODE IO + Riser + PPB + + The location entry contains the location string of the UV Hub + of the CPU physically connected to the corresponding PCI bus. + + The slot entry contains the physical slot number of the + corresponding PCI bus. This value is used to physically locate + PCI cards within a system. + + The ppb_addr entry contains the PCI address string of the + bridged PCI bus. This entry is only present when the PCI bus + object type is 'PPB'. + + The iio_stack entry contains a value describing the IIO stack + number that the corresponding PCI bus object is connected to. From 6043082c96844fa3a047896212e2da0adc1dde81 Mon Sep 17 00:00:00 2001 From: Justin Ernst Date: Wed, 25 Nov 2020 11:54:44 -0600 Subject: [PATCH 295/360] x86/platform/uv: Update MAINTAINERS for uv_sysfs driver Add an entry and email address for the new uv_sysfs driver and its maintainer. Signed-off-by: Justin Ernst Signed-off-by: Borislav Petkov Acked-by: Hans de Goede Acked-by: Steve Wahl Link: https://lkml.kernel.org/r/20201125175444.279074-6-justin.ernst@hpe.com --- MAINTAINERS | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index a008b70f3c16..bcf83e124fb8 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -18354,6 +18354,12 @@ F: include/uapi/linux/uuid.h F: lib/test_uuid.c F: lib/uuid.c +UV SYSFS DRIVER +M: Justin Ernst +L: platform-driver-x86@vger.kernel.org +S: Maintained +F: drivers/platform/x86/uv_sysfs.c + UVESAFB DRIVER M: Michal Januszewski L: linux-fbdev@vger.kernel.org From 83321c335dccba262a57378361d63da96b8166d6 Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Mon, 30 Nov 2020 11:39:00 -0800 Subject: [PATCH 296/360] x86/pci: Fix the function type for check_reserved_t e820__mapped_all() is passed as a callback to is_mmconf_reserved(), which expects a function of type: typedef bool (*check_reserved_t)(u64 start, u64 end, unsigned type); However, e820__mapped_all() accepts enum e820_type as the last argument and this type mismatch trips indirect call checking with Clang's Control-Flow Integrity (CFI). As is_mmconf_reserved() only passes enum e820_type values for the type argument, change the typedef and the unused type argument in is_acpi_reserved() to enum e820_type to fix the type mismatch. Reported-by: Sedat Dilek Suggested-by: Borislav Petkov Signed-off-by: Sami Tolvanen Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201130193900.456726-1-samitolvanen@google.com --- arch/x86/pci/mmconfig-shared.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c index 6fa42e9c4e6f..234998f196d4 100644 --- a/arch/x86/pci/mmconfig-shared.c +++ b/arch/x86/pci/mmconfig-shared.c @@ -425,7 +425,7 @@ static acpi_status find_mboard_resource(acpi_handle handle, u32 lvl, return AE_OK; } -static bool is_acpi_reserved(u64 start, u64 end, unsigned not_used) +static bool is_acpi_reserved(u64 start, u64 end, enum e820_type not_used) { struct resource mcfg_res; @@ -442,7 +442,7 @@ static bool is_acpi_reserved(u64 start, u64 end, unsigned not_used) return mcfg_res.flags; } -typedef bool (*check_reserved_t)(u64 start, u64 end, unsigned type); +typedef bool (*check_reserved_t)(u64 start, u64 end, enum e820_type type); static bool __ref is_mmconf_reserved(check_reserved_t is_reserved, struct pci_mmcfg_region *cfg, From 2838307b019dfec0c309c4e8e589658736cff4c9 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Mon, 30 Nov 2020 17:13:06 -0800 Subject: [PATCH 297/360] x86/build: Remove -m16 workaround for unsupported versions of GCC Revert the following two commits: de3accdaec88 ("x86, build: Build 16-bit code with -m16 where possible") a9cfccee6604 ("x86, build: Change code16gcc.h from a C header to an assembly header") Since 0bddd227f3dc ("Documentation: update for gcc 4.9 requirement") the minimum supported version of GCC is gcc-4.9. It's now safe to remove this code. Signed-off-by: Nick Desaulniers Signed-off-by: Borislav Petkov Reviewed-by: Nathan Chancellor Reviewed-by: David Woodhouse Link: https://lkml.kernel.org/r/20201201011307.3676986-1-ndesaulniers@google.com --- arch/x86/Makefile | 9 +-------- arch/x86/boot/code16gcc.h | 12 ------------ 2 files changed, 1 insertion(+), 20 deletions(-) delete mode 100644 arch/x86/boot/code16gcc.h diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 154259f18b8b..b8910661447f 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -24,14 +24,7 @@ endif # How to compile the 16-bit code. Note we always compile for -march=i386; # that way we can complain to the user if the CPU is insufficient. -# -# The -m16 option is supported by GCC >= 4.9 and clang >= 3.5. For -# older versions of GCC, include an *assembly* header to make sure that -# gcc doesn't play any games behind our back. -CODE16GCC_CFLAGS := -m32 -Wa,$(srctree)/arch/x86/boot/code16gcc.h -M16_CFLAGS := $(call cc-option, -m16, $(CODE16GCC_CFLAGS)) - -REALMODE_CFLAGS := $(M16_CFLAGS) -g -Os -DDISABLE_BRANCH_PROFILING \ +REALMODE_CFLAGS := -m16 -g -Os -DDISABLE_BRANCH_PROFILING \ -Wall -Wstrict-prototypes -march=i386 -mregparm=3 \ -fno-strict-aliasing -fomit-frame-pointer -fno-pic \ -mno-mmx -mno-sse diff --git a/arch/x86/boot/code16gcc.h b/arch/x86/boot/code16gcc.h deleted file mode 100644 index e19fd7536307..000000000000 --- a/arch/x86/boot/code16gcc.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -# -# code16gcc.h -# -# This file is added to the assembler via -Wa when compiling 16-bit C code. -# This is done this way instead via asm() to make sure gcc does not reorder -# things around us. -# -# gcc 4.9+ has a real -m16 option so we can drop this hack long term. -# - - .code16gcc From 19eb86a72df50adcf554f234469bb5b7209b7640 Mon Sep 17 00:00:00 2001 From: Xiaochen Shen Date: Tue, 1 Dec 2020 02:06:58 +0800 Subject: [PATCH 298/360] x86/resctrl: Clean up unused function parameter in rmdir path Commit fd8d9db3559a ("x86/resctrl: Remove superfluous kernfs_get() calls to prevent refcount leak") removed superfluous kernfs_get() calls in rdtgroup_ctrl_remove() and rdtgroup_rmdir_ctrl(). That change resulted in an unused function parameter to these two functions. Clean up the unused function parameter in rdtgroup_ctrl_remove(), rdtgroup_rmdir_mon() and their callers rdtgroup_rmdir_ctrl() and rdtgroup_rmdir(). Signed-off-by: Xiaochen Shen Signed-off-by: Borislav Petkov Reviewed-by: Reinette Chatre Link: https://lkml.kernel.org/r/1606759618-13181-1-git-send-email-xiaochen.shen@intel.com --- arch/x86/kernel/cpu/resctrl/rdtgroup.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c index 05a026d4420b..bcbec8533b74 100644 --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c @@ -3018,8 +3018,7 @@ static int rdtgroup_mkdir(struct kernfs_node *parent_kn, const char *name, return -EPERM; } -static int rdtgroup_rmdir_mon(struct kernfs_node *kn, struct rdtgroup *rdtgrp, - cpumask_var_t tmpmask) +static int rdtgroup_rmdir_mon(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask) { struct rdtgroup *prdtgrp = rdtgrp->mon.parent; int cpu; @@ -3051,8 +3050,7 @@ static int rdtgroup_rmdir_mon(struct kernfs_node *kn, struct rdtgroup *rdtgrp, return 0; } -static int rdtgroup_ctrl_remove(struct kernfs_node *kn, - struct rdtgroup *rdtgrp) +static int rdtgroup_ctrl_remove(struct rdtgroup *rdtgrp) { rdtgrp->flags = RDT_DELETED; list_del(&rdtgrp->rdtgroup_list); @@ -3061,8 +3059,7 @@ static int rdtgroup_ctrl_remove(struct kernfs_node *kn, return 0; } -static int rdtgroup_rmdir_ctrl(struct kernfs_node *kn, struct rdtgroup *rdtgrp, - cpumask_var_t tmpmask) +static int rdtgroup_rmdir_ctrl(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask) { int cpu; @@ -3089,7 +3086,7 @@ static int rdtgroup_rmdir_ctrl(struct kernfs_node *kn, struct rdtgroup *rdtgrp, closid_free(rdtgrp->closid); free_rmid(rdtgrp->mon.rmid); - rdtgroup_ctrl_remove(kn, rdtgrp); + rdtgroup_ctrl_remove(rdtgrp); /* * Free all the child monitor group rmids. @@ -3126,13 +3123,13 @@ static int rdtgroup_rmdir(struct kernfs_node *kn) rdtgrp != &rdtgroup_default) { if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { - ret = rdtgroup_ctrl_remove(kn, rdtgrp); + ret = rdtgroup_ctrl_remove(rdtgrp); } else { - ret = rdtgroup_rmdir_ctrl(kn, rdtgrp, tmpmask); + ret = rdtgroup_rmdir_ctrl(rdtgrp, tmpmask); } } else if (rdtgrp->type == RDTMON_GROUP && is_mon_groups(parent_kn, kn->name)) { - ret = rdtgroup_rmdir_mon(kn, rdtgrp, tmpmask); + ret = rdtgroup_rmdir_mon(rdtgrp, tmpmask); } else { ret = -EPERM; } From e273e6e12ab1db3eb57712bd60655744d0091fa3 Mon Sep 17 00:00:00 2001 From: Gabriele Paoloni Date: Fri, 27 Nov 2020 16:18:16 +0000 Subject: [PATCH 299/360] x86/mce: Move the mce_panic() call and 'kill_it' assignments to the right places Right now, for local MCEs the machine calls panic(), if needed, right after lmce is set. For MCE broadcasting, mce_reign() takes care of calling mce_panic(). Hence: - improve readability by moving the conditional evaluation of tolerant up to when kill_it is set first; - move the mce_panic() call up into the statement where mce_end() fails. [ bp: Massage, remove comment in the mce_end() failure case because it is superfluous; use local ptr 'cfg' in both tests. ] Signed-off-by: Gabriele Paoloni Signed-off-by: Borislav Petkov Reviewed-by: Tony Luck Link: https://lkml.kernel.org/r/20201127161819.3106432-3-gabriele.paoloni@intel.com --- arch/x86/kernel/cpu/mce/core.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index f319bed1c59d..ebaa52a0c024 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1351,8 +1351,7 @@ noinstr void do_machine_check(struct pt_regs *regs) * severity is MCE_AR_SEVERITY we have other options. */ if (!(m.mcgstatus & MCG_STATUS_RIPV)) - kill_it = 1; - + kill_it = (cfg->tolerant == 3) ? 0 : 1; /* * Check if this MCE is signaled to only this logical processor, * on Intel, Zhaoxin only. @@ -1388,6 +1387,9 @@ noinstr void do_machine_check(struct pt_regs *regs) if (mce_end(order) < 0) { if (!no_way_out) no_way_out = worst >= MCE_PANIC_SEVERITY; + + if (no_way_out && cfg->tolerant < 3) + mce_panic("Fatal machine check on current CPU", &m, msg); } } else { /* @@ -1404,15 +1406,6 @@ noinstr void do_machine_check(struct pt_regs *regs) } } - /* - * If tolerant is at an insane level we drop requests to kill - * processes and continue even when there is no way out. - */ - if (cfg->tolerant == 3) - kill_it = 0; - else if (no_way_out) - mce_panic("Fatal machine check on current CPU", &m, msg); - if (worst > 0) irq_work_queue(&mce_irq_work); From 3a866b16fd2360a9c4ebf71cfbf7ebfe968c1409 Mon Sep 17 00:00:00 2001 From: Gabriele Paoloni Date: Fri, 27 Nov 2020 16:18:17 +0000 Subject: [PATCH 300/360] x86/mce: Panic for LMCE only if mca_cfg.tolerant < 3 Right now for LMCE, if no_way_out is set, mce_panic() is called regardless of mca_cfg.tolerant. This is not correct as, if mca_cfg.tolerant = 3, the code should never panic. Add that check. [ bp: use local ptr 'cfg'. ] Signed-off-by: Gabriele Paoloni Signed-off-by: Borislav Petkov Reviewed-by: Tony Luck Link: https://lkml.kernel.org/r/20201127161819.3106432-4-gabriele.paoloni@intel.com --- arch/x86/kernel/cpu/mce/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index ebaa52a0c024..99da2e0b595b 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1368,7 +1368,7 @@ noinstr void do_machine_check(struct pt_regs *regs) * to see it will clear it. */ if (lmce) { - if (no_way_out) + if (no_way_out && cfg->tolerant < 3) mce_panic("Fatal local machine check", &m, msg); } else { order = mce_start(&no_way_out); From d5b38e3d0fdb1a16994b449bc338fb8b26816b07 Mon Sep 17 00:00:00 2001 From: Gabriele Paoloni Date: Fri, 27 Nov 2020 16:18:18 +0000 Subject: [PATCH 301/360] x86/mce: Remove redundant call to irq_work_queue() Currently, __mc_scan_banks() in do_machine_check() does the following callchain: __mc_scan_banks()->mce_log()->irq_work_queue(&mce_irq_work). Hence, the call to irq_work_queue() below after __mc_scan_banks() seems redundant. Just remove it. Signed-off-by: Gabriele Paoloni Signed-off-by: Borislav Petkov Reviewed-by: Tony Luck Link: https://lkml.kernel.org/r/20201127161819.3106432-5-gabriele.paoloni@intel.com --- arch/x86/kernel/cpu/mce/core.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 99da2e0b595b..a9991a96445d 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1406,9 +1406,6 @@ noinstr void do_machine_check(struct pt_regs *regs) } } - if (worst > 0) - irq_work_queue(&mce_irq_work); - if (worst != MCE_AR_SEVERITY && !kill_it) goto out; From e1c06d2366e743475b91045ef0c2ce1bbd028cb6 Mon Sep 17 00:00:00 2001 From: Gabriele Paoloni Date: Fri, 27 Nov 2020 16:18:19 +0000 Subject: [PATCH 302/360] x86/mce: Rename kill_it to kill_current_task Currently, if an MCE happens in user-mode or while the kernel is copying data from user space, 'kill_it' is used to check if execution of the interrupted task can be recovered or not; the flag name however is not very meaningful, hence rename it to match its goal. [ bp: Massage commit message, rename the queue_task_work() arg too. ] Signed-off-by: Gabriele Paoloni Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201127161819.3106432-6-gabriele.paoloni@intel.com --- arch/x86/kernel/cpu/mce/core.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index a9991a96445d..6af6a3c0698f 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -1266,14 +1266,14 @@ static void kill_me_maybe(struct callback_head *cb) } } -static void queue_task_work(struct mce *m, int kill_it) +static void queue_task_work(struct mce *m, int kill_current_task) { current->mce_addr = m->addr; current->mce_kflags = m->kflags; current->mce_ripv = !!(m->mcgstatus & MCG_STATUS_RIPV); current->mce_whole_page = whole_page(m); - if (kill_it) + if (kill_current_task) current->mce_kill_me.func = kill_me_now; else current->mce_kill_me.func = kill_me_maybe; @@ -1321,10 +1321,10 @@ noinstr void do_machine_check(struct pt_regs *regs) int no_way_out = 0; /* - * If kill_it gets set, there might be a way to recover from this + * If kill_current_task is not set, there might be a way to recover from this * error. */ - int kill_it = 0; + int kill_current_task = 0; /* * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES @@ -1351,7 +1351,7 @@ noinstr void do_machine_check(struct pt_regs *regs) * severity is MCE_AR_SEVERITY we have other options. */ if (!(m.mcgstatus & MCG_STATUS_RIPV)) - kill_it = (cfg->tolerant == 3) ? 0 : 1; + kill_current_task = (cfg->tolerant == 3) ? 0 : 1; /* * Check if this MCE is signaled to only this logical processor, * on Intel, Zhaoxin only. @@ -1406,7 +1406,7 @@ noinstr void do_machine_check(struct pt_regs *regs) } } - if (worst != MCE_AR_SEVERITY && !kill_it) + if (worst != MCE_AR_SEVERITY && !kill_current_task) goto out; /* Fault was in user mode and we need to take some action */ @@ -1414,7 +1414,7 @@ noinstr void do_machine_check(struct pt_regs *regs) /* If this triggers there is no way to recover. Die hard. */ BUG_ON(!on_thread_stack() || !user_mode(regs)); - queue_task_work(&m, kill_it); + queue_task_work(&m, kill_current_task); } else { /* @@ -1432,7 +1432,7 @@ noinstr void do_machine_check(struct pt_regs *regs) } if (m.kflags & MCE_IN_KERNEL_COPYIN) - queue_task_work(&m, kill_it); + queue_task_work(&m, kill_current_task); } out: mce_wrmsrl(MSR_IA32_MCG_STATUS, 0); From bab8c183d1d452f5fdc059aef2f0788bd2986231 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 2 Dec 2020 09:27:14 +0100 Subject: [PATCH 303/360] x86/sgx: Fix a typo in kernel-doc markup Fix the following kernel-doc warning: arch/x86/include/uapi/asm/sgx.h:19: warning: expecting prototype \ for enum sgx_epage_flags. Prototype was for enum sgx_page_flags instead [ bp: Launder the commit message. ] Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/ca11a4540d981cbd5f026b6cbc8931aa55654e00.1606897462.git.mchehab+huawei@kernel.org --- arch/x86/include/uapi/asm/sgx.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/include/uapi/asm/sgx.h b/arch/x86/include/uapi/asm/sgx.h index 791e45334a4a..9034f3007c4e 100644 --- a/arch/x86/include/uapi/asm/sgx.h +++ b/arch/x86/include/uapi/asm/sgx.h @@ -9,7 +9,7 @@ #include /** - * enum sgx_epage_flags - page control flags + * enum sgx_page_flags - page control flags * %SGX_PAGE_MEASURE: Measure the page contents with a sequence of * ENCLS[EEXTEND] operations. */ From 5db44cfe2e6ee247a5806e814a328f26d1f21f55 Mon Sep 17 00:00:00 2001 From: Vineeth Vijayan Date: Mon, 25 May 2020 10:36:42 +0200 Subject: [PATCH 304/360] s390/zfcp: remove pm support from zfcp driver As part of removing the power management support from s390 arch, remove PM callbacks from the scsi/zfcp driver. Signed-off-by: Vineeth Vijayan Reviewed-by: Steffen Maier Signed-off-by: Heiko Carstens --- drivers/s390/scsi/zfcp_ccw.c | 71 ++++++------------------------------ drivers/s390/scsi/zfcp_dbf.c | 25 ------------- drivers/s390/scsi/zfcp_def.h | 1 - drivers/s390/scsi/zfcp_ext.h | 1 - 4 files changed, 11 insertions(+), 87 deletions(-) diff --git a/drivers/s390/scsi/zfcp_ccw.c b/drivers/s390/scsi/zfcp_ccw.c index d9fd0a41da64..bdf2cc1ea713 100644 --- a/drivers/s390/scsi/zfcp_ccw.c +++ b/drivers/s390/scsi/zfcp_ccw.c @@ -193,30 +193,6 @@ static int zfcp_ccw_set_online(struct ccw_device *cdev) return 0; } -/** - * zfcp_ccw_offline_sync - shut down adapter and wait for it to finish - * @cdev: pointer to belonging ccw device - * @set: Status flags to set. - * @tag: s390dbf trace record tag - * - * This function gets called by the common i/o layer and sets an adapter - * into state offline. - */ -static int zfcp_ccw_offline_sync(struct ccw_device *cdev, int set, char *tag) -{ - struct zfcp_adapter *adapter = zfcp_ccw_adapter_by_cdev(cdev); - - if (!adapter) - return 0; - - zfcp_erp_set_adapter_status(adapter, set); - zfcp_erp_adapter_shutdown(adapter, 0, tag); - zfcp_erp_wait(adapter); - - zfcp_ccw_adapter_put(adapter); - return 0; -} - /** * zfcp_ccw_set_offline - set_offline function of zfcp driver * @cdev: pointer to belonging ccw device @@ -226,7 +202,17 @@ static int zfcp_ccw_offline_sync(struct ccw_device *cdev, int set, char *tag) */ static int zfcp_ccw_set_offline(struct ccw_device *cdev) { - return zfcp_ccw_offline_sync(cdev, 0, "ccsoff1"); + struct zfcp_adapter *adapter = zfcp_ccw_adapter_by_cdev(cdev); + + if (!adapter) + return 0; + + zfcp_erp_set_adapter_status(adapter, 0); + zfcp_erp_adapter_shutdown(adapter, 0, "ccsoff1"); + zfcp_erp_wait(adapter); + + zfcp_ccw_adapter_put(adapter); + return 0; } /** @@ -246,11 +232,6 @@ static int zfcp_ccw_notify(struct ccw_device *cdev, int event) switch (event) { case CIO_GONE: - if (atomic_read(&adapter->status) & - ZFCP_STATUS_ADAPTER_SUSPENDED) { /* notification ignore */ - zfcp_dbf_hba_basic("ccnigo1", adapter); - break; - } dev_warn(&cdev->dev, "The FCP device has been detached\n"); zfcp_erp_adapter_shutdown(adapter, 0, "ccnoti1"); break; @@ -260,11 +241,6 @@ static int zfcp_ccw_notify(struct ccw_device *cdev, int event) zfcp_erp_adapter_shutdown(adapter, 0, "ccnoti2"); break; case CIO_OPER: - if (atomic_read(&adapter->status) & - ZFCP_STATUS_ADAPTER_SUSPENDED) { /* notification ignore */ - zfcp_dbf_hba_basic("ccniop1", adapter); - break; - } dev_info(&cdev->dev, "The FCP device is operational again\n"); zfcp_erp_set_adapter_status(adapter, ZFCP_STATUS_COMMON_RUNNING); @@ -300,28 +276,6 @@ static void zfcp_ccw_shutdown(struct ccw_device *cdev) zfcp_ccw_adapter_put(adapter); } -static int zfcp_ccw_suspend(struct ccw_device *cdev) -{ - zfcp_ccw_offline_sync(cdev, ZFCP_STATUS_ADAPTER_SUSPENDED, "ccsusp1"); - return 0; -} - -static int zfcp_ccw_thaw(struct ccw_device *cdev) -{ - /* trace records for thaw and final shutdown during suspend - can only be found in system dump until the end of suspend - but not after resume because it's based on the memory image - right after the very first suspend (freeze) callback */ - zfcp_ccw_activate(cdev, 0, "ccthaw1"); - return 0; -} - -static int zfcp_ccw_resume(struct ccw_device *cdev) -{ - zfcp_ccw_activate(cdev, ZFCP_STATUS_ADAPTER_SUSPENDED, "ccresu1"); - return 0; -} - struct ccw_driver zfcp_ccw_driver = { .driver = { .owner = THIS_MODULE, @@ -334,7 +288,4 @@ struct ccw_driver zfcp_ccw_driver = { .set_offline = zfcp_ccw_set_offline, .notify = zfcp_ccw_notify, .shutdown = zfcp_ccw_shutdown, - .freeze = zfcp_ccw_suspend, - .thaw = zfcp_ccw_thaw, - .restore = zfcp_ccw_resume, }; diff --git a/drivers/s390/scsi/zfcp_dbf.c b/drivers/s390/scsi/zfcp_dbf.c index 673e42defb91..ca473b368905 100644 --- a/drivers/s390/scsi/zfcp_dbf.c +++ b/drivers/s390/scsi/zfcp_dbf.c @@ -263,31 +263,6 @@ void zfcp_dbf_hba_def_err(struct zfcp_adapter *adapter, u64 req_id, u16 scount, spin_unlock_irqrestore(&dbf->pay_lock, flags); } -/** - * zfcp_dbf_hba_basic - trace event for basic adapter events - * @tag: identifier for event - * @adapter: pointer to struct zfcp_adapter - */ -void zfcp_dbf_hba_basic(char *tag, struct zfcp_adapter *adapter) -{ - struct zfcp_dbf *dbf = adapter->dbf; - struct zfcp_dbf_hba *rec = &dbf->hba_buf; - static int const level = 1; - unsigned long flags; - - if (unlikely(!debug_level_enabled(dbf->hba, level))) - return; - - spin_lock_irqsave(&dbf->hba_lock, flags); - memset(rec, 0, sizeof(*rec)); - - memcpy(rec->tag, tag, ZFCP_DBF_TAG_LEN); - rec->id = ZFCP_DBF_HBA_BASIC; - - debug_event(dbf->hba, level, rec, sizeof(*rec)); - spin_unlock_irqrestore(&dbf->hba_lock, flags); -} - static void zfcp_dbf_set_common(struct zfcp_dbf_rec *rec, struct zfcp_adapter *adapter, struct zfcp_port *port, diff --git a/drivers/s390/scsi/zfcp_def.h b/drivers/s390/scsi/zfcp_def.h index da8a5ceb615c..5069b555c6c1 100644 --- a/drivers/s390/scsi/zfcp_def.h +++ b/drivers/s390/scsi/zfcp_def.h @@ -70,7 +70,6 @@ #define ZFCP_STATUS_ADAPTER_SIOSL_ISSUED 0x00000004 #define ZFCP_STATUS_ADAPTER_XCONFIG_OK 0x00000008 #define ZFCP_STATUS_ADAPTER_HOST_CON_INIT 0x00000010 -#define ZFCP_STATUS_ADAPTER_SUSPENDED 0x00000040 #define ZFCP_STATUS_ADAPTER_ERP_PENDING 0x00000100 #define ZFCP_STATUS_ADAPTER_LINK_UNPLUGGED 0x00000200 #define ZFCP_STATUS_ADAPTER_DATA_DIV_ENABLED 0x00000400 diff --git a/drivers/s390/scsi/zfcp_ext.h b/drivers/s390/scsi/zfcp_ext.h index 3ef5d74331c3..fdac6350c579 100644 --- a/drivers/s390/scsi/zfcp_ext.h +++ b/drivers/s390/scsi/zfcp_ext.h @@ -49,7 +49,6 @@ extern void zfcp_dbf_hba_fsf_fces(char *tag, const struct zfcp_fsf_req *req, u32 fc_security_new); extern void zfcp_dbf_hba_bit_err(char *, struct zfcp_fsf_req *); extern void zfcp_dbf_hba_def_err(struct zfcp_adapter *, u64, u16, void **); -extern void zfcp_dbf_hba_basic(char *, struct zfcp_adapter *); extern void zfcp_dbf_san_req(char *, struct zfcp_fsf_req *, u32); extern void zfcp_dbf_san_res(char *, struct zfcp_fsf_req *); extern void zfcp_dbf_san_in_els(char *, struct zfcp_fsf_req *); From 127fca609385955a3d8d4f95205e932981b10ea8 Mon Sep 17 00:00:00 2001 From: Vineeth Vijayan Date: Tue, 10 Nov 2020 09:28:33 +0100 Subject: [PATCH 305/360] s390/dasd: remove unused pm related functions The power-management related functions are unused since the 'commit 394216275c7d ("s390: remove broken hibernate / power management support")'. Remove them from dasd drivers. Signed-off-by: Vineeth Vijayan Reviewed-by: Stefan Haberland Signed-off-by: Heiko Carstens --- drivers/s390/block/dasd.c | 93 +-------------------------------- drivers/s390/block/dasd_eckd.c | 94 ---------------------------------- drivers/s390/block/dasd_fba.c | 3 -- drivers/s390/block/dasd_int.h | 10 ---- 4 files changed, 2 insertions(+), 198 deletions(-) diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index eb17fea8075c..0ff054edcad0 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -75,7 +75,6 @@ static int dasd_flush_block_queue(struct dasd_block *); static void dasd_device_tasklet(unsigned long); static void dasd_block_tasklet(unsigned long); static void do_kick_device(struct work_struct *); -static void do_restore_device(struct work_struct *); static void do_reload_device(struct work_struct *); static void do_requeue_requests(struct work_struct *); static void dasd_return_cqr_cb(struct dasd_ccw_req *, void *); @@ -138,7 +137,6 @@ struct dasd_device *dasd_alloc_device(void) INIT_LIST_HEAD(&device->ccw_queue); timer_setup(&device->timer, dasd_device_timeout, 0); INIT_WORK(&device->kick_work, do_kick_device); - INIT_WORK(&device->restore_device, do_restore_device); INIT_WORK(&device->reload_device, do_reload_device); INIT_WORK(&device->requeue_requests, do_requeue_requests); device->state = DASD_STATE_NEW; @@ -620,26 +618,6 @@ void dasd_reload_device(struct dasd_device *device) } EXPORT_SYMBOL(dasd_reload_device); -/* - * dasd_restore_device will schedule a call do do_restore_device to the kernel - * event daemon. - */ -static void do_restore_device(struct work_struct *work) -{ - struct dasd_device *device = container_of(work, struct dasd_device, - restore_device); - device->cdev->drv->restore(device->cdev); - dasd_put_device(device); -} - -void dasd_restore_device(struct dasd_device *device) -{ - dasd_get_device(device); - /* queue call to dasd_restore_device to the kernel event daemon. */ - if (!schedule_work(&device->restore_device)) - dasd_put_device(device); -} - /* * Set the target state for a device and starts the state change. */ @@ -1514,7 +1492,6 @@ int dasd_start_IO(struct dasd_ccw_req *cqr) "start_IO: -EIO device gone, retry"); break; case -EINVAL: - /* most likely caused in power management context */ DBF_DEV_EVENT(DBF_WARNING, device, "%s", "start_IO: -EINVAL device currently " "not accessible"); @@ -2048,7 +2025,7 @@ static void __dasd_device_check_expire(struct dasd_device *device) static int __dasd_device_is_unusable(struct dasd_device *device, struct dasd_ccw_req *cqr) { - int mask = ~(DASD_STOPPED_DC_WAIT | DASD_UNRESUMED_PM | DASD_STOPPED_NOSPC); + int mask = ~(DASD_STOPPED_DC_WAIT | DASD_STOPPED_NOSPC); if (test_bit(DASD_FLAG_OFFLINE, &device->flags) && !test_bit(DASD_FLAG_SAFE_OFFLINE_RUNNING, &device->flags)) { @@ -2112,8 +2089,7 @@ static void __dasd_device_check_path_events(struct dasd_device *device) if (!dasd_path_get_tbvpm(device)) return; - if (device->stopped & - ~(DASD_STOPPED_DC_WAIT | DASD_UNRESUMED_PM)) + if (device->stopped & ~(DASD_STOPPED_DC_WAIT)) return; rc = device->discipline->verify_path(device, dasd_path_get_tbvpm(device)); @@ -3788,11 +3764,6 @@ int dasd_generic_path_operational(struct dasd_device *device) "operational\n"); DBF_DEV_EVENT(DBF_WARNING, device, "%s", "path operational"); dasd_device_remove_stop_bits(device, DASD_STOPPED_DC_WAIT); - if (device->stopped & DASD_UNRESUMED_PM) { - dasd_device_remove_stop_bits(device, DASD_UNRESUMED_PM); - dasd_restore_device(device); - return 1; - } dasd_schedule_device_bh(device); if (device->block) { dasd_schedule_block_bh(device->block); @@ -4052,66 +4023,6 @@ void dasd_schedule_requeue(struct dasd_device *device) } EXPORT_SYMBOL(dasd_schedule_requeue); -int dasd_generic_pm_freeze(struct ccw_device *cdev) -{ - struct dasd_device *device = dasd_device_from_cdev(cdev); - - if (IS_ERR(device)) - return PTR_ERR(device); - - /* mark device as suspended */ - set_bit(DASD_FLAG_SUSPENDED, &device->flags); - - if (device->discipline->freeze) - device->discipline->freeze(device); - - /* disallow new I/O */ - dasd_device_set_stop_bits(device, DASD_STOPPED_PM); - - return dasd_generic_requeue_all_requests(device); -} -EXPORT_SYMBOL_GPL(dasd_generic_pm_freeze); - -int dasd_generic_restore_device(struct ccw_device *cdev) -{ - struct dasd_device *device = dasd_device_from_cdev(cdev); - int rc = 0; - - if (IS_ERR(device)) - return PTR_ERR(device); - - /* allow new IO again */ - dasd_device_remove_stop_bits(device, - (DASD_STOPPED_PM | DASD_UNRESUMED_PM)); - - dasd_schedule_device_bh(device); - - /* - * call discipline restore function - * if device is stopped do nothing e.g. for disconnected devices - */ - if (device->discipline->restore && !(device->stopped)) - rc = device->discipline->restore(device); - if (rc || device->stopped) - /* - * if the resume failed for the DASD we put it in - * an UNRESUMED stop state - */ - device->stopped |= DASD_UNRESUMED_PM; - - if (device->block) { - dasd_schedule_block_bh(device->block); - if (device->block->request_queue) - blk_mq_run_hw_queues(device->block->request_queue, - true); - } - - clear_bit(DASD_FLAG_SUSPENDED, &device->flags); - dasd_put_device(device); - return 0; -} -EXPORT_SYMBOL_GPL(dasd_generic_restore_device); - static struct dasd_ccw_req *dasd_generic_build_rdc(struct dasd_device *device, int rdc_buffer_size, int magic) diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index ad44d22e8859..758ee4153ac1 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -5716,95 +5716,6 @@ static void dasd_eckd_dump_sense(struct dasd_device *device, } } -static int dasd_eckd_pm_freeze(struct dasd_device *device) -{ - /* - * the device should be disconnected from our LCU structure - * on restore we will reconnect it and reread LCU specific - * information like PAV support that might have changed - */ - dasd_alias_remove_device(device); - dasd_alias_disconnect_device_from_lcu(device); - - return 0; -} - -static int dasd_eckd_restore_device(struct dasd_device *device) -{ - struct dasd_eckd_private *private = device->private; - struct dasd_eckd_characteristics temp_rdc_data; - int rc; - struct dasd_uid temp_uid; - unsigned long flags; - unsigned long cqr_flags = 0; - - /* Read Configuration Data */ - rc = dasd_eckd_read_conf(device); - if (rc) { - DBF_EVENT_DEVID(DBF_WARNING, device->cdev, - "Read configuration data failed, rc=%d", rc); - goto out_err; - } - - dasd_eckd_get_uid(device, &temp_uid); - /* Generate device unique id */ - rc = dasd_eckd_generate_uid(device); - spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags); - if (memcmp(&private->uid, &temp_uid, sizeof(struct dasd_uid)) != 0) - dev_err(&device->cdev->dev, "The UID of the DASD has " - "changed\n"); - spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags); - if (rc) - goto out_err; - - /* register lcu with alias handling, enable PAV if this is a new lcu */ - rc = dasd_alias_make_device_known_to_lcu(device); - if (rc) - goto out_err; - - set_bit(DASD_CQR_FLAGS_FAILFAST, &cqr_flags); - dasd_eckd_validate_server(device, cqr_flags); - - /* RE-Read Configuration Data */ - rc = dasd_eckd_read_conf(device); - if (rc) { - DBF_EVENT_DEVID(DBF_WARNING, device->cdev, - "Read configuration data failed, rc=%d", rc); - goto out_err2; - } - - /* Read Feature Codes */ - dasd_eckd_read_features(device); - - /* Read Volume Information */ - dasd_eckd_read_vol_info(device); - - /* Read Extent Pool Information */ - dasd_eckd_read_ext_pool_info(device); - - /* Read Device Characteristics */ - rc = dasd_generic_read_dev_chars(device, DASD_ECKD_MAGIC, - &temp_rdc_data, 64); - if (rc) { - DBF_EVENT_DEVID(DBF_WARNING, device->cdev, - "Read device characteristic failed, rc=%d", rc); - goto out_err2; - } - spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags); - memcpy(&private->rdc_data, &temp_rdc_data, sizeof(temp_rdc_data)); - spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags); - - /* add device to alias management */ - dasd_alias_add_device(device); - - return 0; - -out_err2: - dasd_alias_disconnect_device_from_lcu(device); -out_err: - return -1; -} - static int dasd_eckd_reload_device(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; @@ -6668,9 +6579,6 @@ static struct ccw_driver dasd_eckd_driver = { .notify = dasd_generic_notify, .path_event = dasd_generic_path_event, .shutdown = dasd_generic_shutdown, - .freeze = dasd_generic_pm_freeze, - .thaw = dasd_generic_restore_device, - .restore = dasd_generic_restore_device, .uc_handler = dasd_generic_uc_handler, .int_class = IRQIO_DAS, }; @@ -6702,8 +6610,6 @@ static struct dasd_discipline dasd_eckd_discipline = { .dump_sense_dbf = dasd_eckd_dump_sense_dbf, .fill_info = dasd_eckd_fill_info, .ioctl = dasd_eckd_ioctl, - .freeze = dasd_eckd_pm_freeze, - .restore = dasd_eckd_restore_device, .reload = dasd_eckd_reload_device, .get_uid = dasd_eckd_get_uid, .kick_validate = dasd_eckd_kick_validate_server, diff --git a/drivers/s390/block/dasd_fba.c b/drivers/s390/block/dasd_fba.c index 1a44e321b54e..c027344ee225 100644 --- a/drivers/s390/block/dasd_fba.c +++ b/drivers/s390/block/dasd_fba.c @@ -79,9 +79,6 @@ static struct ccw_driver dasd_fba_driver = { .set_online = dasd_fba_set_online, .notify = dasd_generic_notify, .path_event = dasd_generic_path_event, - .freeze = dasd_generic_pm_freeze, - .thaw = dasd_generic_restore_device, - .restore = dasd_generic_restore_device, .int_class = IRQIO_DAS, }; diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h index fa552f9f1666..7a34161ea5c6 100644 --- a/drivers/s390/block/dasd_int.h +++ b/drivers/s390/block/dasd_int.h @@ -355,10 +355,6 @@ struct dasd_discipline { int (*fill_info) (struct dasd_device *, struct dasd_information2_t *); int (*ioctl) (struct dasd_block *, unsigned int, void __user *); - /* suspend/resume functions */ - int (*freeze) (struct dasd_device *); - int (*restore) (struct dasd_device *); - /* reload device after state change */ int (*reload) (struct dasd_device *); @@ -520,7 +516,6 @@ struct dasd_device { atomic_t tasklet_scheduled; struct tasklet_struct tasklet; struct work_struct kick_work; - struct work_struct restore_device; struct work_struct reload_device; struct work_struct kick_validate; struct work_struct suc_work; @@ -592,8 +587,6 @@ struct dasd_queue { #define DASD_STOPPED_PENDING 4 /* long busy */ #define DASD_STOPPED_DC_WAIT 8 /* disconnected, wait */ #define DASD_STOPPED_SU 16 /* summary unit check handling */ -#define DASD_STOPPED_PM 32 /* pm state transition */ -#define DASD_UNRESUMED_PM 64 /* pm resume failed state */ #define DASD_STOPPED_NOSPC 128 /* no space left */ /* per device flags */ @@ -753,7 +746,6 @@ enum blk_eh_timer_return dasd_times_out(struct request *req, bool reserved); void dasd_enable_device(struct dasd_device *); void dasd_set_target_state(struct dasd_device *, int); void dasd_kick_device(struct dasd_device *); -void dasd_restore_device(struct dasd_device *); void dasd_reload_device(struct dasd_device *); void dasd_schedule_requeue(struct dasd_device *); @@ -785,8 +777,6 @@ int dasd_generic_path_operational(struct dasd_device *); void dasd_generic_shutdown(struct ccw_device *); void dasd_generic_handle_state_change(struct dasd_device *); -int dasd_generic_pm_freeze(struct ccw_device *); -int dasd_generic_restore_device(struct ccw_device *); enum uc_todo dasd_generic_uc_handler(struct ccw_device *, struct irb *); void dasd_generic_path_event(struct ccw_device *, int *); int dasd_generic_verify_path(struct dasd_device *, __u8); From 1cf69b7b51d79eff839c8945cd909e13600f81c7 Mon Sep 17 00:00:00 2001 From: Vineeth Vijayan Date: Mon, 30 Mar 2020 14:36:57 +0200 Subject: [PATCH 306/360] s390: remove pm support from console drivers The power-management functions are unused since the 'commit 394216275c7d ("s390: remove broken hibernate / power management support")'. Remove the unsued pm functions from the console drivers. Signed-off-by: Vineeth Vijayan Signed-off-by: Heiko Carstens --- drivers/s390/char/con3215.c | 85 ++++--------------------------------- drivers/s390/char/con3270.c | 1 - drivers/s390/char/raw3270.c | 78 +++------------------------------- drivers/s390/char/raw3270.h | 1 - 4 files changed, 15 insertions(+), 150 deletions(-) diff --git a/drivers/s390/char/con3215.c b/drivers/s390/char/con3215.c index d8acabbb1ed3..1354c42d95aa 100644 --- a/drivers/s390/char/con3215.c +++ b/drivers/s390/char/con3215.c @@ -289,16 +289,14 @@ static void raw3215_timeout(struct timer_list *t) spin_lock_irqsave(get_ccwdev_lock(raw->cdev), flags); raw->flags &= ~RAW3215_TIMER_RUNS; - if (!tty_port_suspended(&raw->port)) { - raw3215_mk_write_req(raw); - raw3215_start_io(raw); - if ((raw->queued_read || raw->queued_write) && - !(raw->flags & RAW3215_WORKING) && - !(raw->flags & RAW3215_TIMER_RUNS)) { - raw->timer.expires = RAW3215_TIMEOUT + jiffies; - add_timer(&raw->timer); - raw->flags |= RAW3215_TIMER_RUNS; - } + raw3215_mk_write_req(raw); + raw3215_start_io(raw); + if ((raw->queued_read || raw->queued_write) && + !(raw->flags & RAW3215_WORKING) && + !(raw->flags & RAW3215_TIMER_RUNS)) { + raw->timer.expires = RAW3215_TIMEOUT + jiffies; + add_timer(&raw->timer); + raw->flags |= RAW3215_TIMER_RUNS; } spin_unlock_irqrestore(get_ccwdev_lock(raw->cdev), flags); } @@ -311,7 +309,7 @@ static void raw3215_timeout(struct timer_list *t) */ static inline void raw3215_try_io(struct raw3215_info *raw) { - if (!tty_port_initialized(&raw->port) || tty_port_suspended(&raw->port)) + if (!tty_port_initialized(&raw->port)) return; if (raw->queued_read != NULL) raw3215_start_io(raw); @@ -463,26 +461,6 @@ put_tty: tty_kref_put(tty); } -/* - * Drop the oldest line from the output buffer. - */ -static void raw3215_drop_line(struct raw3215_info *raw) -{ - int ix; - char ch; - - BUG_ON(raw->written != 0); - ix = (raw->head - raw->count) & (RAW3215_BUFFER_SIZE - 1); - while (raw->count > 0) { - ch = raw->buffer[ix]; - ix = (ix + 1) & (RAW3215_BUFFER_SIZE - 1); - raw->count--; - if (ch == 0x15) - break; - } - raw->head = ix; -} - /* * Wait until length bytes are available int the output buffer. * Has to be called with the s390irq lock held. Can be called @@ -491,13 +469,6 @@ static void raw3215_drop_line(struct raw3215_info *raw) static void raw3215_make_room(struct raw3215_info *raw, unsigned int length) { while (RAW3215_BUFFER_SIZE - raw->count < length) { - /* While console is frozen for suspend we have no other - * choice but to drop message from the buffer to make - * room for even more messages. */ - if (tty_port_suspended(&raw->port)) { - raw3215_drop_line(raw); - continue; - } /* there might be a request pending */ raw->flags |= RAW3215_FLUSHING; raw3215_mk_write_req(raw); @@ -763,36 +734,6 @@ static int raw3215_set_offline (struct ccw_device *cdev) return 0; } -static int raw3215_pm_stop(struct ccw_device *cdev) -{ - struct raw3215_info *raw; - unsigned long flags; - - /* Empty the output buffer, then prevent new I/O. */ - raw = dev_get_drvdata(&cdev->dev); - spin_lock_irqsave(get_ccwdev_lock(raw->cdev), flags); - raw3215_make_room(raw, RAW3215_BUFFER_SIZE); - tty_port_set_suspended(&raw->port, 1); - spin_unlock_irqrestore(get_ccwdev_lock(raw->cdev), flags); - return 0; -} - -static int raw3215_pm_start(struct ccw_device *cdev) -{ - struct raw3215_info *raw; - unsigned long flags; - - /* Allow I/O again and flush output buffer. */ - raw = dev_get_drvdata(&cdev->dev); - spin_lock_irqsave(get_ccwdev_lock(raw->cdev), flags); - tty_port_set_suspended(&raw->port, 0); - raw->flags |= RAW3215_FLUSHING; - raw3215_try_io(raw); - raw->flags &= ~RAW3215_FLUSHING; - spin_unlock_irqrestore(get_ccwdev_lock(raw->cdev), flags); - return 0; -} - static struct ccw_device_id raw3215_id[] = { { CCW_DEVICE(0x3215, 0) }, { /* end of list */ }, @@ -808,9 +749,6 @@ static struct ccw_driver raw3215_ccw_driver = { .remove = &raw3215_remove, .set_online = &raw3215_set_online, .set_offline = &raw3215_set_offline, - .freeze = &raw3215_pm_stop, - .thaw = &raw3215_pm_start, - .restore = &raw3215_pm_start, .int_class = IRQIO_C15, }; @@ -858,11 +796,6 @@ static void con3215_flush(void) unsigned long flags; raw = raw3215[0]; /* console 3215 is the first one */ - if (tty_port_suspended(&raw->port)) - /* The console is still frozen for suspend. */ - if (ccw_device_force_console(raw->cdev)) - /* Forcing didn't work, no panic message .. */ - return; spin_lock_irqsave(get_ccwdev_lock(raw->cdev), flags); raw3215_make_room(raw, RAW3215_BUFFER_SIZE); spin_unlock_irqrestore(get_ccwdev_lock(raw->cdev), flags); diff --git a/drivers/s390/char/con3270.c b/drivers/s390/char/con3270.c index e17364e13d2f..e21962c0fd94 100644 --- a/drivers/s390/char/con3270.c +++ b/drivers/s390/char/con3270.c @@ -544,7 +544,6 @@ con3270_flush(void) cp = condev; if (!cp->view.dev) return; - raw3270_pm_unfreeze(&cp->view); raw3270_activate_view(&cp->view); spin_lock_irqsave(&cp->view.lock, flags); con3270_wait_write(cp); diff --git a/drivers/s390/char/raw3270.c b/drivers/s390/char/raw3270.c index 63a41b168761..646ec796bb83 100644 --- a/drivers/s390/char/raw3270.c +++ b/drivers/s390/char/raw3270.c @@ -67,7 +67,6 @@ struct raw3270 { #define RAW3270_FLAGS_14BITADDR 0 /* 14-bit buffer addresses */ #define RAW3270_FLAGS_BUSY 1 /* Device busy, leave it alone */ #define RAW3270_FLAGS_CONSOLE 2 /* Device is the console. */ -#define RAW3270_FLAGS_FROZEN 3 /* set if 3270 is frozen for suspend */ /* Semaphore to protect global data of raw3270 (devices, views, etc). */ static DEFINE_MUTEX(raw3270_mutex); @@ -260,8 +259,7 @@ raw3270_view_active(struct raw3270_view *view) { struct raw3270 *rp = view->dev; - return rp && rp->view == view && - !test_bit(RAW3270_FLAGS_FROZEN, &rp->flags); + return rp && rp->view == view; } int @@ -273,8 +271,7 @@ raw3270_start(struct raw3270_view *view, struct raw3270_request *rq) spin_lock_irqsave(get_ccwdev_lock(view->dev->cdev), flags); rp = view->dev; - if (!rp || rp->view != view || - test_bit(RAW3270_FLAGS_FROZEN, &rp->flags)) + if (!rp || rp->view != view) rc = -EACCES; else if (!raw3270_state_ready(rp)) rc = -EBUSY; @@ -291,8 +288,7 @@ raw3270_start_locked(struct raw3270_view *view, struct raw3270_request *rq) int rc; rp = view->dev; - if (!rp || rp->view != view || - test_bit(RAW3270_FLAGS_FROZEN, &rp->flags)) + if (!rp || rp->view != view) rc = -EACCES; else if (!raw3270_state_ready(rp)) rc = -EBUSY; @@ -629,8 +625,7 @@ raw3270_reset(struct raw3270_view *view) int rc; rp = view->dev; - if (!rp || rp->view != view || - test_bit(RAW3270_FLAGS_FROZEN, &rp->flags)) + if (!rp || rp->view != view) rc = -EACCES; else if (!raw3270_state_ready(rp)) rc = -EBUSY; @@ -854,8 +849,6 @@ raw3270_activate_view(struct raw3270_view *view) rc = 0; else if (!raw3270_state_ready(rp)) rc = -EBUSY; - else if (test_bit(RAW3270_FLAGS_FROZEN, &rp->flags)) - rc = -EACCES; else { oldview = NULL; if (rp->view && rp->view->fn->deactivate) { @@ -903,8 +896,7 @@ raw3270_deactivate_view(struct raw3270_view *view) list_del_init(&view->list); list_add_tail(&view->list, &rp->view_list); /* Try to activate another view. */ - if (raw3270_state_ready(rp) && - !test_bit(RAW3270_FLAGS_FROZEN, &rp->flags)) { + if (raw3270_state_ready(rp)) { list_for_each_entry(view, &rp->view_list, list) { rp->view = view; if (view->fn->activate(view) == 0) @@ -999,8 +991,7 @@ raw3270_del_view(struct raw3270_view *view) rp->view = NULL; } list_del_init(&view->list); - if (!rp->view && raw3270_state_ready(rp) && - !test_bit(RAW3270_FLAGS_FROZEN, &rp->flags)) { + if (!rp->view && raw3270_state_ready(rp)) { /* Try to activate another view. */ list_for_each_entry(nv, &rp->view_list, list) { if (nv->fn->activate(nv) == 0) { @@ -1215,60 +1206,6 @@ raw3270_set_offline (struct ccw_device *cdev) return 0; } -static int raw3270_pm_stop(struct ccw_device *cdev) -{ - struct raw3270 *rp; - struct raw3270_view *view; - unsigned long flags; - - rp = dev_get_drvdata(&cdev->dev); - if (!rp) - return 0; - spin_lock_irqsave(get_ccwdev_lock(rp->cdev), flags); - if (rp->view && rp->view->fn->deactivate) - rp->view->fn->deactivate(rp->view); - if (!test_bit(RAW3270_FLAGS_CONSOLE, &rp->flags)) { - /* - * Release tty and fullscreen for all non-console - * devices. - */ - list_for_each_entry(view, &rp->view_list, list) { - if (view->fn->release) - view->fn->release(view); - } - } - set_bit(RAW3270_FLAGS_FROZEN, &rp->flags); - spin_unlock_irqrestore(get_ccwdev_lock(rp->cdev), flags); - return 0; -} - -static int raw3270_pm_start(struct ccw_device *cdev) -{ - struct raw3270 *rp; - unsigned long flags; - - rp = dev_get_drvdata(&cdev->dev); - if (!rp) - return 0; - spin_lock_irqsave(get_ccwdev_lock(rp->cdev), flags); - clear_bit(RAW3270_FLAGS_FROZEN, &rp->flags); - if (rp->view && rp->view->fn->activate) - rp->view->fn->activate(rp->view); - spin_unlock_irqrestore(get_ccwdev_lock(rp->cdev), flags); - return 0; -} - -void raw3270_pm_unfreeze(struct raw3270_view *view) -{ -#ifdef CONFIG_TN3270_CONSOLE - struct raw3270 *rp; - - rp = view->dev; - if (rp && test_bit(RAW3270_FLAGS_FROZEN, &rp->flags)) - ccw_device_force_console(rp->cdev); -#endif -} - static struct ccw_device_id raw3270_id[] = { { CCW_DEVICE(0x3270, 0) }, { CCW_DEVICE(0x3271, 0) }, @@ -1294,9 +1231,6 @@ static struct ccw_driver raw3270_ccw_driver = { .remove = &raw3270_remove, .set_online = &raw3270_set_online, .set_offline = &raw3270_set_offline, - .freeze = &raw3270_pm_stop, - .thaw = &raw3270_pm_start, - .restore = &raw3270_pm_start, .int_class = IRQIO_C70, }; diff --git a/drivers/s390/char/raw3270.h b/drivers/s390/char/raw3270.h index 8d979e0ee605..c6645167cd2b 100644 --- a/drivers/s390/char/raw3270.h +++ b/drivers/s390/char/raw3270.h @@ -199,7 +199,6 @@ struct raw3270_notifier { int raw3270_register_notifier(struct raw3270_notifier *); void raw3270_unregister_notifier(struct raw3270_notifier *); -void raw3270_pm_unfreeze(struct raw3270_view *); /* * Little memory allocator for string objects. From 6b532eec75ef2b470a38aeda6479da8689ce7f69 Mon Sep 17 00:00:00 2001 From: Vineeth Vijayan Date: Mon, 28 Sep 2020 16:16:45 +0200 Subject: [PATCH 307/360] s390/cio: remove pm support from eadm-sch drivers The 'commit 394216275c7d ("s390: remove broken hibernate / power management support")' removes the pm-support. Remove the unused functions in eadm-sch drivers. Signed-off-by: Vineeth Vijayan Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/eadm_sch.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/drivers/s390/cio/eadm_sch.c b/drivers/s390/cio/eadm_sch.c index 53468ae64b99..c8964e0a23e7 100644 --- a/drivers/s390/cio/eadm_sch.c +++ b/drivers/s390/cio/eadm_sch.c @@ -306,16 +306,6 @@ static void eadm_subchannel_shutdown(struct subchannel *sch) eadm_quiesce(sch); } -static int eadm_subchannel_freeze(struct subchannel *sch) -{ - return cio_disable_subchannel(sch); -} - -static int eadm_subchannel_restore(struct subchannel *sch) -{ - return cio_enable_subchannel(sch, (u32)(unsigned long)sch); -} - /** * eadm_subchannel_sch_event - process subchannel event * @sch: subchannel @@ -369,9 +359,6 @@ static struct css_driver eadm_subchannel_driver = { .remove = eadm_subchannel_remove, .shutdown = eadm_subchannel_shutdown, .sch_event = eadm_subchannel_sch_event, - .freeze = eadm_subchannel_freeze, - .thaw = eadm_subchannel_restore, - .restore = eadm_subchannel_restore, }; static int __init eadm_sch_init(void) From 2f6ea6fb88ab9d517644a098fc670b4d5dd1735e Mon Sep 17 00:00:00 2001 From: Vineeth Vijayan Date: Mon, 28 Sep 2020 21:28:20 +0200 Subject: [PATCH 308/360] s390/tape: remove unsupported PM functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The power-management related functions are unused since 'commit 394216275c7d ("s390: remove broken hibernate / power management support")'. Remove them from tape drivers. Signed-off-by: Vineeth Vijayan Reviewed-by: Jan Höppner Signed-off-by: Heiko Carstens --- drivers/s390/char/tape.h | 1 - drivers/s390/char/tape_34xx.c | 1 - drivers/s390/char/tape_3590.c | 1 - drivers/s390/char/tape_core.c | 50 ----------------------------------- 4 files changed, 53 deletions(-) diff --git a/drivers/s390/char/tape.h b/drivers/s390/char/tape.h index e2c60475dfa8..4e5d5efa978f 100644 --- a/drivers/s390/char/tape.h +++ b/drivers/s390/char/tape.h @@ -264,7 +264,6 @@ extern void tape_state_set(struct tape_device *, enum tape_state); extern int tape_generic_online(struct tape_device *, struct tape_discipline *); extern int tape_generic_offline(struct ccw_device *); -extern int tape_generic_pm_suspend(struct ccw_device *); /* Externals from tape_devmap.c */ extern int tape_generic_probe(struct ccw_device *); diff --git a/drivers/s390/char/tape_34xx.c b/drivers/s390/char/tape_34xx.c index 6d73ee3f827a..7ada994d4592 100644 --- a/drivers/s390/char/tape_34xx.c +++ b/drivers/s390/char/tape_34xx.c @@ -1191,7 +1191,6 @@ static struct ccw_driver tape_34xx_driver = { .remove = tape_generic_remove, .set_online = tape_34xx_online, .set_offline = tape_generic_offline, - .freeze = tape_generic_pm_suspend, .int_class = IRQIO_TAP, }; diff --git a/drivers/s390/char/tape_3590.c b/drivers/s390/char/tape_3590.c index 4554cdf4d6bd..ecf8c5006a0e 100644 --- a/drivers/s390/char/tape_3590.c +++ b/drivers/s390/char/tape_3590.c @@ -1651,7 +1651,6 @@ static struct ccw_driver tape_3590_driver = { .remove = tape_generic_remove, .set_offline = tape_generic_offline, .set_online = tape_3590_online, - .freeze = tape_generic_pm_suspend, .int_class = IRQIO_TAP, }; diff --git a/drivers/s390/char/tape_core.c b/drivers/s390/char/tape_core.c index 380e6a67719c..a6d2a4792185 100644 --- a/drivers/s390/char/tape_core.c +++ b/drivers/s390/char/tape_core.c @@ -427,55 +427,6 @@ tape_cleanup_device(struct tape_device *device) tape_med_state_set(device, MS_UNKNOWN); } -/* - * Suspend device. - * - * Called by the common I/O layer if the drive should be suspended on user - * request. We refuse to suspend if the device is loaded or in use for the - * following reason: - * While the Linux guest is suspended, it might be logged off which causes - * devices to be detached. Tape devices are automatically rewound and unloaded - * during DETACH processing (unless the tape device was attached with the - * NOASSIGN or MULTIUSER option). After rewind/unload, there is no way to - * resume the original state of the tape device, since we would need to - * manually re-load the cartridge which was active at suspend time. - */ -int tape_generic_pm_suspend(struct ccw_device *cdev) -{ - struct tape_device *device; - - device = dev_get_drvdata(&cdev->dev); - if (!device) { - return -ENODEV; - } - - DBF_LH(3, "(%08x): tape_generic_pm_suspend(%p)\n", - device->cdev_id, device); - - if (device->medium_state != MS_UNLOADED) { - pr_err("A cartridge is loaded in tape device %s, " - "refusing to suspend\n", dev_name(&cdev->dev)); - return -EBUSY; - } - - spin_lock_irq(get_ccwdev_lock(device->cdev)); - switch (device->tape_state) { - case TS_INIT: - case TS_NOT_OPER: - case TS_UNUSED: - spin_unlock_irq(get_ccwdev_lock(device->cdev)); - break; - default: - pr_err("Tape device %s is busy, refusing to " - "suspend\n", dev_name(&cdev->dev)); - spin_unlock_irq(get_ccwdev_lock(device->cdev)); - return -EBUSY; - } - - DBF_LH(3, "(%08x): Drive suspended.\n", device->cdev_id); - return 0; -} - /* * Set device offline. * @@ -1360,7 +1311,6 @@ EXPORT_SYMBOL(tape_generic_remove); EXPORT_SYMBOL(tape_generic_probe); EXPORT_SYMBOL(tape_generic_online); EXPORT_SYMBOL(tape_generic_offline); -EXPORT_SYMBOL(tape_generic_pm_suspend); EXPORT_SYMBOL(tape_put_device); EXPORT_SYMBOL(tape_get_device); EXPORT_SYMBOL(tape_state_verbose); From 74cee7f3b0ea7fd6231f037ce37be6d9c6d2b4b6 Mon Sep 17 00:00:00 2001 From: Vineeth Vijayan Date: Mon, 28 Sep 2020 21:29:01 +0200 Subject: [PATCH 309/360] s390/vmur: remove unused pm related functions The Power-management related functions are unused since 'commit 394216275c7d ("s390: remove broken hibernate / power management support")'. Remove them from the unit-record device driver. Signed-off-by: Vineeth Vijayan Signed-off-by: Heiko Carstens --- drivers/s390/char/vmur.c | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/drivers/s390/char/vmur.c b/drivers/s390/char/vmur.c index cbde65ab2170..1bbf27b98cf6 100644 --- a/drivers/s390/char/vmur.c +++ b/drivers/s390/char/vmur.c @@ -62,7 +62,6 @@ static int ur_probe(struct ccw_device *cdev); static void ur_remove(struct ccw_device *cdev); static int ur_set_online(struct ccw_device *cdev); static int ur_set_offline(struct ccw_device *cdev); -static int ur_pm_suspend(struct ccw_device *cdev); static struct ccw_driver ur_driver = { .driver = { @@ -74,7 +73,6 @@ static struct ccw_driver ur_driver = { .remove = ur_remove, .set_online = ur_set_online, .set_offline = ur_set_offline, - .freeze = ur_pm_suspend, .int_class = IRQIO_VMR, }; @@ -164,28 +162,6 @@ static void urdev_put(struct urdev *urd) urdev_free(urd); } -/* - * State and contents of ur devices can be changed by class D users issuing - * CP commands such as PURGE or TRANSFER, while the Linux guest is suspended. - * Also the Linux guest might be logged off, which causes all active spool - * files to be closed. - * So we cannot guarantee that spool files are still the same when the Linux - * guest is resumed. In order to avoid unpredictable results at resume time - * we simply refuse to suspend if a ur device node is open. - */ -static int ur_pm_suspend(struct ccw_device *cdev) -{ - struct urdev *urd = dev_get_drvdata(&cdev->dev); - - TRACE("ur_pm_suspend: cdev=%p\n", cdev); - if (urd->open_flag) { - pr_err("Unit record device %s is busy, %s refusing to " - "suspend.\n", dev_name(&cdev->dev), ur_banner); - return -EBUSY; - } - return 0; -} - /* * Low-level functions to do I/O to a ur device. * alloc_chan_prog From bfa11151d223016ceca008dce7f1e55ca78c21bb Mon Sep 17 00:00:00 2001 From: Vineeth Vijayan Date: Thu, 2 Apr 2020 10:35:40 +0200 Subject: [PATCH 310/360] s390/cio: remove pm support from chsc subchannel driver As part of removing the broken pm-support from s390 arch, remove the pm-calls from chsc subchannel driver. The power-management functions are unused since the 'commit 394216275c7d ("s390: remove broken hibernate / power management support")'. Signed-off-by: Vineeth Vijayan Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/chsc_sch.c | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/drivers/s390/cio/chsc_sch.c b/drivers/s390/cio/chsc_sch.c index 8f080d3fd380..c42405c620b5 100644 --- a/drivers/s390/cio/chsc_sch.c +++ b/drivers/s390/cio/chsc_sch.c @@ -120,31 +120,6 @@ static void chsc_subchannel_shutdown(struct subchannel *sch) cio_disable_subchannel(sch); } -static int chsc_subchannel_prepare(struct subchannel *sch) -{ - int cc; - struct schib schib; - /* - * Don't allow suspend while the subchannel is not idle - * since we don't have a way to clear the subchannel and - * cannot disable it with a request running. - */ - cc = stsch(sch->schid, &schib); - if (!cc && scsw_stctl(&schib.scsw)) - return -EAGAIN; - return 0; -} - -static int chsc_subchannel_freeze(struct subchannel *sch) -{ - return cio_disable_subchannel(sch); -} - -static int chsc_subchannel_restore(struct subchannel *sch) -{ - return cio_enable_subchannel(sch, (u32)(unsigned long)sch); -} - static struct css_device_id chsc_subchannel_ids[] = { { .match_flags = 0x1, .type =SUBCHANNEL_TYPE_CHSC, }, { /* end of list */ }, @@ -161,10 +136,6 @@ static struct css_driver chsc_subchannel_driver = { .probe = chsc_subchannel_probe, .remove = chsc_subchannel_remove, .shutdown = chsc_subchannel_shutdown, - .prepare = chsc_subchannel_prepare, - .freeze = chsc_subchannel_freeze, - .thaw = chsc_subchannel_restore, - .restore = chsc_subchannel_restore, }; static int __init chsc_init_dbfs(void) From 796cfabde5b23e1e506994fd11d2609ec2818e8d Mon Sep 17 00:00:00 2001 From: Vineeth Vijayan Date: Thu, 2 Apr 2020 10:34:02 +0200 Subject: [PATCH 311/360] s390/cio: remove pm support from IO subchannel drivers The power-management functions are unused since the 'commit 394216275c7d ("s390: remove broken hibernate / power management support")'. Remove these unused pm callbacks from io-subchannel drivers. Signed-off-by: Vineeth Vijayan Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/device.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c index b29fe8d50baf..9b8ebb29e007 100644 --- a/drivers/s390/cio/device.c +++ b/drivers/s390/cio/device.c @@ -149,19 +149,6 @@ static struct css_device_id io_subchannel_ids[] = { { /* end of list */ }, }; -static int io_subchannel_prepare(struct subchannel *sch) -{ - struct ccw_device *cdev; - /* - * Don't allow suspend while a ccw device registration - * is still outstanding. - */ - cdev = sch_get_cdev(sch); - if (cdev && !device_is_registered(&cdev->dev)) - return -EAGAIN; - return 0; -} - static int io_subchannel_settle(void) { int ret; @@ -186,7 +173,6 @@ static struct css_driver io_subchannel_driver = { .probe = io_subchannel_probe, .remove = io_subchannel_remove, .shutdown = io_subchannel_shutdown, - .prepare = io_subchannel_prepare, .settle = io_subchannel_settle, }; From ef2eea78a6f363a58c909b7a0fd002df7cc52991 Mon Sep 17 00:00:00 2001 From: Vineeth Vijayan Date: Thu, 12 Nov 2020 16:26:29 +0100 Subject: [PATCH 312/360] s390/cio: remove pm support from css-bus driver The power-management functions are unused since the 'commit 394216275c7d ("s390: remove broken hibernate / power management support")'. Remove the unused pm callbacks from css-bus driver. Signed-off-by: Vineeth Vijayan Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- drivers/s390/cio/css.c | 130 +---------------------------------------- drivers/s390/cio/css.h | 10 ---- 2 files changed, 1 insertion(+), 139 deletions(-) diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index cca1a7c4bb33..94c6470de635 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -1044,59 +1043,6 @@ static struct notifier_block css_reboot_notifier = { .notifier_call = css_reboot_event, }; -/* - * Since the css devices are neither on a bus nor have a class - * nor have a special device type, we cannot stop/restart channel - * path measurements via the normal suspend/resume callbacks, but have - * to use notifiers. - */ -static int css_power_event(struct notifier_block *this, unsigned long event, - void *ptr) -{ - struct channel_subsystem *css; - int ret; - - switch (event) { - case PM_HIBERNATION_PREPARE: - case PM_SUSPEND_PREPARE: - ret = NOTIFY_DONE; - for_each_css(css) { - mutex_lock(&css->mutex); - if (!css->cm_enabled) { - mutex_unlock(&css->mutex); - continue; - } - ret = __chsc_do_secm(css, 0); - ret = notifier_from_errno(ret); - mutex_unlock(&css->mutex); - } - break; - case PM_POST_HIBERNATION: - case PM_POST_SUSPEND: - ret = NOTIFY_DONE; - for_each_css(css) { - mutex_lock(&css->mutex); - if (!css->cm_enabled) { - mutex_unlock(&css->mutex); - continue; - } - ret = __chsc_do_secm(css, 1); - ret = notifier_from_errno(ret); - mutex_unlock(&css->mutex); - } - /* search for subchannels, which appeared during hibernation */ - css_schedule_reprobe(); - break; - default: - ret = NOTIFY_DONE; - } - return ret; - -} -static struct notifier_block css_power_notifier = { - .notifier_call = css_power_event, -}; - #define CIO_DMA_GFP (GFP_KERNEL | __GFP_ZERO) static struct gen_pool *cio_dma_pool; @@ -1242,12 +1188,9 @@ static int __init css_bus_init(void) ret = register_reboot_notifier(&css_reboot_notifier); if (ret) goto out_unregister; - ret = register_pm_notifier(&css_power_notifier); - if (ret) - goto out_unregister_rn; ret = cio_dma_pool_init(); if (ret) - goto out_unregister_pmn; + goto out_unregister_rn; airq_init(); css_init_done = 1; @@ -1255,8 +1198,6 @@ static int __init css_bus_init(void) isc_register(IO_SCH_ISC); return 0; -out_unregister_pmn: - unregister_pm_notifier(&css_power_notifier); out_unregister_rn: unregister_reboot_notifier(&css_reboot_notifier); out_unregister: @@ -1456,74 +1397,6 @@ static int css_uevent(struct device *dev, struct kobj_uevent_env *env) return ret; } -static int css_pm_prepare(struct device *dev) -{ - struct subchannel *sch = to_subchannel(dev); - struct css_driver *drv; - - if (mutex_is_locked(&sch->reg_mutex)) - return -EAGAIN; - if (!sch->dev.driver) - return 0; - drv = to_cssdriver(sch->dev.driver); - /* Notify drivers that they may not register children. */ - return drv->prepare ? drv->prepare(sch) : 0; -} - -static void css_pm_complete(struct device *dev) -{ - struct subchannel *sch = to_subchannel(dev); - struct css_driver *drv; - - if (!sch->dev.driver) - return; - drv = to_cssdriver(sch->dev.driver); - if (drv->complete) - drv->complete(sch); -} - -static int css_pm_freeze(struct device *dev) -{ - struct subchannel *sch = to_subchannel(dev); - struct css_driver *drv; - - if (!sch->dev.driver) - return 0; - drv = to_cssdriver(sch->dev.driver); - return drv->freeze ? drv->freeze(sch) : 0; -} - -static int css_pm_thaw(struct device *dev) -{ - struct subchannel *sch = to_subchannel(dev); - struct css_driver *drv; - - if (!sch->dev.driver) - return 0; - drv = to_cssdriver(sch->dev.driver); - return drv->thaw ? drv->thaw(sch) : 0; -} - -static int css_pm_restore(struct device *dev) -{ - struct subchannel *sch = to_subchannel(dev); - struct css_driver *drv; - - css_update_ssd_info(sch); - if (!sch->dev.driver) - return 0; - drv = to_cssdriver(sch->dev.driver); - return drv->restore ? drv->restore(sch) : 0; -} - -static const struct dev_pm_ops css_pm_ops = { - .prepare = css_pm_prepare, - .complete = css_pm_complete, - .freeze = css_pm_freeze, - .thaw = css_pm_thaw, - .restore = css_pm_restore, -}; - static struct bus_type css_bus_type = { .name = "css", .match = css_bus_match, @@ -1531,7 +1404,6 @@ static struct bus_type css_bus_type = { .remove = css_remove, .shutdown = css_shutdown, .uevent = css_uevent, - .pm = &css_pm_ops, }; /** diff --git a/drivers/s390/cio/css.h b/drivers/s390/cio/css.h index 3f322ea0f498..2eddfc47f687 100644 --- a/drivers/s390/cio/css.h +++ b/drivers/s390/cio/css.h @@ -72,11 +72,6 @@ struct chp_link; * @probe: function called on probe * @remove: function called on remove * @shutdown: called at device shutdown - * @prepare: prepare for pm state transition - * @complete: undo work done in @prepare - * @freeze: callback for freezing during hibernation snapshotting - * @thaw: undo work done in @freeze - * @restore: callback for restoring after hibernation * @settle: wait for asynchronous work to finish */ struct css_driver { @@ -88,11 +83,6 @@ struct css_driver { int (*probe)(struct subchannel *); int (*remove)(struct subchannel *); void (*shutdown)(struct subchannel *); - int (*prepare) (struct subchannel *); - void (*complete) (struct subchannel *); - int (*freeze)(struct subchannel *); - int (*thaw) (struct subchannel *); - int (*restore)(struct subchannel *); int (*settle)(void); }; From 8cc0dcfdc1c0e0be107d0288f9c0cf1f4201be62 Mon Sep 17 00:00:00 2001 From: Vineeth Vijayan Date: Fri, 20 Nov 2020 09:36:38 +0100 Subject: [PATCH 313/360] s390/cio: remove pm support from ccw bus driver As part of removing broken pm-support from s390 arch, remove the pm callbacks from ccw-bus driver.The power-management functions are unused since the 'commit 394216275c7d ("s390: remove broken hibernate / power management support")'. Signed-off-by: Vineeth Vijayan Reviewed-by: Peter Oberparleiter Signed-off-by: Heiko Carstens --- arch/s390/include/asm/ccwdev.h | 10 -- drivers/s390/cio/cmf.c | 5 - drivers/s390/cio/device.c | 247 +-------------------------------- drivers/s390/cio/device.h | 1 - drivers/s390/cio/device_fsm.c | 6 - drivers/s390/cio/io_sch.h | 1 - 6 files changed, 2 insertions(+), 268 deletions(-) diff --git a/arch/s390/include/asm/ccwdev.h b/arch/s390/include/asm/ccwdev.h index 069709b8e9e7..e3e2ab0acf83 100644 --- a/arch/s390/include/asm/ccwdev.h +++ b/arch/s390/include/asm/ccwdev.h @@ -124,11 +124,6 @@ enum uc_todo { * @notify: notify driver of device state changes * @path_event: notify driver of channel path events * @shutdown: called at device shutdown - * @prepare: prepare for pm state transition - * @complete: undo work done in @prepare - * @freeze: callback for freezing during hibernation snapshotting - * @thaw: undo work done in @freeze - * @restore: callback for restoring after hibernation * @uc_handler: callback for unit check handler * @driver: embedded device driver structure * @int_class: interruption class to use for accounting interrupts @@ -142,11 +137,6 @@ struct ccw_driver { int (*notify) (struct ccw_device *, int); void (*path_event) (struct ccw_device *, int *); void (*shutdown) (struct ccw_device *); - int (*prepare) (struct ccw_device *); - void (*complete) (struct ccw_device *); - int (*freeze)(struct ccw_device *); - int (*thaw) (struct ccw_device *); - int (*restore)(struct ccw_device *); enum uc_todo (*uc_handler) (struct ccw_device *, struct irb *); struct device_driver driver; enum interruption_class int_class; diff --git a/drivers/s390/cio/cmf.c b/drivers/s390/cio/cmf.c index 72dd2471ec1e..b7b590646d58 100644 --- a/drivers/s390/cio/cmf.c +++ b/drivers/s390/cio/cmf.c @@ -1109,11 +1109,6 @@ static ssize_t cmb_enable_store(struct device *dev, } DEVICE_ATTR_RW(cmb_enable); -int ccw_set_cmf(struct ccw_device *cdev, int enable) -{ - return cmbops->set(cdev, enable ? 2 : 0); -} - /** * enable_cmf() - switch on the channel measurement for a specific device * @cdev: The ccw device to be enabled diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c index 9b8ebb29e007..5fbc549786ab 100644 --- a/drivers/s390/cio/device.c +++ b/drivers/s390/cio/device.c @@ -1408,7 +1408,7 @@ static enum io_sch_action sch_get_action(struct subchannel *sch) } if (device_is_disconnected(cdev)) return IO_SCH_REPROBE; - if (cdev->online && !cdev->private->flags.resuming) + if (cdev->online) return IO_SCH_VERIFY; if (cdev->private->state == DEV_STATE_NOT_OPER) return IO_SCH_UNREG_ATTACH; @@ -1500,11 +1500,6 @@ static int io_subchannel_sch_event(struct subchannel *sch, int process) break; case IO_SCH_UNREG_ATTACH: spin_lock_irqsave(sch->lock, flags); - if (cdev->private->flags.resuming) { - /* Device will be handled later. */ - rc = 0; - goto out_unlock; - } sch_set_cdev(sch, NULL); spin_unlock_irqrestore(sch->lock, flags); /* Unregister ccw device. */ @@ -1517,7 +1512,7 @@ static int io_subchannel_sch_event(struct subchannel *sch, int process) switch (action) { case IO_SCH_ORPH_UNREG: case IO_SCH_UNREG: - if (!cdev || !cdev->private->flags.resuming) + if (!cdev) css_sch_device_unregister(sch); break; case IO_SCH_ORPH_ATTACH: @@ -1676,14 +1671,6 @@ void ccw_device_wait_idle(struct ccw_device *cdev) udelay_simple(100); } } - -static int ccw_device_pm_restore(struct device *dev); - -int ccw_device_force_console(struct ccw_device *cdev) -{ - return ccw_device_pm_restore(&cdev->dev); -} -EXPORT_SYMBOL_GPL(ccw_device_force_console); #endif /** @@ -1784,235 +1771,6 @@ static void ccw_device_shutdown(struct device *dev) __disable_cmf(cdev); } -static int ccw_device_pm_prepare(struct device *dev) -{ - struct ccw_device *cdev = to_ccwdev(dev); - - if (work_pending(&cdev->private->todo_work)) - return -EAGAIN; - /* Fail while device is being set online/offline. */ - if (atomic_read(&cdev->private->onoff)) - return -EAGAIN; - - if (cdev->online && cdev->drv && cdev->drv->prepare) - return cdev->drv->prepare(cdev); - - return 0; -} - -static void ccw_device_pm_complete(struct device *dev) -{ - struct ccw_device *cdev = to_ccwdev(dev); - - if (cdev->online && cdev->drv && cdev->drv->complete) - cdev->drv->complete(cdev); -} - -static int ccw_device_pm_freeze(struct device *dev) -{ - struct ccw_device *cdev = to_ccwdev(dev); - struct subchannel *sch = to_subchannel(cdev->dev.parent); - int ret, cm_enabled; - - /* Fail suspend while device is in transistional state. */ - if (!dev_fsm_final_state(cdev)) - return -EAGAIN; - if (!cdev->online) - return 0; - if (cdev->drv && cdev->drv->freeze) { - ret = cdev->drv->freeze(cdev); - if (ret) - return ret; - } - - spin_lock_irq(sch->lock); - cm_enabled = cdev->private->cmb != NULL; - spin_unlock_irq(sch->lock); - if (cm_enabled) { - /* Don't have the css write on memory. */ - ret = ccw_set_cmf(cdev, 0); - if (ret) - return ret; - } - /* From here on, disallow device driver I/O. */ - spin_lock_irq(sch->lock); - ret = cio_disable_subchannel(sch); - spin_unlock_irq(sch->lock); - - return ret; -} - -static int ccw_device_pm_thaw(struct device *dev) -{ - struct ccw_device *cdev = to_ccwdev(dev); - struct subchannel *sch = to_subchannel(cdev->dev.parent); - int ret, cm_enabled; - - if (!cdev->online) - return 0; - - spin_lock_irq(sch->lock); - /* Allow device driver I/O again. */ - ret = cio_enable_subchannel(sch, (u32)(addr_t)sch); - cm_enabled = cdev->private->cmb != NULL; - spin_unlock_irq(sch->lock); - if (ret) - return ret; - - if (cm_enabled) { - ret = ccw_set_cmf(cdev, 1); - if (ret) - return ret; - } - - if (cdev->drv && cdev->drv->thaw) - ret = cdev->drv->thaw(cdev); - - return ret; -} - -static void __ccw_device_pm_restore(struct ccw_device *cdev) -{ - struct subchannel *sch = to_subchannel(cdev->dev.parent); - - spin_lock_irq(sch->lock); - if (cio_is_console(sch->schid)) { - cio_enable_subchannel(sch, (u32)(addr_t)sch); - goto out_unlock; - } - /* - * While we were sleeping, devices may have gone or become - * available again. Kick re-detection. - */ - cdev->private->flags.resuming = 1; - cdev->private->path_new_mask = LPM_ANYPATH; - css_sched_sch_todo(sch, SCH_TODO_EVAL); - spin_unlock_irq(sch->lock); - css_wait_for_slow_path(); - - /* cdev may have been moved to a different subchannel. */ - sch = to_subchannel(cdev->dev.parent); - spin_lock_irq(sch->lock); - if (cdev->private->state != DEV_STATE_ONLINE && - cdev->private->state != DEV_STATE_OFFLINE) - goto out_unlock; - - ccw_device_recognition(cdev); - spin_unlock_irq(sch->lock); - wait_event(cdev->private->wait_q, dev_fsm_final_state(cdev) || - cdev->private->state == DEV_STATE_DISCONNECTED); - spin_lock_irq(sch->lock); - -out_unlock: - cdev->private->flags.resuming = 0; - spin_unlock_irq(sch->lock); -} - -static int resume_handle_boxed(struct ccw_device *cdev) -{ - cdev->private->state = DEV_STATE_BOXED; - if (ccw_device_notify(cdev, CIO_BOXED) == NOTIFY_OK) - return 0; - ccw_device_sched_todo(cdev, CDEV_TODO_UNREG); - return -ENODEV; -} - -static int resume_handle_disc(struct ccw_device *cdev) -{ - cdev->private->state = DEV_STATE_DISCONNECTED; - if (ccw_device_notify(cdev, CIO_GONE) == NOTIFY_OK) - return 0; - ccw_device_sched_todo(cdev, CDEV_TODO_UNREG); - return -ENODEV; -} - -static int ccw_device_pm_restore(struct device *dev) -{ - struct ccw_device *cdev = to_ccwdev(dev); - struct subchannel *sch; - int ret = 0; - - __ccw_device_pm_restore(cdev); - sch = to_subchannel(cdev->dev.parent); - spin_lock_irq(sch->lock); - if (cio_is_console(sch->schid)) - goto out_restore; - - /* check recognition results */ - switch (cdev->private->state) { - case DEV_STATE_OFFLINE: - case DEV_STATE_ONLINE: - cdev->private->flags.donotify = 0; - break; - case DEV_STATE_BOXED: - ret = resume_handle_boxed(cdev); - if (ret) - goto out_unlock; - goto out_restore; - default: - ret = resume_handle_disc(cdev); - if (ret) - goto out_unlock; - goto out_restore; - } - /* check if the device type has changed */ - if (!ccw_device_test_sense_data(cdev)) { - ccw_device_update_sense_data(cdev); - ccw_device_sched_todo(cdev, CDEV_TODO_REBIND); - ret = -ENODEV; - goto out_unlock; - } - if (!cdev->online) - goto out_unlock; - - if (ccw_device_online(cdev)) { - ret = resume_handle_disc(cdev); - if (ret) - goto out_unlock; - goto out_restore; - } - spin_unlock_irq(sch->lock); - wait_event(cdev->private->wait_q, dev_fsm_final_state(cdev)); - spin_lock_irq(sch->lock); - - if (ccw_device_notify(cdev, CIO_OPER) == NOTIFY_BAD) { - ccw_device_sched_todo(cdev, CDEV_TODO_UNREG); - ret = -ENODEV; - goto out_unlock; - } - - /* reenable cmf, if needed */ - if (cdev->private->cmb) { - spin_unlock_irq(sch->lock); - ret = ccw_set_cmf(cdev, 1); - spin_lock_irq(sch->lock); - if (ret) { - CIO_MSG_EVENT(2, "resume: cdev 0.%x.%04x: cmf failed " - "(rc=%d)\n", cdev->private->dev_id.ssid, - cdev->private->dev_id.devno, ret); - ret = 0; - } - } - -out_restore: - spin_unlock_irq(sch->lock); - if (cdev->online && cdev->drv && cdev->drv->restore) - ret = cdev->drv->restore(cdev); - return ret; - -out_unlock: - spin_unlock_irq(sch->lock); - return ret; -} - -static const struct dev_pm_ops ccw_pm_ops = { - .prepare = ccw_device_pm_prepare, - .complete = ccw_device_pm_complete, - .freeze = ccw_device_pm_freeze, - .thaw = ccw_device_pm_thaw, - .restore = ccw_device_pm_restore, -}; - static struct bus_type ccw_bus_type = { .name = "ccw", .match = ccw_bus_match, @@ -2020,7 +1778,6 @@ static struct bus_type ccw_bus_type = { .probe = ccw_device_probe, .remove = ccw_device_remove, .shutdown = ccw_device_shutdown, - .pm = &ccw_pm_ops, }; /** diff --git a/drivers/s390/cio/device.h b/drivers/s390/cio/device.h index 853b6a8ca095..24b2fce69590 100644 --- a/drivers/s390/cio/device.h +++ b/drivers/s390/cio/device.h @@ -143,6 +143,5 @@ void retry_set_schib(struct ccw_device *cdev); void cmf_retry_copy_block(struct ccw_device *); int cmf_reenable(struct ccw_device *); void cmf_reactivate(void); -int ccw_set_cmf(struct ccw_device *cdev, int enable); extern struct device_attribute dev_attr_cmb_enable; #endif diff --git a/drivers/s390/cio/device_fsm.c b/drivers/s390/cio/device_fsm.c index 8fc267324ebb..6420b197bb05 100644 --- a/drivers/s390/cio/device_fsm.c +++ b/drivers/s390/cio/device_fsm.c @@ -224,12 +224,6 @@ ccw_device_recog_done(struct ccw_device *cdev, int state) wake_up(&cdev->private->wait_q); return; } - if (cdev->private->flags.resuming) { - cdev->private->state = state; - cdev->private->flags.recog_done = 1; - wake_up(&cdev->private->wait_q); - return; - } switch (state) { case DEV_STATE_NOT_OPER: break; diff --git a/drivers/s390/cio/io_sch.h b/drivers/s390/cio/io_sch.h index c03b4a19974e..85a11c1836e5 100644 --- a/drivers/s390/cio/io_sch.h +++ b/drivers/s390/cio/io_sch.h @@ -160,7 +160,6 @@ struct ccw_device_private { unsigned int donotify:1; /* call notify function */ unsigned int recog_done:1; /* dev. recog. complete */ unsigned int fake_irb:2; /* deliver faked irb */ - unsigned int resuming:1; /* recognition while resume */ unsigned int pgroup:1; /* pathgroup is set up */ unsigned int mpath:1; /* multipathing is set up */ unsigned int pgid_unknown:1;/* unknown pgid state */ From 18d047bd89b8c1f9ba3c9b2d2f7309c953b3ce97 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 2 Dec 2020 17:44:07 +0300 Subject: [PATCH 314/360] x86/platform/uv: Fix an error code in uv_hubs_init() Return -ENOMEM on allocation failure instead of returning random stack memory contents. Fixes: 4fc2cf1f2daf ("x86/platform/uv: Add new uv_sysfs platform driver") Signed-off-by: Dan Carpenter Signed-off-by: Borislav Petkov Reviewed-by: Justin Ernst Link: https://lkml.kernel.org/r/X8eoN/jMAJb3H3iv@mwanda --- drivers/platform/x86/uv_sysfs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/platform/x86/uv_sysfs.c b/drivers/platform/x86/uv_sysfs.c index 54c342579f1c..e17ce8c4cdad 100644 --- a/drivers/platform/x86/uv_sysfs.c +++ b/drivers/platform/x86/uv_sysfs.c @@ -248,6 +248,7 @@ static int uv_hubs_init(void) uv_hubs[i] = kzalloc(sizeof(*uv_hubs[i]), GFP_KERNEL); if (!uv_hubs[i]) { i--; + ret = -ENOMEM; goto err_hubs; } From 0c683e9de0c78ee53e220eac9ee3604ca662737a Mon Sep 17 00:00:00 2001 From: Zou Wei Date: Mon, 30 Nov 2020 19:11:53 +0800 Subject: [PATCH 315/360] x86/platform/uv: Make uv_pcibus_kset and uv_hubs_kset static Fix the following sparse warnings: drivers/platform/x86/uv_sysfs.c:22:13: warning: symbol \ 'uv_pcibus_kset' was not declared. Should it be static? drivers/platform/x86/uv_sysfs.c:23:13: warning: symbol \ 'uv_hubs_kset' was not declared. Should it be static? Signed-off-by: Zou Wei Signed-off-by: Borislav Petkov Acked-by: Hans de Goede Link: https://lkml.kernel.org/r/1606734713-43919-1-git-send-email-zou_wei@huawei.com --- drivers/platform/x86/uv_sysfs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/platform/x86/uv_sysfs.c b/drivers/platform/x86/uv_sysfs.c index e17ce8c4cdad..c27f5ffd43bb 100644 --- a/drivers/platform/x86/uv_sysfs.c +++ b/drivers/platform/x86/uv_sysfs.c @@ -19,8 +19,8 @@ #define INVALID_CNODE -1 struct kobject *sgi_uv_kobj; -struct kset *uv_pcibus_kset; -struct kset *uv_hubs_kset; +static struct kset *uv_pcibus_kset; +static struct kset *uv_hubs_kset; static struct uv_bios_hub_info *hub_buf; static struct uv_bios_port_info **port_buf; static struct uv_hub **uv_hubs; From a4b9c48b96517ff4780b22a784e7537eac5dc21b Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Thu, 3 Dec 2020 20:35:27 +0200 Subject: [PATCH 316/360] x86/sgx: Return -EINVAL on a zero length buffer in sgx_ioc_enclave_add_pages() The sgx_enclave_add_pages.length field is documented as * @length: length of the data (multiple of the page size) Fail with -EINVAL, when the caller gives a zero length buffer of data to be added as pages to an enclave. Right now 'ret' is returned as uninitialized in that case. [ bp: Flesh out commit message. ] Fixes: c6d26d370767 ("x86/sgx: Add SGX_IOC_ENCLAVE_ADD_PAGES") Reported-by: Dan Carpenter Signed-off-by: Jarkko Sakkinen Signed-off-by: Borislav Petkov Link: https://lore.kernel.org/linux-sgx/X8ehQssnslm194ld@mwanda/ Link: https://lkml.kernel.org/r/20201203183527.139317-1-jarkko@kernel.org --- arch/x86/kernel/cpu/sgx/ioctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/cpu/sgx/ioctl.c b/arch/x86/kernel/cpu/sgx/ioctl.c index c206aee80a04..90a5caf76939 100644 --- a/arch/x86/kernel/cpu/sgx/ioctl.c +++ b/arch/x86/kernel/cpu/sgx/ioctl.c @@ -428,7 +428,7 @@ static long sgx_ioc_enclave_add_pages(struct sgx_encl *encl, void __user *arg) !IS_ALIGNED(add_arg.src, PAGE_SIZE)) return -EINVAL; - if (add_arg.length & (PAGE_SIZE - 1)) + if (!add_arg.length || add_arg.length & (PAGE_SIZE - 1)) return -EINVAL; if (add_arg.offset + add_arg.length - PAGE_SIZE >= encl->size) From 17858b140bf49961b71d4e73f1c3ea9bc8e7dda0 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 24 Nov 2020 11:47:19 +0100 Subject: [PATCH 317/360] crypto: ecdh - avoid unaligned accesses in ecdh_set_secret() ecdh_set_secret() casts a void* pointer to a const u64* in order to feed it into ecc_is_key_valid(). This is not generally permitted by the C standard, and leads to actual misalignment faults on ARMv6 cores. In some cases, these are fixed up in software, but this still leads to performance hits that are entirely avoidable. So let's copy the key into the ctx buffer first, which we will do anyway in the common case, and which guarantees correct alignment. Cc: Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/ecdh.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crypto/ecdh.c b/crypto/ecdh.c index b0232d6ab4ce..d56b8603dec9 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -53,12 +53,13 @@ static int ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, return ecc_gen_privkey(ctx->curve_id, ctx->ndigits, ctx->private_key); - if (ecc_is_key_valid(ctx->curve_id, ctx->ndigits, - (const u64 *)params.key, params.key_size) < 0) - return -EINVAL; - memcpy(ctx->private_key, params.key, params.key_size); + if (ecc_is_key_valid(ctx->curve_id, ctx->ndigits, + ctx->private_key, params.key_size) < 0) { + memzero_explicit(ctx->private_key, params.key_size); + return -EINVAL; + } return 0; } From f3456b9fd269c6d0c973b136c5449d46b2510f4b Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 26 Nov 2020 08:49:07 +0100 Subject: [PATCH 318/360] crypto: arm/aes-ce - work around Cortex-A57/A72 silion errata ARM Cortex-A57 and Cortex-A72 cores running in 32-bit mode are affected by silicon errata #1742098 and #1655431, respectively, where the second instruction of a AES instruction pair may execute twice if an interrupt is taken right after the first instruction consumes an input register of which a single 32-bit lane has been updated the last time it was modified. This is not such a rare occurrence as it may seem: in counter mode, only the least significant 32-bit word is incremented in the absence of a carry, which makes our counter mode implementation susceptible to these errata. So let's shuffle the counter assignments around a bit so that the most recent updates when the AES instruction pair executes are 128-bit wide. [0] ARM-EPM-049219 v23 Cortex-A57 MPCore Software Developers Errata Notice [1] ARM-EPM-012079 v11.0 Cortex-A72 MPCore Software Developers Errata Notice Cc: # v5.4+ Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- arch/arm/crypto/aes-ce-core.S | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/arch/arm/crypto/aes-ce-core.S b/arch/arm/crypto/aes-ce-core.S index 4d1707388d94..312428d83eed 100644 --- a/arch/arm/crypto/aes-ce-core.S +++ b/arch/arm/crypto/aes-ce-core.S @@ -386,20 +386,32 @@ ENTRY(ce_aes_ctr_encrypt) .Lctrloop4x: subs r4, r4, #4 bmi .Lctr1x - add r6, r6, #1 + + /* + * NOTE: the sequence below has been carefully tweaked to avoid + * a silicon erratum that exists in Cortex-A57 (#1742098) and + * Cortex-A72 (#1655431) cores, where AESE/AESMC instruction pairs + * may produce an incorrect result if they take their input from a + * register of which a single 32-bit lane has been updated the last + * time it was modified. To work around this, the lanes of registers + * q0-q3 below are not manipulated individually, and the different + * counter values are prepared by successive manipulations of q7. + */ + add ip, r6, #1 vmov q0, q7 + rev ip, ip + add lr, r6, #2 + vmov s31, ip @ set lane 3 of q1 via q7 + add ip, r6, #3 + rev lr, lr vmov q1, q7 - rev ip, r6 - add r6, r6, #1 + vmov s31, lr @ set lane 3 of q2 via q7 + rev ip, ip vmov q2, q7 - vmov s7, ip - rev ip, r6 - add r6, r6, #1 + vmov s31, ip @ set lane 3 of q3 via q7 + add r6, r6, #4 vmov q3, q7 - vmov s11, ip - rev ip, r6 - add r6, r6, #1 - vmov s15, ip + vld1.8 {q4-q5}, [r1]! vld1.8 {q6}, [r1]! vld1.8 {q15}, [r1]! From ce0d5d63e897cc7c3a8fd043c7942fc6a78ec6f4 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 27 Nov 2020 16:43:18 +1100 Subject: [PATCH 319/360] crypto: lib/blake2s - Move selftest prototype into header file This patch fixes a missing prototype warning on blake2s_selftest. Reported-by: kernel test robot Signed-off-by: Herbert Xu --- include/crypto/internal/blake2s.h | 2 ++ lib/crypto/blake2s-selftest.c | 2 +- lib/crypto/blake2s.c | 2 -- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/crypto/internal/blake2s.h b/include/crypto/internal/blake2s.h index 74ff77032e52..6e376ae6b6b5 100644 --- a/include/crypto/internal/blake2s.h +++ b/include/crypto/internal/blake2s.h @@ -16,6 +16,8 @@ void blake2s_compress_generic(struct blake2s_state *state,const u8 *block, void blake2s_compress_arch(struct blake2s_state *state,const u8 *block, size_t nblocks, const u32 inc); +bool blake2s_selftest(void); + static inline void blake2s_set_lastblock(struct blake2s_state *state) { state->f[0] = -1; diff --git a/lib/crypto/blake2s-selftest.c b/lib/crypto/blake2s-selftest.c index 79ef404a990d..5d9ea53be973 100644 --- a/lib/crypto/blake2s-selftest.c +++ b/lib/crypto/blake2s-selftest.c @@ -3,7 +3,7 @@ * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. */ -#include +#include #include /* diff --git a/lib/crypto/blake2s.c b/lib/crypto/blake2s.c index 41025a30c524..6a4b6b78d630 100644 --- a/lib/crypto/blake2s.c +++ b/lib/crypto/blake2s.c @@ -17,8 +17,6 @@ #include #include -bool blake2s_selftest(void); - void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen) { const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen; From a1315dcb7b6a7d3a78df848eed5b331a4b3ec28a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 27 Nov 2020 16:49:40 +1100 Subject: [PATCH 320/360] hwrng: ks-sa - Add dependency on IOMEM and OF This patch adds a dependency for KEYSTONE on HAS_IOMEM and OF to prevent COMPILE_TEST build failures. Reported-by: kernel test robot Signed-off-by: Herbert Xu --- drivers/char/hw_random/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index eb0d51a78b44..17c1df8c909a 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -495,6 +495,7 @@ config HW_RANDOM_NPCM config HW_RANDOM_KEYSTONE depends on ARCH_KEYSTONE || COMPILE_TEST + depends on HAS_IOMEM && OF default HW_RANDOM tristate "TI Keystone NETCP SA Hardware random number generator" help From f2d4576a9d38f0a35bc1a5436f5e6e15b6c46aa4 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 27 Nov 2020 17:23:29 +1100 Subject: [PATCH 321/360] crypto: cpt - Fix sparse warnings in cptpf This patch fixes a few sparse warnings that were missed in the last round. Signed-off-by: Herbert Xu --- drivers/crypto/cavium/cpt/cptpf_main.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c index 24d63bdc5dd2..711b1acdd4e0 100644 --- a/drivers/crypto/cavium/cpt/cptpf_main.c +++ b/drivers/crypto/cavium/cpt/cptpf_main.c @@ -244,7 +244,7 @@ cpt_init_fail: struct ucode_header { u8 version[CPT_UCODE_VERSION_SZ]; - u32 code_length; + __be32 code_length; u32 data_length; u64 sram_address; }; @@ -288,10 +288,10 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae) /* Byte swap 64-bit */ for (j = 0; j < (mcode->code_size / 8); j++) - ((u64 *)mcode->code)[j] = cpu_to_be64(((u64 *)mcode->code)[j]); + ((__be64 *)mcode->code)[j] = cpu_to_be64(((u64 *)mcode->code)[j]); /* MC needs 16-bit swap */ for (j = 0; j < (mcode->code_size / 2); j++) - ((u16 *)mcode->code)[j] = cpu_to_be16(((u16 *)mcode->code)[j]); + ((__be16 *)mcode->code)[j] = cpu_to_be16(((u16 *)mcode->code)[j]); dev_dbg(dev, "mcode->code_size = %u\n", mcode->code_size); dev_dbg(dev, "mcode->is_ae = %u\n", mcode->is_ae); From 032d049ea0f45b45c21f3f02b542aa18bc6b6428 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 27 Nov 2020 10:44:52 +0100 Subject: [PATCH 322/360] crypto: aesni - Use TEST %reg,%reg instead of CMP $0,%reg CMP $0,%reg can't set overflow flag, so we can use shorter TEST %reg,%reg instruction when only zero and sign flags are checked (E,L,LE,G,GE conditions). Signed-off-by: Uros Bizjak Cc: Herbert Xu Cc: Borislav Petkov Cc: "H. Peter Anvin" Signed-off-by: Herbert Xu --- arch/x86/crypto/aesni-intel_asm.S | 20 ++++++++++---------- arch/x86/crypto/aesni-intel_avx-x86_64.S | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index 1852b19a73a0..d1436c37008b 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -318,7 +318,7 @@ _initial_blocks_\@: # Main loop - Encrypt/Decrypt remaining blocks - cmp $0, %r13 + test %r13, %r13 je _zero_cipher_left_\@ sub $64, %r13 je _four_cipher_left_\@ @@ -437,7 +437,7 @@ _multiple_of_16_bytes_\@: mov PBlockLen(%arg2), %r12 - cmp $0, %r12 + test %r12, %r12 je _partial_done\@ GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6 @@ -474,7 +474,7 @@ _T_8_\@: add $8, %r10 sub $8, %r11 psrldq $8, %xmm0 - cmp $0, %r11 + test %r11, %r11 je _return_T_done_\@ _T_4_\@: movd %xmm0, %eax @@ -482,7 +482,7 @@ _T_4_\@: add $4, %r10 sub $4, %r11 psrldq $4, %xmm0 - cmp $0, %r11 + test %r11, %r11 je _return_T_done_\@ _T_123_\@: movd %xmm0, %eax @@ -619,7 +619,7 @@ _get_AAD_blocks\@: /* read the last <16B of AAD */ _get_AAD_rest\@: - cmp $0, %r11 + test %r11, %r11 je _get_AAD_done\@ READ_PARTIAL_BLOCK %r10, %r11, \TMP1, \TMP7 @@ -640,7 +640,7 @@ _get_AAD_done\@: .macro PARTIAL_BLOCK CYPH_PLAIN_OUT PLAIN_CYPH_IN PLAIN_CYPH_LEN DATA_OFFSET \ AAD_HASH operation mov PBlockLen(%arg2), %r13 - cmp $0, %r13 + test %r13, %r13 je _partial_block_done_\@ # Leave Macro if no partial blocks # Read in input data without over reading cmp $16, \PLAIN_CYPH_LEN @@ -692,7 +692,7 @@ _no_extra_mask_1_\@: pshufb %xmm2, %xmm3 pxor %xmm3, \AAD_HASH - cmp $0, %r10 + test %r10, %r10 jl _partial_incomplete_1_\@ # GHASH computation for the last <16 Byte block @@ -727,7 +727,7 @@ _no_extra_mask_2_\@: pshufb %xmm2, %xmm9 pxor %xmm9, \AAD_HASH - cmp $0, %r10 + test %r10, %r10 jl _partial_incomplete_2_\@ # GHASH computation for the last <16 Byte block @@ -747,7 +747,7 @@ _encode_done_\@: pshufb %xmm2, %xmm9 .endif # output encrypted Bytes - cmp $0, %r10 + test %r10, %r10 jl _partial_fill_\@ mov %r13, %r12 mov $16, %r13 @@ -2720,7 +2720,7 @@ SYM_FUNC_END(aesni_ctr_enc) */ SYM_FUNC_START(aesni_xts_crypt8) FRAME_BEGIN - cmpb $0, %cl + testb %cl, %cl movl $0, %ecx movl $240, %r10d leaq _aesni_enc4, %r11 diff --git a/arch/x86/crypto/aesni-intel_avx-x86_64.S b/arch/x86/crypto/aesni-intel_avx-x86_64.S index 5fee47956f3b..2cf8e94d986a 100644 --- a/arch/x86/crypto/aesni-intel_avx-x86_64.S +++ b/arch/x86/crypto/aesni-intel_avx-x86_64.S @@ -369,7 +369,7 @@ _initial_num_blocks_is_0\@: _initial_blocks_encrypted\@: - cmp $0, %r13 + test %r13, %r13 je _zero_cipher_left\@ sub $128, %r13 @@ -528,7 +528,7 @@ _multiple_of_16_bytes\@: vmovdqu HashKey(arg2), %xmm13 mov PBlockLen(arg2), %r12 - cmp $0, %r12 + test %r12, %r12 je _partial_done\@ #GHASH computation for the last <16 Byte block @@ -573,7 +573,7 @@ _T_8\@: add $8, %r10 sub $8, %r11 vpsrldq $8, %xmm9, %xmm9 - cmp $0, %r11 + test %r11, %r11 je _return_T_done\@ _T_4\@: vmovd %xmm9, %eax @@ -581,7 +581,7 @@ _T_4\@: add $4, %r10 sub $4, %r11 vpsrldq $4, %xmm9, %xmm9 - cmp $0, %r11 + test %r11, %r11 je _return_T_done\@ _T_123\@: vmovd %xmm9, %eax @@ -625,7 +625,7 @@ _get_AAD_blocks\@: cmp $16, %r11 jge _get_AAD_blocks\@ vmovdqu \T8, \T7 - cmp $0, %r11 + test %r11, %r11 je _get_AAD_done\@ vpxor \T7, \T7, \T7 @@ -644,7 +644,7 @@ _get_AAD_rest8\@: vpxor \T1, \T7, \T7 jmp _get_AAD_rest8\@ _get_AAD_rest4\@: - cmp $0, %r11 + test %r11, %r11 jle _get_AAD_rest0\@ mov (%r10), %eax movq %rax, \T1 @@ -749,7 +749,7 @@ _done_read_partial_block_\@: .macro PARTIAL_BLOCK GHASH_MUL CYPH_PLAIN_OUT PLAIN_CYPH_IN PLAIN_CYPH_LEN DATA_OFFSET \ AAD_HASH ENC_DEC mov PBlockLen(arg2), %r13 - cmp $0, %r13 + test %r13, %r13 je _partial_block_done_\@ # Leave Macro if no partial blocks # Read in input data without over reading cmp $16, \PLAIN_CYPH_LEN @@ -801,7 +801,7 @@ _no_extra_mask_1_\@: vpshufb %xmm2, %xmm3, %xmm3 vpxor %xmm3, \AAD_HASH, \AAD_HASH - cmp $0, %r10 + test %r10, %r10 jl _partial_incomplete_1_\@ # GHASH computation for the last <16 Byte block @@ -836,7 +836,7 @@ _no_extra_mask_2_\@: vpshufb %xmm2, %xmm9, %xmm9 vpxor %xmm9, \AAD_HASH, \AAD_HASH - cmp $0, %r10 + test %r10, %r10 jl _partial_incomplete_2_\@ # GHASH computation for the last <16 Byte block @@ -856,7 +856,7 @@ _encode_done_\@: vpshufb %xmm2, %xmm9, %xmm9 .endif # output encrypted Bytes - cmp $0, %r10 + test %r10, %r10 jl _partial_fill_\@ mov %r13, %r12 mov $16, %r13 From 0b837f1ef8822a8df880ffba6778ba42f707b83d Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 27 Nov 2020 10:59:43 +0100 Subject: [PATCH 323/360] crypto: x86/sha512 - Use TEST %reg,%reg instead of CMP $0,%reg CMP $0,%reg can't set overflow flag, so we can use shorter TEST %reg,%reg instruction when only zero and sign flags are checked (E,L,LE,G,GE conditions). Signed-off-by: Uros Bizjak Cc: Herbert Xu Cc: Borislav Petkov Cc: "H. Peter Anvin" Signed-off-by: Herbert Xu --- arch/x86/crypto/sha512-avx-asm.S | 2 +- arch/x86/crypto/sha512-ssse3-asm.S | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/crypto/sha512-avx-asm.S b/arch/x86/crypto/sha512-avx-asm.S index 63470fd6ae32..684d58c8bc4f 100644 --- a/arch/x86/crypto/sha512-avx-asm.S +++ b/arch/x86/crypto/sha512-avx-asm.S @@ -278,7 +278,7 @@ frame_size = frame_GPRSAVE + GPRSAVE_SIZE # "blocks" is the message length in SHA512 blocks ######################################################################## SYM_FUNC_START(sha512_transform_avx) - cmp $0, msglen + test msglen, msglen je nowork # Allocate Stack Space diff --git a/arch/x86/crypto/sha512-ssse3-asm.S b/arch/x86/crypto/sha512-ssse3-asm.S index 7946a1bee85b..50812af0b083 100644 --- a/arch/x86/crypto/sha512-ssse3-asm.S +++ b/arch/x86/crypto/sha512-ssse3-asm.S @@ -280,7 +280,7 @@ frame_size = frame_GPRSAVE + GPRSAVE_SIZE ######################################################################## SYM_FUNC_START(sha512_transform_ssse3) - cmp $0, msglen + test msglen, msglen je nowork # Allocate Stack Space From be169fe3cec9efe2bfff98b3f645bca6cc7d09cd Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Fri, 27 Nov 2020 11:18:12 +0100 Subject: [PATCH 324/360] crypto: x86/poly1305 - Use TEST %reg,%reg instead of CMP $0,%reg CMP $0,%reg can't set overflow flag, so we can use shorter TEST %reg,%reg instruction when only zero and sign flags are checked (E,L,LE,G,GE conditions). Signed-off-by: Uros Bizjak Cc: Herbert Xu Cc: Borislav Petkov Cc: "H. Peter Anvin" Signed-off-by: Herbert Xu --- arch/x86/crypto/poly1305-x86_64-cryptogams.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl index 7d568012cc15..71fae5a09e56 100644 --- a/arch/x86/crypto/poly1305-x86_64-cryptogams.pl +++ b/arch/x86/crypto/poly1305-x86_64-cryptogams.pl @@ -251,7 +251,7 @@ $code.=<<___; mov %rax,8($ctx) mov %rax,16($ctx) - cmp \$0,$inp + test $inp,$inp je .Lno_key ___ $code.=<<___ if (!$kernel); From 1069e97688b21b9c754dc8364ccfb3fea79788bf Mon Sep 17 00:00:00 2001 From: Tom Rix Date: Fri, 27 Nov 2020 08:23:45 -0800 Subject: [PATCH 325/360] crypto: seed - remove trailing semicolon in macro definition The macro use will already have a semicolon. Signed-off-by: Tom Rix Signed-off-by: Herbert Xu --- crypto/seed.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/seed.c b/crypto/seed.c index 5e3bef3a617d..27720140820e 100644 --- a/crypto/seed.c +++ b/crypto/seed.c @@ -322,7 +322,7 @@ static const u32 KC[SEED_NUM_KCONSTANTS] = { SS2[byte(t1, 2)] ^ SS3[byte(t1, 3)]; \ t0 += t1; \ X1 ^= t0; \ - X2 ^= t1; + X2 ^= t1 static int seed_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) From 0464e0ef4f144bd5f2fa6ef6e06ab1e0ae4806f6 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 30 Nov 2020 13:26:20 +0100 Subject: [PATCH 326/360] crypto: aegis128 - avoid spurious references crypto_aegis128_update_simd Geert reports that builds where CONFIG_CRYPTO_AEGIS128_SIMD is not set may still emit references to crypto_aegis128_update_simd(), which cannot be satisfied and therefore break the build. These references only exist in functions that can be optimized away, but apparently, the compiler is not always able to prove this. So add some explicit checks for CONFIG_CRYPTO_AEGIS128_SIMD to help the compiler figure this out. Tested-by: Geert Uytterhoeven Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- crypto/aegis128-core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/aegis128-core.c b/crypto/aegis128-core.c index 2b05f79475d3..89dc1c559689 100644 --- a/crypto/aegis128-core.c +++ b/crypto/aegis128-core.c @@ -89,7 +89,7 @@ static void crypto_aegis128_update_a(struct aegis_state *state, const union aegis_block *msg, bool do_simd) { - if (do_simd) { + if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && do_simd) { crypto_aegis128_update_simd(state, msg); return; } @@ -101,7 +101,7 @@ static void crypto_aegis128_update_a(struct aegis_state *state, static void crypto_aegis128_update_u(struct aegis_state *state, const void *msg, bool do_simd) { - if (do_simd) { + if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) && do_simd) { crypto_aegis128_update_simd(state, msg); return; } From 2c9cfbadfa234b03473f1ef54e6f4772cc07a371 Mon Sep 17 00:00:00 2001 From: Finn Thain Date: Fri, 20 Nov 2020 15:39:56 +1100 Subject: [PATCH 327/360] macintosh/adb-iop: Always wait for reply message from IOP A recent patch incorrectly altered the adb-iop state machine behaviour and introduced a regression that can appear intermittently as a malfunctioning ADB input device. This seems to be caused when reply packets from different ADB commands become mixed up, especially during the adb bus scan. Fix this by unconditionally entering the awaiting_reply state after sending an explicit command, even when the ADB command won't generate a reply from the ADB device. It turns out that the IOP always generates reply messages, even when the ADB command does not produce a reply packet (e.g. ADB Listen command). So it's not really the ADB reply packets that are being mixed up, it's the IOP messages that enclose them. The bug goes like this: 1. CPU sends a message to the IOP, expecting no response because this message contains an ADB Listen command. The ADB command is now considered complete. 2. CPU sends a second message to the IOP, this time expecting a response because this message contains an ADB Talk command. This ADB command needs a reply before it can be completed. 3. adb-iop driver receives an IOP message and assumes that it relates to the Talk command. It's actually an empty one (with flags == ADB_IOP_EXPLICIT|ADB_IOP_TIMEOUT) for the previous command. The Talk command is now considered complete but it gets the wrong reply data. 4. adb-iop driver gets another IOP response message, which contains the actual reply data for the Talk command, but this is dropped (the driver is no longer in awaiting_reply state). Cc: Joshua Thompson Fixes: e2954e5f727f ("macintosh/adb-iop: Implement sending -> idle state transition") Tested-by: Stan Johnson Signed-off-by: Finn Thain Link: https://lore.kernel.org/r/0f0a25855391e7eaa53a50f651aea0124e8525dd.1605847196.git.fthain@telegraphics.com.au Signed-off-by: Geert Uytterhoeven --- drivers/macintosh/adb-iop.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/macintosh/adb-iop.c b/drivers/macintosh/adb-iop.c index f3d1a460fbce..422abd1d48e1 100644 --- a/drivers/macintosh/adb-iop.c +++ b/drivers/macintosh/adb-iop.c @@ -78,10 +78,7 @@ static void adb_iop_complete(struct iop_msg *msg) local_irq_save(flags); - if (current_req->reply_expected) - adb_iop_state = awaiting_reply; - else - adb_iop_done(); + adb_iop_state = awaiting_reply; local_irq_restore(flags); } @@ -89,8 +86,9 @@ static void adb_iop_complete(struct iop_msg *msg) /* * Listen for ADB messages from the IOP. * - * This will be called when unsolicited messages (usually replies to TALK - * commands or autopoll packets) are received. + * This will be called when unsolicited IOP messages are received. + * These IOP messages can carry ADB autopoll responses and also occur + * after explicit ADB commands. */ static void adb_iop_listen(struct iop_msg *msg) @@ -110,8 +108,10 @@ static void adb_iop_listen(struct iop_msg *msg) if (adb_iop_state == awaiting_reply) { struct adb_request *req = current_req; - req->reply_len = amsg->count + 1; - memcpy(req->reply, &amsg->cmd, req->reply_len); + if (req->reply_expected) { + req->reply_len = amsg->count + 1; + memcpy(req->reply, &amsg->cmd, req->reply_len); + } req_done = true; } From 10199e90ee20e68859f8128331ec8d85b036d349 Mon Sep 17 00:00:00 2001 From: Finn Thain Date: Fri, 20 Nov 2020 15:39:56 +1100 Subject: [PATCH 328/360] macintosh/adb-iop: Send correct poll command The behaviour of the IOP firmware is not well documented but we do know that IOP message reply data can be used to issue new ADB commands. Use the message reply to better control autopoll behaviour by sending a Talk Register 0 command after every ADB response, not unlike the algorithm in the via-macii driver. This poll command is addressed to that device which last received a Talk command (explicit or otherwise). Cc: Joshua Thompson Fixes: 32226e817043 ("macintosh/adb-iop: Implement idle -> sending state transition") Tested-by: Stan Johnson Signed-off-by: Finn Thain Link: https://lore.kernel.org/r/58bba4310da4c29b068345a4b36af8a531397ff7.1605847196.git.fthain@telegraphics.com.au Signed-off-by: Geert Uytterhoeven --- drivers/macintosh/adb-iop.c | 38 ++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/drivers/macintosh/adb-iop.c b/drivers/macintosh/adb-iop.c index 422abd1d48e1..0ee327249150 100644 --- a/drivers/macintosh/adb-iop.c +++ b/drivers/macintosh/adb-iop.c @@ -25,6 +25,7 @@ static struct adb_request *current_req; static struct adb_request *last_req; static unsigned int autopoll_devs; +static u8 autopoll_addr; static enum adb_iop_state { idle, @@ -41,6 +42,11 @@ static int adb_iop_autopoll(int); static void adb_iop_poll(void); static int adb_iop_reset_bus(void); +/* ADB command byte structure */ +#define ADDR_MASK 0xF0 +#define OP_MASK 0x0C +#define TALK 0x0C + struct adb_driver adb_iop_driver = { .name = "ISM IOP", .probe = adb_iop_probe, @@ -94,17 +100,24 @@ static void adb_iop_complete(struct iop_msg *msg) static void adb_iop_listen(struct iop_msg *msg) { struct adb_iopmsg *amsg = (struct adb_iopmsg *)msg->message; + u8 addr = (amsg->cmd & ADDR_MASK) >> 4; + u8 op = amsg->cmd & OP_MASK; unsigned long flags; bool req_done = false; local_irq_save(flags); - /* Handle a timeout. Timeout packets seem to occur even after - * we've gotten a valid reply to a TALK, presumably because of - * autopolling. + /* Responses to Talk commands may be unsolicited as they are + * produced when the IOP polls devices. They are mostly timeouts. */ + if (op == TALK && ((1 << addr) & autopoll_devs)) + autopoll_addr = addr; - if (amsg->flags & ADB_IOP_EXPLICIT) { + switch (amsg->flags & (ADB_IOP_EXPLICIT | + ADB_IOP_AUTOPOLL | + ADB_IOP_TIMEOUT)) { + case ADB_IOP_EXPLICIT: + case ADB_IOP_EXPLICIT | ADB_IOP_TIMEOUT: if (adb_iop_state == awaiting_reply) { struct adb_request *req = current_req; @@ -115,12 +128,16 @@ static void adb_iop_listen(struct iop_msg *msg) req_done = true; } - } else if (!(amsg->flags & ADB_IOP_TIMEOUT)) { - adb_input(&amsg->cmd, amsg->count + 1, - amsg->flags & ADB_IOP_AUTOPOLL); + break; + case ADB_IOP_AUTOPOLL: + if (((1 << addr) & autopoll_devs) && + amsg->cmd == ADB_READREG(addr, 0)) + adb_input(&amsg->cmd, amsg->count + 1, 1); + break; } - - msg->reply[0] = autopoll_devs ? ADB_IOP_AUTOPOLL : 0; + msg->reply[0] = autopoll_addr ? ADB_IOP_AUTOPOLL : 0; + msg->reply[1] = 0; + msg->reply[2] = autopoll_addr ? ADB_READREG(autopoll_addr, 0) : 0; iop_complete_message(msg); if (req_done) @@ -233,6 +250,9 @@ static void adb_iop_set_ap_complete(struct iop_msg *msg) struct adb_iopmsg *amsg = (struct adb_iopmsg *)msg->message; autopoll_devs = (amsg->data[1] << 8) | amsg->data[0]; + if (autopoll_devs & (1 << autopoll_addr)) + return; + autopoll_addr = autopoll_devs ? (ffs(autopoll_devs) - 1) : 0; } static int adb_iop_autopoll(int devs) From 2ae92e8b9b7eb042ccb7e9fc7ea9431f211a1bd3 Mon Sep 17 00:00:00 2001 From: Finn Thain Date: Sat, 5 Dec 2020 14:46:27 +1100 Subject: [PATCH 329/360] MAINTAINERS: Update m68k Mac entry Two files under drivers/macintosh are actually m68k-only. I think that patches for these files should be reviewed in the appropriate forum and merged via the appropriate tree, rather than falling to the powerpc maintainers to deal with. Update the "M68K ON APPLE MACINTOSH" section accordingly. Cc: Michael Ellerman Cc: Benjamin Herrenschmidt Cc: Joshua Thompson Cc: linuxppc-dev@lists.ozlabs.org Cc: linux-m68k@lists.linux-m68k.org Signed-off-by: Finn Thain Acked-by: Michael Ellerman Link: https://lore.kernel.org/r/fbac2cd8632bb719f48cd1368910abd310548a0e.1607139987.git.fthain@telegraphics.com.au Signed-off-by: Geert Uytterhoeven --- MAINTAINERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index e73636b75f29..909d3e80679d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10397,6 +10397,8 @@ L: linux-m68k@lists.linux-m68k.org S: Maintained W: http://www.mac.linux-m68k.org/ F: arch/m68k/mac/ +F: drivers/macintosh/adb-iop.c +F: drivers/macintosh/via-macii.c M68K ON HP9000/300 M: Philip Blundell From e36a17f846bc9ebc540a6c53f38421a1b2dadfdb Mon Sep 17 00:00:00 2001 From: Troy Lee Date: Mon, 7 Dec 2020 17:00:11 +0800 Subject: [PATCH 330/360] dt-bindings: edac: aspeed-sdram-edac: Add ast2400/ast2600 support Add Aspeed AST2400 and AST2600 binding for the Aspeed EDAC driver. Signed-off-by: Troy Lee Signed-off-by: Borislav Petkov Acked-by: Joel Stanley Link: https://lkml.kernel.org/r/20201207090013.14145-1-troy_lee@aspeedtech.com --- .../devicetree/bindings/edac/aspeed-sdram-edac.txt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/edac/aspeed-sdram-edac.txt b/Documentation/devicetree/bindings/edac/aspeed-sdram-edac.txt index 6a0f3d90d682..8ca9e0a049d8 100644 --- a/Documentation/devicetree/bindings/edac/aspeed-sdram-edac.txt +++ b/Documentation/devicetree/bindings/edac/aspeed-sdram-edac.txt @@ -1,6 +1,6 @@ -Aspeed AST2500 SoC EDAC node +Aspeed BMC SoC EDAC node -The Aspeed AST2500 SoC supports DDR3 and DDR4 memory with and without ECC (error +The Aspeed BMC SoC supports DDR3 and DDR4 memory with and without ECC (error correction check). The memory controller supports SECDED (single bit error correction, double bit @@ -11,7 +11,10 @@ Note, the bootloader must configure ECC mode in the memory controller. Required properties: -- compatible: should be "aspeed,ast2500-sdram-edac" +- compatible: should be one of + - "aspeed,ast2400-sdram-edac" + - "aspeed,ast2500-sdram-edac" + - "aspeed,ast2600-sdram-edac" - reg: sdram controller register set should be <0x1e6e0000 0x174> - interrupts: should be AVIC interrupt #0 From aac82707fa4593b99053fa9ebb2e73d8103130ed Mon Sep 17 00:00:00 2001 From: Troy Lee Date: Mon, 7 Dec 2020 17:00:12 +0800 Subject: [PATCH 331/360] ARM: dts: aspeed: Add AST2600 EDAC into common devicetree Add Aspeed AST2600 EDAC node into the common devicetree. Signed-off-by: Troy Lee Signed-off-by: Borislav Petkov Reviewed-by: Joel Stanley Link: https://lkml.kernel.org/r/20201207090013.14145-2-troy_lee@aspeedtech.com --- arch/arm/boot/dts/aspeed-g6.dtsi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm/boot/dts/aspeed-g6.dtsi b/arch/arm/boot/dts/aspeed-g6.dtsi index b58220a49cbd..74367ee96f20 100644 --- a/arch/arm/boot/dts/aspeed-g6.dtsi +++ b/arch/arm/boot/dts/aspeed-g6.dtsi @@ -69,6 +69,12 @@ always-on; }; + edac: sdram@1e6e0000 { + compatible = "aspeed,ast2600-sdram-edac", "syscon"; + reg = <0x1e6e0000 0x174>; + interrupts = ; + }; + ahb { compatible = "simple-bus"; #address-cells = <1>; From edfc2d73ca45da19fb76f9b76ecc6e885d74d093 Mon Sep 17 00:00:00 2001 From: Troy Lee Date: Mon, 7 Dec 2020 17:00:13 +0800 Subject: [PATCH 332/360] EDAC/aspeed: Add support for AST2400 and AST2600 Add AST2400 and AST2600 EDAC driver support. Signed-off-by: Troy Lee Signed-off-by: Borislav Petkov Reviewed-by: Stefan Schaeckeler Link: https://lkml.kernel.org/r/20201207090013.14145-3-troy_lee@aspeedtech.com --- drivers/edac/Kconfig | 6 +++--- drivers/edac/aspeed_edac.c | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig index 7a47680d6f07..c410331e8ee8 100644 --- a/drivers/edac/Kconfig +++ b/drivers/edac/Kconfig @@ -515,10 +515,10 @@ config EDAC_QCOM health, you should probably say 'Y' here. config EDAC_ASPEED - tristate "Aspeed AST 2500 SoC" - depends on MACH_ASPEED_G5 + tristate "Aspeed AST BMC SoC" + depends on ARCH_ASPEED help - Support for error detection and correction on the Aspeed AST 2500 SoC. + Support for error detection and correction on the Aspeed AST BMC SoC. First, ECC must be configured in the bootloader. Then, this driver will expose error counters via the EDAC kernel framework. diff --git a/drivers/edac/aspeed_edac.c b/drivers/edac/aspeed_edac.c index fde809efc520..a46da56d6d54 100644 --- a/drivers/edac/aspeed_edac.c +++ b/drivers/edac/aspeed_edac.c @@ -239,7 +239,7 @@ static int init_csrows(struct mem_ctl_info *mci) int rc; /* retrieve info about physical memory from device tree */ - np = of_find_node_by_path("/memory"); + np = of_find_node_by_name(NULL, "memory"); if (!np) { dev_err(mci->pdev, "dt: missing /memory node\n"); return -ENODEV; @@ -375,10 +375,13 @@ static int aspeed_remove(struct platform_device *pdev) static const struct of_device_id aspeed_of_match[] = { + { .compatible = "aspeed,ast2400-sdram-edac" }, { .compatible = "aspeed,ast2500-sdram-edac" }, + { .compatible = "aspeed,ast2600-sdram-edac" }, {}, }; +MODULE_DEVICE_TABLE(of, aspeed_of_match); static struct platform_driver aspeed_driver = { .driver = { @@ -392,5 +395,5 @@ module_platform_driver(aspeed_driver); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Stefan Schaeckeler "); -MODULE_DESCRIPTION("Aspeed AST2500 EDAC driver"); +MODULE_DESCRIPTION("Aspeed BMC SoC EDAC driver"); MODULE_VERSION("1.0"); From 0385979a30dc4abdef2dcebbccef818947c80cb7 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Mon, 7 Dec 2020 15:02:53 +1100 Subject: [PATCH 333/360] EDAC/mv64x60: Remove orphan mv64x60 driver The mv64x60 EDAC driver depends on CONFIG_MV64X60. But that symbol is not user-selectable, and the last code that selected it was removed with the C2K board support in 2018, see: 92c8c16f3457 ("powerpc/embedded6xx: Remove C2K board support") That means the driver is now dead code, so remove it. Suggested-by: Borislav Petkov Signed-off-by: Michael Ellerman Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201207040253.628528-1-mpe@ellerman.id.au --- drivers/edac/Kconfig | 7 - drivers/edac/Makefile | 1 - drivers/edac/mv64x60_edac.c | 883 ------------------------------------ drivers/edac/mv64x60_edac.h | 114 ----- 4 files changed, 1005 deletions(-) delete mode 100644 drivers/edac/mv64x60_edac.c delete mode 100644 drivers/edac/mv64x60_edac.h diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig index c410331e8ee8..2864a32c7d17 100644 --- a/drivers/edac/Kconfig +++ b/drivers/edac/Kconfig @@ -283,13 +283,6 @@ config EDAC_LAYERSCAPE Support for error detection and correction on Freescale memory controllers on Layerscape SoCs. -config EDAC_MV64X60 - tristate "Marvell MV64x60" - depends on MV64X60 - help - Support for error detection and correction on the Marvell - MV64360 and MV64460 chipsets. - config EDAC_PASEMI tristate "PA Semi PWRficient" depends on PPC_PASEMI && PCI diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile index 3a849168780d..28edee9dca66 100644 --- a/drivers/edac/Makefile +++ b/drivers/edac/Makefile @@ -64,7 +64,6 @@ obj-$(CONFIG_EDAC_SKX) += skx_edac.o i10nm_edac-y := skx_common.o i10nm_base.o obj-$(CONFIG_EDAC_I10NM) += i10nm_edac.o -obj-$(CONFIG_EDAC_MV64X60) += mv64x60_edac.o obj-$(CONFIG_EDAC_CELL) += cell_edac.o obj-$(CONFIG_EDAC_PPC4XX) += ppc4xx_edac.o obj-$(CONFIG_EDAC_AMD8111) += amd8111_edac.o diff --git a/drivers/edac/mv64x60_edac.c b/drivers/edac/mv64x60_edac.c deleted file mode 100644 index 3c68bb525d5d..000000000000 --- a/drivers/edac/mv64x60_edac.c +++ /dev/null @@ -1,883 +0,0 @@ -/* - * Marvell MV64x60 Memory Controller kernel module for PPC platforms - * - * Author: Dave Jiang - * - * 2006-2007 (c) MontaVista Software, Inc. This file is licensed under - * the terms of the GNU General Public License version 2. This program - * is licensed "as is" without any warranty of any kind, whether express - * or implied. - * - */ - -#include -#include -#include -#include -#include -#include - -#include "edac_module.h" -#include "mv64x60_edac.h" - -static const char *mv64x60_ctl_name = "MV64x60"; -static int edac_dev_idx; -static int edac_pci_idx; -static int edac_mc_idx; - -/*********************** PCI err device **********************************/ -#ifdef CONFIG_PCI -static void mv64x60_pci_check(struct edac_pci_ctl_info *pci) -{ - struct mv64x60_pci_pdata *pdata = pci->pvt_info; - u32 cause; - - cause = readl(pdata->pci_vbase + MV64X60_PCI_ERROR_CAUSE); - if (!cause) - return; - - printk(KERN_ERR "Error in PCI %d Interface\n", pdata->pci_hose); - printk(KERN_ERR "Cause register: 0x%08x\n", cause); - printk(KERN_ERR "Address Low: 0x%08x\n", - readl(pdata->pci_vbase + MV64X60_PCI_ERROR_ADDR_LO)); - printk(KERN_ERR "Address High: 0x%08x\n", - readl(pdata->pci_vbase + MV64X60_PCI_ERROR_ADDR_HI)); - printk(KERN_ERR "Attribute: 0x%08x\n", - readl(pdata->pci_vbase + MV64X60_PCI_ERROR_ATTR)); - printk(KERN_ERR "Command: 0x%08x\n", - readl(pdata->pci_vbase + MV64X60_PCI_ERROR_CMD)); - writel(~cause, pdata->pci_vbase + MV64X60_PCI_ERROR_CAUSE); - - if (cause & MV64X60_PCI_PE_MASK) - edac_pci_handle_pe(pci, pci->ctl_name); - - if (!(cause & MV64X60_PCI_PE_MASK)) - edac_pci_handle_npe(pci, pci->ctl_name); -} - -static irqreturn_t mv64x60_pci_isr(int irq, void *dev_id) -{ - struct edac_pci_ctl_info *pci = dev_id; - struct mv64x60_pci_pdata *pdata = pci->pvt_info; - u32 val; - - val = readl(pdata->pci_vbase + MV64X60_PCI_ERROR_CAUSE); - if (!val) - return IRQ_NONE; - - mv64x60_pci_check(pci); - - return IRQ_HANDLED; -} - -/* - * Bit 0 of MV64x60_PCIx_ERR_MASK does not exist on the 64360 and because of - * errata FEr-#11 and FEr-##16 for the 64460, it should be 0 on that chip as - * well. IOW, don't set bit 0. - */ - -/* Erratum FEr PCI-#16: clear bit 0 of PCI SERRn Mask reg. */ -static int __init mv64x60_pci_fixup(struct platform_device *pdev) -{ - struct resource *r; - void __iomem *pci_serr; - - r = platform_get_resource(pdev, IORESOURCE_MEM, 1); - if (!r) { - printk(KERN_ERR "%s: Unable to get resource for " - "PCI err regs\n", __func__); - return -ENOENT; - } - - pci_serr = ioremap(r->start, resource_size(r)); - if (!pci_serr) - return -ENOMEM; - - writel(readl(pci_serr) & ~0x1, pci_serr); - iounmap(pci_serr); - - return 0; -} - -static int mv64x60_pci_err_probe(struct platform_device *pdev) -{ - struct edac_pci_ctl_info *pci; - struct mv64x60_pci_pdata *pdata; - struct resource *r; - int res = 0; - - if (!devres_open_group(&pdev->dev, mv64x60_pci_err_probe, GFP_KERNEL)) - return -ENOMEM; - - pci = edac_pci_alloc_ctl_info(sizeof(*pdata), "mv64x60_pci_err"); - if (!pci) - return -ENOMEM; - - pdata = pci->pvt_info; - - pdata->pci_hose = pdev->id; - pdata->name = "mv64x60_pci_err"; - platform_set_drvdata(pdev, pci); - pci->dev = &pdev->dev; - pci->dev_name = dev_name(&pdev->dev); - pci->mod_name = EDAC_MOD_STR; - pci->ctl_name = pdata->name; - - if (edac_op_state == EDAC_OPSTATE_POLL) - pci->edac_check = mv64x60_pci_check; - - pdata->edac_idx = edac_pci_idx++; - - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!r) { - printk(KERN_ERR "%s: Unable to get resource for " - "PCI err regs\n", __func__); - res = -ENOENT; - goto err; - } - - if (!devm_request_mem_region(&pdev->dev, - r->start, - resource_size(r), - pdata->name)) { - printk(KERN_ERR "%s: Error while requesting mem region\n", - __func__); - res = -EBUSY; - goto err; - } - - pdata->pci_vbase = devm_ioremap(&pdev->dev, - r->start, - resource_size(r)); - if (!pdata->pci_vbase) { - printk(KERN_ERR "%s: Unable to setup PCI err regs\n", __func__); - res = -ENOMEM; - goto err; - } - - res = mv64x60_pci_fixup(pdev); - if (res < 0) { - printk(KERN_ERR "%s: PCI fixup failed\n", __func__); - goto err; - } - - writel(0, pdata->pci_vbase + MV64X60_PCI_ERROR_CAUSE); - writel(0, pdata->pci_vbase + MV64X60_PCI_ERROR_MASK); - writel(MV64X60_PCIx_ERR_MASK_VAL, - pdata->pci_vbase + MV64X60_PCI_ERROR_MASK); - - if (edac_pci_add_device(pci, pdata->edac_idx) > 0) { - edac_dbg(3, "failed edac_pci_add_device()\n"); - goto err; - } - - if (edac_op_state == EDAC_OPSTATE_INT) { - pdata->irq = platform_get_irq(pdev, 0); - res = devm_request_irq(&pdev->dev, - pdata->irq, - mv64x60_pci_isr, - 0, - "[EDAC] PCI err", - pci); - if (res < 0) { - printk(KERN_ERR "%s: Unable to request irq %d for " - "MV64x60 PCI ERR\n", __func__, pdata->irq); - res = -ENODEV; - goto err2; - } - printk(KERN_INFO EDAC_MOD_STR " acquired irq %d for PCI Err\n", - pdata->irq); - } - - devres_remove_group(&pdev->dev, mv64x60_pci_err_probe); - - /* get this far and it's successful */ - edac_dbg(3, "success\n"); - - return 0; - -err2: - edac_pci_del_device(&pdev->dev); -err: - edac_pci_free_ctl_info(pci); - devres_release_group(&pdev->dev, mv64x60_pci_err_probe); - return res; -} - -static int mv64x60_pci_err_remove(struct platform_device *pdev) -{ - struct edac_pci_ctl_info *pci = platform_get_drvdata(pdev); - - edac_dbg(0, "\n"); - - edac_pci_del_device(&pdev->dev); - - edac_pci_free_ctl_info(pci); - - return 0; -} - -static struct platform_driver mv64x60_pci_err_driver = { - .probe = mv64x60_pci_err_probe, - .remove = mv64x60_pci_err_remove, - .driver = { - .name = "mv64x60_pci_err", - } -}; - -#endif /* CONFIG_PCI */ - -/*********************** SRAM err device **********************************/ -static void mv64x60_sram_check(struct edac_device_ctl_info *edac_dev) -{ - struct mv64x60_sram_pdata *pdata = edac_dev->pvt_info; - u32 cause; - - cause = readl(pdata->sram_vbase + MV64X60_SRAM_ERR_CAUSE); - if (!cause) - return; - - printk(KERN_ERR "Error in internal SRAM\n"); - printk(KERN_ERR "Cause register: 0x%08x\n", cause); - printk(KERN_ERR "Address Low: 0x%08x\n", - readl(pdata->sram_vbase + MV64X60_SRAM_ERR_ADDR_LO)); - printk(KERN_ERR "Address High: 0x%08x\n", - readl(pdata->sram_vbase + MV64X60_SRAM_ERR_ADDR_HI)); - printk(KERN_ERR "Data Low: 0x%08x\n", - readl(pdata->sram_vbase + MV64X60_SRAM_ERR_DATA_LO)); - printk(KERN_ERR "Data High: 0x%08x\n", - readl(pdata->sram_vbase + MV64X60_SRAM_ERR_DATA_HI)); - printk(KERN_ERR "Parity: 0x%08x\n", - readl(pdata->sram_vbase + MV64X60_SRAM_ERR_PARITY)); - writel(0, pdata->sram_vbase + MV64X60_SRAM_ERR_CAUSE); - - edac_device_handle_ue(edac_dev, 0, 0, edac_dev->ctl_name); -} - -static irqreturn_t mv64x60_sram_isr(int irq, void *dev_id) -{ - struct edac_device_ctl_info *edac_dev = dev_id; - struct mv64x60_sram_pdata *pdata = edac_dev->pvt_info; - u32 cause; - - cause = readl(pdata->sram_vbase + MV64X60_SRAM_ERR_CAUSE); - if (!cause) - return IRQ_NONE; - - mv64x60_sram_check(edac_dev); - - return IRQ_HANDLED; -} - -static int mv64x60_sram_err_probe(struct platform_device *pdev) -{ - struct edac_device_ctl_info *edac_dev; - struct mv64x60_sram_pdata *pdata; - struct resource *r; - int res = 0; - - if (!devres_open_group(&pdev->dev, mv64x60_sram_err_probe, GFP_KERNEL)) - return -ENOMEM; - - edac_dev = edac_device_alloc_ctl_info(sizeof(*pdata), - "sram", 1, NULL, 0, 0, NULL, 0, - edac_dev_idx); - if (!edac_dev) { - devres_release_group(&pdev->dev, mv64x60_sram_err_probe); - return -ENOMEM; - } - - pdata = edac_dev->pvt_info; - pdata->name = "mv64x60_sram_err"; - edac_dev->dev = &pdev->dev; - platform_set_drvdata(pdev, edac_dev); - edac_dev->dev_name = dev_name(&pdev->dev); - - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!r) { - printk(KERN_ERR "%s: Unable to get resource for " - "SRAM err regs\n", __func__); - res = -ENOENT; - goto err; - } - - if (!devm_request_mem_region(&pdev->dev, - r->start, - resource_size(r), - pdata->name)) { - printk(KERN_ERR "%s: Error while request mem region\n", - __func__); - res = -EBUSY; - goto err; - } - - pdata->sram_vbase = devm_ioremap(&pdev->dev, - r->start, - resource_size(r)); - if (!pdata->sram_vbase) { - printk(KERN_ERR "%s: Unable to setup SRAM err regs\n", - __func__); - res = -ENOMEM; - goto err; - } - - /* setup SRAM err registers */ - writel(0, pdata->sram_vbase + MV64X60_SRAM_ERR_CAUSE); - - edac_dev->mod_name = EDAC_MOD_STR; - edac_dev->ctl_name = pdata->name; - - if (edac_op_state == EDAC_OPSTATE_POLL) - edac_dev->edac_check = mv64x60_sram_check; - - pdata->edac_idx = edac_dev_idx++; - - if (edac_device_add_device(edac_dev) > 0) { - edac_dbg(3, "failed edac_device_add_device()\n"); - goto err; - } - - if (edac_op_state == EDAC_OPSTATE_INT) { - pdata->irq = platform_get_irq(pdev, 0); - res = devm_request_irq(&pdev->dev, - pdata->irq, - mv64x60_sram_isr, - 0, - "[EDAC] SRAM err", - edac_dev); - if (res < 0) { - printk(KERN_ERR - "%s: Unable to request irq %d for " - "MV64x60 SRAM ERR\n", __func__, pdata->irq); - res = -ENODEV; - goto err2; - } - - printk(KERN_INFO EDAC_MOD_STR " acquired irq %d for SRAM Err\n", - pdata->irq); - } - - devres_remove_group(&pdev->dev, mv64x60_sram_err_probe); - - /* get this far and it's successful */ - edac_dbg(3, "success\n"); - - return 0; - -err2: - edac_device_del_device(&pdev->dev); -err: - devres_release_group(&pdev->dev, mv64x60_sram_err_probe); - edac_device_free_ctl_info(edac_dev); - return res; -} - -static int mv64x60_sram_err_remove(struct platform_device *pdev) -{ - struct edac_device_ctl_info *edac_dev = platform_get_drvdata(pdev); - - edac_dbg(0, "\n"); - - edac_device_del_device(&pdev->dev); - edac_device_free_ctl_info(edac_dev); - - return 0; -} - -static struct platform_driver mv64x60_sram_err_driver = { - .probe = mv64x60_sram_err_probe, - .remove = mv64x60_sram_err_remove, - .driver = { - .name = "mv64x60_sram_err", - } -}; - -/*********************** CPU err device **********************************/ -static void mv64x60_cpu_check(struct edac_device_ctl_info *edac_dev) -{ - struct mv64x60_cpu_pdata *pdata = edac_dev->pvt_info; - u32 cause; - - cause = readl(pdata->cpu_vbase[1] + MV64x60_CPU_ERR_CAUSE) & - MV64x60_CPU_CAUSE_MASK; - if (!cause) - return; - - printk(KERN_ERR "Error on CPU interface\n"); - printk(KERN_ERR "Cause register: 0x%08x\n", cause); - printk(KERN_ERR "Address Low: 0x%08x\n", - readl(pdata->cpu_vbase[0] + MV64x60_CPU_ERR_ADDR_LO)); - printk(KERN_ERR "Address High: 0x%08x\n", - readl(pdata->cpu_vbase[0] + MV64x60_CPU_ERR_ADDR_HI)); - printk(KERN_ERR "Data Low: 0x%08x\n", - readl(pdata->cpu_vbase[1] + MV64x60_CPU_ERR_DATA_LO)); - printk(KERN_ERR "Data High: 0x%08x\n", - readl(pdata->cpu_vbase[1] + MV64x60_CPU_ERR_DATA_HI)); - printk(KERN_ERR "Parity: 0x%08x\n", - readl(pdata->cpu_vbase[1] + MV64x60_CPU_ERR_PARITY)); - writel(0, pdata->cpu_vbase[1] + MV64x60_CPU_ERR_CAUSE); - - edac_device_handle_ue(edac_dev, 0, 0, edac_dev->ctl_name); -} - -static irqreturn_t mv64x60_cpu_isr(int irq, void *dev_id) -{ - struct edac_device_ctl_info *edac_dev = dev_id; - struct mv64x60_cpu_pdata *pdata = edac_dev->pvt_info; - u32 cause; - - cause = readl(pdata->cpu_vbase[1] + MV64x60_CPU_ERR_CAUSE) & - MV64x60_CPU_CAUSE_MASK; - if (!cause) - return IRQ_NONE; - - mv64x60_cpu_check(edac_dev); - - return IRQ_HANDLED; -} - -static int mv64x60_cpu_err_probe(struct platform_device *pdev) -{ - struct edac_device_ctl_info *edac_dev; - struct resource *r; - struct mv64x60_cpu_pdata *pdata; - int res = 0; - - if (!devres_open_group(&pdev->dev, mv64x60_cpu_err_probe, GFP_KERNEL)) - return -ENOMEM; - - edac_dev = edac_device_alloc_ctl_info(sizeof(*pdata), - "cpu", 1, NULL, 0, 0, NULL, 0, - edac_dev_idx); - if (!edac_dev) { - devres_release_group(&pdev->dev, mv64x60_cpu_err_probe); - return -ENOMEM; - } - - pdata = edac_dev->pvt_info; - pdata->name = "mv64x60_cpu_err"; - edac_dev->dev = &pdev->dev; - platform_set_drvdata(pdev, edac_dev); - edac_dev->dev_name = dev_name(&pdev->dev); - - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!r) { - printk(KERN_ERR "%s: Unable to get resource for " - "CPU err regs\n", __func__); - res = -ENOENT; - goto err; - } - - if (!devm_request_mem_region(&pdev->dev, - r->start, - resource_size(r), - pdata->name)) { - printk(KERN_ERR "%s: Error while requesting mem region\n", - __func__); - res = -EBUSY; - goto err; - } - - pdata->cpu_vbase[0] = devm_ioremap(&pdev->dev, - r->start, - resource_size(r)); - if (!pdata->cpu_vbase[0]) { - printk(KERN_ERR "%s: Unable to setup CPU err regs\n", __func__); - res = -ENOMEM; - goto err; - } - - r = platform_get_resource(pdev, IORESOURCE_MEM, 1); - if (!r) { - printk(KERN_ERR "%s: Unable to get resource for " - "CPU err regs\n", __func__); - res = -ENOENT; - goto err; - } - - if (!devm_request_mem_region(&pdev->dev, - r->start, - resource_size(r), - pdata->name)) { - printk(KERN_ERR "%s: Error while requesting mem region\n", - __func__); - res = -EBUSY; - goto err; - } - - pdata->cpu_vbase[1] = devm_ioremap(&pdev->dev, - r->start, - resource_size(r)); - if (!pdata->cpu_vbase[1]) { - printk(KERN_ERR "%s: Unable to setup CPU err regs\n", __func__); - res = -ENOMEM; - goto err; - } - - /* setup CPU err registers */ - writel(0, pdata->cpu_vbase[1] + MV64x60_CPU_ERR_CAUSE); - writel(0, pdata->cpu_vbase[1] + MV64x60_CPU_ERR_MASK); - writel(0x000000ff, pdata->cpu_vbase[1] + MV64x60_CPU_ERR_MASK); - - edac_dev->mod_name = EDAC_MOD_STR; - edac_dev->ctl_name = pdata->name; - if (edac_op_state == EDAC_OPSTATE_POLL) - edac_dev->edac_check = mv64x60_cpu_check; - - pdata->edac_idx = edac_dev_idx++; - - if (edac_device_add_device(edac_dev) > 0) { - edac_dbg(3, "failed edac_device_add_device()\n"); - goto err; - } - - if (edac_op_state == EDAC_OPSTATE_INT) { - pdata->irq = platform_get_irq(pdev, 0); - res = devm_request_irq(&pdev->dev, - pdata->irq, - mv64x60_cpu_isr, - 0, - "[EDAC] CPU err", - edac_dev); - if (res < 0) { - printk(KERN_ERR - "%s: Unable to request irq %d for MV64x60 " - "CPU ERR\n", __func__, pdata->irq); - res = -ENODEV; - goto err2; - } - - printk(KERN_INFO EDAC_MOD_STR - " acquired irq %d for CPU Err\n", pdata->irq); - } - - devres_remove_group(&pdev->dev, mv64x60_cpu_err_probe); - - /* get this far and it's successful */ - edac_dbg(3, "success\n"); - - return 0; - -err2: - edac_device_del_device(&pdev->dev); -err: - devres_release_group(&pdev->dev, mv64x60_cpu_err_probe); - edac_device_free_ctl_info(edac_dev); - return res; -} - -static int mv64x60_cpu_err_remove(struct platform_device *pdev) -{ - struct edac_device_ctl_info *edac_dev = platform_get_drvdata(pdev); - - edac_dbg(0, "\n"); - - edac_device_del_device(&pdev->dev); - edac_device_free_ctl_info(edac_dev); - return 0; -} - -static struct platform_driver mv64x60_cpu_err_driver = { - .probe = mv64x60_cpu_err_probe, - .remove = mv64x60_cpu_err_remove, - .driver = { - .name = "mv64x60_cpu_err", - } -}; - -/*********************** DRAM err device **********************************/ - -static void mv64x60_mc_check(struct mem_ctl_info *mci) -{ - struct mv64x60_mc_pdata *pdata = mci->pvt_info; - u32 reg; - u32 err_addr; - u32 sdram_ecc; - u32 comp_ecc; - u32 syndrome; - - reg = readl(pdata->mc_vbase + MV64X60_SDRAM_ERR_ADDR); - if (!reg) - return; - - err_addr = reg & ~0x3; - sdram_ecc = readl(pdata->mc_vbase + MV64X60_SDRAM_ERR_ECC_RCVD); - comp_ecc = readl(pdata->mc_vbase + MV64X60_SDRAM_ERR_ECC_CALC); - syndrome = sdram_ecc ^ comp_ecc; - - /* first bit clear in ECC Err Reg, 1 bit error, correctable by HW */ - if (!(reg & 0x1)) - edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1, - err_addr >> PAGE_SHIFT, - err_addr & PAGE_MASK, syndrome, - 0, 0, -1, - mci->ctl_name, ""); - else /* 2 bit error, UE */ - edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1, - err_addr >> PAGE_SHIFT, - err_addr & PAGE_MASK, 0, - 0, 0, -1, - mci->ctl_name, ""); - - /* clear the error */ - writel(0, pdata->mc_vbase + MV64X60_SDRAM_ERR_ADDR); -} - -static irqreturn_t mv64x60_mc_isr(int irq, void *dev_id) -{ - struct mem_ctl_info *mci = dev_id; - struct mv64x60_mc_pdata *pdata = mci->pvt_info; - u32 reg; - - reg = readl(pdata->mc_vbase + MV64X60_SDRAM_ERR_ADDR); - if (!reg) - return IRQ_NONE; - - /* writing 0's to the ECC err addr in check function clears irq */ - mv64x60_mc_check(mci); - - return IRQ_HANDLED; -} - -static void get_total_mem(struct mv64x60_mc_pdata *pdata) -{ - struct device_node *np = NULL; - const unsigned int *reg; - - np = of_find_node_by_type(NULL, "memory"); - if (!np) - return; - - reg = of_get_property(np, "reg", NULL); - - pdata->total_mem = reg[1]; -} - -static void mv64x60_init_csrows(struct mem_ctl_info *mci, - struct mv64x60_mc_pdata *pdata) -{ - struct csrow_info *csrow; - struct dimm_info *dimm; - - u32 devtype; - u32 ctl; - - get_total_mem(pdata); - - ctl = readl(pdata->mc_vbase + MV64X60_SDRAM_CONFIG); - - csrow = mci->csrows[0]; - dimm = csrow->channels[0]->dimm; - - dimm->nr_pages = pdata->total_mem >> PAGE_SHIFT; - dimm->grain = 8; - - dimm->mtype = (ctl & MV64X60_SDRAM_REGISTERED) ? MEM_RDDR : MEM_DDR; - - devtype = (ctl >> 20) & 0x3; - switch (devtype) { - case 0x0: - dimm->dtype = DEV_X32; - break; - case 0x2: /* could be X8 too, but no way to tell */ - dimm->dtype = DEV_X16; - break; - case 0x3: - dimm->dtype = DEV_X4; - break; - default: - dimm->dtype = DEV_UNKNOWN; - break; - } - - dimm->edac_mode = EDAC_SECDED; -} - -static int mv64x60_mc_err_probe(struct platform_device *pdev) -{ - struct mem_ctl_info *mci; - struct edac_mc_layer layers[2]; - struct mv64x60_mc_pdata *pdata; - struct resource *r; - u32 ctl; - int res = 0; - - if (!devres_open_group(&pdev->dev, mv64x60_mc_err_probe, GFP_KERNEL)) - return -ENOMEM; - - layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; - layers[0].size = 1; - layers[0].is_virt_csrow = true; - layers[1].type = EDAC_MC_LAYER_CHANNEL; - layers[1].size = 1; - layers[1].is_virt_csrow = false; - mci = edac_mc_alloc(edac_mc_idx, ARRAY_SIZE(layers), layers, - sizeof(struct mv64x60_mc_pdata)); - if (!mci) { - printk(KERN_ERR "%s: No memory for CPU err\n", __func__); - devres_release_group(&pdev->dev, mv64x60_mc_err_probe); - return -ENOMEM; - } - - pdata = mci->pvt_info; - mci->pdev = &pdev->dev; - platform_set_drvdata(pdev, mci); - pdata->name = "mv64x60_mc_err"; - mci->dev_name = dev_name(&pdev->dev); - pdata->edac_idx = edac_mc_idx++; - - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!r) { - printk(KERN_ERR "%s: Unable to get resource for " - "MC err regs\n", __func__); - res = -ENOENT; - goto err; - } - - if (!devm_request_mem_region(&pdev->dev, - r->start, - resource_size(r), - pdata->name)) { - printk(KERN_ERR "%s: Error while requesting mem region\n", - __func__); - res = -EBUSY; - goto err; - } - - pdata->mc_vbase = devm_ioremap(&pdev->dev, - r->start, - resource_size(r)); - if (!pdata->mc_vbase) { - printk(KERN_ERR "%s: Unable to setup MC err regs\n", __func__); - res = -ENOMEM; - goto err; - } - - ctl = readl(pdata->mc_vbase + MV64X60_SDRAM_CONFIG); - if (!(ctl & MV64X60_SDRAM_ECC)) { - /* Non-ECC RAM? */ - printk(KERN_WARNING "%s: No ECC DIMMs discovered\n", __func__); - res = -ENODEV; - goto err; - } - - edac_dbg(3, "init mci\n"); - mci->mtype_cap = MEM_FLAG_RDDR | MEM_FLAG_DDR; - mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED; - mci->edac_cap = EDAC_FLAG_SECDED; - mci->mod_name = EDAC_MOD_STR; - mci->ctl_name = mv64x60_ctl_name; - - if (edac_op_state == EDAC_OPSTATE_POLL) - mci->edac_check = mv64x60_mc_check; - - mci->ctl_page_to_phys = NULL; - - mci->scrub_mode = SCRUB_SW_SRC; - - mv64x60_init_csrows(mci, pdata); - - /* setup MC registers */ - writel(0, pdata->mc_vbase + MV64X60_SDRAM_ERR_ADDR); - ctl = readl(pdata->mc_vbase + MV64X60_SDRAM_ERR_ECC_CNTL); - ctl = (ctl & 0xff00ffff) | 0x10000; - writel(ctl, pdata->mc_vbase + MV64X60_SDRAM_ERR_ECC_CNTL); - - res = edac_mc_add_mc(mci); - if (res) { - edac_dbg(3, "failed edac_mc_add_mc()\n"); - goto err; - } - - if (edac_op_state == EDAC_OPSTATE_INT) { - /* acquire interrupt that reports errors */ - pdata->irq = platform_get_irq(pdev, 0); - res = devm_request_irq(&pdev->dev, - pdata->irq, - mv64x60_mc_isr, - 0, - "[EDAC] MC err", - mci); - if (res < 0) { - printk(KERN_ERR "%s: Unable to request irq %d for " - "MV64x60 DRAM ERR\n", __func__, pdata->irq); - res = -ENODEV; - goto err2; - } - - printk(KERN_INFO EDAC_MOD_STR " acquired irq %d for MC Err\n", - pdata->irq); - } - - /* get this far and it's successful */ - edac_dbg(3, "success\n"); - - return 0; - -err2: - edac_mc_del_mc(&pdev->dev); -err: - devres_release_group(&pdev->dev, mv64x60_mc_err_probe); - edac_mc_free(mci); - return res; -} - -static int mv64x60_mc_err_remove(struct platform_device *pdev) -{ - struct mem_ctl_info *mci = platform_get_drvdata(pdev); - - edac_dbg(0, "\n"); - - edac_mc_del_mc(&pdev->dev); - edac_mc_free(mci); - return 0; -} - -static struct platform_driver mv64x60_mc_err_driver = { - .probe = mv64x60_mc_err_probe, - .remove = mv64x60_mc_err_remove, - .driver = { - .name = "mv64x60_mc_err", - } -}; - -static struct platform_driver * const drivers[] = { - &mv64x60_mc_err_driver, - &mv64x60_cpu_err_driver, - &mv64x60_sram_err_driver, -#ifdef CONFIG_PCI - &mv64x60_pci_err_driver, -#endif -}; - -static int __init mv64x60_edac_init(void) -{ - - printk(KERN_INFO "Marvell MV64x60 EDAC driver " MV64x60_REVISION "\n"); - printk(KERN_INFO "\t(C) 2006-2007 MontaVista Software\n"); - - /* make sure error reporting method is sane */ - switch (edac_op_state) { - case EDAC_OPSTATE_POLL: - case EDAC_OPSTATE_INT: - break; - default: - edac_op_state = EDAC_OPSTATE_INT; - break; - } - - return platform_register_drivers(drivers, ARRAY_SIZE(drivers)); -} -module_init(mv64x60_edac_init); - -static void __exit mv64x60_edac_exit(void) -{ - platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); -} -module_exit(mv64x60_edac_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Montavista Software, Inc."); -module_param(edac_op_state, int, 0444); -MODULE_PARM_DESC(edac_op_state, - "EDAC Error Reporting state: 0=Poll, 2=Interrupt"); diff --git a/drivers/edac/mv64x60_edac.h b/drivers/edac/mv64x60_edac.h deleted file mode 100644 index c7f209c92a1a..000000000000 --- a/drivers/edac/mv64x60_edac.h +++ /dev/null @@ -1,114 +0,0 @@ -/* - * EDAC defs for Marvell MV64x60 bridge chip - * - * Author: Dave Jiang - * - * 2007 (c) MontaVista Software, Inc. This file is licensed under - * the terms of the GNU General Public License version 2. This program - * is licensed "as is" without any warranty of any kind, whether express - * or implied. - * - */ -#ifndef _MV64X60_EDAC_H_ -#define _MV64X60_EDAC_H_ - -#define MV64x60_REVISION " Ver: 2.0.0" -#define EDAC_MOD_STR "MV64x60_edac" - -#define mv64x60_printk(level, fmt, arg...) \ - edac_printk(level, "MV64x60", fmt, ##arg) - -#define mv64x60_mc_printk(mci, level, fmt, arg...) \ - edac_mc_chipset_printk(mci, level, "MV64x60", fmt, ##arg) - -/* CPU Error Report Registers */ -#define MV64x60_CPU_ERR_ADDR_LO 0x00 /* 0x0070 */ -#define MV64x60_CPU_ERR_ADDR_HI 0x08 /* 0x0078 */ -#define MV64x60_CPU_ERR_DATA_LO 0x00 /* 0x0128 */ -#define MV64x60_CPU_ERR_DATA_HI 0x08 /* 0x0130 */ -#define MV64x60_CPU_ERR_PARITY 0x10 /* 0x0138 */ -#define MV64x60_CPU_ERR_CAUSE 0x18 /* 0x0140 */ -#define MV64x60_CPU_ERR_MASK 0x20 /* 0x0148 */ - -#define MV64x60_CPU_CAUSE_MASK 0x07ffffff - -/* SRAM Error Report Registers */ -#define MV64X60_SRAM_ERR_CAUSE 0x08 /* 0x0388 */ -#define MV64X60_SRAM_ERR_ADDR_LO 0x10 /* 0x0390 */ -#define MV64X60_SRAM_ERR_ADDR_HI 0x78 /* 0x03f8 */ -#define MV64X60_SRAM_ERR_DATA_LO 0x18 /* 0x0398 */ -#define MV64X60_SRAM_ERR_DATA_HI 0x20 /* 0x03a0 */ -#define MV64X60_SRAM_ERR_PARITY 0x28 /* 0x03a8 */ - -/* SDRAM Controller Registers */ -#define MV64X60_SDRAM_CONFIG 0x00 /* 0x1400 */ -#define MV64X60_SDRAM_ERR_DATA_HI 0x40 /* 0x1440 */ -#define MV64X60_SDRAM_ERR_DATA_LO 0x44 /* 0x1444 */ -#define MV64X60_SDRAM_ERR_ECC_RCVD 0x48 /* 0x1448 */ -#define MV64X60_SDRAM_ERR_ECC_CALC 0x4c /* 0x144c */ -#define MV64X60_SDRAM_ERR_ADDR 0x50 /* 0x1450 */ -#define MV64X60_SDRAM_ERR_ECC_CNTL 0x54 /* 0x1454 */ -#define MV64X60_SDRAM_ERR_ECC_ERR_CNT 0x58 /* 0x1458 */ - -#define MV64X60_SDRAM_REGISTERED 0x20000 -#define MV64X60_SDRAM_ECC 0x40000 - -#ifdef CONFIG_PCI -/* - * Bit 0 of MV64x60_PCIx_ERR_MASK does not exist on the 64360 and because of - * errata FEr-#11 and FEr-##16 for the 64460, it should be 0 on that chip as - * well. IOW, don't set bit 0. - */ -#define MV64X60_PCIx_ERR_MASK_VAL 0x00a50c24 - -/* Register offsets from PCIx error address low register */ -#define MV64X60_PCI_ERROR_ADDR_LO 0x00 -#define MV64X60_PCI_ERROR_ADDR_HI 0x04 -#define MV64X60_PCI_ERROR_ATTR 0x08 -#define MV64X60_PCI_ERROR_CMD 0x10 -#define MV64X60_PCI_ERROR_CAUSE 0x18 -#define MV64X60_PCI_ERROR_MASK 0x1c - -#define MV64X60_PCI_ERR_SWrPerr 0x0002 -#define MV64X60_PCI_ERR_SRdPerr 0x0004 -#define MV64X60_PCI_ERR_MWrPerr 0x0020 -#define MV64X60_PCI_ERR_MRdPerr 0x0040 - -#define MV64X60_PCI_PE_MASK (MV64X60_PCI_ERR_SWrPerr | \ - MV64X60_PCI_ERR_SRdPerr | \ - MV64X60_PCI_ERR_MWrPerr | \ - MV64X60_PCI_ERR_MRdPerr) - -struct mv64x60_pci_pdata { - int pci_hose; - void __iomem *pci_vbase; - char *name; - int irq; - int edac_idx; -}; - -#endif /* CONFIG_PCI */ - -struct mv64x60_mc_pdata { - void __iomem *mc_vbase; - int total_mem; - char *name; - int irq; - int edac_idx; -}; - -struct mv64x60_cpu_pdata { - void __iomem *cpu_vbase[2]; - char *name; - int irq; - int edac_idx; -}; - -struct mv64x60_sram_pdata { - void __iomem *sram_vbase; - char *name; - int irq; - int edac_idx; -}; - -#endif From 72ebb5ff806f9a421a2a53cdfe6c4ebbab243bd5 Mon Sep 17 00:00:00 2001 From: Qiujun Huang Date: Thu, 3 Dec 2020 22:50:20 +0800 Subject: [PATCH 334/360] x86/alternative: Update text_poke_bp() kernel-doc comment Update kernel-doc parameter name after c3d6324f841b ("x86/alternatives: Teach text_poke_bp() to emulate instructions") changed the last parameter from @handler to @emulate. [ bp: Make commit message more precise. ] Signed-off-by: Qiujun Huang Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201203145020.2441-1-hqjagain@gmail.com --- arch/x86/kernel/alternative.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 4adbe65afe23..ed3efc5af783 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1365,7 +1365,7 @@ void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const voi * @addr: address to patch * @opcode: opcode of new instruction * @len: length to copy - * @handler: address to jump to when the temporary breakpoint is hit + * @emulate: instruction to be emulated * * Update a single instruction with the vector in the stack, avoiding * dynamically allocated memory. This function should be used when it is From a67fffb017aed93fca42ce7aa5b6aaf54ff912ad Mon Sep 17 00:00:00 2001 From: Mike Travis Date: Fri, 27 Nov 2020 21:42:23 -0600 Subject: [PATCH 335/360] x86/platform/uv: Add kernel interfaces for obtaining system info Add kernel interfaces used to obtain info for the uv_sysfs driver to display. Signed-off-by: Mike Travis Signed-off-by: Borislav Petkov Reviewed-by: Steve Wahl Acked-by: Hans de Goede Link: https://lkml.kernel.org/r/20201128034227.120869-2-mike.travis@hpe.com --- arch/x86/include/asm/uv/bios.h | 2 ++ arch/x86/kernel/apic/x2apic_uv_x.c | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/arch/x86/include/asm/uv/bios.h b/arch/x86/include/asm/uv/bios.h index 01ba080887b3..1b6455f881f9 100644 --- a/arch/x86/include/asm/uv/bios.h +++ b/arch/x86/include/asm/uv/bios.h @@ -200,6 +200,8 @@ extern long sn_partition_id; extern long sn_coherency_id; extern long sn_region_size; extern long system_serial_number; +extern ssize_t uv_get_archtype(char *buf, int len); +extern int uv_get_hubless_system(void); extern struct kobject *sgi_uv_kobj; /* /sys/firmware/sgi_uv */ diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 1b98f8c12b96..48746031b39a 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -502,6 +502,18 @@ enum uv_system_type get_uv_system_type(void) return uv_system_type; } +int uv_get_hubless_system(void) +{ + return uv_hubless_system; +} +EXPORT_SYMBOL_GPL(uv_get_hubless_system); + +ssize_t uv_get_archtype(char *buf, int len) +{ + return scnprintf(buf, len, "%s/%s", uv_archtype, oem_table_id); +} +EXPORT_SYMBOL_GPL(uv_get_archtype); + int is_uv_system(void) { return uv_system_type != UV_NONE; From 612a0063c9ba3aba79b9006faa0edad5f9d41162 Mon Sep 17 00:00:00 2001 From: Mike Travis Date: Fri, 27 Nov 2020 21:42:24 -0600 Subject: [PATCH 336/360] x86/platform/uv: Add sysfs leaves to replace those in procfs Add uv_sysfs leaves to display the info. Signed-off-by: Mike Travis Signed-off-by: Borislav Petkov Reviewed-by: Steve Wahl Acked-by: Hans de Goede Link: https://lkml.kernel.org/r/20201128034227.120869-3-mike.travis@hpe.com --- drivers/platform/x86/uv_sysfs.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/drivers/platform/x86/uv_sysfs.c b/drivers/platform/x86/uv_sysfs.c index c27f5ffd43bb..922d32fb580a 100644 --- a/drivers/platform/x86/uv_sysfs.c +++ b/drivers/platform/x86/uv_sysfs.c @@ -736,17 +736,35 @@ static ssize_t uv_type_show(struct kobject *kobj, return scnprintf(buf, PAGE_SIZE, "%s\n", uv_type_string()); } +static ssize_t uv_archtype_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return uv_get_archtype(buf, PAGE_SIZE); +} + +static ssize_t uv_hub_type_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "0x%x\n", uv_hub_type()); +} + static struct kobj_attribute partition_id_attr = __ATTR(partition_id, 0444, partition_id_show, NULL); static struct kobj_attribute coherence_id_attr = __ATTR(coherence_id, 0444, coherence_id_show, NULL); static struct kobj_attribute uv_type_attr = __ATTR(uv_type, 0444, uv_type_show, NULL); +static struct kobj_attribute uv_archtype_attr = + __ATTR(archtype, 0444, uv_archtype_show, NULL); +static struct kobj_attribute uv_hub_type_attr = + __ATTR(hub_type, 0444, uv_hub_type_show, NULL); static struct attribute *base_attrs[] = { &partition_id_attr.attr, &coherence_id_attr.attr, &uv_type_attr.attr, + &uv_archtype_attr.attr, + &uv_hub_type_attr.attr, NULL, }; From 433e817ae157479844d84b186dd4d165a3f2b06e Mon Sep 17 00:00:00 2001 From: Mike Travis Date: Fri, 27 Nov 2020 21:42:25 -0600 Subject: [PATCH 337/360] x86/platform/uv: Add sysfs hubless leaves Add uv_sysfs hubless leaves for UV hubless systems. Signed-off-by: Mike Travis Signed-off-by: Borislav Petkov Reviewed-by: Steve Wahl Acked-by: Hans de Goede Link: https://lkml.kernel.org/r/20201128034227.120869-4-mike.travis@hpe.com --- drivers/platform/x86/uv_sysfs.c | 52 +++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/drivers/platform/x86/uv_sysfs.c b/drivers/platform/x86/uv_sysfs.c index 922d32fb580a..7badcfa3f384 100644 --- a/drivers/platform/x86/uv_sysfs.c +++ b/drivers/platform/x86/uv_sysfs.c @@ -44,6 +44,8 @@ static const char *uv_type_string(void) return "5.0"; else if (is_uv2_hub()) return "3.0"; + else if (uv_get_hubless_system()) + return "0.1"; else return "unknown"; } @@ -748,6 +750,12 @@ static ssize_t uv_hub_type_show(struct kobject *kobj, return scnprintf(buf, PAGE_SIZE, "0x%x\n", uv_hub_type()); } +static ssize_t uv_hubless_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return scnprintf(buf, PAGE_SIZE, "0x%x\n", uv_get_hubless_system()); +} + static struct kobj_attribute partition_id_attr = __ATTR(partition_id, 0444, partition_id_show, NULL); static struct kobj_attribute coherence_id_attr = @@ -758,6 +766,8 @@ static struct kobj_attribute uv_archtype_attr = __ATTR(archtype, 0444, uv_archtype_show, NULL); static struct kobj_attribute uv_hub_type_attr = __ATTR(hub_type, 0444, uv_hub_type_show, NULL); +static struct kobj_attribute uv_hubless_attr = + __ATTR(hubless, 0444, uv_hubless_show, NULL); static struct attribute *base_attrs[] = { &partition_id_attr.attr, @@ -805,11 +815,36 @@ static int initial_bios_setup(void) return 0; } +static struct attribute *hubless_base_attrs[] = { + &partition_id_attr.attr, + &uv_type_attr.attr, + &uv_archtype_attr.attr, + &uv_hubless_attr.attr, + NULL, +}; + +static struct attribute_group hubless_base_attr_group = { + .attrs = hubless_base_attrs +}; + + +static int __init uv_sysfs_hubless_init(void) +{ + int ret; + + ret = sysfs_create_group(sgi_uv_kobj, &hubless_base_attr_group); + if (ret) { + pr_warn("sysfs_create_group hubless_base_attr_group failed\n"); + kobject_put(sgi_uv_kobj); + } + return ret; +} + static int __init uv_sysfs_init(void) { int ret = 0; - if (!is_uv_system()) + if (!is_uv_system() && !uv_get_hubless_system()) return -ENODEV; num_cnodes = uv_num_possible_blades(); @@ -820,6 +855,10 @@ static int __init uv_sysfs_init(void) pr_warn("kobject_create_and_add sgi_uv failed\n"); return -EINVAL; } + + if (uv_get_hubless_system()) + return uv_sysfs_hubless_init(); + ret = sysfs_create_group(sgi_uv_kobj, &base_attr_group); if (ret) { pr_warn("sysfs_create_group base_attr_group failed\n"); @@ -857,10 +896,19 @@ err_create_group: return ret; } +static void __exit uv_sysfs_hubless_exit(void) +{ + sysfs_remove_group(sgi_uv_kobj, &hubless_base_attr_group); + kobject_put(sgi_uv_kobj); +} + static void __exit uv_sysfs_exit(void) { - if (!is_uv_system()) + if (!is_uv_system()) { + if (uv_get_hubless_system()) + uv_sysfs_hubless_exit(); return; + } pci_topology_exit(); uv_ports_exit(); From 148c277165cdc72d97d1711b9a1e566d66521828 Mon Sep 17 00:00:00 2001 From: Mike Travis Date: Fri, 27 Nov 2020 21:42:26 -0600 Subject: [PATCH 338/360] x86/platform/uv: Add deprecated messages to /proc info leaves Add "deprecated" message to any access to old /proc/sgi_uv/* leaves. [ bp: Do not have a trailing function opening brace and the arguments continuing on the next line and align them on the opening brace. ] Signed-off-by: Mike Travis Signed-off-by: Borislav Petkov Reviewed-by: Steve Wahl Acked-by: Hans de Goede Link: https://lkml.kernel.org/r/20201128034227.120869-5-mike.travis@hpe.com --- arch/x86/kernel/apic/x2apic_uv_x.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c index 48746031b39a..d75e1d98fa7c 100644 --- a/arch/x86/kernel/apic/x2apic_uv_x.c +++ b/arch/x86/kernel/apic/x2apic_uv_x.c @@ -1615,21 +1615,30 @@ static void check_efi_reboot(void) reboot_type = BOOT_ACPI; } -/* Setup user proc fs files */ +/* + * User proc fs file handling now deprecated. + * Recommend using /sys/firmware/sgi_uv/... instead. + */ static int __maybe_unused proc_hubbed_show(struct seq_file *file, void *data) { + pr_notice_once("%s: using deprecated /proc/sgi_uv/hubbed, use /sys/firmware/sgi_uv/hub_type\n", + current->comm); seq_printf(file, "0x%x\n", uv_hubbed_system); return 0; } static int __maybe_unused proc_hubless_show(struct seq_file *file, void *data) { + pr_notice_once("%s: using deprecated /proc/sgi_uv/hubless, use /sys/firmware/sgi_uv/hubless\n", + current->comm); seq_printf(file, "0x%x\n", uv_hubless_system); return 0; } static int __maybe_unused proc_archtype_show(struct seq_file *file, void *data) { + pr_notice_once("%s: using deprecated /proc/sgi_uv/archtype, use /sys/firmware/sgi_uv/archtype\n", + current->comm); seq_printf(file, "%s/%s\n", uv_archtype, oem_table_id); return 0; } From c9624cb7db1c418cbdc8fd2cde6835f83cd0f8a2 Mon Sep 17 00:00:00 2001 From: Mike Travis Date: Fri, 27 Nov 2020 21:42:27 -0600 Subject: [PATCH 339/360] x86/platform/uv: Update sysfs documentation Update sysfs documentation file to include moved /proc leaves. Signed-off-by: Mike Travis Signed-off-by: Borislav Petkov Acked-by: Hans de Goede Link: https://lkml.kernel.org/r/20201128034227.120869-6-mike.travis@hpe.com --- Documentation/ABI/testing/sysfs-firmware-sgi_uv | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-firmware-sgi_uv b/Documentation/ABI/testing/sysfs-firmware-sgi_uv index 351b1f41b6bf..637c668cbe45 100644 --- a/Documentation/ABI/testing/sysfs-firmware-sgi_uv +++ b/Documentation/ABI/testing/sysfs-firmware-sgi_uv @@ -7,10 +7,25 @@ Description: Under that directory are a number of read-only attributes:: + archtype + hub_type + hubless partition_id coherence_id uv_type + The archtype entry contains the UV architecture type that + is used to select arch-dependent addresses and features. + It can be set via the OEM_ID in the ACPI MADT table or by + UVsystab entry both passed from UV BIOS. + + The hub_type entry is used to select the type of hub which is + similar to uv_type but encoded in a binary format. Include + the file uv_hub.h to get the definitions. + + The hubless entry basically is present and set only if there + is no hub. In this case the hub_type entry is not present. + The partition_id entry contains the partition id. UV systems can be partitioned into multiple physical machines, which each partition running a unique copy @@ -24,6 +39,7 @@ Description: The uv_type entry contains the hub revision number. This value can be used to identify the UV system version:: + "0.*" = Hubless UV ('*' is subtype) "3.0" = UV2 "5.0" = UV3 From f77f420d34754b8d08ac6ebf094ff7193023196a Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Sat, 5 Dec 2020 01:19:45 +0100 Subject: [PATCH 340/360] x86/msr: Add a pointer to an URL which contains further details After having collected the majority of reports about MSRs being written by userspace tools and what tools those are, and all newer reports mostly repeating, add an URL where detailed information is gathered and kept up-to-date. Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201205002825.19107-1-bp@alien8.de --- arch/x86/kernel/msr.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c index 95e6b97b7d8b..8a67d1fa8dc5 100644 --- a/arch/x86/kernel/msr.c +++ b/arch/x86/kernel/msr.c @@ -99,8 +99,9 @@ static int filter_write(u32 reg) if (!__ratelimit(&fw_rs)) return 0; - pr_warn("Write to unrecognized MSR 0x%x by %s (pid: %d). Please report to x86@kernel.org.\n", + pr_warn("Write to unrecognized MSR 0x%x by %s (pid: %d).\n", reg, current->comm, current->pid); + pr_warn("See https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git/about for details.\n"); return 0; } From 262bd5724afdefd4c48a260d6100e78cc43ee06b Mon Sep 17 00:00:00 2001 From: Arvind Sankar Date: Fri, 27 Nov 2020 12:13:24 -0500 Subject: [PATCH 341/360] x86/cpu/amd: Remove dead code for TSEG region remapping Commit 26bfa5f89486 ("x86, amd: Cleanup init_amd") moved the code that remaps the TSEG region using 4k pages from init_amd() to bsp_init_amd(). However, bsp_init_amd() is executed well before the direct mapping is actually created: setup_arch() -> early_cpu_init() -> early_identify_cpu() -> this_cpu->c_bsp_init() -> bsp_init_amd() ... -> init_mem_mapping() So the change effectively disabled the 4k remapping, because pfn_range_is_mapped() is always false at this point. It has been over six years since the commit, and no-one seems to have noticed this, so just remove the code. The original code was also incomplete, since it doesn't check how large the TSEG address range actually is, so it might remap only part of it in any case. Hygon has copied the incorrect version, so the code has never run on it since the cpu support was added two years ago. Remove it from there as well. Committer notes: This workaround is incomplete anyway: 1. The code must check MSRC001_0113.TValid (SMM TSeg Mask MSR) first, to check whether the TSeg address range is enabled. 2. The code must check whether the range is not 2M aligned - if it is, there's nothing to work around. 3. In all the BIOSes tested, the TSeg range is in a e820 reserved area and those are not mapped anymore, after 66520ebc2df3 ("x86, mm: Only direct map addresses that are marked as E820_RAM") which means, there's nothing to be worked around either. So let's rip it out. Signed-off-by: Arvind Sankar Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201127171324.1846019-1-nivedita@alum.mit.edu --- arch/x86/kernel/cpu/amd.c | 21 --------------------- arch/x86/kernel/cpu/hygon.c | 20 -------------------- 2 files changed, 41 deletions(-) diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 1f71c7616917..f8ca66f3d861 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -23,7 +23,6 @@ #ifdef CONFIG_X86_64 # include -# include #endif #include "cpu.h" @@ -509,26 +508,6 @@ static void early_init_amd_mc(struct cpuinfo_x86 *c) static void bsp_init_amd(struct cpuinfo_x86 *c) { - -#ifdef CONFIG_X86_64 - if (c->x86 >= 0xf) { - unsigned long long tseg; - - /* - * Split up direct mapping around the TSEG SMM area. - * Don't do it for gbpages because there seems very little - * benefit in doing so. - */ - if (!rdmsrl_safe(MSR_K8_TSEG_ADDR, &tseg)) { - unsigned long pfn = tseg >> PAGE_SHIFT; - - pr_debug("tseg: %010llx\n", tseg); - if (pfn_range_is_mapped(pfn, pfn + 1)) - set_memory_4k((unsigned long)__va(tseg), 1); - } - } -#endif - if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) { if (c->x86 > 0x10 || diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c index dc0840aae26c..ae59115d18f9 100644 --- a/arch/x86/kernel/cpu/hygon.c +++ b/arch/x86/kernel/cpu/hygon.c @@ -14,9 +14,6 @@ #include #include #include -#ifdef CONFIG_X86_64 -# include -#endif #include "cpu.h" @@ -203,23 +200,6 @@ static void early_init_hygon_mc(struct cpuinfo_x86 *c) static void bsp_init_hygon(struct cpuinfo_x86 *c) { -#ifdef CONFIG_X86_64 - unsigned long long tseg; - - /* - * Split up direct mapping around the TSEG SMM area. - * Don't do it for gbpages because there seems very little - * benefit in doing so. - */ - if (!rdmsrl_safe(MSR_K8_TSEG_ADDR, &tseg)) { - unsigned long pfn = tseg >> PAGE_SHIFT; - - pr_debug("tseg: %010llx\n", tseg); - if (pfn_range_is_mapped(pfn, pfn + 1)) - set_memory_4k((unsigned long)__va(tseg), 1); - } -#endif - if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) { u64 val; From 14d4c4fa46eeaa3922e8e1c4aa727eb0a1412804 Mon Sep 17 00:00:00 2001 From: Qinglang Miao Date: Tue, 1 Dec 2020 14:31:50 +0800 Subject: [PATCH 342/360] s390/cio: fix use-after-free in ccw_device_destroy_console Use of sch->dev reference after the put_device() call could trigger the use-after-free bugs. Fix this by simply adjusting the position of put_device. Fixes: 37db8985b211 ("s390/cio: add basic protected virtualization support") Reported-by: Hulk Robot Suggested-by: Cornelia Huck Signed-off-by: Qinglang Miao Reviewed-by: Cornelia Huck Reviewed-by: Vineeth Vijayan [vneethv@linux.ibm.com: Slight modification in the commit-message] Signed-off-by: Vineeth Vijayan Signed-off-by: Heiko Carstens --- drivers/s390/cio/device.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c index 5fbc549786ab..e0005a4fc978 100644 --- a/drivers/s390/cio/device.c +++ b/drivers/s390/cio/device.c @@ -1645,10 +1645,10 @@ void __init ccw_device_destroy_console(struct ccw_device *cdev) struct io_subchannel_private *io_priv = to_io_private(sch); set_io_private(sch, NULL); - put_device(&sch->dev); - put_device(&cdev->dev); dma_free_coherent(&sch->dev, sizeof(*io_priv->dma_area), io_priv->dma_area, io_priv->dma_area_dma); + put_device(&sch->dev); + put_device(&cdev->dev); kfree(io_priv); } From 613775d62ec60202f98d2c5f520e6e9ba6dd4ac4 Mon Sep 17 00:00:00 2001 From: Philipp Rudo Date: Thu, 26 Nov 2020 18:31:08 +0100 Subject: [PATCH 343/360] s390/kexec_file: fix diag308 subcode when loading crash kernel diag308 subcode 0 performes a clear reset which inlcudes the reset of all registers in the system. While this is the preferred behavior when loading a normal kernel via kexec it prevents the crash kernel to store the register values in the dump. To prevent this use subcode 1 when loading a crash kernel instead. Fixes: ee337f5469fd ("s390/kexec_file: Add crash support to image loader") Cc: # 4.17 Signed-off-by: Philipp Rudo Reported-by: Xiaoying Yan Tested-by: Lianbo Jiang Signed-off-by: Heiko Carstens --- arch/s390/purgatory/head.S | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/s390/purgatory/head.S b/arch/s390/purgatory/head.S index 5a10ce34b95d..3d1c31e0cf3d 100644 --- a/arch/s390/purgatory/head.S +++ b/arch/s390/purgatory/head.S @@ -62,14 +62,15 @@ jh 10b .endm -.macro START_NEXT_KERNEL base +.macro START_NEXT_KERNEL base subcode lg %r4,kernel_entry-\base(%r13) lg %r5,load_psw_mask-\base(%r13) ogr %r4,%r5 stg %r4,0(%r0) xgr %r0,%r0 - diag %r0,%r0,0x308 + lghi %r1,\subcode + diag %r0,%r1,0x308 .endm .text @@ -123,7 +124,7 @@ ENTRY(purgatory_start) je .start_crash_kernel /* start normal kernel */ - START_NEXT_KERNEL .base_crash + START_NEXT_KERNEL .base_crash 0 .return_old_kernel: lmg %r6,%r15,gprregs-.base_crash(%r13) @@ -227,7 +228,7 @@ ENTRY(purgatory_start) MEMCPY %r9,%r10,%r11 /* start crash kernel */ - START_NEXT_KERNEL .base_dst + START_NEXT_KERNEL .base_dst 1 load_psw_mask: From 7a84ffc0471f0ff59f9ae10807a77bd666e24fd0 Mon Sep 17 00:00:00 2001 From: Philipp Rudo Date: Wed, 25 Nov 2020 19:08:23 +0100 Subject: [PATCH 344/360] s390/boot: add build-id to decompressor More and more functionality from the early boot phase gets carried over to the decompressor. With this the complexity of the code and thus the chance to introduce bugs increases. In order to be able to debug these early boot bugs the distributions have to package the decompressors vmlinux together with the other debuginfos. However for that the distributions require the vmlinux to contain a build-id. Per default the section containing the build-id is placed first in the section table. So make sure to move it behind the .text section otherwise the image would be unbootable. Signed-off-by: Philipp Rudo Signed-off-by: Heiko Carstens --- arch/s390/boot/compressed/Makefile | 2 +- arch/s390/boot/compressed/vmlinux.lds.S | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/s390/boot/compressed/Makefile b/arch/s390/boot/compressed/Makefile index e0502222706a..de18dab518bb 100644 --- a/arch/s390/boot/compressed/Makefile +++ b/arch/s390/boot/compressed/Makefile @@ -23,7 +23,7 @@ OBJCOPYFLAGS := OBJECTS := $(addprefix $(obj)/,$(obj-y)) OBJECTS_ALL := $(addprefix $(obj)/,$(obj-all)) -LDFLAGS_vmlinux := --oformat $(LD_BFD) -e startup -T +LDFLAGS_vmlinux := --oformat $(LD_BFD) -e startup --build-id=sha1 -T $(obj)/vmlinux: $(obj)/vmlinux.lds $(objtree)/arch/s390/boot/startup.a $(OBJECTS_ALL) FORCE $(call if_changed,ld) diff --git a/arch/s390/boot/compressed/vmlinux.lds.S b/arch/s390/boot/compressed/vmlinux.lds.S index 152ac6f875c0..27a09c1c78f6 100644 --- a/arch/s390/boot/compressed/vmlinux.lds.S +++ b/arch/s390/boot/compressed/vmlinux.lds.S @@ -27,6 +27,7 @@ SECTIONS *(.rodata.*) _erodata = . ; } + NOTES .data : { _data = . ; *(.data) From e259b3fafa7de362b04ecd86e7fa9a9e9273e5fb Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Thu, 3 Dec 2020 22:03:32 +0100 Subject: [PATCH 345/360] s390/idle: add missing mt_cycles calculation During removal of the critical section cleanup the calculation of mt_cycles during idle was removed. This causes invalid accounting on systems with SMT enabled. Fixes: 0b0ed657fe00 ("s390: remove critical section cleanup from entry.S") Cc: # 5.8 Reviewed-by: Heiko Carstens Signed-off-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/kernel/entry.S | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 377f75616693..e7f7aab805e1 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -118,7 +118,7 @@ _LPP_OFFSET = __LC_LPP .macro SWITCH_ASYNC savearea,timer tmhh %r8,0x0001 # interrupting from user ? - jnz 2f + jnz 4f #if IS_ENABLED(CONFIG_KVM) lgr %r14,%r9 larl %r13,.Lsie_gmap @@ -131,9 +131,25 @@ _LPP_OFFSET = __LC_LPP #endif 0: larl %r13,.Lpsw_idle_exit cgr %r13,%r9 - jne 1f + jne 3f - mvc __CLOCK_IDLE_EXIT(8,%r2), __LC_INT_CLOCK + larl %r1,smp_cpu_mtid + llgf %r1,0(%r1) + ltgr %r1,%r1 + jz 2f # no SMT, skip mt_cycles calculation + .insn rsy,0xeb0000000017,%r1,5,__SF_EMPTY+80(%r15) + larl %r3,mt_cycles + ag %r3,__LC_PERCPU_OFFSET + la %r4,__SF_EMPTY+16(%r15) +1: lg %r0,0(%r3) + slg %r0,0(%r4) + alg %r0,64(%r4) + stg %r0,0(%r3) + la %r3,8(%r3) + la %r4,8(%r4) + brct %r1,1b + +2: mvc __CLOCK_IDLE_EXIT(8,%r2), __LC_INT_CLOCK mvc __TIMER_IDLE_EXIT(8,%r2), __LC_ASYNC_ENTER_TIMER # account system time going idle ni __LC_CPU_FLAGS+7,255-_CIF_ENABLED_WAIT @@ -152,17 +168,17 @@ _LPP_OFFSET = __LC_LPP mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2) nihh %r8,0xfcfd # clear wait state and irq bits -1: lg %r14,__LC_ASYNC_STACK # are we already on the target stack? +3: lg %r14,__LC_ASYNC_STACK # are we already on the target stack? slgr %r14,%r15 srag %r14,%r14,STACK_SHIFT - jnz 3f + jnz 5f CHECK_STACK \savearea aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) - j 4f -2: UPDATE_VTIME %r14,%r15,\timer + j 6f +4: UPDATE_VTIME %r14,%r15,\timer BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP -3: lg %r15,__LC_ASYNC_STACK # load async stack -4: la %r11,STACK_FRAME_OVERHEAD(%r15) +5: lg %r15,__LC_ASYNC_STACK # load async stack +6: la %r11,STACK_FRAME_OVERHEAD(%r15) .endm .macro UPDATE_VTIME w1,w2,enter_timer From 454efcf82ea17d7efeb86ebaa20775a21ec87d27 Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Sun, 6 Dec 2020 10:47:47 +0100 Subject: [PATCH 346/360] s390/idle: fix accounting with machine checks When a machine check interrupt is triggered during idle, the code is using the async timer/clock for idle time calculation. It should use the machine check enter timer/clock which is passed to the macro. Fixes: 0b0ed657fe00 ("s390: remove critical section cleanup from entry.S") Cc: # 5.8 Reviewed-by: Heiko Carstens Signed-off-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/kernel/entry.S | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index e7f7aab805e1..55f65f2b7daf 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -116,7 +116,7 @@ _LPP_OFFSET = __LC_LPP #endif .endm - .macro SWITCH_ASYNC savearea,timer + .macro SWITCH_ASYNC savearea,timer,clock tmhh %r8,0x0001 # interrupting from user ? jnz 4f #if IS_ENABLED(CONFIG_KVM) @@ -149,8 +149,8 @@ _LPP_OFFSET = __LC_LPP la %r4,8(%r4) brct %r1,1b -2: mvc __CLOCK_IDLE_EXIT(8,%r2), __LC_INT_CLOCK - mvc __TIMER_IDLE_EXIT(8,%r2), __LC_ASYNC_ENTER_TIMER +2: mvc __CLOCK_IDLE_EXIT(8,%r2), \clock + mvc __TIMER_IDLE_EXIT(8,%r2), \timer # account system time going idle ni __LC_CPU_FLAGS+7,255-_CIF_ENABLED_WAIT @@ -757,7 +757,7 @@ ENTRY(io_int_handler) stmg %r8,%r15,__LC_SAVE_AREA_ASYNC lg %r12,__LC_CURRENT lmg %r8,%r9,__LC_IO_OLD_PSW - SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER + SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER,__LC_INT_CLOCK stmg %r0,%r7,__PT_R0(%r11) # clear user controlled registers to prevent speculative use xgr %r0,%r0 @@ -952,7 +952,7 @@ ENTRY(ext_int_handler) stmg %r8,%r15,__LC_SAVE_AREA_ASYNC lg %r12,__LC_CURRENT lmg %r8,%r9,__LC_EXT_OLD_PSW - SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER + SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER,__LC_INT_CLOCK stmg %r0,%r7,__PT_R0(%r11) # clear user controlled registers to prevent speculative use xgr %r0,%r0 @@ -1183,7 +1183,7 @@ ENTRY(mcck_int_handler) TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID jno .Lmcck_panic 4: ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off - SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_MCCK_ENTER_TIMER + SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_MCCK_ENTER_TIMER,__LC_MCCK_CLOCK .Lmcck_skip: lghi %r14,__LC_GPREGS_SAVE_AREA+64 stmg %r0,%r7,__PT_R0(%r11) From b4d70a6134d2152d692ccc873ff6fa3351631927 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 7 Dec 2020 20:56:38 +0100 Subject: [PATCH 347/360] s390/mm: use invalid asce for user space when switching to init_mm Currently only idle_task_exit() explicitly switches (switch_mm) to init_mm. This causes the kernel asce to be loaded into cr7 and therefore it would be used for potential user space accesses. This is currently no problem since idle_task_exit() is nearly the last thing a CPU executes before it is taken down. However things might change - and therefore make sure that always the invalid asce is used for cr7 when active_mm is init_mm. This makes sure that all potential user space accesses will fail, instead of accessing kernel address space. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens --- arch/s390/include/asm/mmu_context.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index 87a84fc59fc3..5dc49c467319 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -76,7 +76,10 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, { int cpu = smp_processor_id(); - S390_lowcore.user_asce = next->context.asce; + if (next == &init_mm) + S390_lowcore.user_asce = s390_invalid_asce; + else + S390_lowcore.user_asce = next->context.asce; cpumask_set_cpu(cpu, &next->context.cpu_attach_mask); /* Clear previous user-ASCE from CR7 */ __ctl_load(s390_invalid_asce, 7, 7); From b5e438ebd7e808d1d2435159ac4742e01a94b8da Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Tue, 8 Dec 2020 07:35:21 +0100 Subject: [PATCH 348/360] s390/smp: perform initial CPU reset also for SMT siblings Not resetting the SMT siblings might leave them in unpredictable state. One of the observed problems was that the CPU timer wasn't reset and therefore large system time values where accounted during CPU bringup. Cc: # 4.0 Fixes: 10ad34bc76dfb ("s390: add SMT support") Reviewed-by: Heiko Carstens Signed-off-by: Sven Schnelle Signed-off-by: Heiko Carstens --- arch/s390/kernel/smp.c | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 647226e50c80..27c763014114 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -890,24 +890,12 @@ static void __no_sanitize_address smp_start_secondary(void *cpuvoid) /* Upping and downing of CPUs */ int __cpu_up(unsigned int cpu, struct task_struct *tidle) { - struct pcpu *pcpu; - int base, i, rc; + struct pcpu *pcpu = pcpu_devices + cpu; + int rc; - pcpu = pcpu_devices + cpu; if (pcpu->state != CPU_STATE_CONFIGURED) return -EIO; - base = smp_get_base_cpu(cpu); - for (i = 0; i <= smp_cpu_mtid; i++) { - if (base + i < nr_cpu_ids) - if (cpu_online(base + i)) - break; - } - /* - * If this is the first CPU of the core to get online - * do an initial CPU reset. - */ - if (i > smp_cpu_mtid && - pcpu_sigp_retry(pcpu_devices + base, SIGP_INITIAL_CPU_RESET, 0) != + if (pcpu_sigp_retry(pcpu, SIGP_INITIAL_CPU_RESET, 0) != SIGP_CC_ORDER_CODE_ACCEPTED) return -EIO; From ff98cc986ae883eec5f26af72d4e2406612fe683 Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Thu, 3 Dec 2020 15:02:08 +0100 Subject: [PATCH 349/360] s390/crypto: add arch_get_random_long() support The random longs to be pulled by arch_get_random_long() are prepared in an 4K buffer which is filled from the NIST 800-90 compliant s390 drbg. By default the random long buffer is refilled 256 times before the drbg itself needs a reseed. The reseed of the drbg is done with 32 bytes fetched from the high quality (but slow) trng which is assumed to deliver 100% entropy. So the 32 * 8 = 256 bits of entropy are spread over 256 * 4KB = 1MB serving 131072 arch_get_random_long() invocations before reseeded. How often the 4K random long buffer is refilled with the drbg before the drbg is reseeded can be adjusted. There is a module parameter 's390_arch_rnd_long_drbg_reseed' accessible via /sys/module/arch_random/parameters/rndlong_drbg_reseed or as kernel command line parameter arch_random.rndlong_drbg_reseed= This parameter tells how often the drbg fills the 4K buffer before it is re-seeded by fresh entropy from the trng. A value of 16 results in reseeding the drbg at every 16 * 4 KB = 64 KB with 32 bytes of fresh entropy pulled from the trng. So a value of 16 would result in 256 bits entropy per 64 KB. A value of 256 results in 1MB of drbg output before a reseed of the drbg is done. So this would spread the 256 bits of entropy among 1MB. Setting this parameter to 0 forces the reseed to take place every time the 4K buffer is depleted, so the entropy rises to 256 bits entropy per 4K or 0.5 bit entropy per arch_get_random_long(). With setting this parameter to negative values all this effort is disabled, arch_get_random long() returns false and thus indicating that the arch_get_random_long() feature is disabled at all. arch_get_random_long() is used by random.c among others to provide an initial hash value to be mixed with the entropy pool on every random data pull. For about 64 bytes read from /dev/urandom there is one call to arch_get_random_long(). So these additional random long values count for performance of /dev/urandom with measurable but low penalty. Signed-off-by: Harald Freudenberger Reviewed-by: Ingo Franzki Reviewed-by: Juergen Christ Signed-off-by: Heiko Carstens --- arch/s390/crypto/arch_random.c | 110 ++++++++++++++++++++++++++++- arch/s390/include/asm/archrandom.h | 5 +- 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/arch/s390/crypto/arch_random.c b/arch/s390/crypto/arch_random.c index dd95cdbd22ce..7b947728d57e 100644 --- a/arch/s390/crypto/arch_random.c +++ b/arch/s390/crypto/arch_random.c @@ -2,7 +2,7 @@ /* * s390 arch random implementation. * - * Copyright IBM Corp. 2017, 2018 + * Copyright IBM Corp. 2017, 2020 * Author(s): Harald Freudenberger * * The s390_arch_random_generate() function may be called from random.c @@ -33,6 +33,7 @@ #include #include #include +#include #include DEFINE_STATIC_KEY_FALSE(s390_arch_random_available); @@ -99,6 +100,113 @@ static void arch_rng_refill_buffer(struct work_struct *unused) queue_delayed_work(system_long_wq, &arch_rng_work, delay); } +/* + * Here follows the implementation of s390_arch_get_random_long(). + * + * The random longs to be pulled by arch_get_random_long() are + * prepared in an 4K buffer which is filled from the NIST 800-90 + * compliant s390 drbg. By default the random long buffer is refilled + * 256 times before the drbg itself needs a reseed. The reseed of the + * drbg is done with 32 bytes fetched from the high quality (but slow) + * trng which is assumed to deliver 100% entropy. So the 32 * 8 = 256 + * bits of entropy are spread over 256 * 4KB = 1MB serving 131072 + * arch_get_random_long() invocations before reseeded. + * + * How often the 4K random long buffer is refilled with the drbg + * before the drbg is reseeded can be adjusted. There is a module + * parameter 's390_arch_rnd_long_drbg_reseed' accessible via + * /sys/module/arch_random/parameters/rndlong_drbg_reseed + * or as kernel command line parameter + * arch_random.rndlong_drbg_reseed= + * This parameter tells how often the drbg fills the 4K buffer before + * it is re-seeded by fresh entropy from the trng. + * A value of 16 results in reseeding the drbg at every 16 * 4 KB = 64 + * KB with 32 bytes of fresh entropy pulled from the trng. So a value + * of 16 would result in 256 bits entropy per 64 KB. + * A value of 256 results in 1MB of drbg output before a reseed of the + * drbg is done. So this would spread the 256 bits of entropy among 1MB. + * Setting this parameter to 0 forces the reseed to take place every + * time the 4K buffer is depleted, so the entropy rises to 256 bits + * entropy per 4K or 0.5 bit entropy per arch_get_random_long(). With + * setting this parameter to negative values all this effort is + * disabled, arch_get_random long() returns false and thus indicating + * that the arch_get_random_long() feature is disabled at all. + */ + +static unsigned long rndlong_buf[512]; +static DEFINE_SPINLOCK(rndlong_lock); +static int rndlong_buf_index; + +static int rndlong_drbg_reseed = 256; +module_param_named(rndlong_drbg_reseed, rndlong_drbg_reseed, int, 0600); +MODULE_PARM_DESC(rndlong_drbg_reseed, "s390 arch_get_random_long() drbg reseed"); + +static inline void refill_rndlong_buf(void) +{ + static u8 prng_ws[240]; + static int drbg_counter; + + if (--drbg_counter < 0) { + /* need to re-seed the drbg */ + u8 seed[32]; + + /* fetch seed from trng */ + cpacf_trng(NULL, 0, seed, sizeof(seed)); + /* seed drbg */ + memset(prng_ws, 0, sizeof(prng_ws)); + cpacf_prno(CPACF_PRNO_SHA512_DRNG_SEED, + &prng_ws, NULL, 0, seed, sizeof(seed)); + /* re-init counter for drbg */ + drbg_counter = rndlong_drbg_reseed; + } + + /* fill the arch_get_random_long buffer from drbg */ + cpacf_prno(CPACF_PRNO_SHA512_DRNG_GEN, &prng_ws, + (u8 *) rndlong_buf, sizeof(rndlong_buf), + NULL, 0); +} + +bool s390_arch_get_random_long(unsigned long *v) +{ + bool rc = false; + unsigned long flags; + + /* arch_get_random_long() disabled ? */ + if (rndlong_drbg_reseed < 0) + return false; + + /* try to lock the random long lock */ + if (!spin_trylock_irqsave(&rndlong_lock, flags)) + return false; + + if (--rndlong_buf_index >= 0) { + /* deliver next long value from the buffer */ + *v = rndlong_buf[rndlong_buf_index]; + rc = true; + goto out; + } + + /* buffer is depleted and needs refill */ + if (in_interrupt()) { + /* delay refill in interrupt context to next caller */ + rndlong_buf_index = 0; + goto out; + } + + /* refill random long buffer */ + refill_rndlong_buf(); + rndlong_buf_index = ARRAY_SIZE(rndlong_buf); + + /* and provide one random long */ + *v = rndlong_buf[--rndlong_buf_index]; + rc = true; + +out: + spin_unlock_irqrestore(&rndlong_lock, flags); + return rc; +} +EXPORT_SYMBOL(s390_arch_get_random_long); + static int __init s390_arch_random_init(void) { /* all the needed PRNO subfunctions available ? */ diff --git a/arch/s390/include/asm/archrandom.h b/arch/s390/include/asm/archrandom.h index de61ce562052..5dc712fde3c7 100644 --- a/arch/s390/include/asm/archrandom.h +++ b/arch/s390/include/asm/archrandom.h @@ -2,7 +2,7 @@ /* * Kernel interface for the s390 arch_random_* functions * - * Copyright IBM Corp. 2017 + * Copyright IBM Corp. 2017, 2020 * * Author: Harald Freudenberger * @@ -19,10 +19,13 @@ DECLARE_STATIC_KEY_FALSE(s390_arch_random_available); extern atomic64_t s390_arch_random_counter; +bool s390_arch_get_random_long(unsigned long *v); bool s390_arch_random_generate(u8 *buf, unsigned int nbytes); static inline bool __must_check arch_get_random_long(unsigned long *v) { + if (static_branch_likely(&s390_arch_random_available)) + return s390_arch_get_random_long(v); return false; } From 343dbdb7cb8997a2cb0fd804d6563b8a6de8d49b Mon Sep 17 00:00:00 2001 From: Gerald Schaefer Date: Tue, 8 Dec 2020 19:47:15 +0100 Subject: [PATCH 350/360] s390/mm: add support to allocate gigantic hugepages using CMA Commit cf11e85fc08c ("mm: hugetlb: optionally allocate gigantic hugepages using cma") added support for allocating gigantic hugepages using CMA, by specifying the hugetlb_cma= kernel parameter, which will disable any boot-time allocation of gigantic hugepages. This patch enables that option also for s390. Signed-off-by: Gerald Schaefer Signed-off-by: Heiko Carstens --- arch/s390/kernel/setup.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 756936785598..1f16a03be995 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -49,6 +49,7 @@ #include #include #include +#include #include #include @@ -1146,6 +1147,8 @@ void __init setup_arch(char **cmdline_p) setup_memory(); dma_contiguous_reserve(ident_map_size); vmcp_cma_reserve(); + if (MACHINE_HAS_EDAT2) + hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT); check_initrd(); reserve_crashkernel(); From c7e34aa31d6297a3a49a0c0142df858838d4b54f Mon Sep 17 00:00:00 2001 From: Daniele Alessandrelli Date: Thu, 26 Nov 2020 11:51:47 +0000 Subject: [PATCH 351/360] dt-bindings: Add Keem Bay OCS AES bindings Add device-tree bindings for Intel Keem Bay Offload and Crypto Subsystem (OCS) AES crypto driver. Signed-off-by: Daniele Alessandrelli Acked-by: Mark Gross Reviewed-by: Rob Herring Signed-off-by: Herbert Xu --- .../crypto/intel,keembay-ocs-aes.yaml | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml diff --git a/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml new file mode 100644 index 000000000000..ee2c099981b2 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/crypto/intel,keembay-ocs-aes.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Intel Keem Bay OCS AES Device Tree Bindings + +maintainers: + - Daniele Alessandrelli + +description: + The Intel Keem Bay Offload and Crypto Subsystem (OCS) AES engine provides + hardware-accelerated AES/SM4 encryption/decryption. + +properties: + compatible: + const: intel,keembay-ocs-aes + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - clocks + +additionalProperties: false + +examples: + - | + #include + crypto@30008000 { + compatible = "intel,keembay-ocs-aes"; + reg = <0x30008000 0x1000>; + interrupts = ; + clocks = <&scmi_clk 95>; + }; From 88574332451380f4b51f6ca88ab9810e714bfb9b Mon Sep 17 00:00:00 2001 From: Mike Healy Date: Thu, 26 Nov 2020 11:51:48 +0000 Subject: [PATCH 352/360] crypto: keembay - Add support for Keem Bay OCS AES/SM4 Add support for the AES/SM4 crypto engine included in the Offload and Crypto Subsystem (OCS) of the Intel Keem Bay SoC, thus enabling hardware-acceleration for the following transformations: - ecb(aes), cbc(aes), ctr(aes), cts(cbc(aes)), gcm(aes) and cbc(aes); supported for 128-bit and 256-bit keys. - ecb(sm4), cbc(sm4), ctr(sm4), cts(cbc(sm4)), gcm(sm4) and cbc(sm4); supported for 128-bit keys. The driver passes crypto manager self-tests, including the extra tests (CRYPTO_MANAGER_EXTRA_TESTS=y). Signed-off-by: Mike Healy Co-developed-by: Daniele Alessandrelli Signed-off-by: Daniele Alessandrelli Acked-by: Mark Gross Signed-off-by: Herbert Xu --- MAINTAINERS | 10 + drivers/crypto/Kconfig | 2 + drivers/crypto/Makefile | 1 + drivers/crypto/keembay/Kconfig | 39 + drivers/crypto/keembay/Makefile | 5 + drivers/crypto/keembay/keembay-ocs-aes-core.c | 1713 +++++++++++++++++ drivers/crypto/keembay/ocs-aes.c | 1489 ++++++++++++++ drivers/crypto/keembay/ocs-aes.h | 129 ++ 8 files changed, 3388 insertions(+) create mode 100644 drivers/crypto/keembay/Kconfig create mode 100644 drivers/crypto/keembay/Makefile create mode 100644 drivers/crypto/keembay/keembay-ocs-aes-core.c create mode 100644 drivers/crypto/keembay/ocs-aes.c create mode 100644 drivers/crypto/keembay/ocs-aes.h diff --git a/MAINTAINERS b/MAINTAINERS index a835a74d4b29..3f81dd8e4004 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8958,6 +8958,16 @@ M: Deepak Saxena S: Maintained F: drivers/char/hw_random/ixp4xx-rng.c +INTEL KEEM BAY OCS AES/SM4 CRYPTO DRIVER +M: Daniele Alessandrelli +S: Maintained +F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml +F: drivers/crypto/keembay/Kconfig +F: drivers/crypto/keembay/Makefile +F: drivers/crypto/keembay/keembay-ocs-aes-core.c +F: drivers/crypto/keembay/ocs-aes.c +F: drivers/crypto/keembay/ocs-aes.h + INTEL MANAGEMENT ENGINE (mei) M: Tomas Winkler L: linux-kernel@vger.kernel.org diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 129bbdc4993b..808eecef0f32 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -900,4 +900,6 @@ config CRYPTO_DEV_SA2UL used for crypto offload. Select this if you want to use hardware acceleration for cryptographic algorithms on these devices. +source "drivers/crypto/keembay/Kconfig" + endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 53fc115cf459..fff9a70348e1 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -51,3 +51,4 @@ obj-$(CONFIG_CRYPTO_DEV_ARTPEC6) += axis/ obj-$(CONFIG_CRYPTO_DEV_ZYNQMP_AES) += xilinx/ obj-y += hisilicon/ obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/ +obj-y += keembay/ diff --git a/drivers/crypto/keembay/Kconfig b/drivers/crypto/keembay/Kconfig new file mode 100644 index 000000000000..3c16797b25b9 --- /dev/null +++ b/drivers/crypto/keembay/Kconfig @@ -0,0 +1,39 @@ +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + tristate "Support for Intel Keem Bay OCS AES/SM4 HW acceleration" + depends on OF || COMPILE_TEST + select CRYPTO_SKCIPHER + select CRYPTO_AEAD + select CRYPTO_ENGINE + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) AES and + SM4 cihper hardware acceleration for use with Crypto API. + + Provides HW acceleration for the following transformations: + cbc(aes), ctr(aes), ccm(aes), gcm(aes), cbc(sm4), ctr(sm4), ccm(sm4) + and gcm(sm4). + + Optionally, support for the following transformations can also be + enabled: ecb(aes), cts(cbc(aes)), ecb(sm4) and cts(cbc(sm4)). + +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + bool "Support for Intel Keem Bay OCS AES/SM4 ECB HW acceleration" + depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) + AES/SM4 ECB mode hardware acceleration for use with Crypto API. + + Provides OCS version of ecb(aes) and ecb(sm4) + + Intel does not recommend use of ECB mode with AES/SM4. + +config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + bool "Support for Intel Keem Bay OCS AES/SM4 CTS HW acceleration" + depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4 + help + Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) + AES/SM4 CBC with CTS mode hardware acceleration for use with + Crypto API. + + Provides OCS version of cts(cbc(aes)) and cts(cbc(sm4)). + + Intel does not recommend use of CTS mode with AES/SM4. diff --git a/drivers/crypto/keembay/Makefile b/drivers/crypto/keembay/Makefile new file mode 100644 index 000000000000..f21e2c4ab3b3 --- /dev/null +++ b/drivers/crypto/keembay/Makefile @@ -0,0 +1,5 @@ +# +# Makefile for Intel Keem Bay OCS Crypto API Linux drivers +# +obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4) += keembay-ocs-aes.o +keembay-ocs-aes-objs := keembay-ocs-aes-core.o ocs-aes.o diff --git a/drivers/crypto/keembay/keembay-ocs-aes-core.c b/drivers/crypto/keembay/keembay-ocs-aes-core.c new file mode 100644 index 000000000000..b6b25d994af3 --- /dev/null +++ b/drivers/crypto/keembay/keembay-ocs-aes-core.c @@ -0,0 +1,1713 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include "ocs-aes.h" + +#define KMB_OCS_PRIORITY 350 +#define DRV_NAME "keembay-ocs-aes" + +#define OCS_AES_MIN_KEY_SIZE 16 +#define OCS_AES_MAX_KEY_SIZE 32 +#define OCS_AES_KEYSIZE_128 16 +#define OCS_AES_KEYSIZE_192 24 +#define OCS_AES_KEYSIZE_256 32 +#define OCS_SM4_KEY_SIZE 16 + +/** + * struct ocs_aes_tctx - OCS AES Transform context + * @engine_ctx: Engine context. + * @aes_dev: The OCS AES device. + * @key: AES/SM4 key. + * @key_len: The length (in bytes) of @key. + * @cipher: OCS cipher to use (either AES or SM4). + * @sw_cipher: The cipher to use as fallback. + * @use_fallback: Whether or not fallback cipher should be used. + */ +struct ocs_aes_tctx { + struct crypto_engine_ctx engine_ctx; + struct ocs_aes_dev *aes_dev; + u8 key[OCS_AES_KEYSIZE_256]; + unsigned int key_len; + enum ocs_cipher cipher; + union { + struct crypto_sync_skcipher *sk; + struct crypto_aead *aead; + } sw_cipher; + bool use_fallback; +}; + +/** + * struct ocs_aes_rctx - OCS AES Request context. + * @instruction: Instruction to be executed (encrypt / decrypt). + * @mode: Mode to use (ECB, CBC, CTR, CCm, GCM, CTS) + * @src_nents: Number of source SG entries. + * @dst_nents: Number of destination SG entries. + * @src_dma_count: The number of DMA-mapped entries of the source SG. + * @dst_dma_count: The number of DMA-mapped entries of the destination SG. + * @in_place: Whether or not this is an in place request, i.e., + * src_sg == dst_sg. + * @src_dll: OCS DMA linked list for input data. + * @dst_dll: OCS DMA linked list for output data. + * @last_ct_blk: Buffer to hold last cipher text block (only used in CBC + * mode). + * @cts_swap: Whether or not CTS swap must be performed. + * @aad_src_dll: OCS DMA linked list for input AAD data. + * @aad_dst_dll: OCS DMA linked list for output AAD data. + * @in_tag: Buffer to hold input encrypted tag (only used for + * CCM/GCM decrypt). + * @out_tag: Buffer to hold output encrypted / decrypted tag (only + * used for GCM encrypt / decrypt). + */ +struct ocs_aes_rctx { + /* Fields common across all modes. */ + enum ocs_instruction instruction; + enum ocs_mode mode; + int src_nents; + int dst_nents; + int src_dma_count; + int dst_dma_count; + bool in_place; + struct ocs_dll_desc src_dll; + struct ocs_dll_desc dst_dll; + + /* CBC specific */ + u8 last_ct_blk[AES_BLOCK_SIZE]; + + /* CTS specific */ + int cts_swap; + + /* CCM/GCM specific */ + struct ocs_dll_desc aad_src_dll; + struct ocs_dll_desc aad_dst_dll; + u8 in_tag[AES_BLOCK_SIZE]; + + /* GCM specific */ + u8 out_tag[AES_BLOCK_SIZE]; +}; + +/* Driver data. */ +struct ocs_aes_drv { + struct list_head dev_list; + spinlock_t lock; /* Protects dev_list. */ +}; + +static struct ocs_aes_drv ocs_aes = { + .dev_list = LIST_HEAD_INIT(ocs_aes.dev_list), + .lock = __SPIN_LOCK_UNLOCKED(ocs_aes.lock), +}; + +static struct ocs_aes_dev *kmb_ocs_aes_find_dev(struct ocs_aes_tctx *tctx) +{ + struct ocs_aes_dev *aes_dev; + + spin_lock(&ocs_aes.lock); + + if (tctx->aes_dev) { + aes_dev = tctx->aes_dev; + goto exit; + } + + /* Only a single OCS device available */ + aes_dev = list_first_entry(&ocs_aes.dev_list, struct ocs_aes_dev, list); + tctx->aes_dev = aes_dev; + +exit: + spin_unlock(&ocs_aes.lock); + + return aes_dev; +} + +/* + * Ensure key is 128-bit or 256-bit for AES or 128-bit for SM4 and an actual + * key is being passed in. + * + * Return: 0 if key is valid, -EINVAL otherwise. + */ +static int check_key(const u8 *in_key, size_t key_len, enum ocs_cipher cipher) +{ + if (!in_key) + return -EINVAL; + + /* For AES, only 128-byte or 256-byte keys are supported. */ + if (cipher == OCS_AES && (key_len == OCS_AES_KEYSIZE_128 || + key_len == OCS_AES_KEYSIZE_256)) + return 0; + + /* For SM4, only 128-byte keys are supported. */ + if (cipher == OCS_SM4 && key_len == OCS_AES_KEYSIZE_128) + return 0; + + /* Everything else is unsupported. */ + return -EINVAL; +} + +/* Save key into transformation context. */ +static int save_key(struct ocs_aes_tctx *tctx, const u8 *in_key, size_t key_len, + enum ocs_cipher cipher) +{ + int ret; + + ret = check_key(in_key, key_len, cipher); + if (ret) + return ret; + + memcpy(tctx->key, in_key, key_len); + tctx->key_len = key_len; + tctx->cipher = cipher; + + return 0; +} + +/* Set key for symmetric cypher. */ +static int kmb_ocs_sk_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + size_t key_len, enum ocs_cipher cipher) +{ + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + + /* Fallback is used for AES with 192-bit key. */ + tctx->use_fallback = (cipher == OCS_AES && + key_len == OCS_AES_KEYSIZE_192); + + if (!tctx->use_fallback) + return save_key(tctx, in_key, key_len, cipher); + + crypto_sync_skcipher_clear_flags(tctx->sw_cipher.sk, + CRYPTO_TFM_REQ_MASK); + crypto_sync_skcipher_set_flags(tctx->sw_cipher.sk, + tfm->base.crt_flags & + CRYPTO_TFM_REQ_MASK); + + return crypto_sync_skcipher_setkey(tctx->sw_cipher.sk, in_key, key_len); +} + +/* Set key for AEAD cipher. */ +static int kmb_ocs_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + size_t key_len, enum ocs_cipher cipher) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + + /* Fallback is used for AES with 192-bit key. */ + tctx->use_fallback = (cipher == OCS_AES && + key_len == OCS_AES_KEYSIZE_192); + + if (!tctx->use_fallback) + return save_key(tctx, in_key, key_len, cipher); + + crypto_aead_clear_flags(tctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(tctx->sw_cipher.aead, + crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); + + return crypto_aead_setkey(tctx->sw_cipher.aead, in_key, key_len); +} + +/* Swap two AES blocks in SG lists. */ +static void sg_swap_blocks(struct scatterlist *sgl, unsigned int nents, + off_t blk1_offset, off_t blk2_offset) +{ + u8 tmp_buf1[AES_BLOCK_SIZE], tmp_buf2[AES_BLOCK_SIZE]; + + /* + * No easy way to copy within sg list, so copy both blocks to temporary + * buffers first. + */ + sg_pcopy_to_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk1_offset); + sg_pcopy_to_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk2_offset); + sg_pcopy_from_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk2_offset); + sg_pcopy_from_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk1_offset); +} + +/* Initialize request context to default values. */ +static void ocs_aes_init_rctx(struct ocs_aes_rctx *rctx) +{ + /* Zero everything. */ + memset(rctx, 0, sizeof(*rctx)); + + /* Set initial value for DMA addresses. */ + rctx->src_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->dst_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->aad_src_dll.dma_addr = DMA_MAPPING_ERROR; + rctx->aad_dst_dll.dma_addr = DMA_MAPPING_ERROR; +} + +static int kmb_ocs_sk_validate_input(struct skcipher_request *req, + enum ocs_mode mode) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + int iv_size = crypto_skcipher_ivsize(tfm); + + switch (mode) { + case OCS_MODE_ECB: + /* Ensure input length is multiple of block size */ + if (req->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + + return 0; + + case OCS_MODE_CBC: + /* Ensure input length is multiple of block size */ + if (req->cryptlen % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + /* + * NOTE: Since req->cryptlen == 0 case was already handled in + * kmb_ocs_sk_common(), the above two conditions also guarantee + * that: cryptlen >= iv_size + */ + return 0; + + case OCS_MODE_CTR: + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + return 0; + + case OCS_MODE_CTS: + /* Ensure input length >= block size */ + if (req->cryptlen < AES_BLOCK_SIZE) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!req->iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + default: + return -EINVAL; + } +} + +/* + * Called by encrypt() / decrypt() skcipher functions. + * + * Use fallback if needed, otherwise initialize context and enqueue request + * into engine. + */ +static int kmb_ocs_sk_common(struct skcipher_request *req, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct ocs_aes_dev *aes_dev; + int rc; + + if (tctx->use_fallback) { + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, tctx->sw_cipher.sk); + + skcipher_request_set_sync_tfm(subreq, tctx->sw_cipher.sk); + skcipher_request_set_callback(subreq, req->base.flags, NULL, + NULL); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + + if (instruction == OCS_ENCRYPT) + rc = crypto_skcipher_encrypt(subreq); + else + rc = crypto_skcipher_decrypt(subreq); + + skcipher_request_zero(subreq); + + return rc; + } + + /* + * If cryptlen == 0, no processing needed for ECB, CBC and CTR. + * + * For CTS continue: kmb_ocs_sk_validate_input() will return -EINVAL. + */ + if (!req->cryptlen && mode != OCS_MODE_CTS) + return 0; + + rc = kmb_ocs_sk_validate_input(req, mode); + if (rc) + return rc; + + aes_dev = kmb_ocs_aes_find_dev(tctx); + if (!aes_dev) + return -ENODEV; + + if (cipher != tctx->cipher) + return -EINVAL; + + ocs_aes_init_rctx(rctx); + rctx->instruction = instruction; + rctx->mode = mode; + + return crypto_transfer_skcipher_request_to_engine(aes_dev->engine, req); +} + +static void cleanup_ocs_dma_linked_list(struct device *dev, + struct ocs_dll_desc *dll) +{ + if (dll->vaddr) + dma_free_coherent(dev, dll->size, dll->vaddr, dll->dma_addr); + dll->vaddr = NULL; + dll->size = 0; + dll->dma_addr = DMA_MAPPING_ERROR; +} + +static void kmb_ocs_sk_dma_cleanup(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct device *dev = tctx->aes_dev->dev; + + if (rctx->src_dma_count) { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + rctx->src_dma_count = 0; + } + + if (rctx->dst_dma_count) { + dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? + DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + rctx->dst_dma_count = 0; + } + + /* Clean up OCS DMA linked lists */ + cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); +} + +static int kmb_ocs_sk_prepare_inplace(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int iv_size = crypto_skcipher_ivsize(tfm); + int rc; + + /* + * For CBC decrypt, save last block (iv) to last_ct_blk buffer. + * + * Note: if we are here, we already checked that cryptlen >= iv_size + * and iv_size == AES_BLOCK_SIZE (i.e., the size of last_ct_blk); see + * kmb_ocs_sk_validate_input(). + */ + if (rctx->mode == OCS_MODE_CBC && rctx->instruction == OCS_DECRYPT) + scatterwalk_map_and_copy(rctx->last_ct_blk, req->src, + req->cryptlen - iv_size, iv_size, 0); + + /* For CTS decrypt, swap last two blocks, if needed. */ + if (rctx->cts_swap && rctx->instruction == OCS_DECRYPT) + sg_swap_blocks(req->dst, rctx->dst_nents, + req->cryptlen - AES_BLOCK_SIZE, + req->cryptlen - (2 * AES_BLOCK_SIZE)); + + /* src and dst buffers are the same, use bidirectional DMA mapping. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, DMA_BIDIRECTIONAL); + if (rctx->dst_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create DST linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + req->cryptlen, 0); + if (rc) + return rc; + /* + * If descriptor creation was successful, set the src_dll.dma_addr to + * the value of dst_dll.dma_addr, as we do in-place AES operation on + * the src. + */ + rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; + + return 0; +} + +static int kmb_ocs_sk_prepare_notinplace(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int rc; + + rctx->src_nents = sg_nents_for_len(req->src, req->cryptlen); + if (rctx->src_nents < 0) + return -EBADMSG; + + /* Map SRC SG. */ + rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, + rctx->src_nents, DMA_TO_DEVICE); + if (rctx->src_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); + return -ENOMEM; + } + + /* Create SRC linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, &rctx->src_dll, + req->cryptlen, 0); + if (rc) + return rc; + + /* Map DST SG. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, DMA_FROM_DEVICE); + if (rctx->dst_dma_count == 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create DST linked list */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + req->cryptlen, 0); + if (rc) + return rc; + + /* If this is not a CTS decrypt operation with swapping, we are done. */ + if (!(rctx->cts_swap && rctx->instruction == OCS_DECRYPT)) + return 0; + + /* + * Otherwise, we have to copy src to dst (as we cannot modify src). + * Use OCS AES bypass mode to copy src to dst via DMA. + * + * NOTE: for anything other than small data sizes this is rather + * inefficient. + */ + rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, req->cryptlen); + if (rc) + return rc; + + /* + * Now dst == src, so clean up what we did so far and use in_place + * logic. + */ + kmb_ocs_sk_dma_cleanup(req); + rctx->in_place = true; + + return kmb_ocs_sk_prepare_inplace(req); +} + +static int kmb_ocs_sk_run(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_rctx *rctx = skcipher_request_ctx(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct ocs_aes_dev *aes_dev = tctx->aes_dev; + int iv_size = crypto_skcipher_ivsize(tfm); + int rc; + + rctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen); + if (rctx->dst_nents < 0) + return -EBADMSG; + + /* + * If 2 blocks or greater, and multiple of block size swap last two + * blocks to be compatible with other crypto API CTS implementations: + * OCS mode uses CBC-CS2, whereas other crypto API implementations use + * CBC-CS3. + * CBC-CS2 and CBC-CS3 defined by: + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a-add.pdf + */ + rctx->cts_swap = (rctx->mode == OCS_MODE_CTS && + req->cryptlen > AES_BLOCK_SIZE && + req->cryptlen % AES_BLOCK_SIZE == 0); + + rctx->in_place = (req->src == req->dst); + + if (rctx->in_place) + rc = kmb_ocs_sk_prepare_inplace(req); + else + rc = kmb_ocs_sk_prepare_notinplace(req); + + if (rc) + goto error; + + rc = ocs_aes_op(aes_dev, rctx->mode, tctx->cipher, rctx->instruction, + rctx->dst_dll.dma_addr, rctx->src_dll.dma_addr, + req->cryptlen, req->iv, iv_size); + if (rc) + goto error; + + /* Clean-up DMA before further processing output. */ + kmb_ocs_sk_dma_cleanup(req); + + /* For CTS Encrypt, swap last 2 blocks, if needed. */ + if (rctx->cts_swap && rctx->instruction == OCS_ENCRYPT) { + sg_swap_blocks(req->dst, rctx->dst_nents, + req->cryptlen - AES_BLOCK_SIZE, + req->cryptlen - (2 * AES_BLOCK_SIZE)); + return 0; + } + + /* For CBC copy IV to req->IV. */ + if (rctx->mode == OCS_MODE_CBC) { + /* CBC encrypt case. */ + if (rctx->instruction == OCS_ENCRYPT) { + scatterwalk_map_and_copy(req->iv, req->dst, + req->cryptlen - iv_size, + iv_size, 0); + return 0; + } + /* CBC decrypt case. */ + if (rctx->in_place) + memcpy(req->iv, rctx->last_ct_blk, iv_size); + else + scatterwalk_map_and_copy(req->iv, req->src, + req->cryptlen - iv_size, + iv_size, 0); + return 0; + } + /* For all other modes there's nothing to do. */ + + return 0; + +error: + kmb_ocs_sk_dma_cleanup(req); + + return rc; +} + +static int kmb_ocs_aead_validate_input(struct aead_request *req, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + int tag_size = crypto_aead_authsize(tfm); + int iv_size = crypto_aead_ivsize(tfm); + + /* For decrypt crytplen == len(PT) + len(tag). */ + if (instruction == OCS_DECRYPT && req->cryptlen < tag_size) + return -EINVAL; + + /* IV is mandatory. */ + if (!req->iv) + return -EINVAL; + + switch (mode) { + case OCS_MODE_GCM: + if (iv_size != GCM_AES_IV_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CCM: + /* Ensure IV is present and block size in length */ + if (iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + default: + return -EINVAL; + } +} + +/* + * Called by encrypt() / decrypt() aead functions. + * + * Use fallback if needed, otherwise initialize context and enqueue request + * into engine. + */ +static int kmb_ocs_aead_common(struct aead_request *req, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + enum ocs_mode mode) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + struct ocs_aes_dev *dd; + int rc; + + if (tctx->use_fallback) { + struct aead_request *subreq = aead_request_ctx(req); + + aead_request_set_tfm(subreq, tctx->sw_cipher.aead); + aead_request_set_callback(subreq, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_ad(subreq, req->assoclen); + rc = crypto_aead_setauthsize(tctx->sw_cipher.aead, + crypto_aead_authsize(crypto_aead_reqtfm(req))); + if (rc) + return rc; + + return (instruction == OCS_ENCRYPT) ? + crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); + } + + rc = kmb_ocs_aead_validate_input(req, instruction, mode); + if (rc) + return rc; + + dd = kmb_ocs_aes_find_dev(tctx); + if (!dd) + return -ENODEV; + + if (cipher != tctx->cipher) + return -EINVAL; + + ocs_aes_init_rctx(rctx); + rctx->instruction = instruction; + rctx->mode = mode; + + return crypto_transfer_aead_request_to_engine(dd->engine, req); +} + +static void kmb_ocs_aead_dma_cleanup(struct aead_request *req) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + struct device *dev = tctx->aes_dev->dev; + + if (rctx->src_dma_count) { + dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE); + rctx->src_dma_count = 0; + } + + if (rctx->dst_dma_count) { + dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ? + DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + rctx->dst_dma_count = 0; + } + /* Clean up OCS DMA linked lists */ + cleanup_ocs_dma_linked_list(dev, &rctx->src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->aad_src_dll); + cleanup_ocs_dma_linked_list(dev, &rctx->aad_dst_dll); +} + +/** + * kmb_ocs_aead_dma_prepare() - Do DMA mapping for AEAD processing. + * @req: The AEAD request being processed. + * @src_dll_size: Where to store the length of the data mapped into the + * src_dll OCS DMA list. + * + * Do the following: + * - DMA map req->src and req->dst + * - Initialize the following OCS DMA linked lists: rctx->src_dll, + * rctx->dst_dll, rctx->aad_src_dll and rxtc->aad_dst_dll. + * + * Return: 0 on success, negative error code otherwise. + */ +static int kmb_ocs_aead_dma_prepare(struct aead_request *req, u32 *src_dll_size) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + u32 in_size; /* The length of the data to be mapped by src_dll. */ + u32 out_size; /* The length of the data to be mapped by dst_dll. */ + u32 dst_size; /* The length of the data in dst_sg. */ + int rc; + + /* Get number of entries in input data SG list. */ + rctx->src_nents = sg_nents_for_len(req->src, + req->assoclen + req->cryptlen); + if (rctx->src_nents < 0) + return -EBADMSG; + + if (rctx->instruction == OCS_DECRYPT) { + /* + * For decrypt: + * - src sg list is: AAD|CT|tag + * - dst sg list expects: AAD|PT + * + * in_size == len(CT); out_size == len(PT) + */ + + /* req->cryptlen includes both CT and tag. */ + in_size = req->cryptlen - tag_size; + + /* out_size = PT size == CT size */ + out_size = in_size; + + /* len(dst_sg) == len(AAD) + len(PT) */ + dst_size = req->assoclen + out_size; + + /* + * Copy tag from source SG list to 'in_tag' buffer. + * + * Note: this needs to be done here, before DMA mapping src_sg. + */ + sg_pcopy_to_buffer(req->src, rctx->src_nents, rctx->in_tag, + tag_size, req->assoclen + in_size); + + } else { /* OCS_ENCRYPT */ + /* + * For encrypt: + * src sg list is: AAD|PT + * dst sg list expects: AAD|CT|tag + */ + /* in_size == len(PT) */ + in_size = req->cryptlen; + + /* + * In CCM mode the OCS engine appends the tag to the ciphertext, + * but in GCM mode the tag must be read from the tag registers + * and appended manually below + */ + out_size = (rctx->mode == OCS_MODE_CCM) ? in_size + tag_size : + in_size; + /* len(dst_sg) == len(AAD) + len(CT) + len(tag) */ + dst_size = req->assoclen + in_size + tag_size; + } + *src_dll_size = in_size; + + /* Get number of entries in output data SG list. */ + rctx->dst_nents = sg_nents_for_len(req->dst, dst_size); + if (rctx->dst_nents < 0) + return -EBADMSG; + + rctx->in_place = (req->src == req->dst) ? 1 : 0; + + /* Map destination; use bidirectional mapping for in-place case. */ + rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst, + rctx->dst_nents, + rctx->in_place ? DMA_BIDIRECTIONAL : + DMA_FROM_DEVICE); + if (rctx->dst_dma_count == 0 && rctx->dst_nents != 0) { + dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n"); + return -ENOMEM; + } + + /* Create AAD DST list: maps dst[0:AAD_SIZE-1]. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->aad_dst_dll, req->assoclen, + 0); + if (rc) + return rc; + + /* Create DST list: maps dst[AAD_SIZE:out_size] */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, &rctx->dst_dll, + out_size, req->assoclen); + if (rc) + return rc; + + if (rctx->in_place) { + /* If this is not CCM encrypt, we are done. */ + if (!(rctx->mode == OCS_MODE_CCM && + rctx->instruction == OCS_ENCRYPT)) { + /* + * SRC and DST are the same, so re-use the same DMA + * addresses (to avoid allocating new DMA lists + * identical to the dst ones). + */ + rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr; + rctx->aad_src_dll.dma_addr = rctx->aad_dst_dll.dma_addr; + + return 0; + } + /* + * For CCM encrypt the input and output linked lists contain + * different amounts of data, so, we need to create different + * SRC and AAD SRC lists, even for the in-place case. + */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->aad_src_dll, + req->assoclen, 0); + if (rc) + return rc; + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst, + rctx->dst_dma_count, + &rctx->src_dll, in_size, + req->assoclen); + if (rc) + return rc; + + return 0; + } + /* Not in-place case. */ + + /* Map source SG. */ + rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src, + rctx->src_nents, DMA_TO_DEVICE); + if (rctx->src_dma_count == 0 && rctx->src_nents != 0) { + dev_err(tctx->aes_dev->dev, "Failed to map source sg\n"); + return -ENOMEM; + } + + /* Create AAD SRC list. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, + &rctx->aad_src_dll, + req->assoclen, 0); + if (rc) + return rc; + + /* Create SRC list. */ + rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src, + rctx->src_dma_count, + &rctx->src_dll, in_size, + req->assoclen); + if (rc) + return rc; + + if (req->assoclen == 0) + return 0; + + /* Copy AAD from src sg to dst sg using OCS DMA. */ + rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->aad_dst_dll.dma_addr, + rctx->aad_src_dll.dma_addr, req->cryptlen); + if (rc) + dev_err(tctx->aes_dev->dev, + "Failed to copy source AAD to destination AAD\n"); + + return rc; +} + +static int kmb_ocs_aead_run(struct aead_request *req) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req)); + struct ocs_aes_rctx *rctx = aead_request_ctx(req); + u32 in_size; /* The length of the data mapped by src_dll. */ + int rc; + + rc = kmb_ocs_aead_dma_prepare(req, &in_size); + if (rc) + goto exit; + + /* For CCM, we just call the OCS processing and we are done. */ + if (rctx->mode == OCS_MODE_CCM) { + rc = ocs_aes_ccm_op(tctx->aes_dev, tctx->cipher, + rctx->instruction, rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, in_size, + req->iv, + rctx->aad_src_dll.dma_addr, req->assoclen, + rctx->in_tag, tag_size); + goto exit; + } + /* GCM case; invoke OCS processing. */ + rc = ocs_aes_gcm_op(tctx->aes_dev, tctx->cipher, + rctx->instruction, + rctx->dst_dll.dma_addr, + rctx->src_dll.dma_addr, in_size, + req->iv, + rctx->aad_src_dll.dma_addr, req->assoclen, + rctx->out_tag, tag_size); + if (rc) + goto exit; + + /* For GCM decrypt, we have to compare in_tag with out_tag. */ + if (rctx->instruction == OCS_DECRYPT) { + rc = memcmp(rctx->in_tag, rctx->out_tag, tag_size) ? + -EBADMSG : 0; + goto exit; + } + + /* For GCM encrypt, we must manually copy out_tag to DST sg. */ + + /* Clean-up must be called before the sg_pcopy_from_buffer() below. */ + kmb_ocs_aead_dma_cleanup(req); + + /* Copy tag to destination sg after AAD and CT. */ + sg_pcopy_from_buffer(req->dst, rctx->dst_nents, rctx->out_tag, + tag_size, req->assoclen + req->cryptlen); + + /* Return directly as DMA cleanup already done. */ + return 0; + +exit: + kmb_ocs_aead_dma_cleanup(req); + + return rc; +} + +static int kmb_ocs_aes_sk_do_one_request(struct crypto_engine *engine, + void *areq) +{ + struct skcipher_request *req = + container_of(areq, struct skcipher_request, base); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + int err; + + if (!tctx->aes_dev) { + err = -ENODEV; + goto exit; + } + + err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, + tctx->cipher); + if (err) + goto exit; + + err = kmb_ocs_sk_run(req); + +exit: + crypto_finalize_skcipher_request(engine, req, err); + + return 0; +} + +static int kmb_ocs_aes_aead_do_one_request(struct crypto_engine *engine, + void *areq) +{ + struct aead_request *req = container_of(areq, + struct aead_request, base); + struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + int err; + + if (!tctx->aes_dev) + return -ENODEV; + + err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key, + tctx->cipher); + if (err) + goto exit; + + err = kmb_ocs_aead_run(req); + +exit: + crypto_finalize_aead_request(tctx->aes_dev->engine, req, err); + + return 0; +} + +static int kmb_ocs_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_AES); +} + +static int kmb_ocs_aes_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_AES); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +static int kmb_ocs_aes_ecb_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_ECB); +} + +static int kmb_ocs_aes_ecb_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_ECB); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +static int kmb_ocs_aes_cbc_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_aes_cbc_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_aes_ctr_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTR); +} + +static int kmb_ocs_aes_ctr_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTR); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +static int kmb_ocs_aes_cts_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTS); +} + +static int kmb_ocs_aes_cts_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTS); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ + +static int kmb_ocs_aes_gcm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_aes_gcm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_aes_ccm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_aes_ccm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_sm4_set_key(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_SM4); +} + +static int kmb_ocs_sm4_aead_set_key(struct crypto_aead *tfm, const u8 *in_key, + unsigned int key_len) +{ + return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_SM4); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +static int kmb_ocs_sm4_ecb_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_ECB); +} + +static int kmb_ocs_sm4_ecb_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_ECB); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +static int kmb_ocs_sm4_cbc_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_sm4_cbc_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CBC); +} + +static int kmb_ocs_sm4_ctr_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTR); +} + +static int kmb_ocs_sm4_ctr_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTR); +} + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +static int kmb_ocs_sm4_cts_encrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTS); +} + +static int kmb_ocs_sm4_cts_decrypt(struct skcipher_request *req) +{ + return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTS); +} +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ + +static int kmb_ocs_sm4_gcm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_sm4_gcm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_GCM); +} + +static int kmb_ocs_sm4_ccm_encrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CCM); +} + +static int kmb_ocs_sm4_ccm_decrypt(struct aead_request *req) +{ + return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CCM); +} + +static inline int ocs_common_init(struct ocs_aes_tctx *tctx) +{ + tctx->engine_ctx.op.prepare_request = NULL; + tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_sk_do_one_request; + tctx->engine_ctx.op.unprepare_request = NULL; + + return 0; +} + +static int ocs_aes_init_tfm(struct crypto_skcipher *tfm) +{ + const char *alg_name = crypto_tfm_alg_name(&tfm->base); + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + struct crypto_sync_skcipher *blk; + + /* set fallback cipher in case it will be needed */ + blk = crypto_alloc_sync_skcipher(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(blk)) + return PTR_ERR(blk); + + tctx->sw_cipher.sk = blk; + + crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return ocs_common_init(tctx); +} + +static int ocs_sm4_init_tfm(struct crypto_skcipher *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + + crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return ocs_common_init(tctx); +} + +static inline void clear_key(struct ocs_aes_tctx *tctx) +{ + memzero_explicit(tctx->key, OCS_AES_KEYSIZE_256); + + /* Zero key registers if set */ + if (tctx->aes_dev) + ocs_aes_set_key(tctx->aes_dev, OCS_AES_KEYSIZE_256, + tctx->key, OCS_AES); +} + +static void ocs_exit_tfm(struct crypto_skcipher *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm); + + clear_key(tctx); + + if (tctx->sw_cipher.sk) { + crypto_free_sync_skcipher(tctx->sw_cipher.sk); + tctx->sw_cipher.sk = NULL; + } +} + +static inline int ocs_common_aead_init(struct ocs_aes_tctx *tctx) +{ + tctx->engine_ctx.op.prepare_request = NULL; + tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_aead_do_one_request; + tctx->engine_ctx.op.unprepare_request = NULL; + + return 0; +} + +static int ocs_aes_aead_cra_init(struct crypto_aead *tfm) +{ + const char *alg_name = crypto_tfm_alg_name(&tfm->base); + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + struct crypto_aead *blk; + + /* Set fallback cipher in case it will be needed */ + blk = crypto_alloc_aead(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(blk)) + return PTR_ERR(blk); + + tctx->sw_cipher.aead = blk; + + crypto_aead_set_reqsize(tfm, + max(sizeof(struct ocs_aes_rctx), + (sizeof(struct aead_request) + + crypto_aead_reqsize(tctx->sw_cipher.aead)))); + + return ocs_common_aead_init(tctx); +} + +static int kmb_ocs_aead_ccm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + switch (authsize) { + case 4: + case 6: + case 8: + case 10: + case 12: + case 14: + case 16: + return 0; + default: + return -EINVAL; + } +} + +static int kmb_ocs_aead_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + return crypto_gcm_check_authsize(authsize); +} + +static int ocs_sm4_aead_cra_init(struct crypto_aead *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + + crypto_aead_set_reqsize(tfm, sizeof(struct ocs_aes_rctx)); + + return ocs_common_aead_init(tctx); +} + +static void ocs_aead_cra_exit(struct crypto_aead *tfm) +{ + struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm); + + clear_key(tctx); + + if (tctx->sw_cipher.aead) { + crypto_free_aead(tctx->sw_cipher.aead); + tctx->sw_cipher.aead = NULL; + } +} + +static struct skcipher_alg algs[] = { +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_ecb_encrypt, + .decrypt = kmb_ocs_aes_ecb_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_cbc_encrypt, + .decrypt = kmb_ocs_aes_cbc_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, + { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_ctr_encrypt, + .decrypt = kmb_ocs_aes_ctr_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + { + .base.cra_name = "cts(cbc(aes))", + .base.cra_driver_name = "cts-aes-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_AES_MIN_KEY_SIZE, + .max_keysize = OCS_AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_aes_set_key, + .encrypt = kmb_ocs_aes_cts_encrypt, + .decrypt = kmb_ocs_aes_cts_decrypt, + .init = ocs_aes_init_tfm, + .exit = ocs_exit_tfm, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB + { + .base.cra_name = "ecb(sm4)", + .base.cra_driver_name = "ecb-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_ecb_encrypt, + .decrypt = kmb_ocs_sm4_ecb_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + }, +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + { + .base.cra_name = "cbc(sm4)", + .base.cra_driver_name = "cbc-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_cbc_encrypt, + .decrypt = kmb_ocs_sm4_cbc_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + }, + { + .base.cra_name = "ctr(sm4)", + .base.cra_driver_name = "ctr-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_ctr_encrypt, + .decrypt = kmb_ocs_sm4_ctr_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + }, +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS + { + .base.cra_name = "cts(cbc(sm4))", + .base.cra_driver_name = "cts-sm4-keembay-ocs", + .base.cra_priority = KMB_OCS_PRIORITY, + .base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct ocs_aes_tctx), + .base.cra_module = THIS_MODULE, + .base.cra_alignmask = 0, + + .min_keysize = OCS_SM4_KEY_SIZE, + .max_keysize = OCS_SM4_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = kmb_ocs_sm4_set_key, + .encrypt = kmb_ocs_sm4_cts_encrypt, + .decrypt = kmb_ocs_sm4_cts_decrypt, + .init = ocs_sm4_init_tfm, + .exit = ocs_exit_tfm, + } +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ +}; + +static struct aead_alg algs_aead[] = { + { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_aes_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_gcm_setauthsize, + .setkey = kmb_ocs_aes_aead_set_key, + .encrypt = kmb_ocs_aes_gcm_encrypt, + .decrypt = kmb_ocs_aes_gcm_decrypt, + }, + { + .base = { + .cra_name = "ccm(aes)", + .cra_driver_name = "ccm-aes-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_aes_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_ccm_setauthsize, + .setkey = kmb_ocs_aes_aead_set_key, + .encrypt = kmb_ocs_aes_ccm_encrypt, + .decrypt = kmb_ocs_aes_ccm_decrypt, + }, + { + .base = { + .cra_name = "gcm(sm4)", + .cra_driver_name = "gcm-sm4-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_sm4_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = GCM_AES_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_gcm_setauthsize, + .setkey = kmb_ocs_sm4_aead_set_key, + .encrypt = kmb_ocs_sm4_gcm_encrypt, + .decrypt = kmb_ocs_sm4_gcm_decrypt, + }, + { + .base = { + .cra_name = "ccm(sm4)", + .cra_driver_name = "ccm-sm4-keembay-ocs", + .cra_priority = KMB_OCS_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ocs_aes_tctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .init = ocs_sm4_aead_cra_init, + .exit = ocs_aead_cra_exit, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + .setauthsize = kmb_ocs_aead_ccm_setauthsize, + .setkey = kmb_ocs_sm4_aead_set_key, + .encrypt = kmb_ocs_sm4_ccm_encrypt, + .decrypt = kmb_ocs_sm4_ccm_decrypt, + } +}; + +static void unregister_aes_algs(struct ocs_aes_dev *aes_dev) +{ + crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead)); + crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); +} + +static int register_aes_algs(struct ocs_aes_dev *aes_dev) +{ + int ret; + + /* + * If any algorithm fails to register, all preceding algorithms that + * were successfully registered will be automatically unregistered. + */ + ret = crypto_register_aeads(algs_aead, ARRAY_SIZE(algs_aead)); + if (ret) + return ret; + + ret = crypto_register_skciphers(algs, ARRAY_SIZE(algs)); + if (ret) + crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs)); + + return ret; +} + +/* Device tree driver match. */ +static const struct of_device_id kmb_ocs_aes_of_match[] = { + { + .compatible = "intel,keembay-ocs-aes", + }, + {} +}; + +static int kmb_ocs_aes_remove(struct platform_device *pdev) +{ + struct ocs_aes_dev *aes_dev; + + aes_dev = platform_get_drvdata(pdev); + if (!aes_dev) + return -ENODEV; + + unregister_aes_algs(aes_dev); + + spin_lock(&ocs_aes.lock); + list_del(&aes_dev->list); + spin_unlock(&ocs_aes.lock); + + crypto_engine_exit(aes_dev->engine); + + return 0; +} + +static int kmb_ocs_aes_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ocs_aes_dev *aes_dev; + struct resource *aes_mem; + int rc; + + aes_dev = devm_kzalloc(dev, sizeof(*aes_dev), GFP_KERNEL); + if (!aes_dev) + return -ENOMEM; + + aes_dev->dev = dev; + + platform_set_drvdata(pdev, aes_dev); + + rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + if (rc) { + dev_err(dev, "Failed to set 32 bit dma mask %d\n", rc); + return rc; + } + + /* Get base register address. */ + aes_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!aes_mem) { + dev_err(dev, "Could not retrieve io mem resource\n"); + return -ENODEV; + } + + aes_dev->base_reg = devm_ioremap_resource(&pdev->dev, aes_mem); + if (IS_ERR(aes_dev->base_reg)) { + dev_err(dev, "Failed to get base address\n"); + return PTR_ERR(aes_dev->base_reg); + } + + /* Get and request IRQ */ + aes_dev->irq = platform_get_irq(pdev, 0); + if (aes_dev->irq < 0) + return aes_dev->irq; + + rc = devm_request_threaded_irq(dev, aes_dev->irq, ocs_aes_irq_handler, + NULL, 0, "keembay-ocs-aes", aes_dev); + if (rc < 0) { + dev_err(dev, "Could not request IRQ\n"); + return rc; + } + + INIT_LIST_HEAD(&aes_dev->list); + spin_lock(&ocs_aes.lock); + list_add_tail(&aes_dev->list, &ocs_aes.dev_list); + spin_unlock(&ocs_aes.lock); + + init_completion(&aes_dev->irq_completion); + + /* Initialize crypto engine */ + aes_dev->engine = crypto_engine_alloc_init(dev, true); + if (!aes_dev->engine) + goto list_del; + + rc = crypto_engine_start(aes_dev->engine); + if (rc) { + dev_err(dev, "Could not start crypto engine\n"); + goto cleanup; + } + + rc = register_aes_algs(aes_dev); + if (rc) { + dev_err(dev, + "Could not register OCS algorithms with Crypto API\n"); + goto cleanup; + } + + return 0; + +cleanup: + crypto_engine_exit(aes_dev->engine); +list_del: + spin_lock(&ocs_aes.lock); + list_del(&aes_dev->list); + spin_unlock(&ocs_aes.lock); + + return rc; +} + +/* The OCS driver is a platform device. */ +static struct platform_driver kmb_ocs_aes_driver = { + .probe = kmb_ocs_aes_probe, + .remove = kmb_ocs_aes_remove, + .driver = { + .name = DRV_NAME, + .of_match_table = kmb_ocs_aes_of_match, + }, +}; + +module_platform_driver(kmb_ocs_aes_driver); + +MODULE_DESCRIPTION("Intel Keem Bay Offload and Crypto Subsystem (OCS) AES/SM4 Driver"); +MODULE_LICENSE("GPL"); + +MODULE_ALIAS_CRYPTO("cbc-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ctr-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("gcm-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ccm-aes-keembay-ocs"); + +MODULE_ALIAS_CRYPTO("cbc-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ctr-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("gcm-sm4-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ccm-sm4-keembay-ocs"); + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB +MODULE_ALIAS_CRYPTO("ecb-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("ecb-sm4-keembay-ocs"); +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */ + +#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS +MODULE_ALIAS_CRYPTO("cts-aes-keembay-ocs"); +MODULE_ALIAS_CRYPTO("cts-sm4-keembay-ocs"); +#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */ diff --git a/drivers/crypto/keembay/ocs-aes.c b/drivers/crypto/keembay/ocs-aes.c new file mode 100644 index 000000000000..cc286adb1c4a --- /dev/null +++ b/drivers/crypto/keembay/ocs-aes.c @@ -0,0 +1,1489 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +#include "ocs-aes.h" + +#define AES_COMMAND_OFFSET 0x0000 +#define AES_KEY_0_OFFSET 0x0004 +#define AES_KEY_1_OFFSET 0x0008 +#define AES_KEY_2_OFFSET 0x000C +#define AES_KEY_3_OFFSET 0x0010 +#define AES_KEY_4_OFFSET 0x0014 +#define AES_KEY_5_OFFSET 0x0018 +#define AES_KEY_6_OFFSET 0x001C +#define AES_KEY_7_OFFSET 0x0020 +#define AES_IV_0_OFFSET 0x0024 +#define AES_IV_1_OFFSET 0x0028 +#define AES_IV_2_OFFSET 0x002C +#define AES_IV_3_OFFSET 0x0030 +#define AES_ACTIVE_OFFSET 0x0034 +#define AES_STATUS_OFFSET 0x0038 +#define AES_KEY_SIZE_OFFSET 0x0044 +#define AES_IER_OFFSET 0x0048 +#define AES_ISR_OFFSET 0x005C +#define AES_MULTIPURPOSE1_0_OFFSET 0x0200 +#define AES_MULTIPURPOSE1_1_OFFSET 0x0204 +#define AES_MULTIPURPOSE1_2_OFFSET 0x0208 +#define AES_MULTIPURPOSE1_3_OFFSET 0x020C +#define AES_MULTIPURPOSE2_0_OFFSET 0x0220 +#define AES_MULTIPURPOSE2_1_OFFSET 0x0224 +#define AES_MULTIPURPOSE2_2_OFFSET 0x0228 +#define AES_MULTIPURPOSE2_3_OFFSET 0x022C +#define AES_BYTE_ORDER_CFG_OFFSET 0x02C0 +#define AES_TLEN_OFFSET 0x0300 +#define AES_T_MAC_0_OFFSET 0x0304 +#define AES_T_MAC_1_OFFSET 0x0308 +#define AES_T_MAC_2_OFFSET 0x030C +#define AES_T_MAC_3_OFFSET 0x0310 +#define AES_PLEN_OFFSET 0x0314 +#define AES_A_DMA_SRC_ADDR_OFFSET 0x0400 +#define AES_A_DMA_DST_ADDR_OFFSET 0x0404 +#define AES_A_DMA_SRC_SIZE_OFFSET 0x0408 +#define AES_A_DMA_DST_SIZE_OFFSET 0x040C +#define AES_A_DMA_DMA_MODE_OFFSET 0x0410 +#define AES_A_DMA_NEXT_SRC_DESCR_OFFSET 0x0418 +#define AES_A_DMA_NEXT_DST_DESCR_OFFSET 0x041C +#define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET 0x0420 +#define AES_A_DMA_LOG_OFFSET 0x0424 +#define AES_A_DMA_STATUS_OFFSET 0x0428 +#define AES_A_DMA_PERF_CNTR_OFFSET 0x042C +#define AES_A_DMA_MSI_ISR_OFFSET 0x0480 +#define AES_A_DMA_MSI_IER_OFFSET 0x0484 +#define AES_A_DMA_MSI_MASK_OFFSET 0x0488 +#define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET 0x0600 +#define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET 0x0700 + +/* + * AES_A_DMA_DMA_MODE register. + * Default: 0x00000000. + * bit[31] ACTIVE + * This bit activates the DMA. When the DMA finishes, it resets + * this bit to zero. + * bit[30:26] Unused by this driver. + * bit[25] SRC_LINK_LIST_EN + * Source link list enable bit. When the linked list is terminated + * this bit is reset by the DMA. + * bit[24] DST_LINK_LIST_EN + * Destination link list enable bit. When the linked list is + * terminated this bit is reset by the DMA. + * bit[23:0] Unused by this driver. + */ +#define AES_A_DMA_DMA_MODE_ACTIVE BIT(31) +#define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN BIT(25) +#define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN BIT(24) + +/* + * AES_ACTIVE register + * default 0x00000000 + * bit[31:10] Reserved + * bit[9] LAST_ADATA + * bit[8] LAST_GCX + * bit[7:2] Reserved + * bit[1] TERMINATION + * bit[0] TRIGGER + */ +#define AES_ACTIVE_LAST_ADATA BIT(9) +#define AES_ACTIVE_LAST_CCM_GCM BIT(8) +#define AES_ACTIVE_TERMINATION BIT(1) +#define AES_ACTIVE_TRIGGER BIT(0) + +#define AES_DISABLE_INT 0x00000000 +#define AES_DMA_CPD_ERR_INT BIT(8) +#define AES_DMA_OUTBUF_RD_ERR_INT BIT(7) +#define AES_DMA_OUTBUF_WR_ERR_INT BIT(6) +#define AES_DMA_INBUF_RD_ERR_INT BIT(5) +#define AES_DMA_INBUF_WR_ERR_INT BIT(4) +#define AES_DMA_BAD_COMP_INT BIT(3) +#define AES_DMA_SAI_INT BIT(2) +#define AES_DMA_SRC_DONE_INT BIT(0) +#define AES_COMPLETE_INT BIT(1) + +#define AES_DMA_MSI_MASK_CLEAR BIT(0) + +#define AES_128_BIT_KEY 0x00000000 +#define AES_256_BIT_KEY BIT(0) + +#define AES_DEACTIVATE_PERF_CNTR 0x00000000 +#define AES_ACTIVATE_PERF_CNTR BIT(0) + +#define AES_MAX_TAG_SIZE_U32 4 + +#define OCS_LL_DMA_FLAG_TERMINATE BIT(31) + +/* + * There is an inconsistency in the documentation. This is documented as a + * 11-bit value, but it is actually 10-bits. + */ +#define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK 0x3FF + +/* + * During CCM decrypt, the OCS block needs to finish processing the ciphertext + * before the tag is written. For 128-bit mode this required delay is 28 OCS + * clock cycles. For 256-bit mode it is 36 OCS clock cycles. + */ +#define CCM_DECRYPT_DELAY_TAG_CLK_COUNT 36UL + +/* + * During CCM decrypt there must be a delay of at least 42 OCS clock cycles + * between setting the TRIGGER bit in AES_ACTIVE and setting the LAST_CCM_GCM + * bit in the same register (as stated in the OCS databook) + */ +#define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 42UL + +/* See RFC3610 section 2.2 */ +#define L_PRIME_MIN (1) +#define L_PRIME_MAX (7) +/* + * CCM IV format from RFC 3610 section 2.3 + * + * Octet Number Contents + * ------------ --------- + * 0 Flags + * 1 ... 15-L Nonce N + * 16-L ... 15 Counter i + * + * Flags = L' = L - 1 + */ +#define L_PRIME_IDX 0 +#define COUNTER_START(lprime) (16 - ((lprime) + 1)) +#define COUNTER_LEN(lprime) ((lprime) + 1) + +enum aes_counter_mode { + AES_CTR_M_NO_INC = 0, + AES_CTR_M_32_INC = 1, + AES_CTR_M_64_INC = 2, + AES_CTR_M_128_INC = 3, +}; + +/** + * struct ocs_dma_linked_list - OCS DMA linked list entry. + * @src_addr: Source address of the data. + * @src_len: Length of data to be fetched. + * @next: Next dma_list to fetch. + * @ll_flags: Flags (Freeze @ terminate) for the DMA engine. + */ +struct ocs_dma_linked_list { + u32 src_addr; + u32 src_len; + u32 next; + u32 ll_flags; +} __packed; + +/* + * Set endianness of inputs and outputs + * AES_BYTE_ORDER_CFG + * default 0x00000000 + * bit [10] - KEY_HI_LO_SWAP + * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD + * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD + * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD + * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD + * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD + * bit [4] - IV_SWAP_BYTES_IN_DWORD + * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD + * bit [2] - DOUT_SWAP_BYTES_IN_DWORD + * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD + * bit [0] - DOUT_SWAP_BYTES_IN_DWORD + */ +static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET); +} + +/* Trigger AES process start. */ +static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Indicate last bulk of data. */ +static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_TERMINATION, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* + * Set LAST_CCM_GCM in AES_ACTIVE register and clear all other bits. + * + * Called when DMA is programmed to fetch the last batch of data. + * - For AES-CCM it is called for the last batch of Payload data and Ciphertext + * data. + * - For AES-GCM, it is called for the last batch of Plaintext data and + * Ciphertext data. + */ +static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_LAST_CCM_GCM, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Wait for LAST_CCM_GCM bit to be unset. */ +static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev) +{ + u32 aes_active_reg; + + do { + aes_active_reg = ioread32(aes_dev->base_reg + + AES_ACTIVE_OFFSET); + } while (aes_active_reg & AES_ACTIVE_LAST_CCM_GCM); +} + +/* Wait for 10 bits of input occupancy. */ +static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev) +{ + u32 reg; + + do { + reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET); + } while (reg & AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK); +} + + /* + * Set LAST_CCM_GCM and LAST_ADATA bits in AES_ACTIVE register (and clear all + * other bits). + * + * Called when DMA is programmed to fetch the last batch of Associated Data + * (CCM case) or Additional Authenticated Data (GCM case). + */ +static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_ACTIVE_LAST_ADATA | AES_ACTIVE_LAST_CCM_GCM, + aes_dev->base_reg + AES_ACTIVE_OFFSET); +} + +/* Set DMA src and dst transfer size to 0 */ +static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); + iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); +} + +/* Activate DMA for zero-byte transfer case. */ +static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable src linked list */ +static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable dst linked list */ +static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Activate DMA and enable src and dst linked lists */ +static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(AES_A_DMA_DMA_MODE_ACTIVE | + AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN | + AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN, + aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); +} + +/* Reset PERF_CNTR to 0 and activate it */ +static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev) +{ + iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET); + iowrite32(AES_ACTIVATE_PERF_CNTR, + aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); +} + +/* Wait until PERF_CNTR is > delay, then deactivate it */ +static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev, + int delay) +{ + while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay) + ; + iowrite32(AES_DEACTIVATE_PERF_CNTR, + aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); +} + +/* Disable AES and DMA IRQ. */ +static void aes_irq_disable(struct ocs_aes_dev *aes_dev) +{ + u32 isr_val = 0; + + /* Disable interrupts */ + iowrite32(AES_DISABLE_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); + + /* Clear any pending interrupt */ + isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + if (isr_val) + iowrite32(isr_val, + aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + + isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); + if (isr_val) + iowrite32(isr_val, + aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); + + isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET); + if (isr_val) + iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET); +} + +/* Enable AES or DMA IRQ. IRQ is disabled once fired. */ +static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq) +{ + if (irq == AES_COMPLETE_INT) { + /* Ensure DMA error interrupts are enabled */ + iowrite32(AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + /* + * AES_IER + * default 0x00000000 + * bits [31:3] - reserved + * bit [2] - EN_SKS_ERR + * bit [1] - EN_AES_COMPLETE + * bit [0] - reserved + */ + iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET); + return; + } + if (irq == AES_DMA_SRC_DONE_INT) { + /* Ensure AES interrupts are disabled */ + iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); + /* + * DMA_MSI_IER + * default 0x00000000 + * bits [31:9] - reserved + * bit [8] - CPD_ERR_INT_EN + * bit [7] - OUTBUF_RD_ERR_INT_EN + * bit [6] - OUTBUF_WR_ERR_INT_EN + * bit [5] - INBUF_RD_ERR_INT_EN + * bit [4] - INBUF_WR_ERR_INT_EN + * bit [3] - BAD_COMP_INT_EN + * bit [2] - SAI_INT_EN + * bit [1] - DST_DONE_INT_EN + * bit [0] - SRC_DONE_INT_EN + */ + iowrite32(AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT | + AES_DMA_SRC_DONE_INT, + aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); + } +} + +/* Enable and wait for IRQ (either from OCS AES engine or DMA) */ +static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq) +{ + int rc; + + reinit_completion(&aes_dev->irq_completion); + aes_irq_enable(aes_dev, irq); + rc = wait_for_completion_interruptible(&aes_dev->irq_completion); + if (rc) + return rc; + + return aes_dev->dma_err_mask ? -EIO : 0; +} + +/* Configure DMA to OCS, linked list mode */ +static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev, + dma_addr_t dma_list) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); + iowrite32(dma_list, + aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET); +} + +/* Configure DMA from OCS, linked list mode */ +static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev, + dma_addr_t dma_list) +{ + iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); + iowrite32(dma_list, + aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET); +} + +irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id) +{ + struct ocs_aes_dev *aes_dev = dev_id; + u32 aes_dma_isr; + + /* Read DMA ISR status. */ + aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); + + /* Disable and clear interrupts. */ + aes_irq_disable(aes_dev); + + /* Save DMA error status. */ + aes_dev->dma_err_mask = aes_dma_isr & + (AES_DMA_CPD_ERR_INT | + AES_DMA_OUTBUF_RD_ERR_INT | + AES_DMA_OUTBUF_WR_ERR_INT | + AES_DMA_INBUF_RD_ERR_INT | + AES_DMA_INBUF_WR_ERR_INT | + AES_DMA_BAD_COMP_INT | + AES_DMA_SAI_INT); + + /* Signal IRQ completion. */ + complete(&aes_dev->irq_completion); + + return IRQ_HANDLED; +} + +/** + * ocs_aes_set_key() - Write key into OCS AES hardware. + * @aes_dev: The OCS AES device to write the key to. + * @key_size: The size of the key (in bytes). + * @key: The key to write. + * @cipher: The cipher the key is for. + * + * For AES @key_size must be either 16 or 32. For SM4 @key_size must be 16. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key, + enum ocs_cipher cipher) +{ + const u32 *key_u32; + u32 val; + int i; + + /* OCS AES supports 128-bit and 256-bit keys only. */ + if (cipher == OCS_AES && !(key_size == 32 || key_size == 16)) { + dev_err(aes_dev->dev, + "%d-bit keys not supported by AES cipher\n", + key_size * 8); + return -EINVAL; + } + /* OCS SM4 supports 128-bit keys only. */ + if (cipher == OCS_SM4 && key_size != 16) { + dev_err(aes_dev->dev, + "%d-bit keys not supported for SM4 cipher\n", + key_size * 8); + return -EINVAL; + } + + if (!key) + return -EINVAL; + + key_u32 = (const u32 *)key; + + /* Write key to AES_KEY[0-7] registers */ + for (i = 0; i < (key_size / sizeof(u32)); i++) { + iowrite32(key_u32[i], + aes_dev->base_reg + AES_KEY_0_OFFSET + + (i * sizeof(u32))); + } + /* + * Write key size + * bits [31:1] - reserved + * bit [0] - AES_KEY_SIZE + * 0 - 128 bit key + * 1 - 256 bit key + */ + val = (key_size == 16) ? AES_128_BIT_KEY : AES_256_BIT_KEY; + iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET); + + return 0; +} + +/* Write AES_COMMAND */ +static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_mode mode, + enum ocs_instruction instruction) +{ + u32 val; + + /* AES_COMMAND + * default 0x000000CC + * bit [14] - CIPHER_SELECT + * 0 - AES + * 1 - SM4 + * bits [11:8] - OCS_AES_MODE + * 0000 - ECB + * 0001 - CBC + * 0010 - CTR + * 0110 - CCM + * 0111 - GCM + * 1001 - CTS + * bits [7:6] - AES_INSTRUCTION + * 00 - ENCRYPT + * 01 - DECRYPT + * 10 - EXPAND + * 11 - BYPASS + * bits [3:2] - CTR_M_BITS + * 00 - No increment + * 01 - Least significant 32 bits are incremented + * 10 - Least significant 64 bits are incremented + * 11 - Full 128 bits are incremented + */ + val = (cipher << 14) | (mode << 8) | (instruction << 6) | + (AES_CTR_M_128_INC << 2); + iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET); +} + +static void ocs_aes_init(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction) +{ + /* Ensure interrupts are disabled and pending interrupts cleared. */ + aes_irq_disable(aes_dev); + + /* Set endianness recommended by data-sheet. */ + aes_a_set_endianness(aes_dev); + + /* Set AES_COMMAND register. */ + set_ocs_aes_command(aes_dev, cipher, mode, instruction); +} + +/* + * Write the byte length of the last AES/SM4 block of Payload data (without + * zero padding and without the length of the MAC) in register AES_PLEN. + */ +static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev, + u32 size) +{ + u32 val; + + if (size == 0) { + val = 0; + goto exit; + } + + val = size % AES_BLOCK_SIZE; + if (val == 0) + val = AES_BLOCK_SIZE; + +exit: + iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET); +} + +/* + * Validate inputs according to mode. + * If OK return 0; else return -EINVAL. + */ +static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size, + const u8 *iv, u32 iv_size, + dma_addr_t aad_dma_list, u32 aad_size, + const u8 *tag, u32 tag_size, + enum ocs_cipher cipher, enum ocs_mode mode, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list) +{ + /* Ensure cipher, mode and instruction are valid. */ + if (!(cipher == OCS_AES || cipher == OCS_SM4)) + return -EINVAL; + + if (mode != OCS_MODE_ECB && mode != OCS_MODE_CBC && + mode != OCS_MODE_CTR && mode != OCS_MODE_CCM && + mode != OCS_MODE_GCM && mode != OCS_MODE_CTS) + return -EINVAL; + + if (instruction != OCS_ENCRYPT && instruction != OCS_DECRYPT && + instruction != OCS_EXPAND && instruction != OCS_BYPASS) + return -EINVAL; + + /* + * When instruction is OCS_BYPASS, OCS simply copies data from source + * to destination using DMA. + * + * AES mode is irrelevant, but both source and destination DMA + * linked-list must be defined. + */ + if (instruction == OCS_BYPASS) { + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + } + + /* + * For performance reasons switch based on mode to limit unnecessary + * conditionals for each mode + */ + switch (mode) { + case OCS_MODE_ECB: + /* Ensure input length is multiple of block size */ + if (src_size % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + + case OCS_MODE_CBC: + /* Ensure input length is multiple of block size */ + if (src_size % AES_BLOCK_SIZE != 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CTR: + /* Ensure input length of 1 byte or greater */ + if (src_size == 0) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_CTS: + /* Ensure input length >= block size */ + if (src_size < AES_BLOCK_SIZE) + return -EINVAL; + + /* Ensure source and destination linked lists are created */ + if (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + return 0; + + case OCS_MODE_GCM: + /* Ensure IV is present and GCM_AES_IV_SIZE in length */ + if (!iv || iv_size != GCM_AES_IV_SIZE) + return -EINVAL; + + /* + * If input data present ensure source and destination linked + * lists are created + */ + if (src_size && (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR)) + return -EINVAL; + + /* If aad present ensure aad linked list is created */ + if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Ensure tag destination is set */ + if (!tag) + return -EINVAL; + + /* Just ensure that tag_size doesn't cause overflows. */ + if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) + return -EINVAL; + + return 0; + + case OCS_MODE_CCM: + /* Ensure IV is present and block size in length */ + if (!iv || iv_size != AES_BLOCK_SIZE) + return -EINVAL; + + /* 2 <= L <= 8, so 1 <= L' <= 7 */ + if (iv[L_PRIME_IDX] < L_PRIME_MIN || + iv[L_PRIME_IDX] > L_PRIME_MAX) + return -EINVAL; + + /* If aad present ensure aad linked list is created */ + if (aad_size && aad_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* Just ensure that tag_size doesn't cause overflows. */ + if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32))) + return -EINVAL; + + if (instruction == OCS_DECRYPT) { + /* + * If input data present ensure source and destination + * linked lists are created + */ + if (src_size && (src_dma_list == DMA_MAPPING_ERROR || + dst_dma_list == DMA_MAPPING_ERROR)) + return -EINVAL; + + /* Ensure input tag is present */ + if (!tag) + return -EINVAL; + + return 0; + } + + /* Instruction == OCS_ENCRYPT */ + + /* + * Destination linked list always required (for tag even if no + * input data) + */ + if (dst_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + /* If input data present ensure src linked list is created */ + if (src_size && src_dma_list == DMA_MAPPING_ERROR) + return -EINVAL; + + return 0; + + default: + return -EINVAL; + } +} + +/** + * ocs_aes_op() - Perform AES/SM4 operation. + * @aes_dev: The OCS AES device to use. + * @mode: The mode to use (ECB, CBC, CTR, or CTS). + * @cipher: The cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The IV vector. + * @iv_size: The size (in bytes) of @iv. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_op(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + u32 iv_size) +{ + u32 *iv32; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, iv_size, 0, 0, + NULL, 0, cipher, mode, instruction, + dst_dma_list); + if (rc) + return rc; + /* + * ocs_aes_validate_inputs() is a generic check, now ensure mode is not + * GCM or CCM. + */ + if (mode == OCS_MODE_GCM || mode == OCS_MODE_CCM) + return -EINVAL; + + /* Cast IV to u32 array. */ + iv32 = (u32 *)iv; + + ocs_aes_init(aes_dev, mode, cipher, instruction); + + if (mode == OCS_MODE_CTS) { + /* Write the byte length of the last data block to engine. */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + } + + /* ECB is the only mode that doesn't use IV. */ + if (mode != OCS_MODE_ECB) { + iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET); + iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET); + iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET); + iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET); + } + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + /* Configure and activate input / output DMA. */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + + if (mode == OCS_MODE_CTS) { + /* + * For CTS mode, instruct engine to activate ciphertext + * stealing if last block of data is incomplete. + */ + aes_a_set_last_gcx(aes_dev); + } else { + /* For all other modes, just write the 'termination' bit. */ + aes_a_op_termination(aes_dev); + } + + /* Wait for engine to complete processing. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + if (mode == OCS_MODE_CTR) { + /* Read back IV for streaming mode */ + iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET); + iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET); + iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET); + iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET); + } + + return 0; +} + +/* Compute and write J0 to engine registers. */ +static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev, + const u8 *iv) +{ + const u32 *j0 = (u32 *)iv; + + /* + * IV must be 12 bytes; Other sizes not supported as Linux crypto API + * does only expects/allows 12 byte IV for GCM + */ + iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET); + iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET); + iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET); + iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET); +} + +/* Read GCM tag from engine registers. */ +static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev, + u8 *tag, u32 tag_size) +{ + u32 tag_u32[AES_MAX_TAG_SIZE_U32]; + + /* + * The Authentication Tag T is stored in Little Endian order in the + * registers with the most significant bytes stored from AES_T_MAC[3] + * downward. + */ + tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET)); + tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET)); + tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET)); + tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET)); + + memcpy(tag, tag_u32, tag_size); +} + +/** + * ocs_aes_gcm_op() - Perform GCM operation. + * @aes_dev: The OCS AES device to use. + * @cipher: The Cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The input IV vector. + * @aad_dma_list: The OCS DMA list mapping input AAD data. + * @aad_size: The amount of data mapped by @aad_dma_list. + * @out_tag: Where to store computed tag. + * @tag_size: The size (in bytes) of @out_tag. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + const u8 *iv, + dma_addr_t aad_dma_list, + u32 aad_size, + u8 *out_tag, + u32 tag_size) +{ + u64 bit_len; + u32 val; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, + GCM_AES_IV_SIZE, aad_dma_list, + aad_size, out_tag, tag_size, cipher, + OCS_MODE_GCM, instruction, + dst_dma_list); + if (rc) + return rc; + + ocs_aes_init(aes_dev, OCS_MODE_GCM, cipher, instruction); + + /* Compute and write J0 to OCS HW. */ + ocs_aes_gcm_write_j0(aes_dev, iv); + + /* Write out_tag byte length */ + iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); + + /* Write the byte length of the last plaintext / ciphertext block. */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + + /* Write ciphertext bit length */ + bit_len = src_size * 8; + val = bit_len & 0xFFFFFFFF; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET); + val = bit_len >> 32; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET); + + /* Write aad bit length */ + bit_len = aad_size * 8; + val = bit_len & 0xFFFFFFFF; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET); + val = bit_len >> 32; + iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET); + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + /* Process AAD. */ + if (aad_size) { + /* If aad present, configure DMA to feed it to the engine. */ + dma_to_ocs_aes_ll(aes_dev, aad_dma_list); + aes_a_dma_active_src_ll_en(aes_dev); + + /* Instructs engine to pad last block of aad, if needed. */ + aes_a_set_last_gcx_and_adata(aes_dev); + + /* Wait for DMA transfer to complete. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); + if (rc) + return rc; + } else { + aes_a_set_last_gcx_and_adata(aes_dev); + } + + /* Wait until adata (if present) has been processed. */ + aes_a_wait_last_gcx(aes_dev); + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + /* Now process payload. */ + if (src_size) { + /* Configure and activate DMA for both input and output data. */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + } else { + aes_a_dma_set_xfer_size_zero(aes_dev); + aes_a_dma_active(aes_dev); + } + + /* Instruct AES/SMA4 engine payload processing is over. */ + aes_a_set_last_gcx(aes_dev); + + /* Wait for OCS AES engine to complete processing. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + ocs_aes_gcm_read_tag(aes_dev, out_tag, tag_size); + + return 0; +} + +/* Write encrypted tag to AES/SM4 engine. */ +static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev, + const u8 *in_tag, u32 tag_size) +{ + int i; + + /* Ensure DMA input buffer is empty */ + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + /* + * During CCM decrypt, the OCS block needs to finish processing the + * ciphertext before the tag is written. So delay needed after DMA has + * completed writing the ciphertext + */ + aes_a_dma_reset_and_activate_perf_cntr(aes_dev); + aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, + CCM_DECRYPT_DELAY_TAG_CLK_COUNT); + + /* Write encrypted tag to AES/SM4 engine. */ + for (i = 0; i < tag_size; i++) { + iowrite8(in_tag[i], aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); + } +} + +/* + * Write B0 CCM block to OCS AES HW. + * + * Note: B0 format is documented in NIST Special Publication 800-38C + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf + * (see Section A.2.1) + */ +static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev, + const u8 *iv, u32 adata_size, u32 tag_size, + u32 cryptlen) +{ + u8 b0[16]; /* CCM B0 block is 16 bytes long. */ + int i, q; + + /* Initialize B0 to 0. */ + memset(b0, 0, sizeof(b0)); + + /* + * B0[0] is the 'Flags Octet' and has the following structure: + * bit 7: Reserved + * bit 6: Adata flag + * bit 5-3: t value encoded as (t-2)/2 + * bit 2-0: q value encoded as q - 1 + */ + /* If there is AAD data, set the Adata flag. */ + if (adata_size) + b0[0] |= BIT(6); + /* + * t denotes the octet length of T. + * t can only be an element of { 4, 6, 8, 10, 12, 14, 16} and is + * encoded as (t - 2) / 2 + */ + b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3; + /* + * q is the octet length of Q. + * q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as + * q - 1 == iv[0] + */ + b0[0] |= iv[0] & 0x7; + /* + * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q] + * and must be copied to b0[1]..b0[15-q]. + * q == iv[0] + 1 + */ + q = iv[0] + 1; + for (i = 1; i <= 15 - q; i++) + b0[i] = iv[i]; + /* + * The rest of B0 must contain Q, i.e., the message length. + * Q is encoded in q octets, in big-endian order, so to write it, we + * start from the end of B0 and we move backward. + */ + i = sizeof(b0) - 1; + while (q) { + b0[i] = cryptlen & 0xff; + cryptlen >>= 8; + i--; + q--; + } + /* + * If cryptlen is not zero at this point, it means that its original + * value was too big. + */ + if (cryptlen) + return -EOVERFLOW; + /* Now write B0 to OCS AES input buffer. */ + for (i = 0; i < sizeof(b0); i++) + iowrite8(b0[i], aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); + return 0; +} + +/* + * Write adata length to OCS AES HW. + * + * Note: adata len encoding is documented in NIST Special Publication 800-38C + * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf + * (see Section A.2.2) + */ +static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev, + u64 adata_len) +{ + u8 enc_a[10]; /* Maximum encoded size: 10 octets. */ + int i, len; + + /* + * adata_len ('a') is encoded as follows: + * If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets + * (big endian). + * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32, + * i.e., six octets (big endian). + * If 2^32 ≤ a < 2^64 ==> 'a' encoded as 0xff || 0xff || [a]64, + * i.e., ten octets (big endian). + */ + if (adata_len < 65280) { + len = 2; + *(__be16 *)enc_a = cpu_to_be16(adata_len); + } else if (adata_len <= 0xFFFFFFFF) { + len = 6; + *(__be16 *)enc_a = cpu_to_be16(0xfffe); + *(__be32 *)&enc_a[2] = cpu_to_be32(adata_len); + } else { /* adata_len >= 2^32 */ + len = 10; + *(__be16 *)enc_a = cpu_to_be16(0xffff); + *(__be64 *)&enc_a[2] = cpu_to_be64(adata_len); + } + for (i = 0; i < len; i++) + iowrite8(enc_a[i], + aes_dev->base_reg + + AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET); +} + +static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev, + dma_addr_t adata_dma_list, u32 adata_size) +{ + int rc; + + if (!adata_size) { + /* Since no aad the LAST_GCX bit can be set now */ + aes_a_set_last_gcx_and_adata(aes_dev); + goto exit; + } + + /* Adata case. */ + + /* + * Form the encoding of the Associated data length and write it + * to the AES/SM4 input buffer. + */ + ocs_aes_ccm_write_adata_len(aes_dev, adata_size); + + /* Configure the AES/SM4 DMA to fetch the Associated Data */ + dma_to_ocs_aes_ll(aes_dev, adata_dma_list); + + /* Activate DMA to fetch Associated data. */ + aes_a_dma_active_src_ll_en(aes_dev); + + /* Set LAST_GCX and LAST_ADATA in AES ACTIVE register. */ + aes_a_set_last_gcx_and_adata(aes_dev); + + /* Wait for DMA transfer to complete. */ + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); + if (rc) + return rc; + +exit: + /* Wait until adata (if present) has been processed. */ + aes_a_wait_last_gcx(aes_dev); + aes_a_dma_wait_input_buffer_occupancy(aes_dev); + + return 0; +} + +static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size) +{ + if (src_size) { + /* + * Configure and activate DMA for both input and output + * data. + */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + } else { + /* Configure and activate DMA for output data only. */ + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_dst_ll_en(aes_dev); + } + + /* + * Set the LAST GCX bit in AES_ACTIVE Register to instruct + * AES/SM4 engine to pad the last block of data. + */ + aes_a_set_last_gcx(aes_dev); + + /* We are done, wait for IRQ and return. */ + return ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); +} + +static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size) +{ + if (!src_size) { + /* Let engine process 0-length input. */ + aes_a_dma_set_xfer_size_zero(aes_dev); + aes_a_dma_active(aes_dev); + aes_a_set_last_gcx(aes_dev); + + return 0; + } + + /* + * Configure and activate DMA for both input and output + * data. + */ + dma_to_ocs_aes_ll(aes_dev, src_dma_list); + dma_from_ocs_aes_ll(aes_dev, dst_dma_list); + aes_a_dma_active_src_dst_ll_en(aes_dev); + /* + * Set the LAST GCX bit in AES_ACTIVE Register; this allows the + * AES/SM4 engine to differentiate between encrypted data and + * encrypted MAC. + */ + aes_a_set_last_gcx(aes_dev); + /* + * Enable DMA DONE interrupt; once DMA transfer is over, + * interrupt handler will process the MAC/tag. + */ + return ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT); +} + +/* + * Compare Tag to Yr. + * + * Only used at the end of CCM decrypt. If tag == yr, message authentication + * has succeeded. + */ +static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev, + u8 tag_size_bytes) +{ + u32 tag[AES_MAX_TAG_SIZE_U32]; + u32 yr[AES_MAX_TAG_SIZE_U32]; + u8 i; + + /* Read Tag and Yr from AES registers. */ + for (i = 0; i < AES_MAX_TAG_SIZE_U32; i++) { + tag[i] = ioread32(aes_dev->base_reg + + AES_T_MAC_0_OFFSET + (i * sizeof(u32))); + yr[i] = ioread32(aes_dev->base_reg + + AES_MULTIPURPOSE2_0_OFFSET + + (i * sizeof(u32))); + } + + return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0; +} + +/** + * ocs_aes_ccm_op() - Perform CCM operation. + * @aes_dev: The OCS AES device to use. + * @cipher: The Cipher to use (AES or SM4). + * @instruction: The instruction to perform (encrypt or decrypt). + * @dst_dma_list: The OCS DMA list mapping output memory. + * @src_dma_list: The OCS DMA list mapping input payload data. + * @src_size: The amount of data mapped by @src_dma_list. + * @iv: The input IV vector. + * @adata_dma_list: The OCS DMA list mapping input A-data. + * @adata_size: The amount of data mapped by @adata_dma_list. + * @in_tag: Input tag. + * @tag_size: The size (in bytes) of @in_tag. + * + * Note: for encrypt the tag is appended to the ciphertext (in the memory + * mapped by @dst_dma_list). + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + dma_addr_t adata_dma_list, + u32 adata_size, + u8 *in_tag, + u32 tag_size) +{ + u32 *iv_32; + u8 lprime; + int rc; + + rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, + AES_BLOCK_SIZE, adata_dma_list, adata_size, + in_tag, tag_size, cipher, OCS_MODE_CCM, + instruction, dst_dma_list); + if (rc) + return rc; + + ocs_aes_init(aes_dev, OCS_MODE_CCM, cipher, instruction); + + /* + * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt + * auth tag so ensure this is the case + */ + lprime = iv[L_PRIME_IDX]; + memset(&iv[COUNTER_START(lprime)], 0, COUNTER_LEN(lprime)); + + /* + * Nonce is already converted to ctr0 before being passed into this + * function as iv. + */ + iv_32 = (u32 *)iv; + iowrite32(__swab32(iv_32[0]), + aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET); + iowrite32(__swab32(iv_32[1]), + aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET); + iowrite32(__swab32(iv_32[2]), + aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET); + iowrite32(__swab32(iv_32[3]), + aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET); + + /* Write MAC/tag length in register AES_TLEN */ + iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); + /* + * Write the byte length of the last AES/SM4 block of Payload data + * (without zero padding and without the length of the MAC) in register + * AES_PLEN. + */ + ocs_aes_write_last_data_blk_len(aes_dev, src_size); + + /* Set AES_ACTIVE.TRIGGER to start the operation. */ + aes_a_op_trigger(aes_dev); + + aes_a_dma_reset_and_activate_perf_cntr(aes_dev); + + /* Form block B0 and write it to the AES/SM4 input buffer. */ + rc = ocs_aes_ccm_write_b0(aes_dev, iv, adata_size, tag_size, src_size); + if (rc) + return rc; + /* + * Ensure there has been at least CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT + * clock cycles since TRIGGER bit was set + */ + aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev, + CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT); + + /* Process Adata. */ + ocs_aes_ccm_do_adata(aes_dev, adata_dma_list, adata_size); + + /* For Encrypt case we just process the payload and return. */ + if (instruction == OCS_ENCRYPT) { + return ocs_aes_ccm_encrypt_do_payload(aes_dev, dst_dma_list, + src_dma_list, src_size); + } + /* For Decypt we need to process the payload and then the tag. */ + rc = ocs_aes_ccm_decrypt_do_payload(aes_dev, dst_dma_list, + src_dma_list, src_size); + if (rc) + return rc; + + /* Process MAC/tag directly: feed tag to engine and wait for IRQ. */ + ocs_aes_ccm_write_encrypted_tag(aes_dev, in_tag, tag_size); + rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT); + if (rc) + return rc; + + return ccm_compare_tag_to_yr(aes_dev, tag_size); +} + +/** + * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list. + * @aes_dev: The OCS AES device the list will be created for. + * @sg: The SG list OCS DMA linked list will be created from. When + * passed to this function, @sg must have been already mapped + * with dma_map_sg(). + * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the + * value returned by dma_map_sg() when @sg was mapped. + * @dll_desc: The OCS DMA dma_list to use to store information about the + * created linked list. + * @data_size: The size of the data (from the SG list) to be mapped into the + * OCS DMA linked list. + * @data_offset: The offset (within the SG list) of the data to be mapped. + * + * Return: 0 on success, negative error code otherwise. + */ +int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, + struct scatterlist *sg, + int sg_dma_count, + struct ocs_dll_desc *dll_desc, + size_t data_size, size_t data_offset) +{ + struct ocs_dma_linked_list *ll = NULL; + struct scatterlist *sg_tmp; + unsigned int tmp; + int dma_nents; + int i; + + if (!dll_desc || !sg || !aes_dev) + return -EINVAL; + + /* Default values for when no ddl_desc is created. */ + dll_desc->vaddr = NULL; + dll_desc->dma_addr = DMA_MAPPING_ERROR; + dll_desc->size = 0; + + if (data_size == 0) + return 0; + + /* Loop over sg_list until we reach entry at specified offset. */ + while (data_offset >= sg_dma_len(sg)) { + data_offset -= sg_dma_len(sg); + sg_dma_count--; + sg = sg_next(sg); + /* If we reach the end of the list, offset was invalid. */ + if (!sg || sg_dma_count == 0) + return -EINVAL; + } + + /* Compute number of DMA-mapped SG entries to add into OCS DMA list. */ + dma_nents = 0; + tmp = 0; + sg_tmp = sg; + while (tmp < data_offset + data_size) { + /* If we reach the end of the list, data_size was invalid. */ + if (!sg_tmp) + return -EINVAL; + tmp += sg_dma_len(sg_tmp); + dma_nents++; + sg_tmp = sg_next(sg_tmp); + } + if (dma_nents > sg_dma_count) + return -EINVAL; + + /* Allocate the DMA list, one entry for each SG entry. */ + dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents; + dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size, + &dll_desc->dma_addr, GFP_KERNEL); + if (!dll_desc->vaddr) + return -ENOMEM; + + /* Populate DMA linked list entries. */ + ll = dll_desc->vaddr; + for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) { + ll[i].src_addr = sg_dma_address(sg) + data_offset; + ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ? + (sg_dma_len(sg) - data_offset) : data_size; + data_offset = 0; + data_size -= ll[i].src_len; + /* Current element points to the DMA address of the next one. */ + ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1)); + ll[i].ll_flags = 0; + } + /* Terminate last element. */ + ll[i - 1].next = 0; + ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE; + + return 0; +} diff --git a/drivers/crypto/keembay/ocs-aes.h b/drivers/crypto/keembay/ocs-aes.h new file mode 100644 index 000000000000..c035fc48b7ed --- /dev/null +++ b/drivers/crypto/keembay/ocs-aes.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Intel Keem Bay OCS AES Crypto Driver. + * + * Copyright (C) 2018-2020 Intel Corporation + */ + +#ifndef _CRYPTO_OCS_AES_H +#define _CRYPTO_OCS_AES_H + +#include + +enum ocs_cipher { + OCS_AES = 0, + OCS_SM4 = 1, +}; + +enum ocs_mode { + OCS_MODE_ECB = 0, + OCS_MODE_CBC = 1, + OCS_MODE_CTR = 2, + OCS_MODE_CCM = 6, + OCS_MODE_GCM = 7, + OCS_MODE_CTS = 9, +}; + +enum ocs_instruction { + OCS_ENCRYPT = 0, + OCS_DECRYPT = 1, + OCS_EXPAND = 2, + OCS_BYPASS = 3, +}; + +/** + * struct ocs_aes_dev - AES device context. + * @list: List head for insertion into device list hold + * by driver. + * @dev: OCS AES device. + * @irq: IRQ number. + * @base_reg: IO base address of OCS AES. + * @irq_copy_completion: Completion to indicate IRQ has been triggered. + * @dma_err_mask: Error reported by OCS DMA interrupts. + * @engine: Crypto engine for the device. + */ +struct ocs_aes_dev { + struct list_head list; + struct device *dev; + int irq; + void __iomem *base_reg; + struct completion irq_completion; + u32 dma_err_mask; + struct crypto_engine *engine; +}; + +/** + * struct ocs_dll_desc - Descriptor of an OCS DMA Linked List. + * @vaddr: Virtual address of the linked list head. + * @dma_addr: DMA address of the linked list head. + * @size: Size (in bytes) of the linked list. + */ +struct ocs_dll_desc { + void *vaddr; + dma_addr_t dma_addr; + size_t size; +}; + +int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, const u32 key_size, + const u8 *key, const enum ocs_cipher cipher); + +int ocs_aes_op(struct ocs_aes_dev *aes_dev, + enum ocs_mode mode, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + u32 iv_size); + +/** + * ocs_aes_bypass_op() - Use OCS DMA to copy data. + * @aes_dev: The OCS AES device to use. + * @dst_dma_list: The OCS DMA list mapping the memory where input data + * will be copied to. + * @src_dma_list: The OCS DMA list mapping input data. + * @src_size: The amount of data to copy. + */ +static inline int ocs_aes_bypass_op(struct ocs_aes_dev *aes_dev, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, u32 src_size) +{ + return ocs_aes_op(aes_dev, OCS_MODE_ECB, OCS_AES, OCS_BYPASS, + dst_dma_list, src_dma_list, src_size, NULL, 0); +} + +int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + const u8 *iv, + dma_addr_t aad_dma_list, + u32 aad_size, + u8 *out_tag, + u32 tag_size); + +int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev, + enum ocs_cipher cipher, + enum ocs_instruction instruction, + dma_addr_t dst_dma_list, + dma_addr_t src_dma_list, + u32 src_size, + u8 *iv, + dma_addr_t adata_dma_list, + u32 adata_size, + u8 *in_tag, + u32 tag_size); + +int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev, + struct scatterlist *sg, + int sg_dma_count, + struct ocs_dll_desc *dll_desc, + size_t data_size, + size_t data_offset); + +irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id); + +#endif From a320dc2ff80b8f93b2b8acf2e3ead8ff5ad0bcff Mon Sep 17 00:00:00 2001 From: Yejune Deng Date: Tue, 1 Dec 2020 14:50:18 +0800 Subject: [PATCH 353/360] crypto: hisilicon/trng - replace atomic_add_return() a set of atomic_inc_return() looks more neater Signed-off-by: Yejune Deng Signed-off-by: Herbert Xu --- drivers/crypto/hisilicon/trng/trng.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c index a5033cf09e2b..29712685498a 100644 --- a/drivers/crypto/hisilicon/trng/trng.c +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -267,12 +267,12 @@ static int hisi_trng_probe(struct platform_device *pdev) } hisi_trng_add_to_list(trng); - if (atomic_add_return(1, &trng_active_devs) == 1) { + if (atomic_inc_return(&trng_active_devs) == 1) { ret = crypto_register_rng(&hisi_trng_alg); if (ret) { dev_err(&pdev->dev, "failed to register crypto(%d)\n", ret); - atomic_sub_return(1, &trng_active_devs); + atomic_dec_return(&trng_active_devs); goto err_remove_from_list; } } @@ -289,7 +289,7 @@ static int hisi_trng_probe(struct platform_device *pdev) return ret; err_crypto_unregister: - if (atomic_sub_return(1, &trng_active_devs) == 0) + if (atomic_dec_return(&trng_active_devs) == 0) crypto_unregister_rng(&hisi_trng_alg); err_remove_from_list: @@ -305,7 +305,7 @@ static int hisi_trng_remove(struct platform_device *pdev) while (hisi_trng_del_from_list(trng)) ; - if (atomic_sub_return(1, &trng_active_devs) == 0) + if (atomic_dec_return(&trng_active_devs) == 0) crypto_unregister_rng(&hisi_trng_alg); return 0; From d33a23b0532d5d1b5b700e8641661261e7dbef61 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 4 Dec 2020 00:20:04 +0100 Subject: [PATCH 354/360] crypto: atmel-i2c - select CONFIG_BITREVERSE The bitreverse helper is almost always built into the kernel, but in a rare randconfig build it is possible to hit a case in which it is a loadable module while the atmel-i2c driver is built-in: arm-linux-gnueabi-ld: drivers/crypto/atmel-i2c.o: in function `atmel_i2c_checksum': atmel-i2c.c:(.text+0xa0): undefined reference to `byte_rev_table' Add one more 'select' statement to prevent this. Fixes: 11105693fa05 ("crypto: atmel-ecc - introduce Microchip / Atmel ECC driver") Signed-off-by: Arnd Bergmann Signed-off-by: Herbert Xu --- drivers/crypto/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 808eecef0f32..bbd51703e738 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -548,6 +548,7 @@ config CRYPTO_DEV_ATMEL_SHA config CRYPTO_DEV_ATMEL_I2C tristate + select BITREVERSE config CRYPTO_DEV_ATMEL_ECC tristate "Support for Microchip / Atmel ECC hw accelerator" From 67916c9516893528ecce060ada1f58af0ce33d93 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 1 Dec 2020 14:24:49 +0000 Subject: [PATCH 355/360] crypto: qat - add AES-CTR support for QAT GEN4 devices Add support for AES-CTR for QAT GEN4 devices. Also, introduce the capability ICP_ACCEL_CAPABILITIES_AES_V2 and the helper macro HW_CAP_AES_V2, which allow to distinguish between different HW generations. Co-developed-by: Tomasz Kowalik Signed-off-by: Tomasz Kowalik Co-developed-by: Mateusz Polrola Signed-off-by: Mateusz Polrola Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/icp_qat_fw_la.h | 7 +++++++ drivers/crypto/qat/qat_common/icp_qat_hw.h | 17 ++++++++++++++++- drivers/crypto/qat/qat_common/qat_algs.c | 17 ++++++++++++++++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_la.h b/drivers/crypto/qat/qat_common/icp_qat_fw_la.h index 6757ec09d81f..28fa17f14be4 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_la.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_la.h @@ -33,6 +33,9 @@ struct icp_qat_fw_la_bulk_req { struct icp_qat_fw_comn_req_cd_ctrl cd_ctrl; }; +#define ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE 1 +#define QAT_LA_SLICE_TYPE_BITPOS 14 +#define QAT_LA_SLICE_TYPE_MASK 0x3 #define ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS 1 #define ICP_QAT_FW_LA_GCM_IV_LEN_NOT_12_OCTETS 0 #define QAT_FW_LA_ZUC_3G_PROTO_FLAG_BITPOS 12 @@ -179,6 +182,10 @@ struct icp_qat_fw_la_bulk_req { QAT_FIELD_SET(flags, val, QAT_LA_PARTIAL_BITPOS, \ QAT_LA_PARTIAL_MASK) +#define ICP_QAT_FW_LA_SLICE_TYPE_SET(flags, val) \ + QAT_FIELD_SET(flags, val, QAT_LA_SLICE_TYPE_BITPOS, \ + QAT_LA_SLICE_TYPE_MASK) + struct icp_qat_fw_cipher_req_hdr_cd_pars { union { struct { diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/qat/qat_common/icp_qat_hw.h index 4aa5d724e11b..e39e8a2d51a7 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hw.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hw.h @@ -65,6 +65,11 @@ struct icp_qat_hw_auth_config { __u32 reserved; }; +struct icp_qat_hw_ucs_cipher_config { + __u32 val; + __u32 reserved[3]; +}; + enum icp_qat_slice_mask { ICP_ACCEL_MASK_CIPHER_SLICE = BIT(0), ICP_ACCEL_MASK_AUTH_SLICE = BIT(1), @@ -86,6 +91,8 @@ enum icp_qat_capabilities_mask { ICP_ACCEL_CAPABILITIES_RAND = BIT(7), ICP_ACCEL_CAPABILITIES_ZUC = BIT(8), ICP_ACCEL_CAPABILITIES_SHA3 = BIT(9), + /* Bits 10-25 are currently reserved */ + ICP_ACCEL_CAPABILITIES_AES_V2 = BIT(26) }; #define QAT_AUTH_MODE_BITPOS 4 @@ -278,7 +285,15 @@ struct icp_qat_hw_cipher_aes256_f8 { __u8 key[ICP_QAT_HW_AES_256_F8_KEY_SZ]; }; +struct icp_qat_hw_ucs_cipher_aes256_f8 { + struct icp_qat_hw_ucs_cipher_config cipher_config; + __u8 key[ICP_QAT_HW_AES_256_F8_KEY_SZ]; +}; + struct icp_qat_hw_cipher_algo_blk { - struct icp_qat_hw_cipher_aes256_f8 aes; + union { + struct icp_qat_hw_cipher_aes256_f8 aes; + struct icp_qat_hw_ucs_cipher_aes256_f8 ucs_aes; + }; } __aligned(64); #endif diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index b3a68d986417..84d1a3545c3a 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -33,6 +33,10 @@ ICP_QAT_HW_CIPHER_KEY_CONVERT, \ ICP_QAT_HW_CIPHER_DECRYPT) +#define HW_CAP_AES_V2(accel_dev) \ + (GET_HW_DATA(accel_dev)->accel_capabilities_mask & \ + ICP_ACCEL_CAPABILITIES_AES_V2) + static DEFINE_MUTEX(algs_lock); static unsigned int active_devs; @@ -416,12 +420,23 @@ static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx, struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr; struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl; + bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); + int mode = ctx->mode; - memcpy(cd->aes.key, key, keylen); qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; cd_pars->u.s.content_desc_params_sz = sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; + + if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) { + ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE); + keylen = round_up(keylen, 16); + memcpy(cd->ucs_aes.key, key, keylen); + } else { + memcpy(cd->aes.key, key, keylen); + } + /* Cipher CD config setup */ cd_ctrl->cipher_key_sz = keylen >> 3; cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; From 5106dfeaeabea73d5132daab1d89d57b57fa98b7 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 1 Dec 2020 14:24:50 +0000 Subject: [PATCH 356/360] crypto: qat - add AES-XTS support for QAT GEN4 devices Add handling of AES-XTS specific to QAT GEN4 devices. Co-developed-by: Tomaszx Kowalik Signed-off-by: Tomaszx Kowalik Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/qat/qat_common/qat_algs.c | 96 ++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 84d1a3545c3a..31c7a206a629 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -33,6 +33,11 @@ ICP_QAT_HW_CIPHER_KEY_CONVERT, \ ICP_QAT_HW_CIPHER_DECRYPT) +#define QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode) \ + ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \ + ICP_QAT_HW_CIPHER_NO_CONVERT, \ + ICP_QAT_HW_CIPHER_DECRYPT) + #define HW_CAP_AES_V2(accel_dev) \ (GET_HW_DATA(accel_dev)->accel_capabilities_mask & \ ICP_ACCEL_CAPABILITIES_AES_V2) @@ -95,6 +100,7 @@ struct qat_alg_skcipher_ctx { struct icp_qat_fw_la_bulk_req dec_fw_req; struct qat_crypto_instance *inst; struct crypto_skcipher *ftfm; + struct crypto_cipher *tweak; bool fallback; int mode; }; @@ -428,7 +434,16 @@ static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx, cd_pars->u.s.content_desc_params_sz = sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; - if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) { + if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) { + ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE); + + /* Store both XTS keys in CD, only the first key is sent + * to the HW, the second key is used for tweak calculation + */ + memcpy(cd->ucs_aes.key, key, keylen); + keylen = keylen / 2; + } else if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) { ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags, ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE); keylen = round_up(keylen, 16); @@ -458,6 +473,28 @@ static void qat_alg_skcipher_init_enc(struct qat_alg_skcipher_ctx *ctx, enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode); } +static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen, + u8 *key_reverse) +{ + struct crypto_aes_ctx aes_expanded; + int nrounds; + u8 *key; + + aes_expandkey(&aes_expanded, key_forward, keylen); + if (keylen == AES_KEYSIZE_128) { + nrounds = 10; + key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds); + memcpy(key_reverse, key, AES_BLOCK_SIZE); + } else { + /* AES_KEYSIZE_256 */ + nrounds = 14; + key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds); + memcpy(key_reverse, key, AES_BLOCK_SIZE); + memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + } +} + static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx, int alg, const u8 *key, unsigned int keylen, int mode) @@ -465,16 +502,26 @@ static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx, struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd; struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req; struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; + bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); qat_alg_skcipher_init_com(ctx, req, dec_cd, key, keylen); cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr; - if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) + if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) { + /* Key reversing not supported, set no convert */ + dec_cd->aes.cipher_config.val = + QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode); + + /* In-place key reversal */ + qat_alg_xts_reverse_key(dec_cd->ucs_aes.key, keylen / 2, + dec_cd->ucs_aes.key); + } else if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) { dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode); - else + } else { dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode); + } } static int qat_alg_validate_key(int key_len, int *alg, int mode) @@ -1081,8 +1128,33 @@ static int qat_alg_skcipher_xts_setkey(struct crypto_skcipher *tfm, ctx->fallback = false; - return qat_alg_skcipher_setkey(tfm, key, keylen, - ICP_QAT_HW_CIPHER_XTS_MODE); + ret = qat_alg_skcipher_setkey(tfm, key, keylen, + ICP_QAT_HW_CIPHER_XTS_MODE); + if (ret) + return ret; + + if (HW_CAP_AES_V2(ctx->inst->accel_dev)) + ret = crypto_cipher_setkey(ctx->tweak, key + (keylen / 2), + keylen / 2); + + return ret; +} + +static void qat_alg_set_req_iv(struct qat_crypto_request *qat_req) +{ + struct icp_qat_fw_la_cipher_req_params *cipher_param; + struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx; + bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev); + u8 *iv = qat_req->skcipher_req->iv; + + cipher_param = (void *)&qat_req->req.serv_specif_rqpars; + + if (aes_v2_capable && ctx->mode == ICP_QAT_HW_CIPHER_XTS_MODE) + crypto_cipher_encrypt_one(ctx->tweak, + (u8 *)cipher_param->u.cipher_IV_array, + iv); + else + memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE); } static int qat_alg_skcipher_encrypt(struct skcipher_request *req) @@ -1114,7 +1186,8 @@ static int qat_alg_skcipher_encrypt(struct skcipher_request *req) cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = req->cryptlen; cipher_param->cipher_offset = 0; - memcpy(cipher_param->u.cipher_IV_array, req->iv, AES_BLOCK_SIZE); + + qat_alg_set_req_iv(qat_req); do { ret = adf_send_message(ctx->inst->sym_tx, (u32 *)msg); @@ -1182,8 +1255,8 @@ static int qat_alg_skcipher_decrypt(struct skcipher_request *req) cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = req->cryptlen; cipher_param->cipher_offset = 0; - memcpy(cipher_param->u.cipher_IV_array, req->iv, AES_BLOCK_SIZE); + qat_alg_set_req_iv(qat_req); qat_alg_update_iv(qat_req); do { @@ -1293,6 +1366,12 @@ static int qat_alg_skcipher_init_xts_tfm(struct crypto_skcipher *tfm) if (IS_ERR(ctx->ftfm)) return PTR_ERR(ctx->ftfm); + ctx->tweak = crypto_alloc_cipher("aes", 0, 0); + if (IS_ERR(ctx->tweak)) { + crypto_free_skcipher(ctx->ftfm); + return PTR_ERR(ctx->tweak); + } + reqsize = max(sizeof(struct qat_crypto_request), sizeof(struct skcipher_request) + crypto_skcipher_reqsize(ctx->ftfm)); @@ -1335,6 +1414,9 @@ static void qat_alg_skcipher_exit_xts_tfm(struct crypto_skcipher *tfm) if (ctx->ftfm) crypto_free_skcipher(ctx->ftfm); + if (ctx->tweak) + crypto_free_cipher(ctx->tweak); + qat_alg_skcipher_exit_tfm(tfm); } From 93cebeb1c21a65b92636aaa278a32fbc0415ec67 Mon Sep 17 00:00:00 2001 From: Marco Chiappero Date: Tue, 1 Dec 2020 14:24:51 +0000 Subject: [PATCH 357/360] crypto: qat - add capability detection logic in qat_4xxx Add logic to detect device capabilities in qat_4xxx driver. Read fuses and build the device capabilities mask. This will enable services and handling specific to QAT 4xxx devices. Co-developed-by: Tomaszx Kowalik Signed-off-by: Tomaszx Kowalik Signed-off-by: Marco Chiappero Reviewed-by: Giovanni Cabiddu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.c | 24 +++++++++++++++++++ .../crypto/qat/qat_4xxx/adf_4xxx_hw_data.h | 11 +++++++++ drivers/crypto/qat/qat_4xxx/adf_drv.c | 3 +++ 3 files changed, 38 insertions(+) diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c index e7a7c1e3da28..344bfae45bff 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -5,6 +5,7 @@ #include #include #include "adf_4xxx_hw_data.h" +#include "icp_qat_hw.h" struct adf_fw_config { u32 ae_mask; @@ -91,6 +92,28 @@ static void set_msix_default_rttable(struct adf_accel_dev *accel_dev) ADF_CSR_WR(csr, ADF_4XXX_MSIX_RTTABLE_OFFSET(i), i); } +static u32 get_accel_cap(struct adf_accel_dev *accel_dev) +{ + struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 fusectl1; + u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | + ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | + ICP_ACCEL_CAPABILITIES_AUTHENTICATION | + ICP_ACCEL_CAPABILITIES_AES_V2; + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_4XXX_FUSECTL1_OFFSET, &fusectl1); + + if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC; + if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION; + if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE) + capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC; + + return capabilities; +} + static enum dev_sku_info get_sku(struct adf_hw_device_data *self) { return DEV_SKU_1; @@ -189,6 +212,7 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_arb_info = get_arb_info; hw_data->get_admin_info = get_admin_info; + hw_data->get_accel_cap = get_accel_cap; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_4XXX_FW; hw_data->fw_mmp_name = ADF_4XXX_MMP; diff --git a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h index cdde0be886bf..4fe2a776293c 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h +++ b/drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h @@ -69,6 +69,17 @@ #define ADF_4XXX_ASYM_OBJ "qat_4xxx_asym.bin" #define ADF_4XXX_ADMIN_OBJ "qat_4xxx_admin.bin" +/* qat_4xxx fuse bits are different from old GENs, redefine them */ +enum icp_qat_4xxx_slice_mask { + ICP_ACCEL_4XXX_MASK_CIPHER_SLICE = BIT(0), + ICP_ACCEL_4XXX_MASK_AUTH_SLICE = BIT(1), + ICP_ACCEL_4XXX_MASK_PKE_SLICE = BIT(2), + ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE = BIT(3), + ICP_ACCEL_4XXX_MASK_UCS_SLICE = BIT(4), + ICP_ACCEL_4XXX_MASK_EIA3_SLICE = BIT(5), + ICP_ACCEL_4XXX_MASK_SMX_SLICE = BIT(6), +}; + void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data); void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data); diff --git a/drivers/crypto/qat/qat_4xxx/adf_drv.c b/drivers/crypto/qat/qat_4xxx/adf_drv.c index de5a955f406a..a8805c815d16 100644 --- a/drivers/crypto/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/qat/qat_4xxx/adf_drv.c @@ -233,6 +233,9 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); } + /* Get accelerator capabilities mask */ + hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); + /* Find and map all the device's BARS */ bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_4XXX_BAR_MASK; From 9a02fd8b19247e80e2354a227b6e2392e8fae78a Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 7 Dec 2020 13:41:41 +0100 Subject: [PATCH 358/360] x86/ia32_signal: Propagate __user annotation properly Commit 57d563c82925 ("x86: ia32_setup_rt_frame(): consolidate uaccess areas") dropped a __user annotation in a cast when refactoring __put_user() to unsafe_put_user(). Hence, since then, sparse warns in arch/x86/ia32/ia32_signal.c:350:9: warning: cast removes address space '__user' of expression warning: incorrect type in argument 1 (different address spaces) expected void const volatile [noderef] __user *ptr got unsigned long long [usertype] * Add the __user annotation to restore the propagation of address spaces. Signed-off-by: Lukas Bulwahn Signed-off-by: Borislav Petkov Link: https://lkml.kernel.org/r/20201207124141.21859-1-lukas.bulwahn@gmail.com --- arch/x86/ia32/ia32_signal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/ia32/ia32_signal.c b/arch/x86/ia32/ia32_signal.c index 81cf22398cd1..5e3d9b7fd5fb 100644 --- a/arch/x86/ia32/ia32_signal.c +++ b/arch/x86/ia32/ia32_signal.c @@ -347,7 +347,7 @@ int ia32_setup_rt_frame(int sig, struct ksignal *ksig, */ unsafe_put_user(*((u64 *)&code), (u64 __user *)frame->retcode, Efault); unsafe_put_sigcontext32(&frame->uc.uc_mcontext, fp, regs, set, Efault); - unsafe_put_user(*(__u64 *)set, (__u64 *)&frame->uc.uc_sigmask, Efault); + unsafe_put_user(*(__u64 *)set, (__u64 __user *)&frame->uc.uc_sigmask, Efault); user_access_end(); if (__copy_siginfo_to_user32(&frame->info, &ksig->info)) From 228ddee8ed9e24084274954eb526b5798b006f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 16 Nov 2020 20:12:03 +0100 Subject: [PATCH 359/360] drm/amdgpu: fix check order in amdgpu_bo_move MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorder the code to fix checking if blitting is available. Signed-off-by: Christian König Acked-by: Alex Deucher Link: https://patchwork.freedesktop.org/patch/401019/ Signed-off-by: Linus Torvalds --- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 53 +++++++++++-------------- 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index b848f9e97613..4d8f19ab1014 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -551,25 +551,12 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, struct ttm_resource *old_mem = &bo->mem; int r; - if ((old_mem->mem_type == TTM_PL_SYSTEM && - new_mem->mem_type == TTM_PL_VRAM) || - (old_mem->mem_type == TTM_PL_VRAM && - new_mem->mem_type == TTM_PL_SYSTEM)) { - hop->fpfn = 0; - hop->lpfn = 0; - hop->mem_type = TTM_PL_TT; - hop->flags = 0; - return -EMULTIHOP; - } - if (new_mem->mem_type == TTM_PL_TT) { r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem); if (r) return r; } - amdgpu_bo_move_notify(bo, evict, new_mem); - /* Can't move a pinned BO */ abo = ttm_to_amdgpu_bo(bo); if (WARN_ON_ONCE(abo->tbo.pin_count > 0)) @@ -579,24 +566,23 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { ttm_bo_move_null(bo, new_mem); - return 0; + goto out; } if (old_mem->mem_type == TTM_PL_SYSTEM && new_mem->mem_type == TTM_PL_TT) { ttm_bo_move_null(bo, new_mem); - return 0; + goto out; } - if (old_mem->mem_type == TTM_PL_TT && new_mem->mem_type == TTM_PL_SYSTEM) { r = ttm_bo_wait_ctx(bo, ctx); if (r) - goto fail; + return r; amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm); ttm_resource_free(bo, &bo->mem); ttm_bo_assign_mem(bo, new_mem); - return 0; + goto out; } if (old_mem->mem_type == AMDGPU_PL_GDS || @@ -607,27 +593,37 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, new_mem->mem_type == AMDGPU_PL_OA) { /* Nothing to save here */ ttm_bo_move_null(bo, new_mem); - return 0; + goto out; } - if (!adev->mman.buffer_funcs_enabled) { + if (adev->mman.buffer_funcs_enabled) { + if (((old_mem->mem_type == TTM_PL_SYSTEM && + new_mem->mem_type == TTM_PL_VRAM) || + (old_mem->mem_type == TTM_PL_VRAM && + new_mem->mem_type == TTM_PL_SYSTEM))) { + hop->fpfn = 0; + hop->lpfn = 0; + hop->mem_type = TTM_PL_TT; + hop->flags = 0; + return -EMULTIHOP; + } + + r = amdgpu_move_blit(bo, evict, new_mem, old_mem); + } else { r = -ENODEV; - goto memcpy; } - r = amdgpu_move_blit(bo, evict, new_mem, old_mem); if (r) { -memcpy: /* Check that all memory is CPU accessible */ if (!amdgpu_mem_visible(adev, old_mem) || !amdgpu_mem_visible(adev, new_mem)) { pr_err("Move buffer fallback to memcpy unavailable\n"); - goto fail; + return r; } r = ttm_bo_move_memcpy(bo, ctx, new_mem); if (r) - goto fail; + return r; } if (bo->type == ttm_bo_type_device && @@ -639,14 +635,11 @@ memcpy: abo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; } +out: /* update statistics */ atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &adev->num_bytes_moved); + amdgpu_bo_move_notify(bo, evict, new_mem); return 0; -fail: - swap(*new_mem, bo->mem); - amdgpu_bo_move_notify(bo, false, new_mem); - swap(*new_mem, bo->mem); - return r; } /* From 68b111bf74a0d69a905d6be4b91277d79f9bcafb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 25 Nov 2020 15:32:23 +0100 Subject: [PATCH 360/360] drm/radeon: fix check order in radeon_bo_move MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reorder the code to fix checking if blitting is available. Signed-off-by: Christian König Fixes: 28a68f828266 ("drm/radeon/ttm: use multihop") Reviewed-by: Daniel Vetter Reviewed-by: Dave Airlie Link: https://patchwork.freedesktop.org/patch/403847/ Signed-off-by: Linus Torvalds --- drivers/gpu/drm/radeon/radeon_ttm.c | 60 +++++++++++++---------------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index 28b300ed200e..d59ef6e92a40 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -217,27 +217,15 @@ static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, struct ttm_resource *old_mem = &bo->mem; int r; - if ((old_mem->mem_type == TTM_PL_SYSTEM && - new_mem->mem_type == TTM_PL_VRAM) || - (old_mem->mem_type == TTM_PL_VRAM && - new_mem->mem_type == TTM_PL_SYSTEM)) { - hop->fpfn = 0; - hop->lpfn = 0; - hop->mem_type = TTM_PL_TT; - hop->flags = 0; - return -EMULTIHOP; - } - if (new_mem->mem_type == TTM_PL_TT) { r = radeon_ttm_tt_bind(bo->bdev, bo->ttm, new_mem); if (r) return r; } - radeon_bo_move_notify(bo, evict, new_mem); r = ttm_bo_wait_ctx(bo, ctx); if (r) - goto fail; + return r; /* Can't move a pinned BO */ rbo = container_of(bo, struct radeon_bo, tbo); @@ -247,12 +235,12 @@ static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, rdev = radeon_get_rdev(bo->bdev); if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { ttm_bo_move_null(bo, new_mem); - return 0; + goto out; } if (old_mem->mem_type == TTM_PL_SYSTEM && new_mem->mem_type == TTM_PL_TT) { ttm_bo_move_null(bo, new_mem); - return 0; + goto out; } if (old_mem->mem_type == TTM_PL_TT && @@ -260,31 +248,37 @@ static int radeon_bo_move(struct ttm_buffer_object *bo, bool evict, radeon_ttm_tt_unbind(bo->bdev, bo->ttm); ttm_resource_free(bo, &bo->mem); ttm_bo_assign_mem(bo, new_mem); - return 0; + goto out; } - if (!rdev->ring[radeon_copy_ring_index(rdev)].ready || - rdev->asic->copy.copy == NULL) { - /* use memcpy */ - goto memcpy; - } - - r = radeon_move_blit(bo, evict, new_mem, old_mem); - if (r) { -memcpy: - r = ttm_bo_move_memcpy(bo, ctx, new_mem); - if (r) { - goto fail; + if (rdev->ring[radeon_copy_ring_index(rdev)].ready && + rdev->asic->copy.copy != NULL) { + if ((old_mem->mem_type == TTM_PL_SYSTEM && + new_mem->mem_type == TTM_PL_VRAM) || + (old_mem->mem_type == TTM_PL_VRAM && + new_mem->mem_type == TTM_PL_SYSTEM)) { + hop->fpfn = 0; + hop->lpfn = 0; + hop->mem_type = TTM_PL_TT; + hop->flags = 0; + return -EMULTIHOP; } + + r = radeon_move_blit(bo, evict, new_mem, old_mem); + } else { + r = -ENODEV; } + if (r) { + r = ttm_bo_move_memcpy(bo, ctx, new_mem); + if (r) + return r; + } + +out: /* update statistics */ atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &rdev->num_bytes_moved); + radeon_bo_move_notify(bo, evict, new_mem); return 0; -fail: - swap(*new_mem, bo->mem); - radeon_bo_move_notify(bo, false, new_mem); - swap(*new_mem, bo->mem); - return r; } static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_resource *mem)