From ac23d1a964600bb9c14b5048bdf4f18ae13226f4 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 13 Dec 2021 23:03:54 -0500 Subject: [PATCH 01/95] XArray: Document the locking requirement for the xa_state It wasn't obvious to all readers that it's unsafe to reuse an xa_state after dropping the xas_lock() or the rcu_read_lock(). Reported-by: Charan Teja Kalla Signed-off-by: Matthew Wilcox (Oracle) --- Documentation/core-api/xarray.rst | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Documentation/core-api/xarray.rst b/Documentation/core-api/xarray.rst index a137a0e6d068..77e0ece2b1d6 100644 --- a/Documentation/core-api/xarray.rst +++ b/Documentation/core-api/xarray.rst @@ -315,11 +315,15 @@ indeed the normal API is implemented in terms of the advanced API. The advanced API is only available to modules with a GPL-compatible license. The advanced API is based around the xa_state. This is an opaque data -structure which you declare on the stack using the XA_STATE() -macro. This macro initialises the xa_state ready to start walking -around the XArray. It is used as a cursor to maintain the position -in the XArray and let you compose various operations together without -having to restart from the top every time. +structure which you declare on the stack using the XA_STATE() macro. +This macro initialises the xa_state ready to start walking around the +XArray. It is used as a cursor to maintain the position in the XArray +and let you compose various operations together without having to restart +from the top every time. The contents of the xa_state are protected by +the rcu_read_lock() or the xas_lock(). If you need to drop whichever of +those locks is protecting your state and tree, you must call xas_pause() +so that future calls do not rely on the parts of the state which were +left unprotected. The xa_state is also used to store errors. You can call xas_error() to retrieve the error. All operations check whether From 22f56b8e890d4e2835951b437bb6eeebfd1cb18b Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 3 Feb 2022 16:01:39 -0500 Subject: [PATCH 02/95] XArray: Include bitmap.h from xarray.h xas_find_chunk() calls find_next_bit(), which is defined in find.h, included from bitmap.h. Inside the kernel, this isn't a problem because bitmap.h is included from cpumask.h which is dragged in (eventually) by gfp.h. When building the test-suite, that doesn't happen, so we need to include bitmap.h explicitly. Fixes: 4ade0818cf04 ("tools: sync tools/bitmap with mother linux") Reported-by: Liam Howlett Signed-off-by: Matthew Wilcox (Oracle) --- include/linux/xarray.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/xarray.h b/include/linux/xarray.h index d6d5da6ed735..66e28bc1a023 100644 --- a/include/linux/xarray.h +++ b/include/linux/xarray.h @@ -9,6 +9,7 @@ * See Documentation/core-api/xarray.rst for how to use the XArray. */ +#include #include #include #include From f6e64b66629e8fff0997825ade06619f09a06583 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Thu, 10 Feb 2022 11:19:40 +0530 Subject: [PATCH 03/95] RISC-V: Enable CPU_IDLE drivers We force select CPU_PM and provide asm/cpuidle.h so that we can use CPU IDLE drivers for Linux RISC-V kernel. Signed-off-by: Anup Patel Signed-off-by: Anup Patel Reviewed-by: Guo Ren Reviewed-by: Atish Patra Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 7 +++++++ arch/riscv/configs/defconfig | 1 + arch/riscv/configs/rv32_defconfig | 1 + arch/riscv/include/asm/cpuidle.h | 24 ++++++++++++++++++++++++ arch/riscv/kernel/process.c | 3 ++- 5 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 arch/riscv/include/asm/cpuidle.h diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 5adcbd9b5e88..76976d12b463 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -46,6 +46,7 @@ config RISCV select CLONE_BACKWARDS select CLINT_TIMER if !MMU select COMMON_CLK + select CPU_PM if CPU_IDLE select EDAC_SUPPORT select GENERIC_ARCH_TOPOLOGY if SMP select GENERIC_ATOMIC64 if !64BIT @@ -547,4 +548,10 @@ source "kernel/power/Kconfig" endmenu +menu "CPU Power Management" + +source "drivers/cpuidle/Kconfig" + +endmenu + source "arch/riscv/kvm/Kconfig" diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index f120fcc43d0a..a5e0482a4969 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -20,6 +20,7 @@ CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y CONFIG_SMP=y CONFIG_HOTPLUG_CPU=y +CONFIG_CPU_IDLE=y CONFIG_VIRTUALIZATION=y CONFIG_KVM=m CONFIG_JUMP_LABEL=y diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig index 8b56a7f1eb06..d1b87db54d68 100644 --- a/arch/riscv/configs/rv32_defconfig +++ b/arch/riscv/configs/rv32_defconfig @@ -20,6 +20,7 @@ CONFIG_SOC_VIRT=y CONFIG_ARCH_RV32I=y CONFIG_SMP=y CONFIG_HOTPLUG_CPU=y +CONFIG_CPU_IDLE=y CONFIG_VIRTUALIZATION=y CONFIG_KVM=m CONFIG_JUMP_LABEL=y diff --git a/arch/riscv/include/asm/cpuidle.h b/arch/riscv/include/asm/cpuidle.h new file mode 100644 index 000000000000..71fdc607d4bc --- /dev/null +++ b/arch/riscv/include/asm/cpuidle.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2021 Allwinner Ltd + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + */ + +#ifndef _ASM_RISCV_CPUIDLE_H +#define _ASM_RISCV_CPUIDLE_H + +#include +#include + +static inline void cpu_do_idle(void) +{ + /* + * Add mb() here to ensure that all + * IO/MEM accesses are completed prior + * to entering WFI. + */ + mb(); + wait_for_interrupt(); +} + +#endif diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index 03ac3aa611f5..504b496787aa 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -23,6 +23,7 @@ #include #include #include +#include register unsigned long gp_in_global __asm__("gp"); @@ -37,7 +38,7 @@ extern asmlinkage void ret_from_kernel_thread(void); void arch_cpu_idle(void) { - wait_for_interrupt(); + cpu_do_idle(); raw_local_irq_enable(); } From e1de2c93e78740383db1a0ff626d8ca2cdc28aaf Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Thu, 10 Feb 2022 11:19:41 +0530 Subject: [PATCH 04/95] RISC-V: Rename relocate() and make it global The low-level relocate() function enables mmu and relocates execution to link-time addresses. We rename relocate() function to relocate_enable_mmu() function which is more informative. Also, the relocate_enable_mmu() function will be used in the resume path when a CPU wakes-up from a non-retentive suspend so we make it global symbol. Signed-off-by: Anup Patel Signed-off-by: Anup Patel Reviewed-by: Guo Ren Reviewed-by: Atish Patra Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/head.S | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 2363b43312fc..5f4c6b6c4974 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -90,7 +90,8 @@ pe_head_start: .align 2 #ifdef CONFIG_MMU -relocate: + .global relocate_enable_mmu +relocate_enable_mmu: /* Relocate return address */ la a1, kernel_map XIP_FIXUP_OFFSET a1 @@ -185,7 +186,7 @@ secondary_start_sbi: /* Enable virtual memory and relocate to virtual address */ la a0, swapper_pg_dir XIP_FIXUP_OFFSET a0 - call relocate + call relocate_enable_mmu #endif call setup_trap_vector tail smp_callin @@ -329,7 +330,7 @@ clear_bss_done: #ifdef CONFIG_MMU la a0, early_pg_dir XIP_FIXUP_OFFSET a0 - call relocate + call relocate_enable_mmu #endif /* CONFIG_MMU */ call setup_trap_vector From 63b13e64a829e7b12fba81fccbea0d5448fc0c24 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Thu, 10 Feb 2022 11:19:42 +0530 Subject: [PATCH 05/95] RISC-V: Add arch functions for non-retentive suspend entry/exit The hart registers and CSRs are not preserved in non-retentative suspend state so we provide arch specific helper functions which will save/restore hart context upon entry/exit to non-retentive suspend state. These helper functions can be used by cpuidle drivers for non-retentive suspend entry/exit. Signed-off-by: Anup Patel Signed-off-by: Anup Patel Reviewed-by: Guo Ren Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/asm.h | 27 +++++++ arch/riscv/include/asm/suspend.h | 36 +++++++++ arch/riscv/kernel/Makefile | 2 + arch/riscv/kernel/asm-offsets.c | 3 + arch/riscv/kernel/head.S | 21 ----- arch/riscv/kernel/suspend.c | 87 +++++++++++++++++++++ arch/riscv/kernel/suspend_entry.S | 124 ++++++++++++++++++++++++++++++ 7 files changed, 279 insertions(+), 21 deletions(-) create mode 100644 arch/riscv/include/asm/suspend.h create mode 100644 arch/riscv/kernel/suspend.c create mode 100644 arch/riscv/kernel/suspend_entry.S diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index 618d7c5af1a2..48b4baa4d706 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -67,4 +67,31 @@ #error "Unexpected __SIZEOF_SHORT__" #endif +#ifdef __ASSEMBLY__ + +/* Common assembly source macros */ + +#ifdef CONFIG_XIP_KERNEL +.macro XIP_FIXUP_OFFSET reg + REG_L t0, _xip_fixup + add \reg, \reg, t0 +.endm +.macro XIP_FIXUP_FLASH_OFFSET reg + la t1, __data_loc + li t0, XIP_OFFSET_MASK + and t1, t1, t0 + li t1, XIP_OFFSET + sub t0, t0, t1 + sub \reg, \reg, t0 +.endm +_xip_fixup: .dword CONFIG_PHYS_RAM_BASE - CONFIG_XIP_PHYS_ADDR - XIP_OFFSET +#else +.macro XIP_FIXUP_OFFSET reg +.endm +.macro XIP_FIXUP_FLASH_OFFSET reg +.endm +#endif /* CONFIG_XIP_KERNEL */ + +#endif /* __ASSEMBLY__ */ + #endif /* _ASM_RISCV_ASM_H */ diff --git a/arch/riscv/include/asm/suspend.h b/arch/riscv/include/asm/suspend.h new file mode 100644 index 000000000000..8be391c2aecb --- /dev/null +++ b/arch/riscv/include/asm/suspend.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021 Western Digital Corporation or its affiliates. + * Copyright (c) 2022 Ventana Micro Systems Inc. + */ + +#ifndef _ASM_RISCV_SUSPEND_H +#define _ASM_RISCV_SUSPEND_H + +#include + +struct suspend_context { + /* Saved and restored by low-level functions */ + struct pt_regs regs; + /* Saved and restored by high-level functions */ + unsigned long scratch; + unsigned long tvec; + unsigned long ie; +#ifdef CONFIG_MMU + unsigned long satp; +#endif +}; + +/* Low-level CPU suspend entry function */ +int __cpu_suspend_enter(struct suspend_context *context); + +/* High-level CPU suspend which will save context and call finish() */ +int cpu_suspend(unsigned long arg, + int (*finish)(unsigned long arg, + unsigned long entry, + unsigned long context)); + +/* Low-level CPU resume entry function */ +int __cpu_resume_enter(unsigned long hartid, unsigned long context); + +#endif diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index 612556faa527..13fa5733f5e7 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -48,6 +48,8 @@ obj-$(CONFIG_RISCV_BOOT_SPINWAIT) += cpu_ops_spinwait.o obj-$(CONFIG_MODULES) += module.o obj-$(CONFIG_MODULE_SECTIONS) += module-sections.o +obj-$(CONFIG_CPU_PM) += suspend_entry.o suspend.o + obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o obj-$(CONFIG_DYNAMIC_FTRACE) += mcount-dyn.o diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index df0519a64eaf..df9444397908 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -13,6 +13,7 @@ #include #include #include +#include void asm_offsets(void); @@ -113,6 +114,8 @@ void asm_offsets(void) OFFSET(PT_BADADDR, pt_regs, badaddr); OFFSET(PT_CAUSE, pt_regs, cause); + OFFSET(SUSPEND_CONTEXT_REGS, suspend_context, regs); + OFFSET(KVM_ARCH_GUEST_ZERO, kvm_vcpu_arch, guest_context.zero); OFFSET(KVM_ARCH_GUEST_RA, kvm_vcpu_arch, guest_context.ra); OFFSET(KVM_ARCH_GUEST_SP, kvm_vcpu_arch, guest_context.sp); diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S index 5f4c6b6c4974..893b8bb69391 100644 --- a/arch/riscv/kernel/head.S +++ b/arch/riscv/kernel/head.S @@ -16,27 +16,6 @@ #include #include "efi-header.S" -#ifdef CONFIG_XIP_KERNEL -.macro XIP_FIXUP_OFFSET reg - REG_L t0, _xip_fixup - add \reg, \reg, t0 -.endm -.macro XIP_FIXUP_FLASH_OFFSET reg - la t1, __data_loc - li t0, XIP_OFFSET_MASK - and t1, t1, t0 - li t1, XIP_OFFSET - sub t0, t0, t1 - sub \reg, \reg, t0 -.endm -_xip_fixup: .dword CONFIG_PHYS_RAM_BASE - CONFIG_XIP_PHYS_ADDR - XIP_OFFSET -#else -.macro XIP_FIXUP_OFFSET reg -.endm -.macro XIP_FIXUP_FLASH_OFFSET reg -.endm -#endif /* CONFIG_XIP_KERNEL */ - __HEAD ENTRY(_start) /* diff --git a/arch/riscv/kernel/suspend.c b/arch/riscv/kernel/suspend.c new file mode 100644 index 000000000000..9ba24fb8cc93 --- /dev/null +++ b/arch/riscv/kernel/suspend.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021 Western Digital Corporation or its affiliates. + * Copyright (c) 2022 Ventana Micro Systems Inc. + */ + +#include +#include +#include + +static void suspend_save_csrs(struct suspend_context *context) +{ + context->scratch = csr_read(CSR_SCRATCH); + context->tvec = csr_read(CSR_TVEC); + context->ie = csr_read(CSR_IE); + + /* + * No need to save/restore IP CSR (i.e. MIP or SIP) because: + * + * 1. For no-MMU (M-mode) kernel, the bits in MIP are set by + * external devices (such as interrupt controller, timer, etc). + * 2. For MMU (S-mode) kernel, the bits in SIP are set by + * M-mode firmware and external devices (such as interrupt + * controller, etc). + */ + +#ifdef CONFIG_MMU + context->satp = csr_read(CSR_SATP); +#endif +} + +static void suspend_restore_csrs(struct suspend_context *context) +{ + csr_write(CSR_SCRATCH, context->scratch); + csr_write(CSR_TVEC, context->tvec); + csr_write(CSR_IE, context->ie); + +#ifdef CONFIG_MMU + csr_write(CSR_SATP, context->satp); +#endif +} + +int cpu_suspend(unsigned long arg, + int (*finish)(unsigned long arg, + unsigned long entry, + unsigned long context)) +{ + int rc = 0; + struct suspend_context context = { 0 }; + + /* Finisher should be non-NULL */ + if (!finish) + return -EINVAL; + + /* Save additional CSRs*/ + suspend_save_csrs(&context); + + /* + * Function graph tracer state gets incosistent when the kernel + * calls functions that never return (aka finishers) hence disable + * graph tracing during their execution. + */ + pause_graph_tracing(); + + /* Save context on stack */ + if (__cpu_suspend_enter(&context)) { + /* Call the finisher */ + rc = finish(arg, __pa_symbol(__cpu_resume_enter), + (ulong)&context); + + /* + * Should never reach here, unless the suspend finisher + * fails. Successful cpu_suspend() should return from + * __cpu_resume_entry() + */ + if (!rc) + rc = -EOPNOTSUPP; + } + + /* Enable function graph tracer */ + unpause_graph_tracing(); + + /* Restore additional CSRs */ + suspend_restore_csrs(&context); + + return rc; +} diff --git a/arch/riscv/kernel/suspend_entry.S b/arch/riscv/kernel/suspend_entry.S new file mode 100644 index 000000000000..4b07b809a2b8 --- /dev/null +++ b/arch/riscv/kernel/suspend_entry.S @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021 Western Digital Corporation or its affiliates. + * Copyright (c) 2022 Ventana Micro Systems Inc. + */ + +#include +#include +#include +#include + + .text + .altmacro + .option norelax + +ENTRY(__cpu_suspend_enter) + /* Save registers (except A0 and T0-T6) */ + REG_S ra, (SUSPEND_CONTEXT_REGS + PT_RA)(a0) + REG_S sp, (SUSPEND_CONTEXT_REGS + PT_SP)(a0) + REG_S gp, (SUSPEND_CONTEXT_REGS + PT_GP)(a0) + REG_S tp, (SUSPEND_CONTEXT_REGS + PT_TP)(a0) + REG_S s0, (SUSPEND_CONTEXT_REGS + PT_S0)(a0) + REG_S s1, (SUSPEND_CONTEXT_REGS + PT_S1)(a0) + REG_S a1, (SUSPEND_CONTEXT_REGS + PT_A1)(a0) + REG_S a2, (SUSPEND_CONTEXT_REGS + PT_A2)(a0) + REG_S a3, (SUSPEND_CONTEXT_REGS + PT_A3)(a0) + REG_S a4, (SUSPEND_CONTEXT_REGS + PT_A4)(a0) + REG_S a5, (SUSPEND_CONTEXT_REGS + PT_A5)(a0) + REG_S a6, (SUSPEND_CONTEXT_REGS + PT_A6)(a0) + REG_S a7, (SUSPEND_CONTEXT_REGS + PT_A7)(a0) + REG_S s2, (SUSPEND_CONTEXT_REGS + PT_S2)(a0) + REG_S s3, (SUSPEND_CONTEXT_REGS + PT_S3)(a0) + REG_S s4, (SUSPEND_CONTEXT_REGS + PT_S4)(a0) + REG_S s5, (SUSPEND_CONTEXT_REGS + PT_S5)(a0) + REG_S s6, (SUSPEND_CONTEXT_REGS + PT_S6)(a0) + REG_S s7, (SUSPEND_CONTEXT_REGS + PT_S7)(a0) + REG_S s8, (SUSPEND_CONTEXT_REGS + PT_S8)(a0) + REG_S s9, (SUSPEND_CONTEXT_REGS + PT_S9)(a0) + REG_S s10, (SUSPEND_CONTEXT_REGS + PT_S10)(a0) + REG_S s11, (SUSPEND_CONTEXT_REGS + PT_S11)(a0) + + /* Save CSRs */ + csrr t0, CSR_EPC + REG_S t0, (SUSPEND_CONTEXT_REGS + PT_EPC)(a0) + csrr t0, CSR_STATUS + REG_S t0, (SUSPEND_CONTEXT_REGS + PT_STATUS)(a0) + csrr t0, CSR_TVAL + REG_S t0, (SUSPEND_CONTEXT_REGS + PT_BADADDR)(a0) + csrr t0, CSR_CAUSE + REG_S t0, (SUSPEND_CONTEXT_REGS + PT_CAUSE)(a0) + + /* Return non-zero value */ + li a0, 1 + + /* Return to C code */ + ret +END(__cpu_suspend_enter) + +ENTRY(__cpu_resume_enter) + /* Load the global pointer */ + .option push + .option norelax + la gp, __global_pointer$ + .option pop + +#ifdef CONFIG_MMU + /* Save A0 and A1 */ + add t0, a0, zero + add t1, a1, zero + + /* Enable MMU */ + la a0, swapper_pg_dir + XIP_FIXUP_OFFSET a0 + call relocate_enable_mmu + + /* Restore A0 and A1 */ + add a0, t0, zero + add a1, t1, zero +#endif + + /* Make A0 point to suspend context */ + add a0, a1, zero + + /* Restore CSRs */ + REG_L t0, (SUSPEND_CONTEXT_REGS + PT_EPC)(a0) + csrw CSR_EPC, t0 + REG_L t0, (SUSPEND_CONTEXT_REGS + PT_STATUS)(a0) + csrw CSR_STATUS, t0 + REG_L t0, (SUSPEND_CONTEXT_REGS + PT_BADADDR)(a0) + csrw CSR_TVAL, t0 + REG_L t0, (SUSPEND_CONTEXT_REGS + PT_CAUSE)(a0) + csrw CSR_CAUSE, t0 + + /* Restore registers (except A0 and T0-T6) */ + REG_L ra, (SUSPEND_CONTEXT_REGS + PT_RA)(a0) + REG_L sp, (SUSPEND_CONTEXT_REGS + PT_SP)(a0) + REG_L gp, (SUSPEND_CONTEXT_REGS + PT_GP)(a0) + REG_L tp, (SUSPEND_CONTEXT_REGS + PT_TP)(a0) + REG_L s0, (SUSPEND_CONTEXT_REGS + PT_S0)(a0) + REG_L s1, (SUSPEND_CONTEXT_REGS + PT_S1)(a0) + REG_L a1, (SUSPEND_CONTEXT_REGS + PT_A1)(a0) + REG_L a2, (SUSPEND_CONTEXT_REGS + PT_A2)(a0) + REG_L a3, (SUSPEND_CONTEXT_REGS + PT_A3)(a0) + REG_L a4, (SUSPEND_CONTEXT_REGS + PT_A4)(a0) + REG_L a5, (SUSPEND_CONTEXT_REGS + PT_A5)(a0) + REG_L a6, (SUSPEND_CONTEXT_REGS + PT_A6)(a0) + REG_L a7, (SUSPEND_CONTEXT_REGS + PT_A7)(a0) + REG_L s2, (SUSPEND_CONTEXT_REGS + PT_S2)(a0) + REG_L s3, (SUSPEND_CONTEXT_REGS + PT_S3)(a0) + REG_L s4, (SUSPEND_CONTEXT_REGS + PT_S4)(a0) + REG_L s5, (SUSPEND_CONTEXT_REGS + PT_S5)(a0) + REG_L s6, (SUSPEND_CONTEXT_REGS + PT_S6)(a0) + REG_L s7, (SUSPEND_CONTEXT_REGS + PT_S7)(a0) + REG_L s8, (SUSPEND_CONTEXT_REGS + PT_S8)(a0) + REG_L s9, (SUSPEND_CONTEXT_REGS + PT_S9)(a0) + REG_L s10, (SUSPEND_CONTEXT_REGS + PT_S10)(a0) + REG_L s11, (SUSPEND_CONTEXT_REGS + PT_S11)(a0) + + /* Return zero value */ + add a0, zero, zero + + /* Return to C code */ + ret +END(__cpu_resume_enter) From b8200905e48891b4a3134b47f35c730347135eed Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Thu, 10 Feb 2022 11:19:43 +0530 Subject: [PATCH 06/95] RISC-V: Add SBI HSM suspend related defines We add defines related to SBI HSM suspend call and also update HSM states naming as-per latest SBI specification. Signed-off-by: Anup Patel Signed-off-by: Anup Patel Reviewed-by: Guo Ren Reviewed-by: Atish Patra Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/sbi.h | 27 ++++++++++++++++++++++----- arch/riscv/kernel/cpu_ops_sbi.c | 2 +- arch/riscv/kvm/vcpu_sbi_hsm.c | 4 ++-- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index d1c37479d828..06133b4f8e20 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -71,15 +71,32 @@ enum sbi_ext_hsm_fid { SBI_EXT_HSM_HART_START = 0, SBI_EXT_HSM_HART_STOP, SBI_EXT_HSM_HART_STATUS, + SBI_EXT_HSM_HART_SUSPEND, }; -enum sbi_hsm_hart_status { - SBI_HSM_HART_STATUS_STARTED = 0, - SBI_HSM_HART_STATUS_STOPPED, - SBI_HSM_HART_STATUS_START_PENDING, - SBI_HSM_HART_STATUS_STOP_PENDING, +enum sbi_hsm_hart_state { + SBI_HSM_STATE_STARTED = 0, + SBI_HSM_STATE_STOPPED, + SBI_HSM_STATE_START_PENDING, + SBI_HSM_STATE_STOP_PENDING, + SBI_HSM_STATE_SUSPENDED, + SBI_HSM_STATE_SUSPEND_PENDING, + SBI_HSM_STATE_RESUME_PENDING, }; +#define SBI_HSM_SUSP_BASE_MASK 0x7fffffff +#define SBI_HSM_SUSP_NON_RET_BIT 0x80000000 +#define SBI_HSM_SUSP_PLAT_BASE 0x10000000 + +#define SBI_HSM_SUSPEND_RET_DEFAULT 0x00000000 +#define SBI_HSM_SUSPEND_RET_PLATFORM SBI_HSM_SUSP_PLAT_BASE +#define SBI_HSM_SUSPEND_RET_LAST SBI_HSM_SUSP_BASE_MASK +#define SBI_HSM_SUSPEND_NON_RET_DEFAULT SBI_HSM_SUSP_NON_RET_BIT +#define SBI_HSM_SUSPEND_NON_RET_PLATFORM (SBI_HSM_SUSP_NON_RET_BIT | \ + SBI_HSM_SUSP_PLAT_BASE) +#define SBI_HSM_SUSPEND_NON_RET_LAST (SBI_HSM_SUSP_NON_RET_BIT | \ + SBI_HSM_SUSP_BASE_MASK) + enum sbi_ext_srst_fid { SBI_EXT_SRST_RESET = 0, }; diff --git a/arch/riscv/kernel/cpu_ops_sbi.c b/arch/riscv/kernel/cpu_ops_sbi.c index dae29cbfe550..2e16f6732cdf 100644 --- a/arch/riscv/kernel/cpu_ops_sbi.c +++ b/arch/riscv/kernel/cpu_ops_sbi.c @@ -111,7 +111,7 @@ static int sbi_cpu_is_stopped(unsigned int cpuid) rc = sbi_hsm_hart_get_status(hartid); - if (rc == SBI_HSM_HART_STATUS_STOPPED) + if (rc == SBI_HSM_STATE_STOPPED) return 0; return rc; } diff --git a/arch/riscv/kvm/vcpu_sbi_hsm.c b/arch/riscv/kvm/vcpu_sbi_hsm.c index 2e383687fa48..1ac4b2e8e4ec 100644 --- a/arch/riscv/kvm/vcpu_sbi_hsm.c +++ b/arch/riscv/kvm/vcpu_sbi_hsm.c @@ -60,9 +60,9 @@ static int kvm_sbi_hsm_vcpu_get_status(struct kvm_vcpu *vcpu) if (!target_vcpu) return -EINVAL; if (!target_vcpu->arch.power_off) - return SBI_HSM_HART_STATUS_STARTED; + return SBI_HSM_STATE_STARTED; else - return SBI_HSM_HART_STATUS_STOPPED; + return SBI_HSM_STATE_STOPPED; } static int kvm_sbi_ext_hsm_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, From 9d976d6721dfb525b81ce981e1363c70c0975aab Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Thu, 10 Feb 2022 11:19:44 +0530 Subject: [PATCH 07/95] cpuidle: Factor-out power domain related code from PSCI domain driver The generic power domain related code in PSCI domain driver is largely independent of PSCI and can be shared with RISC-V SBI domain driver hence we factor-out this code into dt_idle_genpd.c and dt_idle_genpd.h. Signed-off-by: Anup Patel Signed-off-by: Anup Patel Reviewed-by: Ulf Hansson Signed-off-by: Palmer Dabbelt --- MAINTAINERS | 7 + drivers/cpuidle/Kconfig | 4 + drivers/cpuidle/Kconfig.arm | 1 + drivers/cpuidle/Makefile | 1 + drivers/cpuidle/cpuidle-psci-domain.c | 138 +------------------- drivers/cpuidle/cpuidle-psci.h | 15 ++- drivers/cpuidle/dt_idle_genpd.c | 178 ++++++++++++++++++++++++++ drivers/cpuidle/dt_idle_genpd.h | 50 ++++++++ 8 files changed, 259 insertions(+), 135 deletions(-) create mode 100644 drivers/cpuidle/dt_idle_genpd.c create mode 100644 drivers/cpuidle/dt_idle_genpd.h diff --git a/MAINTAINERS b/MAINTAINERS index ea3e6c914384..0c67ebe7f07e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5051,6 +5051,13 @@ S: Supported F: drivers/cpuidle/cpuidle-psci.h F: drivers/cpuidle/cpuidle-psci-domain.c +CPUIDLE DRIVER - DT IDLE PM DOMAIN +M: Ulf Hansson +L: linux-pm@vger.kernel.org +S: Supported +F: drivers/cpuidle/dt_idle_genpd.c +F: drivers/cpuidle/dt_idle_genpd.h + CRAMFS FILESYSTEM M: Nicolas Pitre S: Maintained diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig index c0aeedd66f02..f1afe7ab6b54 100644 --- a/drivers/cpuidle/Kconfig +++ b/drivers/cpuidle/Kconfig @@ -47,6 +47,10 @@ config CPU_IDLE_GOV_HALTPOLL config DT_IDLE_STATES bool +config DT_IDLE_GENPD + depends on PM_GENERIC_DOMAINS_OF + bool + menu "ARM CPU Idle Drivers" depends on ARM || ARM64 source "drivers/cpuidle/Kconfig.arm" diff --git a/drivers/cpuidle/Kconfig.arm b/drivers/cpuidle/Kconfig.arm index 15d6c46c0a47..be7f512109f7 100644 --- a/drivers/cpuidle/Kconfig.arm +++ b/drivers/cpuidle/Kconfig.arm @@ -27,6 +27,7 @@ config ARM_PSCI_CPUIDLE_DOMAIN bool "PSCI CPU idle Domain" depends on ARM_PSCI_CPUIDLE depends on PM_GENERIC_DOMAINS_OF + select DT_IDLE_GENPD default y help Select this to enable the PSCI based CPUidle driver to use PM domains, diff --git a/drivers/cpuidle/Makefile b/drivers/cpuidle/Makefile index 26bbc5e74123..11a26cef279f 100644 --- a/drivers/cpuidle/Makefile +++ b/drivers/cpuidle/Makefile @@ -6,6 +6,7 @@ obj-y += cpuidle.o driver.o governor.o sysfs.o governors/ obj-$(CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED) += coupled.o obj-$(CONFIG_DT_IDLE_STATES) += dt_idle_states.o +obj-$(CONFIG_DT_IDLE_GENPD) += dt_idle_genpd.o obj-$(CONFIG_ARCH_HAS_CPU_RELAX) += poll_state.o obj-$(CONFIG_HALTPOLL_CPUIDLE) += cpuidle-haltpoll.o diff --git a/drivers/cpuidle/cpuidle-psci-domain.c b/drivers/cpuidle/cpuidle-psci-domain.c index ff2c3f8e4668..755bbdfc5b82 100644 --- a/drivers/cpuidle/cpuidle-psci-domain.c +++ b/drivers/cpuidle/cpuidle-psci-domain.c @@ -47,73 +47,14 @@ static int psci_pd_power_off(struct generic_pm_domain *pd) return 0; } -static int psci_pd_parse_state_nodes(struct genpd_power_state *states, - int state_count) -{ - int i, ret; - u32 psci_state, *psci_state_buf; - - for (i = 0; i < state_count; i++) { - ret = psci_dt_parse_state_node(to_of_node(states[i].fwnode), - &psci_state); - if (ret) - goto free_state; - - psci_state_buf = kmalloc(sizeof(u32), GFP_KERNEL); - if (!psci_state_buf) { - ret = -ENOMEM; - goto free_state; - } - *psci_state_buf = psci_state; - states[i].data = psci_state_buf; - } - - return 0; - -free_state: - i--; - for (; i >= 0; i--) - kfree(states[i].data); - return ret; -} - -static int psci_pd_parse_states(struct device_node *np, - struct genpd_power_state **states, int *state_count) -{ - int ret; - - /* Parse the domain idle states. */ - ret = of_genpd_parse_idle_states(np, states, state_count); - if (ret) - return ret; - - /* Fill out the PSCI specifics for each found state. */ - ret = psci_pd_parse_state_nodes(*states, *state_count); - if (ret) - kfree(*states); - - return ret; -} - -static void psci_pd_free_states(struct genpd_power_state *states, - unsigned int state_count) -{ - int i; - - for (i = 0; i < state_count; i++) - kfree(states[i].data); - kfree(states); -} - static int psci_pd_init(struct device_node *np, bool use_osi) { struct generic_pm_domain *pd; struct psci_pd_provider *pd_provider; struct dev_power_governor *pd_gov; - struct genpd_power_state *states = NULL; int ret = -ENOMEM, state_count = 0; - pd = kzalloc(sizeof(*pd), GFP_KERNEL); + pd = dt_idle_pd_alloc(np, psci_dt_parse_state_node); if (!pd) goto out; @@ -121,22 +62,6 @@ static int psci_pd_init(struct device_node *np, bool use_osi) if (!pd_provider) goto free_pd; - pd->name = kasprintf(GFP_KERNEL, "%pOF", np); - if (!pd->name) - goto free_pd_prov; - - /* - * Parse the domain idle states and let genpd manage the state selection - * for those being compatible with "domain-idle-state". - */ - ret = psci_pd_parse_states(np, &states, &state_count); - if (ret) - goto free_name; - - pd->free_states = psci_pd_free_states; - pd->name = kbasename(pd->name); - pd->states = states; - pd->state_count = state_count; pd->flags |= GENPD_FLAG_IRQ_SAFE | GENPD_FLAG_CPU_DOMAIN; /* Allow power off when OSI has been successfully enabled. */ @@ -149,10 +74,8 @@ static int psci_pd_init(struct device_node *np, bool use_osi) pd_gov = state_count > 0 ? &pm_domain_cpu_gov : NULL; ret = pm_genpd_init(pd, pd_gov, false); - if (ret) { - psci_pd_free_states(states, state_count); - goto free_name; - } + if (ret) + goto free_pd_prov; ret = of_genpd_add_provider_simple(np, pd); if (ret) @@ -166,12 +89,10 @@ static int psci_pd_init(struct device_node *np, bool use_osi) remove_pd: pm_genpd_remove(pd); -free_name: - kfree(pd->name); free_pd_prov: kfree(pd_provider); free_pd: - kfree(pd); + dt_idle_pd_free(pd); out: pr_err("failed to init PM domain ret=%d %pOF\n", ret, np); return ret; @@ -195,30 +116,6 @@ static void psci_pd_remove(void) } } -static int psci_pd_init_topology(struct device_node *np) -{ - struct device_node *node; - struct of_phandle_args child, parent; - int ret; - - for_each_child_of_node(np, node) { - if (of_parse_phandle_with_args(node, "power-domains", - "#power-domain-cells", 0, &parent)) - continue; - - child.np = node; - child.args_count = 0; - ret = of_genpd_add_subdomain(&parent, &child); - of_node_put(parent.np); - if (ret) { - of_node_put(node); - return ret; - } - } - - return 0; -} - static bool psci_pd_try_set_osi_mode(void) { int ret; @@ -282,7 +179,7 @@ static int psci_cpuidle_domain_probe(struct platform_device *pdev) goto no_pd; /* Link genpd masters/subdomains to model the CPU topology. */ - ret = psci_pd_init_topology(np); + ret = dt_idle_pd_init_topology(np); if (ret) goto remove_pd; @@ -314,28 +211,3 @@ static int __init psci_idle_init_domains(void) return platform_driver_register(&psci_cpuidle_domain_driver); } subsys_initcall(psci_idle_init_domains); - -struct device *psci_dt_attach_cpu(int cpu) -{ - struct device *dev; - - dev = dev_pm_domain_attach_by_name(get_cpu_device(cpu), "psci"); - if (IS_ERR_OR_NULL(dev)) - return dev; - - pm_runtime_irq_safe(dev); - if (cpu_online(cpu)) - pm_runtime_get_sync(dev); - - dev_pm_syscore_device(dev, true); - - return dev; -} - -void psci_dt_detach_cpu(struct device *dev) -{ - if (IS_ERR_OR_NULL(dev)) - return; - - dev_pm_domain_detach(dev, false); -} diff --git a/drivers/cpuidle/cpuidle-psci.h b/drivers/cpuidle/cpuidle-psci.h index d8e925e84c27..4e132640ed64 100644 --- a/drivers/cpuidle/cpuidle-psci.h +++ b/drivers/cpuidle/cpuidle-psci.h @@ -10,8 +10,19 @@ void psci_set_domain_state(u32 state); int psci_dt_parse_state_node(struct device_node *np, u32 *state); #ifdef CONFIG_ARM_PSCI_CPUIDLE_DOMAIN -struct device *psci_dt_attach_cpu(int cpu); -void psci_dt_detach_cpu(struct device *dev); + +#include "dt_idle_genpd.h" + +static inline struct device *psci_dt_attach_cpu(int cpu) +{ + return dt_idle_attach_cpu(cpu, "psci"); +} + +static inline void psci_dt_detach_cpu(struct device *dev) +{ + dt_idle_detach_cpu(dev); +} + #else static inline struct device *psci_dt_attach_cpu(int cpu) { return NULL; } static inline void psci_dt_detach_cpu(struct device *dev) { } diff --git a/drivers/cpuidle/dt_idle_genpd.c b/drivers/cpuidle/dt_idle_genpd.c new file mode 100644 index 000000000000..b37165514d4e --- /dev/null +++ b/drivers/cpuidle/dt_idle_genpd.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PM domains for CPUs via genpd. + * + * Copyright (C) 2019 Linaro Ltd. + * Author: Ulf Hansson + * + * Copyright (c) 2021 Western Digital Corporation or its affiliates. + * Copyright (c) 2022 Ventana Micro Systems Inc. + */ + +#define pr_fmt(fmt) "dt-idle-genpd: " fmt + +#include +#include +#include +#include +#include +#include +#include + +#include "dt_idle_genpd.h" + +static int pd_parse_state_nodes( + int (*parse_state)(struct device_node *, u32 *), + struct genpd_power_state *states, int state_count) +{ + int i, ret; + u32 state, *state_buf; + + for (i = 0; i < state_count; i++) { + ret = parse_state(to_of_node(states[i].fwnode), &state); + if (ret) + goto free_state; + + state_buf = kmalloc(sizeof(u32), GFP_KERNEL); + if (!state_buf) { + ret = -ENOMEM; + goto free_state; + } + *state_buf = state; + states[i].data = state_buf; + } + + return 0; + +free_state: + i--; + for (; i >= 0; i--) + kfree(states[i].data); + return ret; +} + +static int pd_parse_states(struct device_node *np, + int (*parse_state)(struct device_node *, u32 *), + struct genpd_power_state **states, + int *state_count) +{ + int ret; + + /* Parse the domain idle states. */ + ret = of_genpd_parse_idle_states(np, states, state_count); + if (ret) + return ret; + + /* Fill out the dt specifics for each found state. */ + ret = pd_parse_state_nodes(parse_state, *states, *state_count); + if (ret) + kfree(*states); + + return ret; +} + +static void pd_free_states(struct genpd_power_state *states, + unsigned int state_count) +{ + int i; + + for (i = 0; i < state_count; i++) + kfree(states[i].data); + kfree(states); +} + +void dt_idle_pd_free(struct generic_pm_domain *pd) +{ + pd_free_states(pd->states, pd->state_count); + kfree(pd->name); + kfree(pd); +} + +struct generic_pm_domain *dt_idle_pd_alloc(struct device_node *np, + int (*parse_state)(struct device_node *, u32 *)) +{ + struct generic_pm_domain *pd; + struct genpd_power_state *states = NULL; + int ret, state_count = 0; + + pd = kzalloc(sizeof(*pd), GFP_KERNEL); + if (!pd) + goto out; + + pd->name = kasprintf(GFP_KERNEL, "%pOF", np); + if (!pd->name) + goto free_pd; + + /* + * Parse the domain idle states and let genpd manage the state selection + * for those being compatible with "domain-idle-state". + */ + ret = pd_parse_states(np, parse_state, &states, &state_count); + if (ret) + goto free_name; + + pd->free_states = pd_free_states; + pd->name = kbasename(pd->name); + pd->states = states; + pd->state_count = state_count; + + pr_debug("alloc PM domain %s\n", pd->name); + return pd; + +free_name: + kfree(pd->name); +free_pd: + kfree(pd); +out: + pr_err("failed to alloc PM domain %pOF\n", np); + return NULL; +} + +int dt_idle_pd_init_topology(struct device_node *np) +{ + struct device_node *node; + struct of_phandle_args child, parent; + int ret; + + for_each_child_of_node(np, node) { + if (of_parse_phandle_with_args(node, "power-domains", + "#power-domain-cells", 0, &parent)) + continue; + + child.np = node; + child.args_count = 0; + ret = of_genpd_add_subdomain(&parent, &child); + of_node_put(parent.np); + if (ret) { + of_node_put(node); + return ret; + } + } + + return 0; +} + +struct device *dt_idle_attach_cpu(int cpu, const char *name) +{ + struct device *dev; + + dev = dev_pm_domain_attach_by_name(get_cpu_device(cpu), name); + if (IS_ERR_OR_NULL(dev)) + return dev; + + pm_runtime_irq_safe(dev); + if (cpu_online(cpu)) + pm_runtime_get_sync(dev); + + dev_pm_syscore_device(dev, true); + + return dev; +} + +void dt_idle_detach_cpu(struct device *dev) +{ + if (IS_ERR_OR_NULL(dev)) + return; + + dev_pm_domain_detach(dev, false); +} diff --git a/drivers/cpuidle/dt_idle_genpd.h b/drivers/cpuidle/dt_idle_genpd.h new file mode 100644 index 000000000000..a95483d08a02 --- /dev/null +++ b/drivers/cpuidle/dt_idle_genpd.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __DT_IDLE_GENPD +#define __DT_IDLE_GENPD + +struct device_node; +struct generic_pm_domain; + +#ifdef CONFIG_DT_IDLE_GENPD + +void dt_idle_pd_free(struct generic_pm_domain *pd); + +struct generic_pm_domain *dt_idle_pd_alloc(struct device_node *np, + int (*parse_state)(struct device_node *, u32 *)); + +int dt_idle_pd_init_topology(struct device_node *np); + +struct device *dt_idle_attach_cpu(int cpu, const char *name); + +void dt_idle_detach_cpu(struct device *dev); + +#else + +static inline void dt_idle_pd_free(struct generic_pm_domain *pd) +{ +} + +static inline struct generic_pm_domain *dt_idle_pd_alloc( + struct device_node *np, + int (*parse_state)(struct device_node *, u32 *)) +{ + return NULL; +} + +static inline int dt_idle_pd_init_topology(struct device_node *np) +{ + return 0; +} + +static inline struct device *dt_idle_attach_cpu(int cpu, const char *name) +{ + return NULL; +} + +static inline void dt_idle_detach_cpu(struct device *dev) +{ +} + +#endif + +#endif From 6abf32f1d9c5009dcccded2c1e7ca899a4ab587b Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Thu, 10 Feb 2022 11:19:45 +0530 Subject: [PATCH 08/95] cpuidle: Add RISC-V SBI CPU idle driver The RISC-V SBI HSM extension provides HSM suspend call which can be used by Linux RISC-V to enter platform specific low-power state. This patch adds a CPU idle driver based on RISC-V SBI calls which will populate idle states from device tree and use SBI calls to entry these idle states. Signed-off-by: Anup Patel Signed-off-by: Anup Patel Acked-by: Atish Patra Signed-off-by: Palmer Dabbelt --- MAINTAINERS | 7 + drivers/cpuidle/Kconfig | 5 + drivers/cpuidle/Kconfig.riscv | 15 + drivers/cpuidle/Makefile | 4 + drivers/cpuidle/cpuidle-riscv-sbi.c | 627 ++++++++++++++++++++++++++++ 5 files changed, 658 insertions(+) create mode 100644 drivers/cpuidle/Kconfig.riscv create mode 100644 drivers/cpuidle/cpuidle-riscv-sbi.c diff --git a/MAINTAINERS b/MAINTAINERS index 0c67ebe7f07e..15c63a717458 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5058,6 +5058,13 @@ S: Supported F: drivers/cpuidle/dt_idle_genpd.c F: drivers/cpuidle/dt_idle_genpd.h +CPUIDLE DRIVER - RISC-V SBI +M: Anup Patel +L: linux-pm@vger.kernel.org +L: linux-riscv@lists.infradead.org +S: Maintained +F: drivers/cpuidle/cpuidle-riscv-sbi.c + CRAMFS FILESYSTEM M: Nicolas Pitre S: Maintained diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig index f1afe7ab6b54..ff71dd662880 100644 --- a/drivers/cpuidle/Kconfig +++ b/drivers/cpuidle/Kconfig @@ -66,6 +66,11 @@ depends on PPC source "drivers/cpuidle/Kconfig.powerpc" endmenu +menu "RISC-V CPU Idle Drivers" +depends on RISCV +source "drivers/cpuidle/Kconfig.riscv" +endmenu + config HALTPOLL_CPUIDLE tristate "Halt poll cpuidle driver" depends on X86 && KVM_GUEST diff --git a/drivers/cpuidle/Kconfig.riscv b/drivers/cpuidle/Kconfig.riscv new file mode 100644 index 000000000000..78518c26af74 --- /dev/null +++ b/drivers/cpuidle/Kconfig.riscv @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# RISC-V CPU Idle drivers +# + +config RISCV_SBI_CPUIDLE + bool "RISC-V SBI CPU idle Driver" + depends on RISCV_SBI + select DT_IDLE_STATES + select CPU_IDLE_MULTIPLE_DRIVERS + select DT_IDLE_GENPD if PM_GENERIC_DOMAINS_OF + help + Select this option to enable RISC-V SBI firmware based CPU idle + driver for RISC-V systems. This drivers also supports hierarchical + DT based layout of the idle state. diff --git a/drivers/cpuidle/Makefile b/drivers/cpuidle/Makefile index 11a26cef279f..d103342b7cfc 100644 --- a/drivers/cpuidle/Makefile +++ b/drivers/cpuidle/Makefile @@ -35,3 +35,7 @@ obj-$(CONFIG_MIPS_CPS_CPUIDLE) += cpuidle-cps.o # POWERPC drivers obj-$(CONFIG_PSERIES_CPUIDLE) += cpuidle-pseries.o obj-$(CONFIG_POWERNV_CPUIDLE) += cpuidle-powernv.o + +############################################################################### +# RISC-V drivers +obj-$(CONFIG_RISCV_SBI_CPUIDLE) += cpuidle-riscv-sbi.o diff --git a/drivers/cpuidle/cpuidle-riscv-sbi.c b/drivers/cpuidle/cpuidle-riscv-sbi.c new file mode 100644 index 000000000000..b459eda2cd37 --- /dev/null +++ b/drivers/cpuidle/cpuidle-riscv-sbi.c @@ -0,0 +1,627 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * RISC-V SBI CPU idle driver. + * + * Copyright (c) 2021 Western Digital Corporation or its affiliates. + * Copyright (c) 2022 Ventana Micro Systems Inc. + */ + +#define pr_fmt(fmt) "cpuidle-riscv-sbi: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dt_idle_states.h" +#include "dt_idle_genpd.h" + +struct sbi_cpuidle_data { + u32 *states; + struct device *dev; +}; + +struct sbi_domain_state { + bool available; + u32 state; +}; + +static DEFINE_PER_CPU_READ_MOSTLY(struct sbi_cpuidle_data, sbi_cpuidle_data); +static DEFINE_PER_CPU(struct sbi_domain_state, domain_state); +static bool sbi_cpuidle_use_osi; +static bool sbi_cpuidle_use_cpuhp; +static bool sbi_cpuidle_pd_allow_domain_state; + +static inline void sbi_set_domain_state(u32 state) +{ + struct sbi_domain_state *data = this_cpu_ptr(&domain_state); + + data->available = true; + data->state = state; +} + +static inline u32 sbi_get_domain_state(void) +{ + struct sbi_domain_state *data = this_cpu_ptr(&domain_state); + + return data->state; +} + +static inline void sbi_clear_domain_state(void) +{ + struct sbi_domain_state *data = this_cpu_ptr(&domain_state); + + data->available = false; +} + +static inline bool sbi_is_domain_state_available(void) +{ + struct sbi_domain_state *data = this_cpu_ptr(&domain_state); + + return data->available; +} + +static int sbi_suspend_finisher(unsigned long suspend_type, + unsigned long resume_addr, + unsigned long opaque) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_HSM, SBI_EXT_HSM_HART_SUSPEND, + suspend_type, resume_addr, opaque, 0, 0, 0); + + return (ret.error) ? sbi_err_map_linux_errno(ret.error) : 0; +} + +static int sbi_suspend(u32 state) +{ + if (state & SBI_HSM_SUSP_NON_RET_BIT) + return cpu_suspend(state, sbi_suspend_finisher); + else + return sbi_suspend_finisher(state, 0, 0); +} + +static int sbi_cpuidle_enter_state(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int idx) +{ + u32 *states = __this_cpu_read(sbi_cpuidle_data.states); + + return CPU_PM_CPU_IDLE_ENTER_PARAM(sbi_suspend, idx, states[idx]); +} + +static int __sbi_enter_domain_idle_state(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int idx, + bool s2idle) +{ + struct sbi_cpuidle_data *data = this_cpu_ptr(&sbi_cpuidle_data); + u32 *states = data->states; + struct device *pd_dev = data->dev; + u32 state; + int ret; + + ret = cpu_pm_enter(); + if (ret) + return -1; + + /* Do runtime PM to manage a hierarchical CPU toplogy. */ + rcu_irq_enter_irqson(); + if (s2idle) + dev_pm_genpd_suspend(pd_dev); + else + pm_runtime_put_sync_suspend(pd_dev); + rcu_irq_exit_irqson(); + + if (sbi_is_domain_state_available()) + state = sbi_get_domain_state(); + else + state = states[idx]; + + ret = sbi_suspend(state) ? -1 : idx; + + rcu_irq_enter_irqson(); + if (s2idle) + dev_pm_genpd_resume(pd_dev); + else + pm_runtime_get_sync(pd_dev); + rcu_irq_exit_irqson(); + + cpu_pm_exit(); + + /* Clear the domain state to start fresh when back from idle. */ + sbi_clear_domain_state(); + return ret; +} + +static int sbi_enter_domain_idle_state(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int idx) +{ + return __sbi_enter_domain_idle_state(dev, drv, idx, false); +} + +static int sbi_enter_s2idle_domain_idle_state(struct cpuidle_device *dev, + struct cpuidle_driver *drv, + int idx) +{ + return __sbi_enter_domain_idle_state(dev, drv, idx, true); +} + +static int sbi_cpuidle_cpuhp_up(unsigned int cpu) +{ + struct device *pd_dev = __this_cpu_read(sbi_cpuidle_data.dev); + + if (pd_dev) + pm_runtime_get_sync(pd_dev); + + return 0; +} + +static int sbi_cpuidle_cpuhp_down(unsigned int cpu) +{ + struct device *pd_dev = __this_cpu_read(sbi_cpuidle_data.dev); + + if (pd_dev) { + pm_runtime_put_sync(pd_dev); + /* Clear domain state to start fresh at next online. */ + sbi_clear_domain_state(); + } + + return 0; +} + +static void sbi_idle_init_cpuhp(void) +{ + int err; + + if (!sbi_cpuidle_use_cpuhp) + return; + + err = cpuhp_setup_state_nocalls(CPUHP_AP_CPU_PM_STARTING, + "cpuidle/sbi:online", + sbi_cpuidle_cpuhp_up, + sbi_cpuidle_cpuhp_down); + if (err) + pr_warn("Failed %d while setup cpuhp state\n", err); +} + +static const struct of_device_id sbi_cpuidle_state_match[] = { + { .compatible = "riscv,idle-state", + .data = sbi_cpuidle_enter_state }, + { }, +}; + +static bool sbi_suspend_state_is_valid(u32 state) +{ + if (state > SBI_HSM_SUSPEND_RET_DEFAULT && + state < SBI_HSM_SUSPEND_RET_PLATFORM) + return false; + if (state > SBI_HSM_SUSPEND_NON_RET_DEFAULT && + state < SBI_HSM_SUSPEND_NON_RET_PLATFORM) + return false; + return true; +} + +static int sbi_dt_parse_state_node(struct device_node *np, u32 *state) +{ + int err = of_property_read_u32(np, "riscv,sbi-suspend-param", state); + + if (err) { + pr_warn("%pOF missing riscv,sbi-suspend-param property\n", np); + return err; + } + + if (!sbi_suspend_state_is_valid(*state)) { + pr_warn("Invalid SBI suspend state %#x\n", *state); + return -EINVAL; + } + + return 0; +} + +static int sbi_dt_cpu_init_topology(struct cpuidle_driver *drv, + struct sbi_cpuidle_data *data, + unsigned int state_count, int cpu) +{ + /* Currently limit the hierarchical topology to be used in OSI mode. */ + if (!sbi_cpuidle_use_osi) + return 0; + + data->dev = dt_idle_attach_cpu(cpu, "sbi"); + if (IS_ERR_OR_NULL(data->dev)) + return PTR_ERR_OR_ZERO(data->dev); + + /* + * Using the deepest state for the CPU to trigger a potential selection + * of a shared state for the domain, assumes the domain states are all + * deeper states. + */ + drv->states[state_count - 1].enter = sbi_enter_domain_idle_state; + drv->states[state_count - 1].enter_s2idle = + sbi_enter_s2idle_domain_idle_state; + sbi_cpuidle_use_cpuhp = true; + + return 0; +} + +static int sbi_cpuidle_dt_init_states(struct device *dev, + struct cpuidle_driver *drv, + unsigned int cpu, + unsigned int state_count) +{ + struct sbi_cpuidle_data *data = per_cpu_ptr(&sbi_cpuidle_data, cpu); + struct device_node *state_node; + struct device_node *cpu_node; + u32 *states; + int i, ret; + + cpu_node = of_cpu_device_node_get(cpu); + if (!cpu_node) + return -ENODEV; + + states = devm_kcalloc(dev, state_count, sizeof(*states), GFP_KERNEL); + if (!states) { + ret = -ENOMEM; + goto fail; + } + + /* Parse SBI specific details from state DT nodes */ + for (i = 1; i < state_count; i++) { + state_node = of_get_cpu_state_node(cpu_node, i - 1); + if (!state_node) + break; + + ret = sbi_dt_parse_state_node(state_node, &states[i]); + of_node_put(state_node); + + if (ret) + return ret; + + pr_debug("sbi-state %#x index %d\n", states[i], i); + } + if (i != state_count) { + ret = -ENODEV; + goto fail; + } + + /* Initialize optional data, used for the hierarchical topology. */ + ret = sbi_dt_cpu_init_topology(drv, data, state_count, cpu); + if (ret < 0) + return ret; + + /* Store states in the per-cpu struct. */ + data->states = states; + +fail: + of_node_put(cpu_node); + + return ret; +} + +static void sbi_cpuidle_deinit_cpu(int cpu) +{ + struct sbi_cpuidle_data *data = per_cpu_ptr(&sbi_cpuidle_data, cpu); + + dt_idle_detach_cpu(data->dev); + sbi_cpuidle_use_cpuhp = false; +} + +static int sbi_cpuidle_init_cpu(struct device *dev, int cpu) +{ + struct cpuidle_driver *drv; + unsigned int state_count = 0; + int ret = 0; + + drv = devm_kzalloc(dev, sizeof(*drv), GFP_KERNEL); + if (!drv) + return -ENOMEM; + + drv->name = "sbi_cpuidle"; + drv->owner = THIS_MODULE; + drv->cpumask = (struct cpumask *)cpumask_of(cpu); + + /* RISC-V architectural WFI to be represented as state index 0. */ + drv->states[0].enter = sbi_cpuidle_enter_state; + drv->states[0].exit_latency = 1; + drv->states[0].target_residency = 1; + drv->states[0].power_usage = UINT_MAX; + strcpy(drv->states[0].name, "WFI"); + strcpy(drv->states[0].desc, "RISC-V WFI"); + + /* + * If no DT idle states are detected (ret == 0) let the driver + * initialization fail accordingly since there is no reason to + * initialize the idle driver if only wfi is supported, the + * default archictectural back-end already executes wfi + * on idle entry. + */ + ret = dt_init_idle_driver(drv, sbi_cpuidle_state_match, 1); + if (ret <= 0) { + pr_debug("HART%ld: failed to parse DT idle states\n", + cpuid_to_hartid_map(cpu)); + return ret ? : -ENODEV; + } + state_count = ret + 1; /* Include WFI state as well */ + + /* Initialize idle states from DT. */ + ret = sbi_cpuidle_dt_init_states(dev, drv, cpu, state_count); + if (ret) { + pr_err("HART%ld: failed to init idle states\n", + cpuid_to_hartid_map(cpu)); + return ret; + } + + ret = cpuidle_register(drv, NULL); + if (ret) + goto deinit; + + cpuidle_cooling_register(drv); + + return 0; +deinit: + sbi_cpuidle_deinit_cpu(cpu); + return ret; +} + +static void sbi_cpuidle_domain_sync_state(struct device *dev) +{ + /* + * All devices have now been attached/probed to the PM domain + * topology, hence it's fine to allow domain states to be picked. + */ + sbi_cpuidle_pd_allow_domain_state = true; +} + +#ifdef CONFIG_DT_IDLE_GENPD + +static int sbi_cpuidle_pd_power_off(struct generic_pm_domain *pd) +{ + struct genpd_power_state *state = &pd->states[pd->state_idx]; + u32 *pd_state; + + if (!state->data) + return 0; + + if (!sbi_cpuidle_pd_allow_domain_state) + return -EBUSY; + + /* OSI mode is enabled, set the corresponding domain state. */ + pd_state = state->data; + sbi_set_domain_state(*pd_state); + + return 0; +} + +struct sbi_pd_provider { + struct list_head link; + struct device_node *node; +}; + +static LIST_HEAD(sbi_pd_providers); + +static int sbi_pd_init(struct device_node *np) +{ + struct generic_pm_domain *pd; + struct sbi_pd_provider *pd_provider; + struct dev_power_governor *pd_gov; + int ret = -ENOMEM, state_count = 0; + + pd = dt_idle_pd_alloc(np, sbi_dt_parse_state_node); + if (!pd) + goto out; + + pd_provider = kzalloc(sizeof(*pd_provider), GFP_KERNEL); + if (!pd_provider) + goto free_pd; + + pd->flags |= GENPD_FLAG_IRQ_SAFE | GENPD_FLAG_CPU_DOMAIN; + + /* Allow power off when OSI is available. */ + if (sbi_cpuidle_use_osi) + pd->power_off = sbi_cpuidle_pd_power_off; + else + pd->flags |= GENPD_FLAG_ALWAYS_ON; + + /* Use governor for CPU PM domains if it has some states to manage. */ + pd_gov = state_count > 0 ? &pm_domain_cpu_gov : NULL; + + ret = pm_genpd_init(pd, pd_gov, false); + if (ret) + goto free_pd_prov; + + ret = of_genpd_add_provider_simple(np, pd); + if (ret) + goto remove_pd; + + pd_provider->node = of_node_get(np); + list_add(&pd_provider->link, &sbi_pd_providers); + + pr_debug("init PM domain %s\n", pd->name); + return 0; + +remove_pd: + pm_genpd_remove(pd); +free_pd_prov: + kfree(pd_provider); +free_pd: + dt_idle_pd_free(pd); +out: + pr_err("failed to init PM domain ret=%d %pOF\n", ret, np); + return ret; +} + +static void sbi_pd_remove(void) +{ + struct sbi_pd_provider *pd_provider, *it; + struct generic_pm_domain *genpd; + + list_for_each_entry_safe(pd_provider, it, &sbi_pd_providers, link) { + of_genpd_del_provider(pd_provider->node); + + genpd = of_genpd_remove_last(pd_provider->node); + if (!IS_ERR(genpd)) + kfree(genpd); + + of_node_put(pd_provider->node); + list_del(&pd_provider->link); + kfree(pd_provider); + } +} + +static int sbi_genpd_probe(struct device_node *np) +{ + struct device_node *node; + int ret = 0, pd_count = 0; + + if (!np) + return -ENODEV; + + /* + * Parse child nodes for the "#power-domain-cells" property and + * initialize a genpd/genpd-of-provider pair when it's found. + */ + for_each_child_of_node(np, node) { + if (!of_find_property(node, "#power-domain-cells", NULL)) + continue; + + ret = sbi_pd_init(node); + if (ret) + goto put_node; + + pd_count++; + } + + /* Bail out if not using the hierarchical CPU topology. */ + if (!pd_count) + goto no_pd; + + /* Link genpd masters/subdomains to model the CPU topology. */ + ret = dt_idle_pd_init_topology(np); + if (ret) + goto remove_pd; + + return 0; + +put_node: + of_node_put(node); +remove_pd: + sbi_pd_remove(); + pr_err("failed to create CPU PM domains ret=%d\n", ret); +no_pd: + return ret; +} + +#else + +static inline int sbi_genpd_probe(struct device_node *np) +{ + return 0; +} + +#endif + +static int sbi_cpuidle_probe(struct platform_device *pdev) +{ + int cpu, ret; + struct cpuidle_driver *drv; + struct cpuidle_device *dev; + struct device_node *np, *pds_node; + + /* Detect OSI support based on CPU DT nodes */ + sbi_cpuidle_use_osi = true; + for_each_possible_cpu(cpu) { + np = of_cpu_device_node_get(cpu); + if (np && + of_find_property(np, "power-domains", NULL) && + of_find_property(np, "power-domain-names", NULL)) { + continue; + } else { + sbi_cpuidle_use_osi = false; + break; + } + } + + /* Populate generic power domains from DT nodes */ + pds_node = of_find_node_by_path("/cpus/power-domains"); + if (pds_node) { + ret = sbi_genpd_probe(pds_node); + of_node_put(pds_node); + if (ret) + return ret; + } + + /* Initialize CPU idle driver for each CPU */ + for_each_possible_cpu(cpu) { + ret = sbi_cpuidle_init_cpu(&pdev->dev, cpu); + if (ret) { + pr_debug("HART%ld: idle driver init failed\n", + cpuid_to_hartid_map(cpu)); + goto out_fail; + } + } + + /* Setup CPU hotplut notifiers */ + sbi_idle_init_cpuhp(); + + pr_info("idle driver registered for all CPUs\n"); + + return 0; + +out_fail: + while (--cpu >= 0) { + dev = per_cpu(cpuidle_devices, cpu); + drv = cpuidle_get_cpu_driver(dev); + cpuidle_unregister(drv); + sbi_cpuidle_deinit_cpu(cpu); + } + + return ret; +} + +static struct platform_driver sbi_cpuidle_driver = { + .probe = sbi_cpuidle_probe, + .driver = { + .name = "sbi-cpuidle", + .sync_state = sbi_cpuidle_domain_sync_state, + }, +}; + +static int __init sbi_cpuidle_init(void) +{ + int ret; + struct platform_device *pdev; + + /* + * The SBI HSM suspend function is only available when: + * 1) SBI version is 0.3 or higher + * 2) SBI HSM extension is available + */ + if ((sbi_spec_version < sbi_mk_version(0, 3)) || + sbi_probe_extension(SBI_EXT_HSM) <= 0) { + pr_info("HSM suspend not available\n"); + return 0; + } + + ret = platform_driver_register(&sbi_cpuidle_driver); + if (ret) + return ret; + + pdev = platform_device_register_simple("sbi-cpuidle", + -1, NULL, 0); + if (IS_ERR(pdev)) { + platform_driver_unregister(&sbi_cpuidle_driver); + return PTR_ERR(pdev); + } + + return 0; +} +device_initcall(sbi_cpuidle_init); From 1bd524f7e8d8f194cd94bc4535df91391d0f1dc8 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Thu, 10 Feb 2022 11:19:46 +0530 Subject: [PATCH 09/95] dt-bindings: Add common bindings for ARM and RISC-V idle states The RISC-V CPU idle states will be described in under the /cpus/idle-states DT node in the same way as ARM CPU idle states. This patch adds common bindings documentation for both ARM and RISC-V idle states. Signed-off-by: Anup Patel Signed-off-by: Anup Patel Reviewed-by: Rob Herring Reviewed-by: Guo Ren Signed-off-by: Palmer Dabbelt --- .../bindings/arm/msm/qcom,idle-state.txt | 2 +- .../devicetree/bindings/arm/psci.yaml | 2 +- .../bindings/{arm => cpu}/idle-states.yaml | 228 ++++++++++++++++-- .../devicetree/bindings/riscv/cpus.yaml | 6 + 4 files changed, 219 insertions(+), 19 deletions(-) rename Documentation/devicetree/bindings/{arm => cpu}/idle-states.yaml (74%) diff --git a/Documentation/devicetree/bindings/arm/msm/qcom,idle-state.txt b/Documentation/devicetree/bindings/arm/msm/qcom,idle-state.txt index 6ce0b212ec6d..606b4b1b709d 100644 --- a/Documentation/devicetree/bindings/arm/msm/qcom,idle-state.txt +++ b/Documentation/devicetree/bindings/arm/msm/qcom,idle-state.txt @@ -81,4 +81,4 @@ Example: }; }; -[1]. Documentation/devicetree/bindings/arm/idle-states.yaml +[1]. Documentation/devicetree/bindings/cpu/idle-states.yaml diff --git a/Documentation/devicetree/bindings/arm/psci.yaml b/Documentation/devicetree/bindings/arm/psci.yaml index 8b77cf83a095..dd83ef278af0 100644 --- a/Documentation/devicetree/bindings/arm/psci.yaml +++ b/Documentation/devicetree/bindings/arm/psci.yaml @@ -101,7 +101,7 @@ properties: bindings in [1]) must specify this property. [1] Kernel documentation - ARM idle states bindings - Documentation/devicetree/bindings/arm/idle-states.yaml + Documentation/devicetree/bindings/cpu/idle-states.yaml patternProperties: "^power-domain-": diff --git a/Documentation/devicetree/bindings/arm/idle-states.yaml b/Documentation/devicetree/bindings/cpu/idle-states.yaml similarity index 74% rename from Documentation/devicetree/bindings/arm/idle-states.yaml rename to Documentation/devicetree/bindings/cpu/idle-states.yaml index 52bce5dbb11f..95506ffb816c 100644 --- a/Documentation/devicetree/bindings/arm/idle-states.yaml +++ b/Documentation/devicetree/bindings/cpu/idle-states.yaml @@ -1,25 +1,30 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- -$id: http://devicetree.org/schemas/arm/idle-states.yaml# +$id: http://devicetree.org/schemas/cpu/idle-states.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: ARM idle states binding description +title: Idle states binding description maintainers: - Lorenzo Pieralisi + - Anup Patel description: |+ ========================================== 1 - Introduction ========================================== - ARM systems contain HW capable of managing power consumption dynamically, - where cores can be put in different low-power states (ranging from simple wfi - to power gating) according to OS PM policies. The CPU states representing the - range of dynamic idle states that a processor can enter at run-time, can be - specified through device tree bindings representing the parameters required to - enter/exit specific idle states on a given processor. + ARM and RISC-V systems contain HW capable of managing power consumption + dynamically, where cores can be put in different low-power states (ranging + from simple wfi to power gating) according to OS PM policies. The CPU states + representing the range of dynamic idle states that a processor can enter at + run-time, can be specified through device tree bindings representing the + parameters required to enter/exit specific idle states on a given processor. + + ========================================== + 2 - ARM idle states + ========================================== According to the Server Base System Architecture document (SBSA, [3]), the power states an ARM CPU can be put into are identified by the following list: @@ -43,8 +48,23 @@ description: |+ The device tree binding definition for ARM idle states is the subject of this document. + ========================================== + 3 - RISC-V idle states + ========================================== + + On RISC-V systems, the HARTs (or CPUs) [6] can be put in platform specific + suspend (or idle) states (ranging from simple WFI, power gating, etc). The + RISC-V SBI v0.3 (or higher) [7] hart state management extension provides a + standard mechanism for OS to request HART state transitions. + + The platform specific suspend (or idle) states of a hart can be either + retentive or non-rententive in nature. A retentive suspend state will + preserve HART registers and CSR values for all privilege modes whereas + a non-retentive suspend state will not preserve HART registers and CSR + values. + =========================================== - 2 - idle-states definitions + 4 - idle-states definitions =========================================== Idle states are characterized for a specific system through a set of @@ -211,10 +231,10 @@ description: |+ properties specification that is the subject of the following sections. =========================================== - 3 - idle-states node + 5 - idle-states node =========================================== - ARM processor idle states are defined within the idle-states node, which is + The processor idle states are defined within the idle-states node, which is a direct child of the cpus node [1] and provides a container where the processor idle states, defined as device tree nodes, are listed. @@ -223,7 +243,7 @@ description: |+ just supports idle_standby, an idle-states node is not required. =========================================== - 4 - References + 6 - References =========================================== [1] ARM Linux Kernel documentation - CPUs bindings @@ -238,9 +258,15 @@ description: |+ [4] ARM Architecture Reference Manuals http://infocenter.arm.com/help/index.jsp - [6] ARM Linux Kernel documentation - Booting AArch64 Linux + [5] ARM Linux Kernel documentation - Booting AArch64 Linux Documentation/arm64/booting.rst + [6] RISC-V Linux Kernel documentation - CPUs bindings + Documentation/devicetree/bindings/riscv/cpus.yaml + + [7] RISC-V Supervisor Binary Interface (SBI) + http://github.com/riscv/riscv-sbi-doc/riscv-sbi.adoc + properties: $nodename: const: idle-states @@ -253,7 +279,7 @@ properties: On ARM 32-bit systems this property is optional This assumes that the "enable-method" property is set to "psci" in the cpu - node[6] that is responsible for setting up CPU idle management in the OS + node[5] that is responsible for setting up CPU idle management in the OS implementation. const: psci @@ -265,8 +291,8 @@ patternProperties: as follows. The idle state entered by executing the wfi instruction (idle_standby - SBSA,[3][4]) is considered standard on all ARM platforms and therefore - must not be listed. + SBSA,[3][4]) is considered standard on all ARM and RISC-V platforms and + therefore must not be listed. In addition to the properties listed above, a state node may require additional properties specific to the entry-method defined in the @@ -275,7 +301,27 @@ patternProperties: properties: compatible: - const: arm,idle-state + enum: + - arm,idle-state + - riscv,idle-state + + arm,psci-suspend-param: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + power_state parameter to pass to the ARM PSCI suspend call. + + Device tree nodes that require usage of PSCI CPU_SUSPEND function + (i.e. idle states node with entry-method property is set to "psci") + must specify this property. + + riscv,sbi-suspend-param: + $ref: /schemas/types.yaml#/definitions/uint32 + description: | + suspend_type parameter to pass to the RISC-V SBI HSM suspend call. + + This property is required in idle state nodes of device tree meant + for RISC-V systems. For more details on the suspend_type parameter + refer the SBI specifiation v0.3 (or higher) [7]. local-timer-stop: description: @@ -317,6 +363,8 @@ patternProperties: description: A string used as a descriptive name for the idle state. + additionalProperties: false + required: - compatible - entry-latency-us @@ -658,4 +706,150 @@ examples: }; }; + - | + // Example 3 (RISC-V 64-bit, 4-cpu systems, two clusters): + + cpus { + #size-cells = <0>; + #address-cells = <1>; + + cpu@0 { + device_type = "cpu"; + compatible = "riscv"; + reg = <0x0>; + riscv,isa = "rv64imafdc"; + mmu-type = "riscv,sv48"; + cpu-idle-states = <&CPU_RET_0_0 &CPU_NONRET_0_0 + &CLUSTER_RET_0 &CLUSTER_NONRET_0>; + + cpu_intc0: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu@1 { + device_type = "cpu"; + compatible = "riscv"; + reg = <0x1>; + riscv,isa = "rv64imafdc"; + mmu-type = "riscv,sv48"; + cpu-idle-states = <&CPU_RET_0_0 &CPU_NONRET_0_0 + &CLUSTER_RET_0 &CLUSTER_NONRET_0>; + + cpu_intc1: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu@10 { + device_type = "cpu"; + compatible = "riscv"; + reg = <0x10>; + riscv,isa = "rv64imafdc"; + mmu-type = "riscv,sv48"; + cpu-idle-states = <&CPU_RET_1_0 &CPU_NONRET_1_0 + &CLUSTER_RET_1 &CLUSTER_NONRET_1>; + + cpu_intc10: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + cpu@11 { + device_type = "cpu"; + compatible = "riscv"; + reg = <0x11>; + riscv,isa = "rv64imafdc"; + mmu-type = "riscv,sv48"; + cpu-idle-states = <&CPU_RET_1_0 &CPU_NONRET_1_0 + &CLUSTER_RET_1 &CLUSTER_NONRET_1>; + + cpu_intc11: interrupt-controller { + #interrupt-cells = <1>; + compatible = "riscv,cpu-intc"; + interrupt-controller; + }; + }; + + idle-states { + CPU_RET_0_0: cpu-retentive-0-0 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x10000000>; + entry-latency-us = <20>; + exit-latency-us = <40>; + min-residency-us = <80>; + }; + + CPU_NONRET_0_0: cpu-nonretentive-0-0 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x90000000>; + entry-latency-us = <250>; + exit-latency-us = <500>; + min-residency-us = <950>; + }; + + CLUSTER_RET_0: cluster-retentive-0 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x11000000>; + local-timer-stop; + entry-latency-us = <50>; + exit-latency-us = <100>; + min-residency-us = <250>; + wakeup-latency-us = <130>; + }; + + CLUSTER_NONRET_0: cluster-nonretentive-0 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x91000000>; + local-timer-stop; + entry-latency-us = <600>; + exit-latency-us = <1100>; + min-residency-us = <2700>; + wakeup-latency-us = <1500>; + }; + + CPU_RET_1_0: cpu-retentive-1-0 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x10000010>; + entry-latency-us = <20>; + exit-latency-us = <40>; + min-residency-us = <80>; + }; + + CPU_NONRET_1_0: cpu-nonretentive-1-0 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x90000010>; + entry-latency-us = <250>; + exit-latency-us = <500>; + min-residency-us = <950>; + }; + + CLUSTER_RET_1: cluster-retentive-1 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x11000010>; + local-timer-stop; + entry-latency-us = <50>; + exit-latency-us = <100>; + min-residency-us = <250>; + wakeup-latency-us = <130>; + }; + + CLUSTER_NONRET_1: cluster-nonretentive-1 { + compatible = "riscv,idle-state"; + riscv,sbi-suspend-param = <0x91000010>; + local-timer-stop; + entry-latency-us = <600>; + exit-latency-us = <1100>; + min-residency-us = <2700>; + wakeup-latency-us = <1500>; + }; + }; + }; + ... diff --git a/Documentation/devicetree/bindings/riscv/cpus.yaml b/Documentation/devicetree/bindings/riscv/cpus.yaml index aa5fb64d57eb..f62f646bc695 100644 --- a/Documentation/devicetree/bindings/riscv/cpus.yaml +++ b/Documentation/devicetree/bindings/riscv/cpus.yaml @@ -99,6 +99,12 @@ properties: - compatible - interrupt-controller + cpu-idle-states: + $ref: '/schemas/types.yaml#/definitions/phandle-array' + description: | + List of phandles to idle state nodes supported + by this hart (see ./idle-states.yaml). + required: - riscv,isa - interrupt-controller From c5179ef1ca0c39dab6955be6b0e3c034cc4164c8 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Thu, 10 Feb 2022 11:19:47 +0530 Subject: [PATCH 10/95] RISC-V: Enable RISC-V SBI CPU Idle driver for QEMU virt machine We enable RISC-V SBI CPU Idle driver for QEMU virt machine to test SBI HSM Supend on QEMU. Signed-off-by: Anup Patel Signed-off-by: Anup Patel Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig.socs | 3 +++ arch/riscv/configs/defconfig | 1 + arch/riscv/configs/rv32_defconfig | 1 + 3 files changed, 5 insertions(+) diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index 6ec44a22278a..f4097a815201 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -36,6 +36,9 @@ config SOC_VIRT select GOLDFISH select RTC_DRV_GOLDFISH if RTC_CLASS select SIFIVE_PLIC + select PM_GENERIC_DOMAINS if PM + select PM_GENERIC_DOMAINS_OF if PM && OF + select RISCV_SBI_CPUIDLE if CPU_IDLE help This enables support for QEMU Virt Machine. diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index a5e0482a4969..b8c882b70b02 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -20,6 +20,7 @@ CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y CONFIG_SMP=y CONFIG_HOTPLUG_CPU=y +CONFIG_PM=y CONFIG_CPU_IDLE=y CONFIG_VIRTUALIZATION=y CONFIG_KVM=m diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig index d1b87db54d68..6f9a7c89bff9 100644 --- a/arch/riscv/configs/rv32_defconfig +++ b/arch/riscv/configs/rv32_defconfig @@ -20,6 +20,7 @@ CONFIG_SOC_VIRT=y CONFIG_ARCH_RV32I=y CONFIG_SMP=y CONFIG_HOTPLUG_CPU=y +CONFIG_PM=y CONFIG_CPU_IDLE=y CONFIG_VIRTUALIZATION=y CONFIG_KVM=m From a4eb44a6435d6d8f9e642407a4a06f65eb90ca04 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 3 Dec 2019 15:22:59 +0000 Subject: [PATCH 11/95] dt: amd-seattle: remove Husky platform The Huskyboard never made it to production, and its successor the Celloboard was only shipped in very limited quantities with ACPI only firmware, so the historical significance of husky.dts is highly questionable. Let's drop it. Signed-off-by: Ard Biesheuvel Signed-off-by: Arnd Bergmann --- arch/arm64/boot/dts/amd/Makefile | 3 +- arch/arm64/boot/dts/amd/husky.dts | 84 ------------------------------- 2 files changed, 1 insertion(+), 86 deletions(-) delete mode 100644 arch/arm64/boot/dts/amd/husky.dts diff --git a/arch/arm64/boot/dts/amd/Makefile b/arch/arm64/boot/dts/amd/Makefile index 6a6093064a32..5e27bc0321e9 100644 --- a/arch/arm64/boot/dts/amd/Makefile +++ b/arch/arm64/boot/dts/amd/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 dtb-$(CONFIG_ARCH_SEATTLE) += amd-overdrive.dtb \ - amd-overdrive-rev-b0.dtb amd-overdrive-rev-b1.dtb \ - husky.dtb + amd-overdrive-rev-b0.dtb amd-overdrive-rev-b1.dtb diff --git a/arch/arm64/boot/dts/amd/husky.dts b/arch/arm64/boot/dts/amd/husky.dts deleted file mode 100644 index 7acde34772cb..000000000000 --- a/arch/arm64/boot/dts/amd/husky.dts +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * DTS file for AMD/Linaro 96Boards Enterprise Edition Server (Husky) Board - * Note: Based-on AMD Seattle Rev.B0 - * - * Copyright (C) 2015 Advanced Micro Devices, Inc. - */ - -/dts-v1/; - -/include/ "amd-seattle-soc.dtsi" - -/ { - model = "Linaro 96Boards Enterprise Edition Server (Husky) Board"; - compatible = "amd,seattle-overdrive", "amd,seattle"; - - chosen { - stdout-path = &serial0; - }; - - psci { - compatible = "arm,psci-0.2"; - method = "smc"; - }; -}; - -&ccp0 { - status = "ok"; - amd,zlib-support = <1>; -}; - -/** - * NOTE: In Rev.B, gpio0 is reserved. - */ -&gpio1 { - status = "ok"; -}; - -&gpio2 { - status = "ok"; -}; - -&gpio3 { - status = "ok"; -}; - -&gpio4 { - status = "ok"; -}; - -&i2c0 { - status = "ok"; -}; - -&i2c1 { - status = "ok"; -}; - -&pcie0 { - status = "ok"; -}; - -&spi0 { - status = "ok"; -}; - -&spi1 { - status = "ok"; - sdcard0: sdcard@0 { - compatible = "mmc-spi-slot"; - reg = <0>; - spi-max-frequency = <20000000>; - voltage-ranges = <3200 3400>; - pl022,hierarchy = <0>; - pl022,interface = <0>; - pl022,com-mode = <0x0>; - pl022,rx-level-trig = <0>; - pl022,tx-level-trig = <0>; - }; -}; - -&smb0 { - /include/ "amd-seattle-xgbe-b.dtsi" -}; From f179eb6b3cc35b451820468a3d09fe169416d4bc Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 3 Dec 2019 15:23:00 +0000 Subject: [PATCH 12/95] dt: amd-seattle: remove Overdrive revision A0 support Support for AMD Seattle silicon revision A0 is no longer relevant, since we no longer have a driver for the network controller, and the PCIe on these boards was very unreliable. So drop the DTS description of the A0 version of the overdrive board. Signed-off-by: Ard Biesheuvel Signed-off-by: Arnd Bergmann --- arch/arm64/boot/dts/amd/Makefile | 3 +- arch/arm64/boot/dts/amd/amd-overdrive.dts | 66 ----------------------- 2 files changed, 1 insertion(+), 68 deletions(-) delete mode 100644 arch/arm64/boot/dts/amd/amd-overdrive.dts diff --git a/arch/arm64/boot/dts/amd/Makefile b/arch/arm64/boot/dts/amd/Makefile index 5e27bc0321e9..68103a8b0ef5 100644 --- a/arch/arm64/boot/dts/amd/Makefile +++ b/arch/arm64/boot/dts/amd/Makefile @@ -1,3 +1,2 @@ # SPDX-License-Identifier: GPL-2.0 -dtb-$(CONFIG_ARCH_SEATTLE) += amd-overdrive.dtb \ - amd-overdrive-rev-b0.dtb amd-overdrive-rev-b1.dtb +dtb-$(CONFIG_ARCH_SEATTLE) += amd-overdrive-rev-b0.dtb amd-overdrive-rev-b1.dtb diff --git a/arch/arm64/boot/dts/amd/amd-overdrive.dts b/arch/arm64/boot/dts/amd/amd-overdrive.dts deleted file mode 100644 index 41b3a6c0993d..000000000000 --- a/arch/arm64/boot/dts/amd/amd-overdrive.dts +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * DTS file for AMD Seattle Overdrive Development Board - * - * Copyright (C) 2014 Advanced Micro Devices, Inc. - */ - -/dts-v1/; - -/include/ "amd-seattle-soc.dtsi" - -/ { - model = "AMD Seattle Development Board (Overdrive)"; - compatible = "amd,seattle-overdrive", "amd,seattle"; - - chosen { - stdout-path = &serial0; - }; -}; - -&ccp0 { - status = "ok"; -}; - -&gpio0 { - status = "ok"; -}; - -&gpio1 { - status = "ok"; -}; - -&i2c0 { - status = "ok"; -}; - -&pcie0 { - status = "ok"; -}; - -&spi0 { - status = "ok"; -}; - -&spi1 { - status = "ok"; - sdcard0: sdcard@0 { - compatible = "mmc-spi-slot"; - reg = <0>; - spi-max-frequency = <20000000>; - voltage-ranges = <3200 3400>; - gpios = <&gpio0 7 0>; - interrupt-parent = <&gpio0>; - interrupts = <7 3>; - pl022,hierarchy = <0>; - pl022,interface = <0>; - pl022,com-mode = <0x0>; - pl022,rx-level-trig = <0>; - pl022,tx-level-trig = <0>; - }; -}; - -&v2m0 { - arm,msi-base-spi = <64>; - arm,msi-num-spis = <256>; -}; From 9bd9a063accf15b8ddae15e17d9d6312a31e4464 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 3 Dec 2019 15:23:01 +0000 Subject: [PATCH 13/95] dt: amd-seattle: upgrade AMD Seattle XGBE to new SMMU binding Upgrade the DT descriptions of the AMD Seattle XGBE network controllers to use the current SMMU bindings. Signed-off-by: Ard Biesheuvel Signed-off-by: Arnd Bergmann --- .../boot/dts/amd/amd-seattle-xgbe-b.dtsi | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/arch/arm64/boot/dts/amd/amd-seattle-xgbe-b.dtsi b/arch/arm64/boot/dts/amd/amd-seattle-xgbe-b.dtsi index d97498361ce3..9259e547e2e8 100644 --- a/arch/arm64/boot/dts/amd/amd-seattle-xgbe-b.dtsi +++ b/arch/arm64/boot/dts/amd/amd-seattle-xgbe-b.dtsi @@ -55,7 +55,7 @@ clocks = <&xgmacclk0_dma_250mhz>, <&xgmacclk0_ptp_250mhz>; clock-names = "dma_clk", "ptp_clk"; phy-mode = "xgmii"; - #stream-id-cells = <16>; + iommus = <&xgmac0_smmu 0x00 0x17>; /* 0-7, 16-23 */ dma-coherent; }; @@ -81,11 +81,11 @@ clocks = <&xgmacclk1_dma_250mhz>, <&xgmacclk1_ptp_250mhz>; clock-names = "dma_clk", "ptp_clk"; phy-mode = "xgmii"; - #stream-id-cells = <16>; + iommus = <&xgmac1_smmu 0x00 0x17>; /* 0-7, 16-23 */ dma-coherent; }; - xgmac0_smmu: smmu@e0600000 { + xgmac0_smmu: iommu@e0600000 { compatible = "arm,mmu-401"; reg = <0 0xe0600000 0 0x10000>; #global-interrupts = <1>; @@ -94,14 +94,11 @@ */ <0 336 4>, <0 336 4>; - - mmu-masters = <&xgmac0 - 0 1 2 3 4 5 6 7 - 16 17 18 19 20 21 22 23 - >; + #iommu-cells = <2>; + dma-coherent; }; - xgmac1_smmu: smmu@e0800000 { + xgmac1_smmu: iommu@e0800000 { compatible = "arm,mmu-401"; reg = <0 0xe0800000 0 0x10000>; #global-interrupts = <1>; @@ -110,9 +107,6 @@ */ <0 335 4>, <0 335 4>; - - mmu-masters = <&xgmac1 - 0 1 2 3 4 5 6 7 - 16 17 18 19 20 21 22 23 - >; + #iommu-cells = <2>; + dma-coherent; }; From acd9208ef8a4e1dc0e1d835fab284ca717a01f93 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 3 Dec 2019 15:23:02 +0000 Subject: [PATCH 14/95] dt: amd-seattle: fix PCIe legacy interrupt routing The AMD Seattle SOC can be configured to expose up to 3 PCIe root ports, each of which is wired to 4 dedicated SPI wired interrupts for legacy INTx support. Update the SOC DT description to reflect this. Fix a stale comment about the size of the MMIO64 resource window while at it. Signed-off-by: Ard Biesheuvel Signed-off-by: Arnd Bergmann --- arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi | 22 ++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi b/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi index b664e7af74eb..9fa6890fca35 100644 --- a/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi +++ b/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi @@ -213,12 +213,22 @@ msi-parent = <&v2m0>; reg = <0 0xf0000000 0 0x10000000>; - interrupt-map-mask = <0xf800 0x0 0x0 0x7>; + interrupt-map-mask = <0xff00 0x0 0x0 0x7>; interrupt-map = - <0x1000 0x0 0x0 0x1 &gic0 0x0 0x0 0x0 0x120 0x1>, - <0x1000 0x0 0x0 0x2 &gic0 0x0 0x0 0x0 0x121 0x1>, - <0x1000 0x0 0x0 0x3 &gic0 0x0 0x0 0x0 0x122 0x1>, - <0x1000 0x0 0x0 0x4 &gic0 0x0 0x0 0x0 0x123 0x1>; + <0x1100 0x0 0x0 0x1 &gic0 0x0 0x0 0x0 0x120 0x1>, + <0x1100 0x0 0x0 0x2 &gic0 0x0 0x0 0x0 0x121 0x1>, + <0x1100 0x0 0x0 0x3 &gic0 0x0 0x0 0x0 0x122 0x1>, + <0x1100 0x0 0x0 0x4 &gic0 0x0 0x0 0x0 0x123 0x1>, + + <0x1200 0x0 0x0 0x1 &gic0 0x0 0x0 0x0 0x124 0x1>, + <0x1200 0x0 0x0 0x2 &gic0 0x0 0x0 0x0 0x125 0x1>, + <0x1200 0x0 0x0 0x3 &gic0 0x0 0x0 0x0 0x126 0x1>, + <0x1200 0x0 0x0 0x4 &gic0 0x0 0x0 0x0 0x127 0x1>, + + <0x1300 0x0 0x0 0x1 &gic0 0x0 0x0 0x0 0x128 0x1>, + <0x1300 0x0 0x0 0x2 &gic0 0x0 0x0 0x0 0x129 0x1>, + <0x1300 0x0 0x0 0x3 &gic0 0x0 0x0 0x0 0x12a 0x1>, + <0x1300 0x0 0x0 0x4 &gic0 0x0 0x0 0x0 0x12b 0x1>; dma-coherent; dma-ranges = <0x43000000 0x0 0x0 0x0 0x0 0x100 0x0>; @@ -227,7 +237,7 @@ <0x01000000 0x00 0x00000000 0x00 0xefff0000 0x00 0x00010000>, /* 32-bit MMIO (size=2G) */ <0x02000000 0x00 0x40000000 0x00 0x40000000 0x00 0x80000000>, - /* 64-bit MMIO (size= 124G) */ + /* 64-bit MMIO (size= 508G) */ <0x03000000 0x01 0x00000000 0x01 0x00000000 0x7f 0x00000000>; }; From dd5c160655e24c1a86b1f23e870b69f6aaa80646 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 3 Dec 2019 15:23:03 +0000 Subject: [PATCH 15/95] dt: amd-seattle: add a description of the PCIe SMMU Add a description of the SMMU that covers the PCIe host bridge on AMD Seattle. Signed-off-by: Ard Biesheuvel Signed-off-by: Arnd Bergmann --- arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi b/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi index 9fa6890fca35..124e58a76be0 100644 --- a/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi +++ b/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi @@ -239,6 +239,16 @@ <0x02000000 0x00 0x40000000 0x00 0x40000000 0x00 0x80000000>, /* 64-bit MMIO (size= 508G) */ <0x03000000 0x01 0x00000000 0x01 0x00000000 0x7f 0x00000000>; + iommu-map = <0x0 &pcie_smmu 0x0 0x10000>; + }; + + pcie_smmu: iommu@e0a00000 { + compatible = "arm,mmu-401"; + reg = <0 0xe0a00000 0 0x10000>; + #global-interrupts = <1>; + interrupts = <0 333 4>, <0 333 4>; + #iommu-cells = <1>; + dma-coherent; }; /* Perf CCN504 PMU */ From 429863e767a084e5de6934d768946cda5793b691 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 3 Dec 2019 15:23:04 +0000 Subject: [PATCH 16/95] dt: amd-seattle: add description of the SATA/CCP SMMUs Add descriptions of the SMMUs that cover the SATA controller(s) on the AMD Seattle SOC. The CCP crypto accelerator shares its SMMU with the second SATA controller, which is only enabled on B1 silicon. Signed-off-by: Ard Biesheuvel Signed-off-by: Arnd Bergmann --- arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi | 26 ++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi b/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi index 124e58a76be0..547a6bf10f5e 100644 --- a/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi +++ b/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi @@ -70,6 +70,7 @@ reg = <0 0xe0300000 0 0xf0000>; interrupts = <0 355 4>; clocks = <&sataclk_333mhz>; + iommus = <&sata0_smmu 0x0 0x1f>; dma-coherent; }; @@ -80,6 +81,27 @@ reg = <0 0xe0d00000 0 0xf0000>; interrupts = <0 354 4>; clocks = <&sataclk_333mhz>; + iommus = <&sata1_smmu 0x0e>, + <&sata1_smmu 0x0f>, + <&sata1_smmu 0x1e>; + dma-coherent; + }; + + sata0_smmu: iommu@e0200000 { + compatible = "arm,mmu-401"; + reg = <0 0xe0200000 0 0x10000>; + #global-interrupts = <1>; + interrupts = <0 332 4>, <0 332 4>; + #iommu-cells = <2>; + dma-coherent; + }; + + sata1_smmu: iommu@e0c00000 { + compatible = "arm,mmu-401"; + reg = <0 0xe0c00000 0 0x10000>; + #global-interrupts = <1>; + interrupts = <0 331 4>, <0 331 4>; + #iommu-cells = <1>; dma-coherent; }; @@ -201,6 +223,10 @@ reg = <0 0xe0100000 0 0x10000>; interrupts = <0 3 4>; dma-coherent; + iommus = <&sata1_smmu 0x00>, + <&sata1_smmu 0x02>, + <&sata1_smmu 0x40>, + <&sata1_smmu 0x42>; }; pcie0: pcie@f0000000 { From e0db1597a71a7de661a107090554f8b187e77a08 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 3 Dec 2019 15:23:05 +0000 Subject: [PATCH 17/95] dt: amd-seattle: disable IPMI controller and some GPIO blocks on B0 Disable some peripherals that are not usable on B0 silicon based Overdrives. Signed-off-by: Ard Biesheuvel Signed-off-by: Arnd Bergmann --- arch/arm64/boot/dts/amd/amd-overdrive-rev-b0.dts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/arch/arm64/boot/dts/amd/amd-overdrive-rev-b0.dts b/arch/arm64/boot/dts/amd/amd-overdrive-rev-b0.dts index 8e341be9a399..4997a74aeb67 100644 --- a/arch/arm64/boot/dts/amd/amd-overdrive-rev-b0.dts +++ b/arch/arm64/boot/dts/amd/amd-overdrive-rev-b0.dts @@ -36,14 +36,6 @@ status = "ok"; }; -&gpio2 { - status = "ok"; -}; - -&gpio3 { - status = "ok"; -}; - &gpio4 { status = "ok"; }; @@ -79,10 +71,6 @@ }; }; -&ipmi_kcs { - status = "ok"; -}; - &smb0 { /include/ "amd-seattle-xgbe-b.dtsi" }; From 6a2f0b2d3b74e3e4002dc4143887637cd216e531 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 3 Dec 2019 15:23:06 +0000 Subject: [PATCH 18/95] dt: amd-seattle: add a description of the CPUs and caches Add a DT description of the CPU and cache hierarchy as found on the AMD Seattle SOC. Given the tight coupling of the PMU with the CPUs, move the PMU node into the cpu .dtsi file as well, and add the missing affinity description. Signed-off-by: Ard Biesheuvel Signed-off-by: Arnd Bergmann --- .../boot/dts/amd/amd-overdrive-rev-b0.dts | 1 + .../boot/dts/amd/amd-overdrive-rev-b1.dts | 1 + arch/arm64/boot/dts/amd/amd-seattle-cpus.dtsi | 224 ++++++++++++++++++ arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi | 12 - 4 files changed, 226 insertions(+), 12 deletions(-) create mode 100644 arch/arm64/boot/dts/amd/amd-seattle-cpus.dtsi diff --git a/arch/arm64/boot/dts/amd/amd-overdrive-rev-b0.dts b/arch/arm64/boot/dts/amd/amd-overdrive-rev-b0.dts index 4997a74aeb67..c290d1ce2b03 100644 --- a/arch/arm64/boot/dts/amd/amd-overdrive-rev-b0.dts +++ b/arch/arm64/boot/dts/amd/amd-overdrive-rev-b0.dts @@ -9,6 +9,7 @@ /dts-v1/; /include/ "amd-seattle-soc.dtsi" +/include/ "amd-seattle-cpus.dtsi" / { model = "AMD Seattle (Rev.B0) Development Board (Overdrive)"; diff --git a/arch/arm64/boot/dts/amd/amd-overdrive-rev-b1.dts b/arch/arm64/boot/dts/amd/amd-overdrive-rev-b1.dts index 92cef05c6b74..e0926f6bb7c3 100644 --- a/arch/arm64/boot/dts/amd/amd-overdrive-rev-b1.dts +++ b/arch/arm64/boot/dts/amd/amd-overdrive-rev-b1.dts @@ -9,6 +9,7 @@ /dts-v1/; /include/ "amd-seattle-soc.dtsi" +/include/ "amd-seattle-cpus.dtsi" / { model = "AMD Seattle (Rev.B1) Development Board (Overdrive)"; diff --git a/arch/arm64/boot/dts/amd/amd-seattle-cpus.dtsi b/arch/arm64/boot/dts/amd/amd-seattle-cpus.dtsi new file mode 100644 index 000000000000..93688a0b6820 --- /dev/null +++ b/arch/arm64/boot/dts/amd/amd-seattle-cpus.dtsi @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0 + +/ { + cpus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + cpu-map { + cluster0 { + core0 { + cpu = <&CPU0>; + }; + core1 { + cpu = <&CPU1>; + }; + }; + cluster1 { + core0 { + cpu = <&CPU2>; + }; + core1 { + cpu = <&CPU3>; + }; + }; + cluster2 { + core0 { + cpu = <&CPU4>; + }; + core1 { + cpu = <&CPU5>; + }; + }; + cluster3 { + core0 { + cpu = <&CPU6>; + }; + core1 { + cpu = <&CPU7>; + }; + }; + }; + + CPU0: cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x0>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_0>; + + }; + + CPU1: cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x1>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_0>; + }; + + CPU2: cpu@100 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x100>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_1>; + }; + + CPU3: cpu@101 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x101>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_1>; + }; + + CPU4: cpu@200 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x200>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_2>; + }; + + CPU5: cpu@201 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x201>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_2>; + }; + + CPU6: cpu@300 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x300>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_3>; + }; + + CPU7: cpu@301 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x301>; + enable-method = "psci"; + + i-cache-size = <0xC000>; + i-cache-line-size = <64>; + i-cache-sets = <256>; + d-cache-size = <0x8000>; + d-cache-line-size = <64>; + d-cache-sets = <256>; + l2-cache = <&L2_3>; + }; + }; + + L2_0: l2-cache0 { + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-unified; + next-level-cache = <&L3>; + }; + + L2_1: l2-cache1 { + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-unified; + next-level-cache = <&L3>; + }; + + L2_2: l2-cache2 { + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-unified; + next-level-cache = <&L3>; + }; + + L2_3: l2-cache3 { + cache-size = <0x100000>; + cache-line-size = <64>; + cache-sets = <1024>; + cache-unified; + next-level-cache = <&L3>; + }; + + L3: l3-cache { + cache-level = <3>; + cache-size = <0x800000>; + cache-line-size = <64>; + cache-sets = <8192>; + cache-unified; + }; + + pmu { + compatible = "arm,cortex-a57-pmu"; + interrupts = <0x0 0x7 0x4>, + <0x0 0x8 0x4>, + <0x0 0x9 0x4>, + <0x0 0xa 0x4>, + <0x0 0xb 0x4>, + <0x0 0xc 0x4>, + <0x0 0xd 0x4>, + <0x0 0xe 0x4>; + interrupt-affinity = <&CPU0>, + <&CPU1>, + <&CPU2>, + <&CPU3>, + <&CPU4>, + <&CPU5>, + <&CPU6>, + <&CPU7>; + }; +}; diff --git a/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi b/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi index 547a6bf10f5e..690020589d41 100644 --- a/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi +++ b/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi @@ -38,18 +38,6 @@ <1 10 0xff04>; }; - pmu { - compatible = "arm,armv8-pmuv3"; - interrupts = <0 7 4>, - <0 8 4>, - <0 9 4>, - <0 10 4>, - <0 11 4>, - <0 12 4>, - <0 13 4>, - <0 14 4>; - }; - smb0: smb { compatible = "simple-bus"; #address-cells = <2>; From 895ae58da4a2360d9c2d255cd9fc8de64e265022 Mon Sep 17 00:00:00 2001 From: Juergen Christ Date: Wed, 15 Dec 2021 19:23:29 +0100 Subject: [PATCH 19/95] s390/zcrypt: Add admask to zcdn Zcrypt custom devices now support control domain masks. Users can set and modify this mask to allow custom devices to access certain control domains. Signed-off-by: Juergen Christ Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/ap_bus.h | 1 + drivers/s390/crypto/zcrypt_api.c | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index 8fd5a17bdf99..6a65885f5f43 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -315,6 +315,7 @@ struct ap_perms { unsigned long ioctlm[BITS_TO_LONGS(AP_IOCTLS)]; unsigned long apm[BITS_TO_LONGS(AP_DEVICES)]; unsigned long aqm[BITS_TO_LONGS(AP_DOMAINS)]; + unsigned long adm[BITS_TO_LONGS(AP_DOMAINS)]; }; extern struct ap_perms ap_perms; extern struct mutex ap_perms_mutex; diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 80e2a306709a..5cfe5cf44ba1 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -285,10 +285,53 @@ static ssize_t aqmask_store(struct device *dev, static DEVICE_ATTR_RW(aqmask); +static ssize_t admask_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + int i, rc; + struct zcdn_device *zcdndev = to_zcdn_dev(dev); + + if (mutex_lock_interruptible(&ap_perms_mutex)) + return -ERESTARTSYS; + + buf[0] = '0'; + buf[1] = 'x'; + for (i = 0; i < sizeof(zcdndev->perms.adm) / sizeof(long); i++) + snprintf(buf + 2 + 2 * i * sizeof(long), + PAGE_SIZE - 2 - 2 * i * sizeof(long), + "%016lx", zcdndev->perms.adm[i]); + buf[2 + 2 * i * sizeof(long)] = '\n'; + buf[2 + 2 * i * sizeof(long) + 1] = '\0'; + rc = 2 + 2 * i * sizeof(long) + 1; + + mutex_unlock(&ap_perms_mutex); + + return rc; +} + +static ssize_t admask_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + int rc; + struct zcdn_device *zcdndev = to_zcdn_dev(dev); + + rc = ap_parse_mask_str(buf, zcdndev->perms.adm, + AP_DOMAINS, &ap_perms_mutex); + if (rc) + return rc; + + return count; +} + +static DEVICE_ATTR_RW(admask); + static struct attribute *zcdn_dev_attrs[] = { &dev_attr_ioctlmask.attr, &dev_attr_apmask.attr, &dev_attr_aqmask.attr, + &dev_attr_admask.attr, NULL }; From cfd68b33094e1a92249850ff3c3c92ae9112a541 Mon Sep 17 00:00:00 2001 From: Juergen Christ Date: Wed, 9 Mar 2022 14:30:05 +0100 Subject: [PATCH 20/95] s390/zcrypt: Filter admin CPRBs on custom devices Add a filter for custom devices to check for allowed control domains of admin CPRBs. This filter only applies to custom devices and not to the main device. Signed-off-by: Juergen Christ Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/zcrypt_api.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 5cfe5cf44ba1..aa6dc3c0c353 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -923,11 +923,22 @@ static long _zcrypt_send_cprb(bool userspace, struct ap_perms *perms, if (rc) goto out; + tdom = *domain; + if (perms != &ap_perms && tdom < AP_DOMAINS) { + if (ap_msg.flags & AP_MSG_FLAG_ADMIN) { + if (!test_bit_inv(tdom, perms->adm)) { + rc = -ENODEV; + goto out; + } + } else if ((ap_msg.flags & AP_MSG_FLAG_USAGE) == 0) { + rc = -EOPNOTSUPP; + goto out; + } + } /* * If a valid target domain is set and this domain is NOT a usage * domain but a control only domain, autoselect target domain. */ - tdom = *domain; if (tdom < AP_DOMAINS && !ap_test_config_usage_domain(tdom) && ap_test_config_ctrl_domain(tdom)) @@ -1105,6 +1116,18 @@ static long _zcrypt_send_ep11_cprb(bool userspace, struct ap_perms *perms, if (rc) goto out_free; + if (perms != &ap_perms && domain < AUTOSEL_DOM) { + if (ap_msg.flags & AP_MSG_FLAG_ADMIN) { + if (!test_bit_inv(domain, perms->adm)) { + rc = -ENODEV; + goto out_free; + } + } else if ((ap_msg.flags & AP_MSG_FLAG_USAGE) == 0) { + rc = -EOPNOTSUPP; + goto out_free; + } + } + pref_zc = NULL; pref_zq = NULL; spin_lock(&zcrypt_list_lock); From f4272c03a32576de7bd1543bed764196d3d0dd2f Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Mon, 14 Mar 2022 12:53:50 +0100 Subject: [PATCH 21/95] s390/pkey: fix typos in comments Various spelling mistakes in comments. Detected with the help of Coccinelle. Signed-off-by: Julia Lawall Signed-off-by: Harald Freudenberger Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/pkey_api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index cf23ce1b1146..7f69ca695fc2 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -155,7 +155,7 @@ static int pkey_skey2pkey(const u8 *key, struct pkey_protkey *pkey) /* * The cca_xxx2protkey call may fail when a card has been * addressed where the master key was changed after last fetch - * of the mkvp into the cache. Try 3 times: First witout verify + * of the mkvp into the cache. Try 3 times: First without verify * then with verify and last round with verify and old master * key verification pattern match not ignored. */ From 0f210fb39e4a79a3687964c2502784db8075e5e8 Mon Sep 17 00:00:00 2001 From: Haowen Bai Date: Tue, 15 Mar 2022 16:01:04 +0800 Subject: [PATCH 22/95] s390: crypto: Use min_t() instead of doing it manually Fix following coccicheck warning: drivers/s390/crypto/zcrypt_ep11misc.c:1112:25-26: WARNING opportunity for min() Signed-off-by: Haowen Bai Signed-off-by: Harald Freudenberger Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/zcrypt_ep11misc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/crypto/zcrypt_ep11misc.c b/drivers/s390/crypto/zcrypt_ep11misc.c index 9ce5a71da69b..98d33f932b0b 100644 --- a/drivers/s390/crypto/zcrypt_ep11misc.c +++ b/drivers/s390/crypto/zcrypt_ep11misc.c @@ -1109,7 +1109,7 @@ static int ep11_wrapkey(u16 card, u16 domain, if (kb->head.type == TOKTYPE_NON_CCA && kb->head.version == TOKVER_EP11_AES) { has_header = true; - keysize = kb->head.len < keysize ? kb->head.len : keysize; + keysize = min_t(size_t, kb->head.len, keysize); } /* request cprb and payload */ From 6ffbeb3fac9d1518109b4adfbe605028883fb181 Mon Sep 17 00:00:00 2001 From: Russell Currey Date: Fri, 4 Mar 2022 19:01:09 +1000 Subject: [PATCH 23/95] s390: fix typo in syscall_wrapper.h Looks like this endif comment was erroneously unchanged when copied over from the x86 version. Signed-off-by: Russell Currey Link: https://lore.kernel.org/r/20220304090109.29386-1-ruscur@russell.cc Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/syscall_wrapper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/include/asm/syscall_wrapper.h b/arch/s390/include/asm/syscall_wrapper.h index ad2c996e7e93..fde7e6b1df48 100644 --- a/arch/s390/include/asm/syscall_wrapper.h +++ b/arch/s390/include/asm/syscall_wrapper.h @@ -162,4 +162,4 @@ __diag_pop(); \ static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)) -#endif /* _ASM_X86_SYSCALL_WRAPPER_H */ +#endif /* _ASM_S390_SYSCALL_WRAPPER_H */ From 081397d13f9cfcc10e5bc903559419f6bfc31cca Mon Sep 17 00:00:00 2001 From: Haowen Bai Date: Tue, 15 Mar 2022 16:53:41 +0800 Subject: [PATCH 24/95] s390/tape: use bitwise instead of arithmetic operator for flags This silences the following coccinelle warning: drivers/s390/char/tape_34xx.c:360:38-39: WARNING: sum of probable bitmasks, consider | Signed-off-by: Haowen Bai Link: https://lore.kernel.org/r/1647334421-29989-1-git-send-email-baihaowen@meizu.com Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- drivers/s390/char/tape_34xx.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/s390/char/tape_34xx.c b/drivers/s390/char/tape_34xx.c index 7ada994d4592..38cc1565d6ae 100644 --- a/drivers/s390/char/tape_34xx.c +++ b/drivers/s390/char/tape_34xx.c @@ -354,10 +354,10 @@ tape_34xx_unit_check(struct tape_device *device, struct tape_request *request, if (( sense[0] == SENSE_DATA_CHECK || sense[0] == SENSE_EQUIPMENT_CHECK || - sense[0] == SENSE_EQUIPMENT_CHECK + SENSE_DEFERRED_UNIT_CHECK + sense[0] == (SENSE_EQUIPMENT_CHECK | SENSE_DEFERRED_UNIT_CHECK) ) && ( sense[1] == SENSE_DRIVE_ONLINE || - sense[1] == SENSE_BEGINNING_OF_TAPE + SENSE_WRITE_MODE + sense[1] == (SENSE_BEGINNING_OF_TAPE | SENSE_WRITE_MODE) )) { switch (request->op) { /* From f09354ffd84eef3c88efa8ba6df05efe50cfd16a Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 16 Mar 2022 19:13:20 +0100 Subject: [PATCH 25/95] s390/traps: improve panic message for translation-specification exception There are many different types of translation exceptions but only a translation-specification exception leads to a kernel panic since it indicates corrupted page tables, which must never happen. Improve the panic message so it is a bit more obvious what this is about. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/kernel/traps.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 674c65019434..1d2aa448d103 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -141,10 +141,10 @@ static inline void do_fp_trap(struct pt_regs *regs, __u32 fpc) do_trap(regs, SIGFPE, si_code, "floating point exception"); } -static void translation_exception(struct pt_regs *regs) +static void translation_specification_exception(struct pt_regs *regs) { /* May never happen. */ - panic("Translation exception"); + panic("Translation-Specification Exception"); } static void illegal_op(struct pt_regs *regs) @@ -368,7 +368,7 @@ static void (*pgm_check_table[128])(struct pt_regs *regs) = { [0x0f] = hfp_divide_exception, [0x10] = do_dat_exception, [0x11] = do_dat_exception, - [0x12] = translation_exception, + [0x12] = translation_specification_exception, [0x13] = special_op_exception, [0x14] = default_trap_handler, [0x15] = operand_exception, From 4afeb670710efa5cd5ed8b1d9f2d22d6ce332bcc Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sun, 13 Mar 2022 22:04:10 +0100 Subject: [PATCH 26/95] s390/alternatives: use instructions instead of byte patterns Use readable nop instructions within the code which generates the padding areas, instead of unreadable byte patterns. The generated code is identical. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/alternative-asm.h | 12 +++++++++--- arch/s390/include/asm/alternative.h | 15 +++++++++++---- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/arch/s390/include/asm/alternative-asm.h b/arch/s390/include/asm/alternative-asm.h index 955d620db23e..221d790361b9 100644 --- a/arch/s390/include/asm/alternative-asm.h +++ b/arch/s390/include/asm/alternative-asm.h @@ -37,9 +37,15 @@ * a 2-byte nop if the size of the area is not divisible by 6. */ .macro alt_pad_fill bytes - .fill ( \bytes ) / 6, 6, 0xc0040000 - .fill ( \bytes ) % 6 / 4, 4, 0x47000000 - .fill ( \bytes ) % 6 % 4 / 2, 2, 0x0700 + .rept ( \bytes ) / 6 + jgnop . + .endr + .rept ( \bytes ) % 6 / 4 + nop + .endr + .rept ( \bytes ) % 6 % 4 / 2 + nopr + .endr .endm /* diff --git a/arch/s390/include/asm/alternative.h b/arch/s390/include/asm/alternative.h index d3880ca764ee..6c72495c8dbb 100644 --- a/arch/s390/include/asm/alternative.h +++ b/arch/s390/include/asm/alternative.h @@ -71,11 +71,18 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end); ".if " oldinstr_pad_len(num) " > 6\n" \ "\tjg " e_oldinstr_pad_end "f\n" \ "6620:\n" \ - "\t.fill (" oldinstr_pad_len(num) " - (6620b-662b)) / 2, 2, 0x0700\n" \ + "\t.rept (" oldinstr_pad_len(num) " - (6620b-662b)) / 2\n" \ + "\tnopr\n" \ ".else\n" \ - "\t.fill " oldinstr_pad_len(num) " / 6, 6, 0xc0040000\n" \ - "\t.fill " oldinstr_pad_len(num) " %% 6 / 4, 4, 0x47000000\n" \ - "\t.fill " oldinstr_pad_len(num) " %% 6 %% 4 / 2, 2, 0x0700\n" \ + "\t.rept " oldinstr_pad_len(num) " / 6\n" \ + "\t.jgnop .\n" \ + "\t.endr\n" \ + "\t.rept " oldinstr_pad_len(num) " %% 6 / 4\n" \ + "\tnop\n" \ + "\t.endr\n" \ + "\t.rept " oldinstr_pad_len(num) " %% 6 %% 4 / 2\n" \ + "\tnopr\n" \ + ".endr\n" \ ".endif\n" #define OLDINSTR(oldinstr, num) \ From 6982dba181deba54c5ccb301aaed6f6ec14c6310 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sun, 13 Mar 2022 22:39:27 +0100 Subject: [PATCH 27/95] s390/alternatives: use insn format for new instructions Use insn format with instruction format specifier instead of plain longs. This way it is also more obvious that code instead of data is generated. The generated code is identical. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/spinlock.h | 2 +- arch/s390/kernel/entry.S | 10 +++++----- arch/s390/lib/spinlock.c | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/s390/include/asm/spinlock.h b/arch/s390/include/asm/spinlock.h index 888a2f1c9ee3..24a54443c865 100644 --- a/arch/s390/include/asm/spinlock.h +++ b/arch/s390/include/asm/spinlock.h @@ -78,7 +78,7 @@ static inline void arch_spin_unlock(arch_spinlock_t *lp) { typecheck(int, lp->lock); asm_inline volatile( - ALTERNATIVE("", ".long 0xb2fa0070", 49) /* NIAI 7 */ + ALTERNATIVE("", ".insn rre,0xb2fa0000,7,0", 49) /* NIAI 7 */ " sth %1,%0\n" : "=R" (((unsigned short *) &lp->lock)[1]) : "d" (0) : "cc", "memory"); diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index a601a518b569..59b69c8ab5e1 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -121,22 +121,22 @@ _LPP_OFFSET = __LC_LPP .endm .macro BPOFF - ALTERNATIVE "", ".long 0xb2e8c000", 82 + ALTERNATIVE "", ".insn rrf,0xb2e80000,0,0,12,0", 82 .endm .macro BPON - ALTERNATIVE "", ".long 0xb2e8d000", 82 + ALTERNATIVE "", ".insn rrf,0xb2e80000,0,0,13,0", 82 .endm .macro BPENTER tif_ptr,tif_mask - ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .long 0xb2e8d000", \ + ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .insn rrf,0xb2e80000,0,0,13,0", \ "", 82 .endm .macro BPEXIT tif_ptr,tif_mask TSTMSK \tif_ptr,\tif_mask - ALTERNATIVE "jz .+8; .long 0xb2e8c000", \ - "jnz .+8; .long 0xb2e8d000", 82 + ALTERNATIVE "jz .+8; .insn rrf,0xb2e80000,0,0,12,0", \ + "jnz .+8; .insn rrf,0xb2e80000,0,0,13,0", 82 .endm /* diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c index 692dc84cd19c..5e7ea8b111e8 100644 --- a/arch/s390/lib/spinlock.c +++ b/arch/s390/lib/spinlock.c @@ -75,7 +75,7 @@ static inline int arch_load_niai4(int *lock) int owner; asm_inline volatile( - ALTERNATIVE("", ".long 0xb2fa0040", 49) /* NIAI 4 */ + ALTERNATIVE("", ".insn rre,0xb2fa0000,4,0", 49) /* NIAI 4 */ " l %0,%1\n" : "=d" (owner) : "Q" (*lock) : "memory"); return owner; @@ -86,7 +86,7 @@ static inline int arch_cmpxchg_niai8(int *lock, int old, int new) int expected = old; asm_inline volatile( - ALTERNATIVE("", ".long 0xb2fa0080", 49) /* NIAI 8 */ + ALTERNATIVE("", ".insn rre,0xb2fa0000,8,0", 49) /* NIAI 8 */ " cs %0,%3,%1\n" : "=d" (old), "=Q" (*lock) : "0" (old), "d" (new), "Q" (*lock) From 2d6c0008be64bd813008d2a796108e89edec1030 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 14 Mar 2022 20:45:32 +0100 Subject: [PATCH 28/95] s390/ap: use insn format for new instructions Use insn format with instruction format specifier instead of plain longs. This way it is also more obvious that code instead of data is generated. The generated code is identical. Reviewed-by: Harald Freudenberger Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ap.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/s390/include/asm/ap.h b/arch/s390/include/asm/ap.h index ae75da592ccb..628bcc0fe84b 100644 --- a/arch/s390/include/asm/ap.h +++ b/arch/s390/include/asm/ap.h @@ -63,7 +63,7 @@ static inline bool ap_instructions_available(void) " lgr 0,%[reg0]\n" /* qid into gr0 */ " lghi 1,0\n" /* 0 into gr1 */ " lghi 2,0\n" /* 0 into gr2 */ - " .long 0xb2af0000\n" /* PQAP(TAPQ) */ + " .insn rre,0xb2af0000,0,0\n" /* PQAP(TAPQ) */ "0: la %[reg1],1\n" /* 1 into reg1 */ "1:\n" EX_TABLE(0b, 1b) @@ -88,7 +88,7 @@ static inline struct ap_queue_status ap_tapq(ap_qid_t qid, unsigned long *info) asm volatile( " lgr 0,%[qid]\n" /* qid into gr0 */ " lghi 2,0\n" /* 0 into gr2 */ - " .long 0xb2af0000\n" /* PQAP(TAPQ) */ + " .insn rre,0xb2af0000,0,0\n" /* PQAP(TAPQ) */ " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ " lgr %[reg2],2\n" /* gr2 into reg2 */ : [reg1] "=&d" (reg1), [reg2] "=&d" (reg2) @@ -129,7 +129,7 @@ static inline struct ap_queue_status ap_rapq(ap_qid_t qid) asm volatile( " lgr 0,%[reg0]\n" /* qid arg into gr0 */ - " .long 0xb2af0000\n" /* PQAP(RAPQ) */ + " .insn rre,0xb2af0000,0,0\n" /* PQAP(RAPQ) */ " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ : [reg1] "=&d" (reg1) : [reg0] "d" (reg0) @@ -150,7 +150,7 @@ static inline struct ap_queue_status ap_zapq(ap_qid_t qid) asm volatile( " lgr 0,%[reg0]\n" /* qid arg into gr0 */ - " .long 0xb2af0000\n" /* PQAP(ZAPQ) */ + " .insn rre,0xb2af0000,0,0\n" /* PQAP(ZAPQ) */ " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ : [reg1] "=&d" (reg1) : [reg0] "d" (reg0) @@ -192,7 +192,7 @@ static inline int ap_qci(struct ap_config_info *config) asm volatile( " lgr 0,%[reg0]\n" /* QCI fc into gr0 */ " lgr 2,%[reg2]\n" /* ptr to config into gr2 */ - " .long 0xb2af0000\n" /* PQAP(QCI) */ + " .insn rre,0xb2af0000,0,0\n" /* PQAP(QCI) */ "0: la %[reg1],0\n" /* good case, QCI fc available */ "1:\n" EX_TABLE(0b, 1b) @@ -249,7 +249,7 @@ static inline struct ap_queue_status ap_aqic(ap_qid_t qid, " lgr 0,%[reg0]\n" /* qid param into gr0 */ " lgr 1,%[reg1]\n" /* irq ctrl into gr1 */ " lgr 2,%[reg2]\n" /* ni addr into gr2 */ - " .long 0xb2af0000\n" /* PQAP(AQIC) */ + " .insn rre,0xb2af0000,0,0\n" /* PQAP(AQIC) */ " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ : [reg1] "+&d" (reg1) : [reg0] "d" (reg0), [reg2] "d" (reg2) @@ -299,7 +299,7 @@ static inline struct ap_queue_status ap_qact(ap_qid_t qid, int ifbit, asm volatile( " lgr 0,%[reg0]\n" /* qid param into gr0 */ " lgr 1,%[reg1]\n" /* qact in info into gr1 */ - " .long 0xb2af0000\n" /* PQAP(QACT) */ + " .insn rre,0xb2af0000,0,0\n" /* PQAP(QACT) */ " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ " lgr %[reg2],2\n" /* qact out info into reg2 */ : [reg1] "+&d" (reg1), [reg2] "=&d" (reg2) From a7e196f579cdcc9cf723d5b1ea05575de2fe72f6 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 14 Mar 2022 20:48:57 +0100 Subject: [PATCH 29/95] s390/ap: adjust whitespace Adjust indentation of inline assemblies, so all comments start at the same position. Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ap.h | 46 +++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/arch/s390/include/asm/ap.h b/arch/s390/include/asm/ap.h index 628bcc0fe84b..b515cfa62bd9 100644 --- a/arch/s390/include/asm/ap.h +++ b/arch/s390/include/asm/ap.h @@ -60,11 +60,11 @@ static inline bool ap_instructions_available(void) unsigned long reg1 = 0; asm volatile( - " lgr 0,%[reg0]\n" /* qid into gr0 */ - " lghi 1,0\n" /* 0 into gr1 */ - " lghi 2,0\n" /* 0 into gr2 */ + " lgr 0,%[reg0]\n" /* qid into gr0 */ + " lghi 1,0\n" /* 0 into gr1 */ + " lghi 2,0\n" /* 0 into gr2 */ " .insn rre,0xb2af0000,0,0\n" /* PQAP(TAPQ) */ - "0: la %[reg1],1\n" /* 1 into reg1 */ + "0: la %[reg1],1\n" /* 1 into reg1 */ "1:\n" EX_TABLE(0b, 1b) : [reg1] "+&d" (reg1) @@ -86,11 +86,11 @@ static inline struct ap_queue_status ap_tapq(ap_qid_t qid, unsigned long *info) unsigned long reg2; asm volatile( - " lgr 0,%[qid]\n" /* qid into gr0 */ - " lghi 2,0\n" /* 0 into gr2 */ + " lgr 0,%[qid]\n" /* qid into gr0 */ + " lghi 2,0\n" /* 0 into gr2 */ " .insn rre,0xb2af0000,0,0\n" /* PQAP(TAPQ) */ - " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ - " lgr %[reg2],2\n" /* gr2 into reg2 */ + " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ + " lgr %[reg2],2\n" /* gr2 into reg2 */ : [reg1] "=&d" (reg1), [reg2] "=&d" (reg2) : [qid] "d" (qid) : "cc", "0", "1", "2"); @@ -128,9 +128,9 @@ static inline struct ap_queue_status ap_rapq(ap_qid_t qid) struct ap_queue_status reg1; asm volatile( - " lgr 0,%[reg0]\n" /* qid arg into gr0 */ + " lgr 0,%[reg0]\n" /* qid arg into gr0 */ " .insn rre,0xb2af0000,0,0\n" /* PQAP(RAPQ) */ - " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ + " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ : [reg1] "=&d" (reg1) : [reg0] "d" (reg0) : "cc", "0", "1"); @@ -149,9 +149,9 @@ static inline struct ap_queue_status ap_zapq(ap_qid_t qid) struct ap_queue_status reg1; asm volatile( - " lgr 0,%[reg0]\n" /* qid arg into gr0 */ + " lgr 0,%[reg0]\n" /* qid arg into gr0 */ " .insn rre,0xb2af0000,0,0\n" /* PQAP(ZAPQ) */ - " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ + " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ : [reg1] "=&d" (reg1) : [reg0] "d" (reg0) : "cc", "0", "1"); @@ -190,10 +190,10 @@ static inline int ap_qci(struct ap_config_info *config) struct ap_config_info *reg2 = config; asm volatile( - " lgr 0,%[reg0]\n" /* QCI fc into gr0 */ - " lgr 2,%[reg2]\n" /* ptr to config into gr2 */ + " lgr 0,%[reg0]\n" /* QCI fc into gr0 */ + " lgr 2,%[reg2]\n" /* ptr to config into gr2 */ " .insn rre,0xb2af0000,0,0\n" /* PQAP(QCI) */ - "0: la %[reg1],0\n" /* good case, QCI fc available */ + "0: la %[reg1],0\n" /* good case, QCI fc available */ "1:\n" EX_TABLE(0b, 1b) : [reg1] "+&d" (reg1) @@ -246,11 +246,11 @@ static inline struct ap_queue_status ap_aqic(ap_qid_t qid, reg1.qirqctrl = qirqctrl; asm volatile( - " lgr 0,%[reg0]\n" /* qid param into gr0 */ - " lgr 1,%[reg1]\n" /* irq ctrl into gr1 */ - " lgr 2,%[reg2]\n" /* ni addr into gr2 */ + " lgr 0,%[reg0]\n" /* qid param into gr0 */ + " lgr 1,%[reg1]\n" /* irq ctrl into gr1 */ + " lgr 2,%[reg2]\n" /* ni addr into gr2 */ " .insn rre,0xb2af0000,0,0\n" /* PQAP(AQIC) */ - " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ + " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ : [reg1] "+&d" (reg1) : [reg0] "d" (reg0), [reg2] "d" (reg2) : "cc", "0", "1", "2"); @@ -297,11 +297,11 @@ static inline struct ap_queue_status ap_qact(ap_qid_t qid, int ifbit, reg1.value = apinfo->val; asm volatile( - " lgr 0,%[reg0]\n" /* qid param into gr0 */ - " lgr 1,%[reg1]\n" /* qact in info into gr1 */ + " lgr 0,%[reg0]\n" /* qid param into gr0 */ + " lgr 1,%[reg1]\n" /* qact in info into gr1 */ " .insn rre,0xb2af0000,0,0\n" /* PQAP(QACT) */ - " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ - " lgr %[reg2],2\n" /* qact out info into reg2 */ + " lgr %[reg1],1\n" /* gr1 (status) into reg1 */ + " lgr %[reg2],2\n" /* qact out info into reg2 */ : [reg1] "+&d" (reg1), [reg2] "=&d" (reg2) : [reg0] "d" (reg0) : "cc", "0", "1", "2"); From 0b452520477efbe4370cb747572c3c63c7cb3176 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Sat, 5 Mar 2022 18:57:44 +0100 Subject: [PATCH 30/95] s390/test_unwind: extend kretprobe test Verify unwinding from kretprobed function. Reviewed-by: Tobias Huschle Signed-off-by: Vasily Gorbik --- arch/s390/lib/test_unwind.c | 49 +++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/arch/s390/lib/test_unwind.c b/arch/s390/lib/test_unwind.c index c01f02887de4..468e1ce66a33 100644 --- a/arch/s390/lib/test_unwind.c +++ b/arch/s390/lib/test_unwind.c @@ -132,36 +132,50 @@ static struct unwindme *unwindme; #define UWM_PGM 0x40 /* Unwind from program check handler */ #define UWM_KPROBE_ON_FTRACE 0x80 /* Unwind from kprobe handler called via ftrace. */ #define UWM_FTRACE 0x100 /* Unwind from ftrace handler. */ -#define UWM_KRETPROBE 0x200 /* Unwind kretprobe handlers. */ +#define UWM_KRETPROBE 0x200 /* Unwind through kretprobed function. */ +#define UWM_KRETPROBE_HANDLER 0x400 /* Unwind from kretprobe handler. */ -static __always_inline unsigned long get_psw_addr(void) +static __always_inline struct pt_regs fake_pt_regs(void) { - unsigned long psw_addr; + struct pt_regs regs; + + memset(®s, 0, sizeof(regs)); + regs.gprs[15] = current_stack_pointer(); asm volatile( "basr %[psw_addr],0\n" - : [psw_addr] "=d" (psw_addr)); - return psw_addr; + : [psw_addr] "=d" (regs.psw.addr)); + return regs; } static int kretprobe_ret_handler(struct kretprobe_instance *ri, struct pt_regs *regs) { struct unwindme *u = unwindme; + if (!(u->flags & UWM_KRETPROBE_HANDLER)) + return 0; + u->ret = test_unwind(NULL, (u->flags & UWM_REGS) ? regs : NULL, (u->flags & UWM_SP) ? u->sp : 0); return 0; } -static noinline notrace void test_unwind_kretprobed_func(void) +static noinline notrace int test_unwind_kretprobed_func(struct unwindme *u) { - asm volatile(" nop\n"); + struct pt_regs regs; + + if (!(u->flags & UWM_KRETPROBE)) + return 0; + + regs = fake_pt_regs(); + return test_unwind(NULL, (u->flags & UWM_REGS) ? ®s : NULL, + (u->flags & UWM_SP) ? u->sp : 0); } -static noinline void test_unwind_kretprobed_func_caller(void) +static noinline int test_unwind_kretprobed_func_caller(struct unwindme *u) { - test_unwind_kretprobed_func(); + return test_unwind_kretprobed_func(u); } static int test_unwind_kretprobe(struct unwindme *u) @@ -187,10 +201,12 @@ static int test_unwind_kretprobe(struct unwindme *u) return -EINVAL; } - test_unwind_kretprobed_func_caller(); + ret = test_unwind_kretprobed_func_caller(u); unregister_kretprobe(&my_kretprobe); unwindme = NULL; - return u->ret; + if (u->flags & UWM_KRETPROBE_HANDLER) + ret = u->ret; + return ret; } static int kprobe_pre_handler(struct kprobe *p, struct pt_regs *regs) @@ -304,16 +320,13 @@ static noinline int unwindme_func4(struct unwindme *u) return 0; } else if (u->flags & (UWM_PGM | UWM_KPROBE_ON_FTRACE)) { return test_unwind_kprobe(u); - } else if (u->flags & (UWM_KRETPROBE)) { + } else if (u->flags & (UWM_KRETPROBE | UWM_KRETPROBE_HANDLER)) { return test_unwind_kretprobe(u); } else if (u->flags & UWM_FTRACE) { return test_unwind_ftrace(u); } else { - struct pt_regs regs; + struct pt_regs regs = fake_pt_regs(); - memset(®s, 0, sizeof(regs)); - regs.psw.addr = get_psw_addr(); - regs.gprs[15] = current_stack_pointer(); return test_unwind(NULL, (u->flags & UWM_REGS) ? ®s : NULL, (u->flags & UWM_SP) ? u->sp : 0); @@ -452,6 +465,10 @@ static const struct test_params param_list[] = { TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_SP), TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_REGS), TEST_WITH_FLAGS(UWM_KRETPROBE | UWM_SP | UWM_REGS), + TEST_WITH_FLAGS(UWM_KRETPROBE_HANDLER), + TEST_WITH_FLAGS(UWM_KRETPROBE_HANDLER | UWM_SP), + TEST_WITH_FLAGS(UWM_KRETPROBE_HANDLER | UWM_REGS), + TEST_WITH_FLAGS(UWM_KRETPROBE_HANDLER | UWM_SP | UWM_REGS), }; /* From 09bc20c8fb35cf1afed1612b287e9ddbe6a7d73c Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Sat, 5 Mar 2022 15:02:07 +0100 Subject: [PATCH 31/95] s390/kprobes: enable kretprobes framepointer verification Use regs->gprs[15] for framepointer verification. This enables additional sanity checks for nested kretprobes. Reviewed-by: Tobias Huschle Signed-off-by: Vasily Gorbik --- arch/s390/kernel/kprobes.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index e32c14fd1282..0032bdbe8e3f 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -284,11 +284,11 @@ NOKPROBE_SYMBOL(pop_kprobe); void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs) { - ri->ret_addr = (kprobe_opcode_t *) regs->gprs[14]; - ri->fp = NULL; + ri->ret_addr = (kprobe_opcode_t *)regs->gprs[14]; + ri->fp = (void *)regs->gprs[15]; /* Replace the return addr with trampoline addr */ - regs->gprs[14] = (unsigned long) &__kretprobe_trampoline; + regs->gprs[14] = (unsigned long)&__kretprobe_trampoline; } NOKPROBE_SYMBOL(arch_prepare_kretprobe); @@ -385,7 +385,7 @@ NOKPROBE_SYMBOL(arch_kretprobe_fixup_return); */ void trampoline_probe_handler(struct pt_regs *regs) { - kretprobe_trampoline_handler(regs, NULL); + kretprobe_trampoline_handler(regs, (void *)regs->gprs[15]); } NOKPROBE_SYMBOL(trampoline_probe_handler); From d81675b60d0959cfa3727f03d5b90558fb457011 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Wed, 28 Apr 2021 16:29:40 +0200 Subject: [PATCH 32/95] s390/unwind: recover kretprobe modified return address in stacktrace Based on commit cd9bc2c92588 ("arm64: Recover kretprobe modified return address in stacktrace"). """ Since the kretprobe replaces the function return address with the __kretprobe_trampoline on the stack, stack unwinder shows it instead of the correct return address. This checks whether the next return address is the __kretprobe_trampoline(), and if so, try to find the correct return address from the kretprobe instance list. """ Original patch series: https://lore.kernel.org/all/163163030719.489837.2236069935502195491.stgit@devnote2/ Reviewed-by: Tobias Huschle Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/unwind.h | 13 +++++++++++++ arch/s390/kernel/unwind_bc.c | 8 ++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/arch/s390/include/asm/unwind.h b/arch/s390/include/asm/unwind.h index 5ebf534ef753..0bf06f1682d8 100644 --- a/arch/s390/include/asm/unwind.h +++ b/arch/s390/include/asm/unwind.h @@ -4,6 +4,8 @@ #include #include +#include +#include #include #include @@ -36,10 +38,21 @@ struct unwind_state { struct pt_regs *regs; unsigned long sp, ip; int graph_idx; + struct llist_node *kr_cur; bool reliable; bool error; }; +/* Recover the return address modified by kretprobe and ftrace_graph. */ +static inline unsigned long unwind_recover_ret_addr(struct unwind_state *state, + unsigned long ip) +{ + ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, ip, NULL); + if (is_kretprobe_trampoline(ip)) + ip = kretprobe_find_ret_addr(state->task, (void *)state->sp, &state->kr_cur); + return ip; +} + void __unwind_start(struct unwind_state *state, struct task_struct *task, struct pt_regs *regs, unsigned long first_frame); bool unwind_next_frame(struct unwind_state *state); diff --git a/arch/s390/kernel/unwind_bc.c b/arch/s390/kernel/unwind_bc.c index 707fd99f6734..984970389bab 100644 --- a/arch/s390/kernel/unwind_bc.c +++ b/arch/s390/kernel/unwind_bc.c @@ -103,13 +103,11 @@ bool unwind_next_frame(struct unwind_state *state) if (sp & 0x7) goto out_err; - ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, ip, (void *) sp); - /* Update unwind state */ state->sp = sp; - state->ip = ip; state->regs = regs; state->reliable = reliable; + state->ip = unwind_recover_ret_addr(state, ip); return true; out_err: @@ -161,12 +159,10 @@ void __unwind_start(struct unwind_state *state, struct task_struct *task, ip = READ_ONCE_NOCHECK(sf->gprs[8]); } - ip = ftrace_graph_ret_addr(state->task, &state->graph_idx, ip, NULL); - /* Update unwind state */ state->sp = sp; - state->ip = ip; state->reliable = true; + state->ip = unwind_recover_ret_addr(state, ip); if (!first_frame) return; From 708b13763920ab3c191b31da555f2d90eef4c618 Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Sat, 5 Mar 2022 13:16:07 +0100 Subject: [PATCH 33/95] s390/unwind: avoid duplicated unwinding entries for kretprobes Currently when unwinding starts from pt_regs or encounters pt_regs along the way unwinder tries to yield 2 unwinding entries: 1. (reliable) ip1: pt_regs->psw.addr, sp1: regs->gprs[15] 2. (non-reliable) ip2: sp1->gprs[8] (r14), sp2: regs->gprs[15] In case of kretprobes those are identical and serves no other purpose than causing confusion over duplicated entries and cause kprobes tests to fail. So, skip a duplicate non-reliable entry in this case. With that kretprobes and unwinder implementation now comply with ARCH_CORRECT_STACKTRACE_ON_KRETPROBE. Reviewed-by: Tobias Huschle Signed-off-by: Vasily Gorbik --- arch/s390/Kconfig | 1 + arch/s390/kernel/unwind_bc.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 9b80e8bed3f6..77b5a03de13a 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -58,6 +58,7 @@ config S390 select ALTERNATE_USER_ADDRESS_SPACE select ARCH_32BIT_USTAT_F_TINODE select ARCH_BINFMT_ELF_STATE + select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM select ARCH_ENABLE_MEMORY_HOTREMOVE select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 diff --git a/arch/s390/kernel/unwind_bc.c b/arch/s390/kernel/unwind_bc.c index 984970389bab..0ece156fdd7c 100644 --- a/arch/s390/kernel/unwind_bc.c +++ b/arch/s390/kernel/unwind_bc.c @@ -64,8 +64,8 @@ bool unwind_next_frame(struct unwind_state *state) ip = READ_ONCE_NOCHECK(sf->gprs[8]); reliable = false; regs = NULL; - if (!__kernel_text_address(ip)) { - /* skip bogus %r14 */ + /* skip bogus %r14 or if is the same as regs->psw.addr */ + if (!__kernel_text_address(ip) || state->ip == unwind_recover_ret_addr(state, ip)) { state->regs = NULL; return unwind_next_frame(state); } From afacda5faabdfe0f0e91e89db3ff1dc0b46f669a Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Sat, 5 Mar 2022 19:10:49 +0100 Subject: [PATCH 34/95] s390/test_unwind: verify __kretprobe_trampoline is replaced Reviewed-by: Tobias Huschle Signed-off-by: Vasily Gorbik --- arch/s390/lib/test_unwind.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/s390/lib/test_unwind.c b/arch/s390/lib/test_unwind.c index 468e1ce66a33..9bb067321ab4 100644 --- a/arch/s390/lib/test_unwind.c +++ b/arch/s390/lib/test_unwind.c @@ -47,7 +47,7 @@ static void print_backtrace(char *bt) static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs, unsigned long sp) { - int frame_count, prev_is_func2, seen_func2_func1; + int frame_count, prev_is_func2, seen_func2_func1, seen_kretprobe_trampoline; const int max_frames = 128; struct unwind_state state; size_t bt_pos = 0; @@ -63,6 +63,7 @@ static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs, frame_count = 0; prev_is_func2 = 0; seen_func2_func1 = 0; + seen_kretprobe_trampoline = 0; unwind_for_each_frame(&state, task, regs, sp) { unsigned long addr = unwind_get_return_address(&state); char sym[KSYM_SYMBOL_LEN]; @@ -88,6 +89,8 @@ static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs, if (prev_is_func2 && str_has_prefix(sym, "unwindme_func1")) seen_func2_func1 = 1; prev_is_func2 = str_has_prefix(sym, "unwindme_func2"); + if (str_has_prefix(sym, "__kretprobe_trampoline+0x0/")) + seen_kretprobe_trampoline = 1; } /* Check the results. */ @@ -103,6 +106,10 @@ static noinline int test_unwind(struct task_struct *task, struct pt_regs *regs, kunit_err(current_test, "Maximum number of frames exceeded\n"); ret = -EINVAL; } + if (seen_kretprobe_trampoline) { + kunit_err(current_test, "__kretprobe_trampoline+0x0 in unwinding results\n"); + ret = -EINVAL; + } if (ret || force_bt) print_backtrace(bt); kfree(bt); From dc2ab23b992c9d5dab93b9bf01b10b10465e537e Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Thu, 17 Mar 2022 15:03:01 +0100 Subject: [PATCH 35/95] s390/smp: cleanup target CPU callback starting Macro mem_assign_absolute() is used to initialize a target CPU lowcore callback parameters. But despite the macro name it writes to the absolute lowcore only if the target CPU is offline. In case the CPU is online the macro does implicitly write to the normal memory. That behaviour is correct, but extremely subtle. Sacrifice few program bits in favour of clarity and distinguish between online vs offline CPUs and normal vs absolute lowcore pointer. Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/smp.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 127da1850b06..b7e56fabe387 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -326,10 +326,17 @@ static void pcpu_delegate(struct pcpu *pcpu, /* Stop target cpu (if func returns this stops the current cpu). */ pcpu_sigp_retry(pcpu, SIGP_STOP, 0); /* Restart func on the target cpu and stop the current cpu. */ - mem_assign_absolute(lc->restart_stack, stack); - mem_assign_absolute(lc->restart_fn, (unsigned long) func); - mem_assign_absolute(lc->restart_data, (unsigned long) data); - mem_assign_absolute(lc->restart_source, source_cpu); + if (lc) { + lc->restart_stack = stack; + lc->restart_fn = (unsigned long)func; + lc->restart_data = (unsigned long)data; + lc->restart_source = source_cpu; + } else { + mem_assign_absolute(lc->restart_stack, stack); + mem_assign_absolute(lc->restart_fn, (unsigned long)func); + mem_assign_absolute(lc->restart_data, (unsigned long)data); + mem_assign_absolute(lc->restart_source, source_cpu); + } __bpon(); asm volatile( "0: sigp 0,%0,%2 # sigp restart to target cpu\n" From 9097fc793f74ef9c677f8c4aed0c24f6f07f0133 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Thu, 17 Mar 2022 15:03:02 +0100 Subject: [PATCH 36/95] s390/smp: cleanup control register update routines Get rid of duplicate code and redundant data. Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/ctl_reg.h | 16 ++++++++++----- arch/s390/kernel/smp.c | 36 +++++++++++---------------------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/arch/s390/include/asm/ctl_reg.h b/arch/s390/include/asm/ctl_reg.h index c800199a376b..82388da3f95f 100644 --- a/arch/s390/include/asm/ctl_reg.h +++ b/arch/s390/include/asm/ctl_reg.h @@ -74,8 +74,17 @@ static __always_inline void __ctl_clear_bit(unsigned int cr, unsigned int bit) __ctl_load(reg, cr, cr); } -void smp_ctl_set_bit(int cr, int bit); -void smp_ctl_clear_bit(int cr, int bit); +void smp_ctl_set_clear_bit(int cr, int bit, bool set); + +static inline void ctl_set_bit(int cr, int bit) +{ + smp_ctl_set_clear_bit(cr, bit, true); +} + +static inline void ctl_clear_bit(int cr, int bit) +{ + smp_ctl_set_clear_bit(cr, bit, false); +} union ctlreg0 { unsigned long val; @@ -130,8 +139,5 @@ union ctlreg15 { }; }; -#define ctl_set_bit(cr, bit) smp_ctl_set_bit(cr, bit) -#define ctl_clear_bit(cr, bit) smp_ctl_clear_bit(cr, bit) - #endif /* __ASSEMBLY__ */ #endif /* __ASM_CTL_REG_H */ diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index b7e56fabe387..cc6971a09aaf 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -577,39 +577,27 @@ static void smp_ctl_bit_callback(void *info) } static DEFINE_SPINLOCK(ctl_lock); -static unsigned long ctlreg; -/* - * Set a bit in a control register of all cpus - */ -void smp_ctl_set_bit(int cr, int bit) +void smp_ctl_set_clear_bit(int cr, int bit, bool set) { - struct ec_creg_mask_parms parms = { 1UL << bit, -1UL, cr }; + struct ec_creg_mask_parms parms = { .cr = cr, }; + u64 ctlreg; + if (set) { + parms.orval = 1UL << bit; + parms.andval = -1UL; + } else { + parms.orval = 0; + parms.andval = ~(1UL << bit); + } spin_lock(&ctl_lock); memcpy_absolute(&ctlreg, &S390_lowcore.cregs_save_area[cr], sizeof(ctlreg)); - __set_bit(bit, &ctlreg); + ctlreg = (ctlreg & parms.andval) | parms.orval; memcpy_absolute(&S390_lowcore.cregs_save_area[cr], &ctlreg, sizeof(ctlreg)); spin_unlock(&ctl_lock); on_each_cpu(smp_ctl_bit_callback, &parms, 1); } -EXPORT_SYMBOL(smp_ctl_set_bit); - -/* - * Clear a bit in a control register of all cpus - */ -void smp_ctl_clear_bit(int cr, int bit) -{ - struct ec_creg_mask_parms parms = { 0, ~(1UL << bit), cr }; - - spin_lock(&ctl_lock); - memcpy_absolute(&ctlreg, &S390_lowcore.cregs_save_area[cr], sizeof(ctlreg)); - __clear_bit(bit, &ctlreg); - memcpy_absolute(&S390_lowcore.cregs_save_area[cr], &ctlreg, sizeof(ctlreg)); - spin_unlock(&ctl_lock); - on_each_cpu(smp_ctl_bit_callback, &parms, 1); -} -EXPORT_SYMBOL(smp_ctl_clear_bit); +EXPORT_SYMBOL(smp_ctl_set_clear_bit); #ifdef CONFIG_CRASH_DUMP From ed0192bc644f3553d64a5cb461bdd0b1fbae3fdf Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Thu, 17 Mar 2022 15:03:03 +0100 Subject: [PATCH 37/95] s390/maccess: rework absolute lowcore accessors Macro mem_assign_absolute() is able to access the whole memory, but is only used and makes sense when updating the absolute lowcore. Instead, introduce get_abs_lowcore() and put_abs_lowcore() macros that limit access to absolute lowcore addresses only. Suggested-by: Heiko Carstens Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/processor.h | 17 ++++++++++++----- arch/s390/kernel/ipl.c | 4 ++-- arch/s390/kernel/machine_kexec.c | 2 +- arch/s390/kernel/os_info.c | 2 +- arch/s390/kernel/setup.c | 19 ++++++++++--------- arch/s390/kernel/smp.c | 12 ++++++------ 6 files changed, 32 insertions(+), 24 deletions(-) diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 84ec63145325..eee8d96fb38e 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -319,11 +319,18 @@ extern void (*s390_base_pgm_handler_fn)(struct pt_regs *regs); extern int memcpy_real(void *, unsigned long, size_t); extern void memcpy_absolute(void *, void *, size_t); -#define mem_assign_absolute(dest, val) do { \ - __typeof__(dest) __tmp = (val); \ - \ - BUILD_BUG_ON(sizeof(__tmp) != sizeof(val)); \ - memcpy_absolute(&(dest), &__tmp, sizeof(__tmp)); \ +#define put_abs_lowcore(member, x) do { \ + unsigned long __abs_address = offsetof(struct lowcore, member); \ + __typeof__(((struct lowcore *)0)->member) __tmp = (x); \ + \ + memcpy_absolute(__va(__abs_address), &__tmp, sizeof(__tmp)); \ +} while (0) + +#define get_abs_lowcore(x, member) do { \ + unsigned long __abs_address = offsetof(struct lowcore, member); \ + __typeof__(((struct lowcore *)0)->member) *__ptr = &(x); \ + \ + memcpy_absolute(__ptr, __va(__abs_address), sizeof(*__ptr)); \ } while (0) extern int s390_isolate_bp(void); diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 28ae7df26c4a..1cc85b8ff42e 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -1646,8 +1646,8 @@ static void dump_reipl_run(struct shutdown_trigger *trigger) csum = (__force unsigned int) csum_partial(reipl_block_actual, reipl_block_actual->hdr.len, 0); - mem_assign_absolute(S390_lowcore.ipib, ipib); - mem_assign_absolute(S390_lowcore.ipib_checksum, csum); + put_abs_lowcore(ipib, ipib); + put_abs_lowcore(ipib_checksum, csum); dump_run(trigger); } diff --git a/arch/s390/kernel/machine_kexec.c b/arch/s390/kernel/machine_kexec.c index 088d57a3083f..b2ef014a9287 100644 --- a/arch/s390/kernel/machine_kexec.c +++ b/arch/s390/kernel/machine_kexec.c @@ -226,7 +226,7 @@ void arch_crash_save_vmcoreinfo(void) vmcoreinfo_append_str("SAMODE31=%lx\n", __samode31); vmcoreinfo_append_str("EAMODE31=%lx\n", __eamode31); vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset()); - mem_assign_absolute(S390_lowcore.vmcore_info, paddr_vmcoreinfo_note()); + put_abs_lowcore(vmcore_info, paddr_vmcoreinfo_note()); } void machine_shutdown(void) diff --git a/arch/s390/kernel/os_info.c b/arch/s390/kernel/os_info.c index 6b5b64e67eee..1acc2e05d70f 100644 --- a/arch/s390/kernel/os_info.c +++ b/arch/s390/kernel/os_info.c @@ -63,7 +63,7 @@ void __init os_info_init(void) os_info.version_minor = OS_INFO_VERSION_MINOR; os_info.magic = OS_INFO_MAGIC; os_info.csum = os_info_csum(&os_info); - mem_assign_absolute(S390_lowcore.os_info, __pa(ptr)); + put_abs_lowcore(os_info, __pa(ptr)); } #ifdef CONFIG_CRASH_DUMP diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index 84e23fcc1106..d860ac300919 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -481,11 +481,11 @@ static void __init setup_lowcore_dat_off(void) lc->mcck_stack = mcck_stack + STACK_INIT_OFFSET; /* Setup absolute zero lowcore */ - mem_assign_absolute(S390_lowcore.restart_stack, lc->restart_stack); - mem_assign_absolute(S390_lowcore.restart_fn, lc->restart_fn); - mem_assign_absolute(S390_lowcore.restart_data, lc->restart_data); - mem_assign_absolute(S390_lowcore.restart_source, lc->restart_source); - mem_assign_absolute(S390_lowcore.restart_psw, lc->restart_psw); + put_abs_lowcore(restart_stack, lc->restart_stack); + put_abs_lowcore(restart_fn, lc->restart_fn); + put_abs_lowcore(restart_data, lc->restart_data); + put_abs_lowcore(restart_source, lc->restart_source); + put_abs_lowcore(restart_psw, lc->restart_psw); lc->spinlock_lockval = arch_spin_lockval(0); lc->spinlock_index = 0; @@ -501,6 +501,7 @@ static void __init setup_lowcore_dat_off(void) static void __init setup_lowcore_dat_on(void) { struct lowcore *lc = lowcore_ptr[0]; + int cr; __ctl_clear_bit(0, 28); S390_lowcore.external_new_psw.mask |= PSW_MASK_DAT; @@ -509,10 +510,10 @@ static void __init setup_lowcore_dat_on(void) S390_lowcore.io_new_psw.mask |= PSW_MASK_DAT; __ctl_store(S390_lowcore.cregs_save_area, 0, 15); __ctl_set_bit(0, 28); - mem_assign_absolute(S390_lowcore.restart_flags, RESTART_FLAG_CTLREGS); - mem_assign_absolute(S390_lowcore.program_new_psw, lc->program_new_psw); - memcpy_absolute(&S390_lowcore.cregs_save_area, lc->cregs_save_area, - sizeof(S390_lowcore.cregs_save_area)); + put_abs_lowcore(restart_flags, RESTART_FLAG_CTLREGS); + put_abs_lowcore(program_new_psw, lc->program_new_psw); + for (cr = 0; cr < ARRAY_SIZE(lc->cregs_save_area); cr++) + put_abs_lowcore(cregs_save_area[cr], lc->cregs_save_area[cr]); } static struct resource code_resource = { diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index cc6971a09aaf..51529c33c7bf 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -332,10 +332,10 @@ static void pcpu_delegate(struct pcpu *pcpu, lc->restart_data = (unsigned long)data; lc->restart_source = source_cpu; } else { - mem_assign_absolute(lc->restart_stack, stack); - mem_assign_absolute(lc->restart_fn, (unsigned long)func); - mem_assign_absolute(lc->restart_data, (unsigned long)data); - mem_assign_absolute(lc->restart_source, source_cpu); + put_abs_lowcore(restart_stack, stack); + put_abs_lowcore(restart_fn, (unsigned long)func); + put_abs_lowcore(restart_data, (unsigned long)data); + put_abs_lowcore(restart_source, source_cpu); } __bpon(); asm volatile( @@ -591,9 +591,9 @@ void smp_ctl_set_clear_bit(int cr, int bit, bool set) parms.andval = ~(1UL << bit); } spin_lock(&ctl_lock); - memcpy_absolute(&ctlreg, &S390_lowcore.cregs_save_area[cr], sizeof(ctlreg)); + get_abs_lowcore(ctlreg, cregs_save_area[cr]); ctlreg = (ctlreg & parms.andval) | parms.orval; - memcpy_absolute(&S390_lowcore.cregs_save_area[cr], &ctlreg, sizeof(ctlreg)); + put_abs_lowcore(cregs_save_area[cr], ctlreg); spin_unlock(&ctl_lock); on_each_cpu(smp_ctl_bit_callback, &parms, 1); } From 7107822004198f60dd46d102694a666db784d28b Mon Sep 17 00:00:00 2001 From: Tony Krowiak Date: Fri, 18 Mar 2022 16:17:35 -0400 Subject: [PATCH 38/95] s390/vfio-ap: fix kernel doc and signature of group notifier functions The vfio_ap device driver registers a group notifier function to handle the VFIO_GROUP_NOTIFY_SET_KVM event signalling the KVM pointer has been set or cleared. There are two helper functions invoked by the handler function: One called when the KVM pointer has been set, and the other when the pointer is cleared. The kernel doc for both of these functions contains a comment introduced by commit 0cc00c8d4050 (s390/vfio-ap: fix circular lockdep when setting/clearing crypto masks) that is no longer valid. This patch removes this comment from the kernel doc of each helper function. Commit 86956e70761b (s390/vfio-ap: replace open coded locks for VFIO_GROUP_NOTIFY_SET_KVM notification) added a parameter to the signature of the helper function that handles the event indicating the KVM pointer has been cleared. The parameter added was the KVM pointer itself. One of the function's primary purposes is to clear the KVM pointer from the ap_matrix_mdev instance in which it is stored. Since the callers of this function derive the KVM pointer passed to the function from the ap_matrix_mdev object itself, it is completely unnecessary to include this parameter in the function's signature since it can simply be retrieved from the ap_matrix_mdev object which is also passed in. This patch removes the KVM pointer from the function's signature. Signed-off-by: Tony Krowiak Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/vfio_ap_ops.c | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index 7dc26365e29a..6e08d04b605d 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -1189,13 +1189,6 @@ static const struct attribute_group *vfio_ap_mdev_attr_groups[] = { * @matrix_mdev: a mediated matrix device * @kvm: reference to KVM instance * - * Note: The matrix_dev->lock must be taken prior to calling - * this function; however, the lock will be temporarily released while the - * guest's AP configuration is set to avoid a potential lockdep splat. - * The kvm->lock is taken to set the guest's AP configuration which, under - * certain circumstances, will result in a circular lock dependency if this is - * done under the @matrix_mdev->lock. - * * Return: 0 if no other mediated matrix device has a reference to @kvm; * otherwise, returns an -EPERM. */ @@ -1269,18 +1262,11 @@ static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb, * by @matrix_mdev. * * @matrix_mdev: a matrix mediated device - * @kvm: the pointer to the kvm structure being unset. - * - * Note: The matrix_dev->lock must be taken prior to calling - * this function; however, the lock will be temporarily released while the - * guest's AP configuration is cleared to avoid a potential lockdep splat. - * The kvm->lock is taken to clear the guest's AP configuration which, under - * certain circumstances, will result in a circular lock dependency if this is - * done under the @matrix_mdev->lock. */ -static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev, - struct kvm *kvm) +static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) { + struct kvm *kvm = matrix_mdev->kvm; + if (kvm && kvm->arch.crypto.crycbd) { down_write(&kvm->arch.crypto.pqap_hook_rwsem); kvm->arch.crypto.pqap_hook = NULL; @@ -1311,7 +1297,7 @@ static int vfio_ap_mdev_group_notifier(struct notifier_block *nb, matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier); if (!data) - vfio_ap_mdev_unset_kvm(matrix_mdev, matrix_mdev->kvm); + vfio_ap_mdev_unset_kvm(matrix_mdev); else if (vfio_ap_mdev_set_kvm(matrix_mdev, data)) notify_rc = NOTIFY_DONE; @@ -1448,7 +1434,7 @@ static void vfio_ap_mdev_close_device(struct vfio_device *vdev) &matrix_mdev->iommu_notifier); vfio_unregister_notifier(vdev->dev, VFIO_GROUP_NOTIFY, &matrix_mdev->group_notifier); - vfio_ap_mdev_unset_kvm(matrix_mdev, matrix_mdev->kvm); + vfio_ap_mdev_unset_kvm(matrix_mdev); } static int vfio_ap_mdev_get_device_info(unsigned long arg) From 97f32e11730e05927df196bcc8e30d8f2bcbb181 Mon Sep 17 00:00:00 2001 From: Jakob Koschel Date: Sat, 19 Mar 2022 21:35:28 +0100 Subject: [PATCH 39/95] s390/zcrypt: fix using the correct variable for sizeof() While the original code is valid, it is not the obvious choice for the sizeof() call and in preparation to limit the scope of the list iterator variable the sizeof should be changed to the size of the variable being allocated. Signed-off-by: Jakob Koschel Signed-off-by: Harald Freudenberger Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/zcrypt_card.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/crypto/zcrypt_card.c b/drivers/s390/crypto/zcrypt_card.c index 3e259befd30a..fcbd537530e8 100644 --- a/drivers/s390/crypto/zcrypt_card.c +++ b/drivers/s390/crypto/zcrypt_card.c @@ -90,7 +90,7 @@ static ssize_t online_store(struct device *dev, list_for_each_entry(zq, &zc->zqueues, list) maxzqs++; if (maxzqs > 0) - zq_uelist = kcalloc(maxzqs + 1, sizeof(zq), GFP_ATOMIC); + zq_uelist = kcalloc(maxzqs + 1, sizeof(*zq_uelist), GFP_ATOMIC); list_for_each_entry(zq, &zc->zqueues, list) if (zcrypt_queue_force_online(zq, online)) if (zq_uelist) { From 4f45c37ffd100522f890fc7f8bd71a3b8f79b6be Mon Sep 17 00:00:00 2001 From: Yu Liao Date: Tue, 22 Mar 2022 11:00:57 +0800 Subject: [PATCH 40/95] s390: cleanup timer API use cleanup the s390's use of the timer API - del_timer() contains timer_pending() condition - mod_timer(timer, expires) is equivalent to: del_timer(timer); timer->expires = expires; add_timer(timer); If the timer is inactive it will be activated, using add_timer() on condition !timer_pending(&private->timer) is redundant. Just cleanup, no logic change. Signed-off-by: Yu Liao Link: https://lore.kernel.org/r/20220322030057.1243196-1-liaoyu15@huawei.com Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- drivers/s390/char/sclp.c | 4 +--- drivers/s390/char/sclp_con.c | 3 +-- drivers/s390/char/sclp_vt220.c | 6 ++---- drivers/s390/cio/device_fsm.c | 12 +++--------- drivers/s390/cio/eadm_sch.c | 12 +++--------- 5 files changed, 10 insertions(+), 27 deletions(-) diff --git a/drivers/s390/char/sclp.c b/drivers/s390/char/sclp.c index f0763e36b861..cb2491761958 100644 --- a/drivers/s390/char/sclp.c +++ b/drivers/s390/char/sclp.c @@ -745,9 +745,7 @@ sclp_sync_wait(void) /* Loop until driver state indicates finished request */ while (sclp_running_state != sclp_running_state_idle) { /* Check for expired request timer */ - if (timer_pending(&sclp_request_timer) && - get_tod_clock_fast() > timeout && - del_timer(&sclp_request_timer)) + if (get_tod_clock_fast() > timeout && del_timer(&sclp_request_timer)) sclp_request_timer.function(&sclp_request_timer); cpu_relax(); } diff --git a/drivers/s390/char/sclp_con.c b/drivers/s390/char/sclp_con.c index de028868c6f4..fe5ee2646fcf 100644 --- a/drivers/s390/char/sclp_con.c +++ b/drivers/s390/char/sclp_con.c @@ -109,8 +109,7 @@ static void sclp_console_sync_queue(void) unsigned long flags; spin_lock_irqsave(&sclp_con_lock, flags); - if (timer_pending(&sclp_con_timer)) - del_timer(&sclp_con_timer); + del_timer(&sclp_con_timer); while (sclp_con_queue_running) { spin_unlock_irqrestore(&sclp_con_lock, flags); sclp_sync_wait(); diff --git a/drivers/s390/char/sclp_vt220.c b/drivers/s390/char/sclp_vt220.c index 7bc4e4a10937..3b4e7e5d9b71 100644 --- a/drivers/s390/char/sclp_vt220.c +++ b/drivers/s390/char/sclp_vt220.c @@ -231,8 +231,7 @@ sclp_vt220_emit_current(void) list_add_tail(&sclp_vt220_current_request->list, &sclp_vt220_outqueue); sclp_vt220_current_request = NULL; - if (timer_pending(&sclp_vt220_timer)) - del_timer(&sclp_vt220_timer); + del_timer(&sclp_vt220_timer); } sclp_vt220_flush_later = 0; } @@ -776,8 +775,7 @@ static void __sclp_vt220_flush_buffer(void) sclp_vt220_emit_current(); spin_lock_irqsave(&sclp_vt220_lock, flags); - if (timer_pending(&sclp_vt220_timer)) - del_timer(&sclp_vt220_timer); + del_timer(&sclp_vt220_timer); while (sclp_vt220_queue_running) { spin_unlock_irqrestore(&sclp_vt220_lock, flags); sclp_sync_wait(); diff --git a/drivers/s390/cio/device_fsm.c b/drivers/s390/cio/device_fsm.c index 05e136cfb8be..6d63b968309a 100644 --- a/drivers/s390/cio/device_fsm.c +++ b/drivers/s390/cio/device_fsm.c @@ -113,16 +113,10 @@ ccw_device_timeout(struct timer_list *t) void ccw_device_set_timeout(struct ccw_device *cdev, int expires) { - if (expires == 0) { + if (expires == 0) del_timer(&cdev->private->timer); - return; - } - if (timer_pending(&cdev->private->timer)) { - if (mod_timer(&cdev->private->timer, jiffies + expires)) - return; - } - cdev->private->timer.expires = jiffies + expires; - add_timer(&cdev->private->timer); + else + mod_timer(&cdev->private->timer, jiffies + expires); } int diff --git a/drivers/s390/cio/eadm_sch.c b/drivers/s390/cio/eadm_sch.c index 8b463681a149..ab6a7495180a 100644 --- a/drivers/s390/cio/eadm_sch.c +++ b/drivers/s390/cio/eadm_sch.c @@ -112,16 +112,10 @@ static void eadm_subchannel_set_timeout(struct subchannel *sch, int expires) { struct eadm_private *private = get_eadm_private(sch); - if (expires == 0) { + if (expires == 0) del_timer(&private->timer); - return; - } - if (timer_pending(&private->timer)) { - if (mod_timer(&private->timer, jiffies + expires)) - return; - } - private->timer.expires = jiffies + expires; - add_timer(&private->timer); + else + mod_timer(&private->timer, jiffies + expires); } static void eadm_subchannel_irq(struct subchannel *sch) From 7277b4216a1162157de06fd93cd54426c47d47b8 Mon Sep 17 00:00:00 2001 From: Alexander Gordeev Date: Thu, 24 Mar 2022 20:02:24 +0100 Subject: [PATCH 41/95] s390/smp: use physical address for SIGP_SET_PREFIX command Signal processor SIGP_SET_PREFIX command expects physical address of the lowcore to be installed, but instead the virtual address is provided. Note: this does not fix a bug currently, since virtual and physical addresses are identical. Reviewed-by: Heiko Carstens Signed-off-by: Alexander Gordeev Signed-off-by: Vasily Gorbik --- arch/s390/kernel/smp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 51529c33c7bf..30c91d565933 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -213,7 +213,7 @@ static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) if (nmi_alloc_mcesa(&lc->mcesad)) goto out; lowcore_ptr[cpu] = lc; - pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, (u32)(unsigned long) lc); + pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, __pa(lc)); return 0; out: From c122383d221dfa2f41cfe5e672540595de986fde Mon Sep 17 00:00:00 2001 From: Niklas Schnelle Date: Mon, 20 Sep 2021 09:32:21 +0200 Subject: [PATCH 42/95] s390/pci: improve zpci_dev reference counting Currently zpci_dev uses kref based reference counting but only accounts for one original reference plus one reference from an added pci_dev to its underlying zpci_dev. Counting just the original reference worked until the pci_dev reference was added in commit 2a671f77ee49 ("s390/pci: fix use after free of zpci_dev") because once a zpci_dev goes away, i.e. enters the reserved state, it would immediately get released. However with the pci_dev reference this is no longer the case and the zpci_dev may still appear in multiple availability events indicating that it was reserved. This was solved by detecting when the zpci_dev is already on its way out but still hanging around. This has however shown some light on how unusual our zpci_dev reference counting is. Improve upon this by modelling zpci_dev reference counting on pci_dev. Analogous to pci_get_slot() increment the reference count in get_zdev_by_fid(). Thus all users of get_zdev_by_fid() must drop the reference once they are done with the zpci_dev. Similar to pci_scan_single_device(), zpci_create_device() returns the device with an initial count of 1 and the device added to the zpci_list (analogous to the PCI bus' device_list). In turn users of zpci_create_device() must only drop the reference once the device is gone from the point of view of the zPCI subsystem, it might still be referenced by the common PCI subsystem though. Reviewed-by: Matthew Rosato Signed-off-by: Niklas Schnelle Signed-off-by: Vasily Gorbik --- arch/s390/pci/pci.c | 1 + arch/s390/pci/pci_bus.h | 3 ++- arch/s390/pci/pci_clp.c | 9 +++++++-- arch/s390/pci/pci_event.c | 7 ++++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 792f8e0f2178..5bcd9228db5f 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -69,6 +69,7 @@ struct zpci_dev *get_zdev_by_fid(u32 fid) list_for_each_entry(tmp, &zpci_list, entry) { if (tmp->fid == fid) { zdev = tmp; + zpci_zdev_get(zdev); break; } } diff --git a/arch/s390/pci/pci_bus.h b/arch/s390/pci/pci_bus.h index e359d2686178..ecef3a9e16c0 100644 --- a/arch/s390/pci/pci_bus.h +++ b/arch/s390/pci/pci_bus.h @@ -19,7 +19,8 @@ void zpci_bus_remove_device(struct zpci_dev *zdev, bool set_error); void zpci_release_device(struct kref *kref); static inline void zpci_zdev_put(struct zpci_dev *zdev) { - kref_put(&zdev->kref, zpci_release_device); + if (zdev) + kref_put(&zdev->kref, zpci_release_device); } static inline void zpci_zdev_get(struct zpci_dev *zdev) diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c index 63f3e057c168..1057d7af4a55 100644 --- a/arch/s390/pci/pci_clp.c +++ b/arch/s390/pci/pci_clp.c @@ -23,6 +23,8 @@ #include #include +#include "pci_bus.h" + bool zpci_unique_uid; void update_uid_checking(bool new) @@ -404,8 +406,11 @@ static void __clp_add(struct clp_fh_list_entry *entry, void *data) return; zdev = get_zdev_by_fid(entry->fid); - if (!zdev) - zpci_create_device(entry->fid, entry->fh, entry->config_state); + if (zdev) { + zpci_zdev_put(zdev); + return; + } + zpci_create_device(entry->fid, entry->fh, entry->config_state); } int clp_scan_pci_devices(void) diff --git a/arch/s390/pci/pci_event.c b/arch/s390/pci/pci_event.c index 2e3e5b278925..ea9db5cea64e 100644 --- a/arch/s390/pci/pci_event.c +++ b/arch/s390/pci/pci_event.c @@ -269,7 +269,7 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf) pdev ? pci_name(pdev) : "n/a", ccdf->pec, ccdf->fid); if (!pdev) - return; + goto no_pdev; switch (ccdf->pec) { case 0x003a: /* Service Action or Error Recovery Successful */ @@ -286,6 +286,8 @@ static void __zpci_event_error(struct zpci_ccdf_err *ccdf) break; } pci_dev_put(pdev); +no_pdev: + zpci_zdev_put(zdev); } void zpci_event_error(void *data) @@ -314,6 +316,7 @@ static void zpci_event_hard_deconfigured(struct zpci_dev *zdev, u32 fh) static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) { struct zpci_dev *zdev = get_zdev_by_fid(ccdf->fid); + bool existing_zdev = !!zdev; enum zpci_state state; zpci_dbg(3, "avl fid:%x, fh:%x, pec:%x\n", @@ -378,6 +381,8 @@ static void __zpci_event_availability(struct zpci_ccdf_avail *ccdf) default: break; } + if (existing_zdev) + zpci_zdev_put(zdev); } void zpci_event_availability(void *data) From 7dcfe50f58d28e0e2ba79e9e4333888bcf9442a4 Mon Sep 17 00:00:00 2001 From: Niklas Schnelle Date: Mon, 20 Sep 2021 09:47:29 +0200 Subject: [PATCH 43/95] s390/pci: rename get_zdev_by_bus() to zdev_from_bus() Getting a zpci_dev via get_zdev_by_bus() uses the long lived reference held in zbus->function[devfn]. This is accounted for in pcibios_add_device() and pcibios_release_device(). Therefore there is no need to increment the reference count in get_zdev_by_bus() as is done for get_zdev_by_fid(). Instead callers must not access the device after pcibios_release_device() was called which is necessary for common PCI code anyway. With this though the very similar naming may be misleading so rename get_zdev_by_bus() to zdev_from_bus() emphasizing that we are directly referencing the zdev via the bus. Reviewed-by: Matthew Rosato Signed-off-by: Niklas Schnelle Signed-off-by: Vasily Gorbik --- arch/s390/pci/pci.c | 4 ++-- arch/s390/pci/pci_bus.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 5bcd9228db5f..e563cb65c0c4 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -400,7 +400,7 @@ EXPORT_SYMBOL(pci_iounmap); static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val) { - struct zpci_dev *zdev = get_zdev_by_bus(bus, devfn); + struct zpci_dev *zdev = zdev_from_bus(bus, devfn); return (zdev) ? zpci_cfg_load(zdev, where, val, size) : -ENODEV; } @@ -408,7 +408,7 @@ static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val) { - struct zpci_dev *zdev = get_zdev_by_bus(bus, devfn); + struct zpci_dev *zdev = zdev_from_bus(bus, devfn); return (zdev) ? zpci_cfg_store(zdev, where, val, size) : -ENODEV; } diff --git a/arch/s390/pci/pci_bus.h b/arch/s390/pci/pci_bus.h index ecef3a9e16c0..e96c9860e064 100644 --- a/arch/s390/pci/pci_bus.h +++ b/arch/s390/pci/pci_bus.h @@ -33,8 +33,8 @@ void zpci_free_domain(int domain); int zpci_setup_bus_resources(struct zpci_dev *zdev, struct list_head *resources); -static inline struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus, - unsigned int devfn) +static inline struct zpci_dev *zdev_from_bus(struct pci_bus *bus, + unsigned int devfn) { struct zpci_bus *zbus = bus->sysdata; From faf79934e65aff90284725518a5ec3c2241c65ae Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Mon, 28 Mar 2022 23:27:54 +0200 Subject: [PATCH 44/95] s390/alternatives: avoid using jgnop mnemonic jgnop mnemonic is only available since binutils 2.36, kernel minimal required version is 2.23. Stick to brcl to avoid build errors. Reported-by: Nathan Chancellor Fixes: 4afeb670710e ("s390/alternatives: use instructions instead of byte patterns") Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/alternative-asm.h | 2 +- arch/s390/include/asm/alternative.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/s390/include/asm/alternative-asm.h b/arch/s390/include/asm/alternative-asm.h index 221d790361b9..bb3837d7387c 100644 --- a/arch/s390/include/asm/alternative-asm.h +++ b/arch/s390/include/asm/alternative-asm.h @@ -38,7 +38,7 @@ */ .macro alt_pad_fill bytes .rept ( \bytes ) / 6 - jgnop . + brcl 0,0 .endr .rept ( \bytes ) % 6 / 4 nop diff --git a/arch/s390/include/asm/alternative.h b/arch/s390/include/asm/alternative.h index 6c72495c8dbb..3f2856ed6808 100644 --- a/arch/s390/include/asm/alternative.h +++ b/arch/s390/include/asm/alternative.h @@ -75,7 +75,7 @@ void apply_alternatives(struct alt_instr *start, struct alt_instr *end); "\tnopr\n" \ ".else\n" \ "\t.rept " oldinstr_pad_len(num) " / 6\n" \ - "\t.jgnop .\n" \ + "\t.brcl 0,0\n" \ "\t.endr\n" \ "\t.rept " oldinstr_pad_len(num) " %% 6 / 4\n" \ "\tnop\n" \ From 3e3c658055c002900982513e289398a1aad4a488 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 28 Mar 2022 19:25:11 -0400 Subject: [PATCH 45/95] XArray: Fix xas_create_range() when multi-order entry present If there is already an entry present that is of order >= XA_CHUNK_SHIFT when we call xas_create_range(), xas_create_range() will misinterpret that entry as a node and dereference xa_node->parent, generally leading to a crash that looks something like this: general protection fault, probably for non-canonical address 0xdffffc0000000001: 0000 [#1] PREEMPT SMP KASAN KASAN: null-ptr-deref in range [0x0000000000000008-0x000000000000000f] CPU: 0 PID: 32 Comm: khugepaged Not tainted 5.17.0-rc8-syzkaller-00003-g56e337f2cf13 #0 RIP: 0010:xa_parent_locked include/linux/xarray.h:1207 [inline] RIP: 0010:xas_create_range+0x2d9/0x6e0 lib/xarray.c:725 It's deterministically reproducable once you know what the problem is, but producing it in a live kernel requires khugepaged to hit a race. While the problem has been present since xas_create_range() was introduced, I'm not aware of a way to hit it before the page cache was converted to use multi-index entries. Fixes: 6b24ca4a1a8d ("mm: Use multi-index entries in the page cache") Reported-by: syzbot+0d2b0bf32ca5cfd09f2e@syzkaller.appspotmail.com Signed-off-by: Matthew Wilcox (Oracle) --- lib/test_xarray.c | 22 ++++++++++++++++++++++ lib/xarray.c | 2 ++ 2 files changed, 24 insertions(+) diff --git a/lib/test_xarray.c b/lib/test_xarray.c index 8b1c318189ce..e77d4856442c 100644 --- a/lib/test_xarray.c +++ b/lib/test_xarray.c @@ -1463,6 +1463,25 @@ unlock: XA_BUG_ON(xa, !xa_empty(xa)); } +static noinline void check_create_range_5(struct xarray *xa, + unsigned long index, unsigned int order) +{ + XA_STATE_ORDER(xas, xa, index, order); + unsigned int i; + + xa_store_order(xa, index, order, xa_mk_index(index), GFP_KERNEL); + + for (i = 0; i < order + 10; i++) { + do { + xas_lock(&xas); + xas_create_range(&xas); + xas_unlock(&xas); + } while (xas_nomem(&xas, GFP_KERNEL)); + } + + xa_destroy(xa); +} + static noinline void check_create_range(struct xarray *xa) { unsigned int order; @@ -1490,6 +1509,9 @@ static noinline void check_create_range(struct xarray *xa) check_create_range_4(xa, (3U << order) + 1, order); check_create_range_4(xa, (3U << order) - 1, order); check_create_range_4(xa, (1U << 24) + 1, order); + + check_create_range_5(xa, 0, order); + check_create_range_5(xa, (1U << order), order); } check_create_range_3(); diff --git a/lib/xarray.c b/lib/xarray.c index 6f47f6375808..757644617b9b 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -722,6 +722,8 @@ void xas_create_range(struct xa_state *xas) for (;;) { struct xa_node *node = xas->xa_node; + if (node->shift >= shift) + break; xas->xa_node = xa_parent_locked(xas->xa, node); xas->xa_offset = node->offset - 1; if (node->offset != 0) From 60210a3d86dc57ce4a76a366e7841dda746a33f7 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Mon, 21 Mar 2022 18:26:17 -0700 Subject: [PATCH 46/95] riscv module: remove (NOLOAD) On ELF, (NOLOAD) sets the section type to SHT_NOBITS[1]. It is conceptually inappropriate for .plt, .got, and .got.plt sections which are always SHT_PROGBITS. In GNU ld, if PLT entries are needed, .plt will be SHT_PROGBITS anyway and (NOLOAD) will be essentially ignored. In ld.lld, since https://reviews.llvm.org/D118840 ("[ELF] Support (TYPE=) to customize the output section type"), ld.lld will report a `section type mismatch` error (later changed to a warning). Just remove (NOLOAD) to fix the warning. [1] https://lld.llvm.org/ELF/linker_script.html As of today, "The section should be marked as not loadable" on https://sourceware.org/binutils/docs/ld/Output-Section-Type.html is outdated for ELF. Link: https://github.com/ClangBuiltLinux/linux/issues/1597 Fixes: ab1ef68e5401 ("RISC-V: Add sections of PLT and GOT for kernel module") Reported-by: Nathan Chancellor Signed-off-by: Fangrui Song Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/module.lds.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/riscv/include/asm/module.lds.h b/arch/riscv/include/asm/module.lds.h index 4254ff2ff049..1075beae1ac6 100644 --- a/arch/riscv/include/asm/module.lds.h +++ b/arch/riscv/include/asm/module.lds.h @@ -2,8 +2,8 @@ /* Copyright (C) 2017 Andes Technology Corporation */ #ifdef CONFIG_MODULE_SECTIONS SECTIONS { - .plt (NOLOAD) : { BYTE(0) } - .got (NOLOAD) : { BYTE(0) } - .got.plt (NOLOAD) : { BYTE(0) } + .plt : { BYTE(0) } + .got : { BYTE(0) } + .got.plt : { BYTE(0) } } #endif From fdecfea09328b33fd08a4d418237cce9fd176d69 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 23 Feb 2022 22:04:11 -0800 Subject: [PATCH 47/95] riscv: Rename "sp_in_global" to "current_stack_pointer" To follow the existing per-arch conventions, rename "sp_in_global" to "current_stack_pointer". This will let it be used in non-arch places (like HARDENED_USERCOPY). Signed-off-by: Kees Cook Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 1 + arch/riscv/include/asm/current.h | 2 ++ arch/riscv/kernel/stacktrace.c | 6 ++---- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index ea8ec8a960bd..73f36acb71ba 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -16,6 +16,7 @@ config RISCV select ARCH_ENABLE_HUGEPAGE_MIGRATION if HUGETLB_PAGE && MIGRATION select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 select ARCH_HAS_BINFMT_FLAT + select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VM_PGTABLE select ARCH_HAS_DEBUG_VIRTUAL if MMU select ARCH_HAS_DEBUG_WX diff --git a/arch/riscv/include/asm/current.h b/arch/riscv/include/asm/current.h index 1de233d8e8de..21774d868c65 100644 --- a/arch/riscv/include/asm/current.h +++ b/arch/riscv/include/asm/current.h @@ -33,6 +33,8 @@ static __always_inline struct task_struct *get_current(void) #define current get_current() +register unsigned long current_stack_pointer __asm__("sp"); + #endif /* __ASSEMBLY__ */ #endif /* _ASM_RISCV_CURRENT_H */ diff --git a/arch/riscv/kernel/stacktrace.c b/arch/riscv/kernel/stacktrace.c index 14d2b53ec322..08d11a53f39e 100644 --- a/arch/riscv/kernel/stacktrace.c +++ b/arch/riscv/kernel/stacktrace.c @@ -14,8 +14,6 @@ #include -register unsigned long sp_in_global __asm__("sp"); - #ifdef CONFIG_FRAME_POINTER void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, @@ -30,7 +28,7 @@ void notrace walk_stackframe(struct task_struct *task, struct pt_regs *regs, pc = instruction_pointer(regs); } else if (task == NULL || task == current) { fp = (unsigned long)__builtin_frame_address(0); - sp = sp_in_global; + sp = current_stack_pointer; pc = (unsigned long)walk_stackframe; } else { /* task blocked in __switch_to */ @@ -78,7 +76,7 @@ void notrace walk_stackframe(struct task_struct *task, sp = user_stack_pointer(regs); pc = instruction_pointer(regs); } else if (task == NULL || task == current) { - sp = sp_in_global; + sp = current_stack_pointer; pc = (unsigned long)walk_stackframe; } else { /* task blocked in __switch_to */ From 6846d656106add3aeefcd6eda0dc885787deaa6e Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Tue, 8 Mar 2022 14:28:05 +0100 Subject: [PATCH 48/95] riscv: dts: canaan: Fix SPI3 bus width According to the K210 Standalone SDK Programming guide: https://canaan-creative.com/wp-content/uploads/2020/03/kendryte_standalone_programming_guide_20190311144158_en.pdf Section 15.4.3.3: SPI0 and SPI1 supports: standard, dual, quad and octal transfers. SPI3 supports: standard, dual and quad transfers (octal is not supported). In order to support quad transfers (Quad SPI), SPI3 must have four IO wires connected to the SPI flash. Update the device tree to specify the correct bus width. Tested on maix bit, maix dock and maixduino, which all have the same SPI flash (gd25lq128d) connected to SPI3. maix go is untested, but it would not make sense for this k210 board to be designed differently. Signed-off-by: Niklas Cassel Reviewed-by: Damien Le Moal Fixes: 8f5b0e79f3e5 ("riscv: Add SiPeed MAIXDUINO board device tree") Fixes: 8194f08bda18 ("riscv: Add SiPeed MAIX GO board device tree") Fixes: a40f920964c4 ("riscv: Add SiPeed MAIX DOCK board device tree") Fixes: 97c279bcf813 ("riscv: Add SiPeed MAIX BiT board device tree") Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts | 2 ++ arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts | 2 ++ arch/riscv/boot/dts/canaan/sipeed_maix_go.dts | 2 ++ arch/riscv/boot/dts/canaan/sipeed_maixduino.dts | 2 ++ 4 files changed, 8 insertions(+) diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts index 984872f3d3a9..b9e30df127fe 100644 --- a/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_bit.dts @@ -203,6 +203,8 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <50000000>; + spi-tx-bus-width = <4>; + spi-rx-bus-width = <4>; m25p,fast-read; broken-flash-reset; }; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts index 7ba99b4da304..8d23401b0bbb 100644 --- a/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_dock.dts @@ -205,6 +205,8 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <50000000>; + spi-tx-bus-width = <4>; + spi-rx-bus-width = <4>; m25p,fast-read; broken-flash-reset; }; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts b/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts index be9b12c9b374..24fd83b43d9d 100644 --- a/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts +++ b/arch/riscv/boot/dts/canaan/sipeed_maix_go.dts @@ -213,6 +213,8 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <50000000>; + spi-tx-bus-width = <4>; + spi-rx-bus-width = <4>; m25p,fast-read; broken-flash-reset; }; diff --git a/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts b/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts index 031c0c28f819..25341f38292a 100644 --- a/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts +++ b/arch/riscv/boot/dts/canaan/sipeed_maixduino.dts @@ -178,6 +178,8 @@ compatible = "jedec,spi-nor"; reg = <0>; spi-max-frequency = <50000000>; + spi-tx-bus-width = <4>; + spi-rx-bus-width = <4>; m25p,fast-read; broken-flash-reset; }; From 2b2b574ac587ec5bd7716a356492a85ab8b0ce9f Mon Sep 17 00:00:00 2001 From: Nikita Shubin Date: Fri, 11 Mar 2022 09:58:15 +0300 Subject: [PATCH 49/95] riscv: Fix fill_callchain return value perf_callchain_store return 0 on success, -1 otherwise, fix fill_callchain to return correct bool value. Fixes: dbeb90b0c1eb ("riscv: Add perf callchain support") Signed-off-by: Nikita Shubin Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/perf_callchain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c index 55faa4991b87..3348a61de7d9 100644 --- a/arch/riscv/kernel/perf_callchain.c +++ b/arch/riscv/kernel/perf_callchain.c @@ -68,7 +68,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, static bool fill_callchain(void *entry, unsigned long pc) { - return perf_callchain_store(entry, pc); + return perf_callchain_store(entry, pc) == 0; } void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, From b81d591386c3a50b96dddcf663628ea0df0bf2b3 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Mon, 14 Mar 2022 10:06:52 +0100 Subject: [PATCH 50/95] riscv: Increase stack size under KASAN KASAN requires more stack space because of compiler instrumentation. Increase stack size as other arches do. Signed-off-by: Dmitry Vyukov Reported-by: syzbot+0600986d88e2d4d7ebb8@syzkaller.appspotmail.com Fixes: 8ad8b72721d0 ("riscv: Add KASAN support") Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/thread_info.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 60da0dcacf14..74d888c8d631 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -11,11 +11,17 @@ #include #include +#ifdef CONFIG_KASAN +#define KASAN_STACK_ORDER 1 +#else +#define KASAN_STACK_ORDER 0 +#endif + /* thread information allocation */ #ifdef CONFIG_64BIT -#define THREAD_SIZE_ORDER (2) +#define THREAD_SIZE_ORDER (2 + KASAN_STACK_ORDER) #else -#define THREAD_SIZE_ORDER (1) +#define THREAD_SIZE_ORDER (1 + KASAN_STACK_ORDER) #endif #define THREAD_SIZE (PAGE_SIZE << THREAD_SIZE_ORDER) From 8a122a66c77034d7dc8f692e1b5c7a3b60c374b3 Mon Sep 17 00:00:00 2001 From: Atish Patra Date: Fri, 18 Mar 2022 19:26:33 -0700 Subject: [PATCH 51/95] RISC-V: Fix a comment typo in riscv_of_parent_hartid() This fixes a typo in a comment that is both obvious and went unnoticed. Signed-off-by: Atish Patra Fixes: a9b202606c69 ("RISC-V: Improve /proc/cpuinfo output for ISA extensions") Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/cpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index d2a936195295..4c303cdc8039 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -73,7 +73,7 @@ int riscv_of_parent_hartid(struct device_node *node) * Here are the ordering rules of extension naming defined by RISC-V * specification : * 1. All extensions should be separated from other multi-letter extensions - * from other multi-letter extensions by an underscore. + * by an underscore. * 2. The first letter following the 'Z' conventionally indicates the most * closely related alphabetical extension category, IMAFDQLCBKJTPVH. * If multiple 'Z' extensions are named, they should be ordered first From 3ed4bb77156da0bc732847c8c9df92454c1fbeea Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 31 Mar 2022 08:27:09 -0400 Subject: [PATCH 52/95] XArray: Update the LRU list in xas_split() When splitting a value entry, we may need to add the new nodes to the LRU list and remove the parent node from the LRU list. The WARN_ON checks in shadow_lru_isolate() catch this oversight. This bug was latent until we stopped splitting folios in shrink_page_list() with commit 820c4e2e6f51 ("mm/vmscan: Free non-shmem folios without splitting them"). That allows the creation of large shadow entries, and subsequently when trying to page in a small page, we will split the large shadow entry in __filemap_add_folio(). Fixes: 8fc75643c5e1 ("XArray: add xas_split") Reported-by: Hugh Dickins Signed-off-by: Matthew Wilcox (Oracle) --- lib/xarray.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/xarray.c b/lib/xarray.c index 757644617b9b..88ca87435e3d 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1081,6 +1081,7 @@ void xas_split(struct xa_state *xas, void *entry, unsigned int order) xa_mk_node(child)); if (xa_is_value(curr)) values--; + xas_update(xas, child); } else { unsigned int canon = offset - xas->xa_sibs; @@ -1095,6 +1096,7 @@ void xas_split(struct xa_state *xas, void *entry, unsigned int order) } while (offset-- > xas->xa_offset); node->nr_values += values; + xas_update(xas, node); } EXPORT_SYMBOL_GPL(xas_split); #endif From f1de125766d6f377a4b5d5821bc12928f929a4eb Mon Sep 17 00:00:00 2001 From: Atish Patra Date: Wed, 23 Mar 2022 10:17:26 -0700 Subject: [PATCH 53/95] RISC-V: Declare per cpu boot data as static The per cpu boot data is only used within the cpu_ops_sbi.c. It can be delcared as static. Fixes: 9a2451f18663 ("RISC-V: Avoid using per cpu array for ordered booting") Reported-by: kernel test robot Signed-off-by: Atish Patra Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/cpu_ops_sbi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kernel/cpu_ops_sbi.c b/arch/riscv/kernel/cpu_ops_sbi.c index 2e16f6732cdf..4f5a6f84e2a4 100644 --- a/arch/riscv/kernel/cpu_ops_sbi.c +++ b/arch/riscv/kernel/cpu_ops_sbi.c @@ -21,7 +21,7 @@ const struct cpu_operations cpu_ops_sbi; * be invoked from multiple threads in parallel. Define a per cpu data * to handle that. */ -DEFINE_PER_CPU(struct sbi_hart_boot_data, boot_data); +static DEFINE_PER_CPU(struct sbi_hart_boot_data, boot_data); static int sbi_hsm_hart_start(unsigned long hartid, unsigned long saddr, unsigned long priv) From 617487600b94b83c9733d8e3cb55a3f09beee194 Mon Sep 17 00:00:00 2001 From: Wu Caize Date: Thu, 24 Mar 2022 10:09:21 +0800 Subject: [PATCH 54/95] RISC-V: module: fix apply_r_riscv_rcv_branch_rela typo This function name was spelled incorrectly, likely to do a typo. Signed-off-by: Wu Caize Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/riscv/kernel/module.c b/arch/riscv/kernel/module.c index 4a48287513c3..c29cef90d1dd 100644 --- a/arch/riscv/kernel/module.c +++ b/arch/riscv/kernel/module.c @@ -69,7 +69,7 @@ static int apply_r_riscv_jal_rela(struct module *me, u32 *location, return 0; } -static int apply_r_riscv_rcv_branch_rela(struct module *me, u32 *location, +static int apply_r_riscv_rvc_branch_rela(struct module *me, u32 *location, Elf_Addr v) { ptrdiff_t offset = (void *)v - (void *)location; @@ -301,7 +301,7 @@ static int (*reloc_handlers_rela[]) (struct module *me, u32 *location, [R_RISCV_64] = apply_r_riscv_64_rela, [R_RISCV_BRANCH] = apply_r_riscv_branch_rela, [R_RISCV_JAL] = apply_r_riscv_jal_rela, - [R_RISCV_RVC_BRANCH] = apply_r_riscv_rcv_branch_rela, + [R_RISCV_RVC_BRANCH] = apply_r_riscv_rvc_branch_rela, [R_RISCV_RVC_JUMP] = apply_r_riscv_rvc_jump_rela, [R_RISCV_PCREL_HI20] = apply_r_riscv_pcrel_hi20_rela, [R_RISCV_PCREL_LO12_I] = apply_r_riscv_pcrel_lo12_i_rela, From 2e7451fb57632454aaccc8ca1aac167751df9fca Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Thu, 24 Mar 2022 11:00:20 +0530 Subject: [PATCH 55/95] RISC-V: Enable profiling by default Let us enable profiling by default in RV32 and RV64 defconfigs so that we can use RISC-V PMU drivers on various RISC-V platforms. Signed-off-by: Anup Patel Reviewed-by: Atish Patra Signed-off-by: Palmer Dabbelt --- arch/riscv/configs/defconfig | 1 + arch/riscv/configs/rv32_defconfig | 1 + 2 files changed, 2 insertions(+) diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 9249ef7bcee7..50acf6363173 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -15,6 +15,7 @@ CONFIG_CHECKPOINT_RESTORE=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y # CONFIG_SYSFS_SYSCALL is not set +CONFIG_PROFILING=y CONFIG_SOC_MICROCHIP_POLARFIRE=y CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig index 17037d5f3ba3..36aa6ff5a001 100644 --- a/arch/riscv/configs/rv32_defconfig +++ b/arch/riscv/configs/rv32_defconfig @@ -15,6 +15,7 @@ CONFIG_CHECKPOINT_RESTORE=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y # CONFIG_SYSFS_SYSCALL is not set +CONFIG_PROFILING=y CONFIG_SOC_SIFIVE=y CONFIG_SOC_VIRT=y CONFIG_ARCH_RV32I=y From 8cf8df89678aa5ab3eb206ccc7c8dee7c528bf50 Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Thu, 31 Mar 2022 20:17:37 +0300 Subject: [PATCH 56/95] ARM: OMAP2+: Fix regression for smc calls for vmap stack Commit 9c46929e7989 ("ARM: implement THREAD_INFO_IN_TASK for uniprocessor systems") started triggering an issue with smc calls hanging on boot as VMAP_STACK is now enabled by default. Based on discussions on the #armlinux irc channel, Arnd noticed that omaps are using __pa() for stack for smc calls. This does not work with vmap stack. Let's fix the issue by changing the param arrays to use static param[5] for each function for __pa() to work. This consumes a bit more memory compared to adding a single static buffer, but avoids potential races with the smc calls initializing the shared buffer. For omap_secure_dispatcher(), we need to use a cpu specific buffer as there's nothing currently ensuring it only gets called from cpu0. Fixes: 9c46929e7989 ("ARM: implement THREAD_INFO_IN_TASK for uniprocessor systems") Link: https://lore.kernel.org/r/20220331171737.48211-1-tony@atomide.com Suggested-by: Ard Biesheuvel Suggested-by: Arnd Bergmann Signed-off-by: Tony Lindgren Reviewed-by: Ard Biesheuvel Signed-off-by: Arnd Bergmann --- arch/arm/mach-omap2/omap-secure.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/arch/arm/mach-omap2/omap-secure.c b/arch/arm/mach-omap2/omap-secure.c index 0659ab4cb0af..11677fc2968f 100644 --- a/arch/arm/mach-omap2/omap-secure.c +++ b/arch/arm/mach-omap2/omap-secure.c @@ -59,8 +59,13 @@ static void __init omap_optee_init_check(void) u32 omap_secure_dispatcher(u32 idx, u32 flag, u32 nargs, u32 arg1, u32 arg2, u32 arg3, u32 arg4) { + static u32 buf[NR_CPUS][5]; + u32 *param; + int cpu; u32 ret; - u32 param[5]; + + cpu = get_cpu(); + param = buf[cpu]; param[0] = nargs; param[1] = arg1; @@ -76,6 +81,8 @@ u32 omap_secure_dispatcher(u32 idx, u32 flag, u32 nargs, u32 arg1, u32 arg2, outer_clean_range(__pa(param), __pa(param + 5)); ret = omap_smc2(idx, flag, __pa(param)); + put_cpu(); + return ret; } @@ -119,8 +126,8 @@ phys_addr_t omap_secure_ram_mempool_base(void) #if defined(CONFIG_ARCH_OMAP3) && defined(CONFIG_PM) u32 omap3_save_secure_ram(void __iomem *addr, int size) { + static u32 param[5]; u32 ret; - u32 param[5]; if (size != OMAP3_SAVE_SECURE_RAM_SZ) return OMAP3_SAVE_SECURE_RAM_SZ; @@ -153,8 +160,8 @@ u32 omap3_save_secure_ram(void __iomem *addr, int size) u32 rx51_secure_dispatcher(u32 idx, u32 process, u32 flag, u32 nargs, u32 arg1, u32 arg2, u32 arg3, u32 arg4) { + static u32 param[5]; u32 ret; - u32 param[5]; param[0] = nargs+1; /* RX-51 needs number of arguments + 1 */ param[1] = arg1; From 31d3687d6017c7ce6061695361598d9cda70807a Mon Sep 17 00:00:00 2001 From: Kuldeep Singh Date: Sat, 26 Mar 2022 09:53:09 +0530 Subject: [PATCH 57/95] ARM: dts: spear13xx: Update SPI dma properties Reorder dmas and dma-names property for spi controller node to make it compliant with bindings. Fixes: 6e8887f60f60 ("ARM: SPEAr13xx: Pass generic DW DMAC platform data from DT") Signed-off-by: Kuldeep Singh Acked-by: Viresh Kumar Link: https://lore.kernel.org/r/20220326042313.97862-2-singh.kuldeep87k@gmail.com' Signed-off-by: Arnd Bergmann --- arch/arm/boot/dts/spear13xx.dtsi | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm/boot/dts/spear13xx.dtsi b/arch/arm/boot/dts/spear13xx.dtsi index c87b881b2c8b..913553367687 100644 --- a/arch/arm/boot/dts/spear13xx.dtsi +++ b/arch/arm/boot/dts/spear13xx.dtsi @@ -284,9 +284,9 @@ #size-cells = <0>; interrupts = <0 31 0x4>; status = "disabled"; - dmas = <&dwdma0 4 0 0>, - <&dwdma0 5 0 0>; - dma-names = "tx", "rx"; + dmas = <&dwdma0 5 0 0>, + <&dwdma0 4 0 0>; + dma-names = "rx", "tx"; }; rtc@e0580000 { From 583d6b0062640def86f3265aa1042ecb6672516e Mon Sep 17 00:00:00 2001 From: Kuldeep Singh Date: Sat, 26 Mar 2022 09:53:10 +0530 Subject: [PATCH 58/95] ARM: dts: spear1340: Update serial node properties Reorder dma and dma-names property for serial node to make it compliant with bindings. Fixes: 6e8887f60f60 ("ARM: SPEAr13xx: Pass generic DW DMAC platform data from DT") Signed-off-by: Kuldeep Singh Acked-by: Viresh Kumar Link: https://lore.kernel.org/r/20220326042313.97862-3-singh.kuldeep87k@gmail.com' Signed-off-by: Arnd Bergmann --- arch/arm/boot/dts/spear1340.dtsi | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm/boot/dts/spear1340.dtsi b/arch/arm/boot/dts/spear1340.dtsi index 827e887afbda..13e1bdb3ddbf 100644 --- a/arch/arm/boot/dts/spear1340.dtsi +++ b/arch/arm/boot/dts/spear1340.dtsi @@ -134,9 +134,9 @@ reg = <0xb4100000 0x1000>; interrupts = <0 105 0x4>; status = "disabled"; - dmas = <&dwdma0 12 0 1>, - <&dwdma0 13 1 0>; - dma-names = "tx", "rx"; + dmas = <&dwdma0 13 0 1>, + <&dwdma0 12 1 0>; + dma-names = "rx", "tx"; }; thermal@e07008c4 { From aa4df840d1c5eab2bb33695efe4409b3e5526749 Mon Sep 17 00:00:00 2001 From: Kuldeep Singh Date: Sat, 26 Mar 2022 09:53:11 +0530 Subject: [PATCH 59/95] arm64: dts: ls1043a: Update i2c dma properties Reorder dmas and dma-names properties for i2c controller node to make it compliant with bindings. Fixes: 6d453cd22357 ("arm64: dts: add Freescale LS1043a SoC support") Signed-off-by: Kuldeep Singh CC: soc@kernel.org Link: https://lore.kernel.org/r/20220326042313.97862-4-singh.kuldeep87k@gmail.com' Signed-off-by: Arnd Bergmann --- arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi index 01b01e320411..35d1939e690b 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi @@ -536,9 +536,9 @@ clock-names = "i2c"; clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(1)>; - dmas = <&edma0 1 39>, - <&edma0 1 38>; - dma-names = "tx", "rx"; + dmas = <&edma0 1 38>, + <&edma0 1 39>; + dma-names = "rx", "tx"; status = "disabled"; }; From eeeb4f1075d71d67083c75f69247206e9b3d9f4a Mon Sep 17 00:00:00 2001 From: Kuldeep Singh Date: Sat, 26 Mar 2022 09:53:12 +0530 Subject: [PATCH 60/95] arm64: dts: ls1046a: Update i2c node dma properties Reorder dmas and dma-names properties for i2c controller node to make it compliant with bindings. Fixes: 8126d88162a5 ("arm64: dts: add QorIQ LS1046A SoC support") Signed-off-by: Kuldeep Singh CC: soc@kernel.org Link: https://lore.kernel.org/r/20220326042313.97862-5-singh.kuldeep87k@gmail.com' Signed-off-by: Arnd Bergmann --- arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi index 687fea6d8afa..4e7bd04d9798 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi @@ -499,9 +499,9 @@ interrupts = ; clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL QORIQ_CLK_PLL_DIV(2)>; - dmas = <&edma0 1 39>, - <&edma0 1 38>; - dma-names = "tx", "rx"; + dmas = <&edma0 1 38>, + <&edma0 1 39>; + dma-names = "rx", "tx"; status = "disabled"; }; From e634ff7733ba3c03a18fb47cc355700a5a01014f Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Mon, 28 Mar 2022 15:04:17 -0700 Subject: [PATCH 61/95] riscv: cpu.c: don't use kernel-doc markers for comments Repair kernel-doc build warnings caused by using "/**" kernel-doc markers for comments that are not in kernel-doc format: cpu.c:89: warning: cannot understand function prototype: 'struct riscv_isa_ext_data isa_ext_arr[] = ' cpu.c:114: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst Signed-off-by: Randy Dunlap Reported-by: kernel test robot Reviewed-by: Atish Patra Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/cpu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index 4c303cdc8039..ccb617791e56 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -69,7 +69,7 @@ int riscv_of_parent_hartid(struct device_node *node) .uprop = #UPROP, \ .isa_ext_id = EXTID, \ } -/** +/* * Here are the ordering rules of extension naming defined by RISC-V * specification : * 1. All extensions should be separated from other multi-letter extensions @@ -110,7 +110,7 @@ static void print_isa_ext(struct seq_file *f) } } -/** +/* * These are the only valid base (single letter) ISA extensions as per the spec. * It also specifies the canonical order in which it appears in the spec. * Some of the extension may just be a place holder for now (B, K, P, J). From 8933e7f2e37549cf54ad1133476bda8ec05b68ea Mon Sep 17 00:00:00 2001 From: Bagas Sanjaya Date: Tue, 29 Mar 2022 15:44:28 +0700 Subject: [PATCH 62/95] Documentation: riscv: remove non-existent directory from table of contents This subdir no longer exists: it was there to document some of the quirks and paths forward for handling them, but those have been dealt with as of Atish's recent patch set. Fixes: 23b1f18326ec ("Documentation: riscv: Remove the old documentation") Link: https://lore.kernel.org/linux-next/20220329133412.591d6882@canb.auug.org.au/ Reported-by: Stephen Rothwell Signed-off-by: Bagas Sanjaya [Palmer: commit text] Signed-off-by: Palmer Dabbelt --- Documentation/riscv/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Documentation/riscv/index.rst b/Documentation/riscv/index.rst index ea915c196048..e23b876ad6eb 100644 --- a/Documentation/riscv/index.rst +++ b/Documentation/riscv/index.rst @@ -7,7 +7,6 @@ RISC-V architecture boot-image-header vm-layout - pmu patch-acceptance features From 6f562570b9c5d6a3e30d87aec60a9d8f22a3203c Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Wed, 30 Mar 2022 16:47:37 -0700 Subject: [PATCH 63/95] RISC-V: defconfig: Drop redundant SBI HVC and earlycon As of 3938d5a2f936 ("riscv: default to CONFIG_RISCV_SBI_V01=n") we no longer default to enabling SBI-0.1 support, so these dependent configs no longer have any effect. Remove them to avoid clutter. Signed-off-by: Palmer Dabbelt --- arch/riscv/configs/defconfig | 2 -- arch/riscv/configs/rv32_defconfig | 2 -- 2 files changed, 4 deletions(-) diff --git a/arch/riscv/configs/defconfig b/arch/riscv/configs/defconfig index 50acf6363173..30e3017f22bc 100644 --- a/arch/riscv/configs/defconfig +++ b/arch/riscv/configs/defconfig @@ -67,8 +67,6 @@ CONFIG_INPUT_MOUSEDEV=y CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_OF_PLATFORM=y -CONFIG_SERIAL_EARLYCON_RISCV_SBI=y -CONFIG_HVC_RISCV_SBI=y CONFIG_VIRTIO_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_HW_RANDOM_VIRTIO=y diff --git a/arch/riscv/configs/rv32_defconfig b/arch/riscv/configs/rv32_defconfig index 36aa6ff5a001..7e5efdc3829d 100644 --- a/arch/riscv/configs/rv32_defconfig +++ b/arch/riscv/configs/rv32_defconfig @@ -65,8 +65,6 @@ CONFIG_INPUT_MOUSEDEV=y CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_OF_PLATFORM=y -CONFIG_SERIAL_EARLYCON_RISCV_SBI=y -CONFIG_HVC_RISCV_SBI=y CONFIG_VIRTIO_CONSOLE=y CONFIG_HW_RANDOM=y CONFIG_HW_RANDOM_VIRTIO=y From 1464d00b27b2e29a5556f6a4099cf083886e883f Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Wed, 30 Mar 2022 16:47:38 -0700 Subject: [PATCH 64/95] RISC-V: K210 defconfigs: Drop redundant MEMBARRIER=n As of 93917ad50972 ("RISC-V: Add support for restartable sequence") we have support for restartable sequences, which default to enabled. These select MEMBARRIER, so disabling it is now redundant. Signed-off-by: Palmer Dabbelt --- arch/riscv/configs/nommu_k210_defconfig | 1 - arch/riscv/configs/nommu_k210_sdcard_defconfig | 1 - arch/riscv/configs/nommu_virt_defconfig | 1 - 3 files changed, 3 deletions(-) diff --git a/arch/riscv/configs/nommu_k210_defconfig b/arch/riscv/configs/nommu_k210_defconfig index 3f42ed87dde8..2438fa39f8ae 100644 --- a/arch/riscv/configs/nommu_k210_defconfig +++ b/arch/riscv/configs/nommu_k210_defconfig @@ -21,7 +21,6 @@ CONFIG_CC_OPTIMIZE_FOR_SIZE=y # CONFIG_AIO is not set # CONFIG_IO_URING is not set # CONFIG_ADVISE_SYSCALLS is not set -# CONFIG_MEMBARRIER is not set # CONFIG_KALLSYMS is not set CONFIG_EMBEDDED=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/riscv/configs/nommu_k210_sdcard_defconfig b/arch/riscv/configs/nommu_k210_sdcard_defconfig index af64b95e88cc..9a133e63ae5b 100644 --- a/arch/riscv/configs/nommu_k210_sdcard_defconfig +++ b/arch/riscv/configs/nommu_k210_sdcard_defconfig @@ -13,7 +13,6 @@ CONFIG_CC_OPTIMIZE_FOR_SIZE=y # CONFIG_AIO is not set # CONFIG_IO_URING is not set # CONFIG_ADVISE_SYSCALLS is not set -# CONFIG_MEMBARRIER is not set # CONFIG_KALLSYMS is not set CONFIG_EMBEDDED=y # CONFIG_VM_EVENT_COUNTERS is not set diff --git a/arch/riscv/configs/nommu_virt_defconfig b/arch/riscv/configs/nommu_virt_defconfig index e1c9864b6237..5269fbb6b4fc 100644 --- a/arch/riscv/configs/nommu_virt_defconfig +++ b/arch/riscv/configs/nommu_virt_defconfig @@ -19,7 +19,6 @@ CONFIG_EXPERT=y # CONFIG_AIO is not set # CONFIG_IO_URING is not set # CONFIG_ADVISE_SYSCALLS is not set -# CONFIG_MEMBARRIER is not set # CONFIG_KALLSYMS is not set # CONFIG_VM_EVENT_COUNTERS is not set # CONFIG_COMPAT_BRK is not set From ebf921a9fac38560e0fc3a4381e163a6969efd5a Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 22 Jan 2022 15:46:22 -0500 Subject: [PATCH 65/95] readahead: Remove read_cache_pages() With no remaining users, remove this function and the related infrastructure. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Al Viro Acked-by: Al Viro --- include/linux/pagemap.h | 2 -- mm/readahead.c | 76 ----------------------------------------- 2 files changed, 78 deletions(-) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index a8d0b327b066..993994cd943a 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -752,8 +752,6 @@ struct page *read_cache_page(struct address_space *, pgoff_t index, filler_t *filler, void *data); extern struct page * read_cache_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); -extern int read_cache_pages(struct address_space *mapping, - struct list_head *pages, filler_t *filler, void *data); static inline struct page *read_mapping_page(struct address_space *mapping, pgoff_t index, struct file *file) diff --git a/mm/readahead.c b/mm/readahead.c index d3a47546d17d..9097af639beb 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -142,82 +142,6 @@ file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping) } EXPORT_SYMBOL_GPL(file_ra_state_init); -/* - * see if a page needs releasing upon read_cache_pages() failure - * - the caller of read_cache_pages() may have set PG_private or PG_fscache - * before calling, such as the NFS fs marking pages that are cached locally - * on disk, thus we need to give the fs a chance to clean up in the event of - * an error - */ -static void read_cache_pages_invalidate_page(struct address_space *mapping, - struct page *page) -{ - if (page_has_private(page)) { - if (!trylock_page(page)) - BUG(); - page->mapping = mapping; - folio_invalidate(page_folio(page), 0, PAGE_SIZE); - page->mapping = NULL; - unlock_page(page); - } - put_page(page); -} - -/* - * release a list of pages, invalidating them first if need be - */ -static void read_cache_pages_invalidate_pages(struct address_space *mapping, - struct list_head *pages) -{ - struct page *victim; - - while (!list_empty(pages)) { - victim = lru_to_page(pages); - list_del(&victim->lru); - read_cache_pages_invalidate_page(mapping, victim); - } -} - -/** - * read_cache_pages - populate an address space with some pages & start reads against them - * @mapping: the address_space - * @pages: The address of a list_head which contains the target pages. These - * pages have their ->index populated and are otherwise uninitialised. - * @filler: callback routine for filling a single page. - * @data: private data for the callback routine. - * - * Hides the details of the LRU cache etc from the filesystems. - * - * Returns: %0 on success, error return by @filler otherwise - */ -int read_cache_pages(struct address_space *mapping, struct list_head *pages, - int (*filler)(void *, struct page *), void *data) -{ - struct page *page; - int ret = 0; - - while (!list_empty(pages)) { - page = lru_to_page(pages); - list_del(&page->lru); - if (add_to_page_cache_lru(page, mapping, page->index, - readahead_gfp_mask(mapping))) { - read_cache_pages_invalidate_page(mapping, page); - continue; - } - put_page(page); - - ret = filler(data, page); - if (unlikely(ret)) { - read_cache_pages_invalidate_pages(mapping, pages); - break; - } - task_io_account_read(PAGE_SIZE); - } - return ret; -} - -EXPORT_SYMBOL(read_cache_pages); - static void read_pages(struct readahead_control *rac, struct list_head *pages, bool skip_page) { From 704528d895dd3e7b173e672116b4eb2b0a0fceb0 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 23 Mar 2022 21:29:04 -0400 Subject: [PATCH 66/95] fs: Remove ->readpages address space operation All filesystems have now been converted to use ->readahead, so remove the ->readpages operation and fix all the comments that used to refer to it. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Al Viro Acked-by: Al Viro --- Documentation/filesystems/fsverity.rst | 6 +++--- Documentation/filesystems/locking.rst | 6 ------ Documentation/filesystems/vfs.rst | 11 ----------- fs/btrfs/reflink.c | 4 ++-- fs/cifs/cifssmb.c | 2 +- fs/cifs/inode.c | 2 +- fs/crypto/crypto.c | 2 +- fs/ext4/readpage.c | 2 +- fs/f2fs/data.c | 4 ++-- fs/fuse/fuse_i.h | 2 +- fs/verity/verify.c | 4 ++-- include/linux/fs.h | 6 ------ include/linux/fsverity.h | 2 +- mm/filemap.c | 2 +- mm/readahead.c | 15 ++------------- 15 files changed, 18 insertions(+), 52 deletions(-) diff --git a/Documentation/filesystems/fsverity.rst b/Documentation/filesystems/fsverity.rst index 1d831e3cbcb3..8cc536d08f51 100644 --- a/Documentation/filesystems/fsverity.rst +++ b/Documentation/filesystems/fsverity.rst @@ -549,7 +549,7 @@ Pagecache ~~~~~~~~~ For filesystems using Linux's pagecache, the ``->readpage()`` and -``->readpages()`` methods must be modified to verify pages before they +``->readahead()`` methods must be modified to verify pages before they are marked Uptodate. Merely hooking ``->read_iter()`` would be insufficient, since ``->read_iter()`` is not used for memory maps. @@ -611,7 +611,7 @@ workqueue, and then the workqueue work does the decryption or verification. Finally, pages where no decryption or verity error occurred are marked Uptodate, and the pages are unlocked. -Files on ext4 and f2fs may contain holes. Normally, ``->readpages()`` +Files on ext4 and f2fs may contain holes. Normally, ``->readahead()`` simply zeroes holes and sets the corresponding pages Uptodate; no bios are issued. To prevent this case from bypassing fs-verity, these filesystems use fsverity_verify_page() to verify hole pages. @@ -778,7 +778,7 @@ weren't already directly answered in other parts of this document. - To prevent bypassing verification, pages must not be marked Uptodate until they've been verified. Currently, each filesystem is responsible for marking pages Uptodate via - ``->readpages()``. Therefore, currently it's not possible for + ``->readahead()``. Therefore, currently it's not possible for the VFS to do the verification on its own. Changing this would require significant changes to the VFS and all filesystems. diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst index 2998cec9af4b..c26d854275a0 100644 --- a/Documentation/filesystems/locking.rst +++ b/Documentation/filesystems/locking.rst @@ -241,8 +241,6 @@ prototypes:: int (*writepages)(struct address_space *, struct writeback_control *); bool (*dirty_folio)(struct address_space *, struct folio *folio); void (*readahead)(struct readahead_control *); - int (*readpages)(struct file *filp, struct address_space *mapping, - struct list_head *pages, unsigned nr_pages); int (*write_begin)(struct file *, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata); @@ -274,7 +272,6 @@ readpage: yes, unlocks shared writepages: dirty_folio maybe readahead: yes, unlocks shared -readpages: no shared write_begin: locks the page exclusive write_end: yes, unlocks exclusive bmap: @@ -300,9 +297,6 @@ completion. ->readahead() unlocks the pages that I/O is attempted on like ->readpage(). -->readpages() populates the pagecache with the passed pages and starts -I/O against them. They come unlocked upon I/O completion. - ->writepage() is used for two purposes: for "memory cleansing" and for "sync". These are quite different operations and the behaviour may differ depending upon the mode. diff --git a/Documentation/filesystems/vfs.rst b/Documentation/filesystems/vfs.rst index 4f14edf93941..794bd1a66bfb 100644 --- a/Documentation/filesystems/vfs.rst +++ b/Documentation/filesystems/vfs.rst @@ -726,8 +726,6 @@ cache in your filesystem. The following members are defined: int (*writepages)(struct address_space *, struct writeback_control *); bool (*dirty_folio)(struct address_space *, struct folio *); void (*readahead)(struct readahead_control *); - int (*readpages)(struct file *filp, struct address_space *mapping, - struct list_head *pages, unsigned nr_pages); int (*write_begin)(struct file *, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata); @@ -817,15 +815,6 @@ cache in your filesystem. The following members are defined: completes successfully. Setting PageError on any page will be ignored; simply unlock the page if an I/O error occurs. -``readpages`` - called by the VM to read pages associated with the address_space - object. This is essentially just a vector version of readpage. - Instead of just one page, several pages are requested. - readpages is only used for read-ahead, so read errors are - ignored. If anything goes wrong, feel free to give up. - This interface is deprecated and will be removed by the end of - 2020; implement readahead instead. - ``write_begin`` Called by the generic buffered write code to ask the filesystem to prepare to write len bytes at the given offset in the file. diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index 04a88bfe4fcf..998e3f180d90 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -645,7 +645,7 @@ static int btrfs_extent_same_range(struct inode *src, u64 loff, u64 len, int ret; /* - * Lock destination range to serialize with concurrent readpages() and + * Lock destination range to serialize with concurrent readahead() and * source range to serialize with relocation. */ btrfs_double_extent_lock(src, loff, dst, dst_loff, len); @@ -739,7 +739,7 @@ static noinline int btrfs_clone_files(struct file *file, struct file *file_src, } /* - * Lock destination range to serialize with concurrent readpages() and + * Lock destination range to serialize with concurrent readahead() and * source range to serialize with relocation. */ btrfs_double_extent_lock(src, off, inode, destoff, len); diff --git a/fs/cifs/cifssmb.c b/fs/cifs/cifssmb.c index 071e2f21a7db..bc3ded4f34f6 100644 --- a/fs/cifs/cifssmb.c +++ b/fs/cifs/cifssmb.c @@ -597,7 +597,7 @@ CIFSSMBNegotiate(const unsigned int xid, set_credits(server, server->maxReq); /* probably no need to store and check maxvcs */ server->maxBuf = le32_to_cpu(pSMBr->MaxBufferSize); - /* set up max_read for readpages check */ + /* set up max_read for readahead check */ server->max_read = server->maxBuf; server->max_rw = le32_to_cpu(pSMBr->MaxRawSize); cifs_dbg(NOISY, "Max buf = %d\n", ses->server->maxBuf); diff --git a/fs/cifs/inode.c b/fs/cifs/inode.c index 60d853c92f6a..2f9e7d2f81b6 100644 --- a/fs/cifs/inode.c +++ b/fs/cifs/inode.c @@ -49,7 +49,7 @@ static void cifs_set_ops(struct inode *inode) inode->i_fop = &cifs_file_ops; } - /* check if server can support readpages */ + /* check if server can support readahead */ if (cifs_sb_master_tcon(cifs_sb)->ses->server->max_read < PAGE_SIZE + MAX_CIFS_HDR_SIZE) inode->i_data.a_ops = &cifs_addr_ops_smallbuf; diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c index 4fcca79f39ae..526a4c1bed99 100644 --- a/fs/crypto/crypto.c +++ b/fs/crypto/crypto.c @@ -248,7 +248,7 @@ EXPORT_SYMBOL(fscrypt_encrypt_block_inplace); * which must still be locked and not uptodate. Normally, blocksize == * PAGE_SIZE and the whole page is decrypted at once. * - * This is for use by the filesystem's ->readpages() method. + * This is for use by the filesystem's ->readahead() method. * * Return: 0 on success; -errno on failure */ diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c index 1aa26d6634fc..af491e170c4a 100644 --- a/fs/ext4/readpage.c +++ b/fs/ext4/readpage.c @@ -109,7 +109,7 @@ static void verity_work(struct work_struct *work) struct bio *bio = ctx->bio; /* - * fsverity_verify_bio() may call readpages() again, and although verity + * fsverity_verify_bio() may call readahead() again, and although verity * will be disabled for that, decryption may still be needed, causing * another bio_post_read_ctx to be allocated. So to guarantee that * mempool_alloc() never deadlocks we must free the current ctx first. diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index f8fcbe91059b..c92920c8661d 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -164,7 +164,7 @@ static void f2fs_verify_bio(struct work_struct *work) bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS); /* - * fsverity_verify_bio() may call readpages() again, and while verity + * fsverity_verify_bio() may call readahead() again, and while verity * will be disabled for this, decryption and/or decompression may still * be needed, resulting in another bio_post_read_ctx being allocated. * So to prevent deadlocks we need to release the current ctx to the @@ -2392,7 +2392,7 @@ static void f2fs_readahead(struct readahead_control *rac) if (!f2fs_is_compress_backend_ready(inode)) return; - /* If the file has inline data, skip readpages */ + /* If the file has inline data, skip readahead */ if (f2fs_has_inline_data(inode)) return; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index eac4984cc753..488b460e046f 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -627,7 +627,7 @@ struct fuse_conn { /** Connection successful. Only set in INIT */ unsigned conn_init:1; - /** Do readpages asynchronously? Only set in INIT */ + /** Do readahead asynchronously? Only set in INIT */ unsigned async_read:1; /** Return an unique read error after abort. Only set in INIT */ diff --git a/fs/verity/verify.c b/fs/verity/verify.c index 0adb970f4e73..14e2fb49cff5 100644 --- a/fs/verity/verify.c +++ b/fs/verity/verify.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Data verification functions, i.e. hooks for ->readpages() + * Data verification functions, i.e. hooks for ->readahead() * * Copyright 2019 Google LLC */ @@ -214,7 +214,7 @@ EXPORT_SYMBOL_GPL(fsverity_verify_page); * that fail verification are set to the Error state. Verification is skipped * for pages already in the Error state, e.g. due to fscrypt decryption failure. * - * This is a helper function for use by the ->readpages() method of filesystems + * This is a helper function for use by the ->readahead() method of filesystems * that issue bios to read data directly into the page cache. Filesystems that * populate the page cache without issuing bios (e.g. non block-based * filesystems) must instead call fsverity_verify_page() directly on each page. diff --git a/include/linux/fs.h b/include/linux/fs.h index 183160872133..7c81887cc7e8 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -370,12 +370,6 @@ struct address_space_operations { /* Mark a folio dirty. Return true if this dirtied it */ bool (*dirty_folio)(struct address_space *, struct folio *); - /* - * Reads in the requested pages. Unlike ->readpage(), this is - * PURELY used for read-ahead!. - */ - int (*readpages)(struct file *filp, struct address_space *mapping, - struct list_head *pages, unsigned nr_pages); void (*readahead)(struct readahead_control *); int (*write_begin)(struct file *, struct address_space *mapping, diff --git a/include/linux/fsverity.h b/include/linux/fsverity.h index b568b3c7d095..a7afc800bd8d 100644 --- a/include/linux/fsverity.h +++ b/include/linux/fsverity.h @@ -221,7 +221,7 @@ static inline void fsverity_enqueue_verify_work(struct work_struct *work) * * This checks whether ->i_verity_info has been set. * - * Filesystems call this from ->readpages() to check whether the pages need to + * Filesystems call this from ->readahead() to check whether the pages need to * be verified or not. Don't use IS_VERITY() for this purpose; it's subject to * a race condition where the file is being read concurrently with * FS_IOC_ENABLE_VERITY completing. (S_VERITY is set before ->i_verity_info.) diff --git a/mm/filemap.c b/mm/filemap.c index 647d72bf23b6..d904cd7e4181 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -2538,7 +2538,7 @@ static int filemap_create_folio(struct file *file, * the page cache as the locked folio would then be enough to * synchronize with hole punching. But there are code paths * such as filemap_update_page() filling in partially uptodate - * pages or ->readpages() that need to hold invalidate_lock + * pages or ->readahead() that need to hold invalidate_lock * while mapping blocks for IO so let's hold the lock here as * well to keep locking rules simple. */ diff --git a/mm/readahead.c b/mm/readahead.c index 9097af639beb..297bd0719cda 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -170,13 +170,6 @@ static void read_pages(struct readahead_control *rac, struct list_head *pages, unlock_page(page); put_page(page); } - } else if (aops->readpages) { - aops->readpages(rac->file, rac->mapping, pages, - readahead_count(rac)); - /* Clean up the remaining pages */ - put_pages_list(pages); - rac->_index += rac->_nr_pages; - rac->_nr_pages = 0; } else { while ((page = readahead_page(rac))) { aops->readpage(rac->file, page); @@ -253,10 +246,7 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, folio = filemap_alloc_folio(gfp_mask, 0); if (!folio) break; - if (mapping->a_ops->readpages) { - folio->index = index + i; - list_add(&folio->lru, &page_pool); - } else if (filemap_add_folio(mapping, folio, index + i, + if (filemap_add_folio(mapping, folio, index + i, gfp_mask) < 0) { folio_put(folio); read_pages(ractl, &page_pool, true); @@ -318,8 +308,7 @@ void force_page_cache_ra(struct readahead_control *ractl, struct backing_dev_info *bdi = inode_to_bdi(mapping->host); unsigned long max_pages, index; - if (unlikely(!mapping->a_ops->readpage && !mapping->a_ops->readpages && - !mapping->a_ops->readahead)) + if (unlikely(!mapping->a_ops->readpage && !mapping->a_ops->readahead)) return; /* From dfd8b4fc76d5f7ae5663328b791c4acf222c4d39 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 31 Mar 2022 05:35:23 -0700 Subject: [PATCH 67/95] mm: remove the pages argument to read_pages This is always an empty list or NULL with the removal of the ->readahead support, so remove it. Signed-off-by: Christoph Hellwig Reviewed-by: Al Viro Acked-by: Al Viro Signed-off-by: Matthew Wilcox (Oracle) --- mm/readahead.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/mm/readahead.c b/mm/readahead.c index 297bd0719cda..05207a663801 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -142,8 +142,7 @@ file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping) } EXPORT_SYMBOL_GPL(file_ra_state_init); -static void read_pages(struct readahead_control *rac, struct list_head *pages, - bool skip_page) +static void read_pages(struct readahead_control *rac, bool skip_page) { const struct address_space_operations *aops = rac->mapping->a_ops; struct page *page; @@ -179,7 +178,6 @@ static void read_pages(struct readahead_control *rac, struct list_head *pages, blk_finish_plug(&plug); - BUG_ON(pages && !list_empty(pages)); BUG_ON(readahead_count(rac)); out: @@ -206,7 +204,6 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, { struct address_space *mapping = ractl->mapping; unsigned long index = readahead_index(ractl); - LIST_HEAD(page_pool); gfp_t gfp_mask = readahead_gfp_mask(mapping); unsigned long i; @@ -238,7 +235,7 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, * have a stable reference to this page, and it's * not worth getting one just for that. */ - read_pages(ractl, &page_pool, true); + read_pages(ractl, true); i = ractl->_index + ractl->_nr_pages - index - 1; continue; } @@ -249,7 +246,7 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, if (filemap_add_folio(mapping, folio, index + i, gfp_mask) < 0) { folio_put(folio); - read_pages(ractl, &page_pool, true); + read_pages(ractl, true); i = ractl->_index + ractl->_nr_pages - index - 1; continue; } @@ -263,7 +260,7 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, * uptodate then the caller will launch readpage again, and * will then handle the error. */ - read_pages(ractl, &page_pool, false); + read_pages(ractl, false); filemap_invalidate_unlock_shared(mapping); memalloc_nofs_restore(nofs); } @@ -537,7 +534,7 @@ void page_cache_ra_order(struct readahead_control *ractl, ra->async_size += index - limit - 1; } - read_pages(ractl, NULL, false); + read_pages(ractl, false); /* * If there were already pages in the page cache, then we may have From b4e089d705eef82364945abae325cd241c80e107 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 31 Mar 2022 05:35:55 -0700 Subject: [PATCH 68/95] mm: remove the skip_page argument to read_pages The skip_page argument to read_pages controls if rac->_index is incremented before returning from the function. Just open code that in the callers. Signed-off-by: Christoph Hellwig Reviewed-by: Al Viro Acked-by: Al Viro Signed-off-by: Matthew Wilcox (Oracle) --- mm/readahead.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/mm/readahead.c b/mm/readahead.c index 05207a663801..2e5c695b303d 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -142,14 +142,14 @@ file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping) } EXPORT_SYMBOL_GPL(file_ra_state_init); -static void read_pages(struct readahead_control *rac, bool skip_page) +static void read_pages(struct readahead_control *rac) { const struct address_space_operations *aops = rac->mapping->a_ops; struct page *page; struct blk_plug plug; if (!readahead_count(rac)) - goto out; + return; blk_start_plug(&plug); @@ -179,10 +179,6 @@ static void read_pages(struct readahead_control *rac, bool skip_page) blk_finish_plug(&plug); BUG_ON(readahead_count(rac)); - -out: - if (skip_page) - rac->_index++; } /** @@ -235,7 +231,8 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, * have a stable reference to this page, and it's * not worth getting one just for that. */ - read_pages(ractl, true); + read_pages(ractl); + ractl->_index++; i = ractl->_index + ractl->_nr_pages - index - 1; continue; } @@ -246,7 +243,8 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, if (filemap_add_folio(mapping, folio, index + i, gfp_mask) < 0) { folio_put(folio); - read_pages(ractl, true); + read_pages(ractl); + ractl->_index++; i = ractl->_index + ractl->_nr_pages - index - 1; continue; } @@ -260,7 +258,7 @@ void page_cache_ra_unbounded(struct readahead_control *ractl, * uptodate then the caller will launch readpage again, and * will then handle the error. */ - read_pages(ractl, false); + read_pages(ractl); filemap_invalidate_unlock_shared(mapping); memalloc_nofs_restore(nofs); } @@ -534,7 +532,7 @@ void page_cache_ra_order(struct readahead_control *ractl, ra->async_size += index - limit - 1; } - read_pages(ractl, false); + read_pages(ractl); /* * If there were already pages in the page cache, then we may have From 1e4702806faca1551733f58be17ea11a9d214e91 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Thu, 31 Mar 2022 15:02:34 -0400 Subject: [PATCH 69/95] readahead: Update comments - Refer to folios where appropriate, not pages (Matthew Wilcox) - Eliminate references to the internal PG_readhead - Use "readahead" consistently - not "read-ahead" or "read ahead" (mostly Neil Brown) - Clarify some sections that, on reflection, weren't very clear (Neil Brown) - Minor punctuation/spelling fixes (Neil Brown) Signed-off-by: Matthew Wilcox (Oracle) --- mm/readahead.c | 92 ++++++++++++++++++++++++-------------------------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/mm/readahead.c b/mm/readahead.c index 2e5c695b303d..8e3775829513 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -13,29 +13,29 @@ * * Readahead is used to read content into the page cache before it is * explicitly requested by the application. Readahead only ever - * attempts to read pages that are not yet in the page cache. If a - * page is present but not up-to-date, readahead will not try to read + * attempts to read folios that are not yet in the page cache. If a + * folio is present but not up-to-date, readahead will not try to read * it. In that case a simple ->readpage() will be requested. * * Readahead is triggered when an application read request (whether a - * systemcall or a page fault) finds that the requested page is not in + * system call or a page fault) finds that the requested folio is not in * the page cache, or that it is in the page cache and has the - * %PG_readahead flag set. This flag indicates that the page was loaded - * as part of a previous read-ahead request and now that it has been - * accessed, it is time for the next read-ahead. + * readahead flag set. This flag indicates that the folio was read + * as part of a previous readahead request and now that it has been + * accessed, it is time for the next readahead. * * Each readahead request is partly synchronous read, and partly async - * read-ahead. This is reflected in the struct file_ra_state which - * contains ->size being to total number of pages, and ->async_size - * which is the number of pages in the async section. The first page in - * this async section will have %PG_readahead set as a trigger for a - * subsequent read ahead. Once a series of sequential reads has been + * readahead. This is reflected in the struct file_ra_state which + * contains ->size being the total number of pages, and ->async_size + * which is the number of pages in the async section. The readahead + * flag will be set on the first folio in this async section to trigger + * a subsequent readahead. Once a series of sequential reads has been * established, there should be no need for a synchronous component and - * all read ahead request will be fully asynchronous. + * all readahead request will be fully asynchronous. * - * When either of the triggers causes a readahead, three numbers need to - * be determined: the start of the region, the size of the region, and - * the size of the async tail. + * When either of the triggers causes a readahead, three numbers need + * to be determined: the start of the region to read, the size of the + * region, and the size of the async tail. * * The start of the region is simply the first page address at or after * the accessed address, which is not currently populated in the page @@ -45,14 +45,14 @@ * was explicitly requested from the determined request size, unless * this would be less than zero - then zero is used. NOTE THIS * CALCULATION IS WRONG WHEN THE START OF THE REGION IS NOT THE ACCESSED - * PAGE. + * PAGE. ALSO THIS CALCULATION IS NOT USED CONSISTENTLY. * * The size of the region is normally determined from the size of the * previous readahead which loaded the preceding pages. This may be * discovered from the struct file_ra_state for simple sequential reads, * or from examining the state of the page cache when multiple * sequential reads are interleaved. Specifically: where the readahead - * was triggered by the %PG_readahead flag, the size of the previous + * was triggered by the readahead flag, the size of the previous * readahead is assumed to be the number of pages from the triggering * page to the start of the new readahead. In these cases, the size of * the previous readahead is scaled, often doubled, for the new @@ -65,52 +65,52 @@ * larger than the current request, and it is not scaled up, unless it * is at the start of file. * - * In general read ahead is accelerated at the start of the file, as + * In general readahead is accelerated at the start of the file, as * reads from there are often sequential. There are other minor - * adjustments to the read ahead size in various special cases and these + * adjustments to the readahead size in various special cases and these * are best discovered by reading the code. * - * The above calculation determines the readahead, to which any requested - * read size may be added. + * The above calculation, based on the previous readahead size, + * determines the size of the readahead, to which any requested read + * size may be added. * * Readahead requests are sent to the filesystem using the ->readahead() * address space operation, for which mpage_readahead() is a canonical * implementation. ->readahead() should normally initiate reads on all - * pages, but may fail to read any or all pages without causing an IO + * folios, but may fail to read any or all folios without causing an I/O * error. The page cache reading code will issue a ->readpage() request - * for any page which ->readahead() does not provided, and only an error + * for any folio which ->readahead() did not read, and only an error * from this will be final. * - * ->readahead() will generally call readahead_page() repeatedly to get - * each page from those prepared for read ahead. It may fail to read a - * page by: + * ->readahead() will generally call readahead_folio() repeatedly to get + * each folio from those prepared for readahead. It may fail to read a + * folio by: * - * * not calling readahead_page() sufficiently many times, effectively - * ignoring some pages, as might be appropriate if the path to + * * not calling readahead_folio() sufficiently many times, effectively + * ignoring some folios, as might be appropriate if the path to * storage is congested. * - * * failing to actually submit a read request for a given page, + * * failing to actually submit a read request for a given folio, * possibly due to insufficient resources, or * * * getting an error during subsequent processing of a request. * - * In the last two cases, the page should be unlocked to indicate that - * the read attempt has failed. In the first case the page will be - * unlocked by the caller. + * In the last two cases, the folio should be unlocked by the filesystem + * to indicate that the read attempt has failed. In the first case the + * folio will be unlocked by the VFS. * - * Those pages not in the final ``async_size`` of the request should be + * Those folios not in the final ``async_size`` of the request should be * considered to be important and ->readahead() should not fail them due * to congestion or temporary resource unavailability, but should wait * for necessary resources (e.g. memory or indexing information) to - * become available. Pages in the final ``async_size`` may be + * become available. Folios in the final ``async_size`` may be * considered less urgent and failure to read them is more acceptable. - * In this case it is best to use delete_from_page_cache() to remove the - * pages from the page cache as is automatically done for pages that - * were not fetched with readahead_page(). This will allow a - * subsequent synchronous read ahead request to try them again. If they + * In this case it is best to use filemap_remove_folio() to remove the + * folios from the page cache as is automatically done for folios that + * were not fetched with readahead_folio(). This will allow a + * subsequent synchronous readahead request to try them again. If they * are left in the page cache, then they will be read individually using - * ->readpage(). - * + * ->readpage() which may be less efficient. */ #include @@ -157,7 +157,7 @@ static void read_pages(struct readahead_control *rac) aops->readahead(rac); /* * Clean up the remaining pages. The sizes in ->ra - * maybe be used to size next read-ahead, so make sure + * may be used to size the next readahead, so make sure * they accurately reflect what happened. */ while ((page = readahead_page(rac))) { @@ -420,7 +420,7 @@ static pgoff_t count_history_pages(struct address_space *mapping, } /* - * page cache context based read-ahead + * page cache context based readahead */ static int try_context_readahead(struct address_space *mapping, struct file_ra_state *ra, @@ -671,9 +671,9 @@ void page_cache_sync_ra(struct readahead_control *ractl, bool do_forced_ra = ractl->file && (ractl->file->f_mode & FMODE_RANDOM); /* - * Even if read-ahead is disabled, issue this request as read-ahead + * Even if readahead is disabled, issue this request as readahead * as we'll need it to satisfy the requested range. The forced - * read-ahead will do the right thing and limit the read to just the + * readahead will do the right thing and limit the read to just the * requested range, which we'll set to 1 page for this case. */ if (!ractl->ra->ra_pages || blk_cgroup_congested()) { @@ -689,7 +689,6 @@ void page_cache_sync_ra(struct readahead_control *ractl, return; } - /* do read-ahead */ ondemand_readahead(ractl, NULL, req_count); } EXPORT_SYMBOL_GPL(page_cache_sync_ra); @@ -697,7 +696,7 @@ EXPORT_SYMBOL_GPL(page_cache_sync_ra); void page_cache_async_ra(struct readahead_control *ractl, struct folio *folio, unsigned long req_count) { - /* no read-ahead */ + /* no readahead */ if (!ractl->ra->ra_pages) return; @@ -712,7 +711,6 @@ void page_cache_async_ra(struct readahead_control *ractl, if (blk_cgroup_congested()) return; - /* do read-ahead */ ondemand_readahead(ractl, folio, req_count); } EXPORT_SYMBOL_GPL(page_cache_async_ra); From 2756c818e5cbcc4c9d13d7399273e86068e95156 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Fri, 4 Mar 2022 14:05:23 -0500 Subject: [PATCH 70/95] iomap: Simplify is_partially_uptodate a little Remove the unnecessary variable 'len' and fix a comment to refer to the folio instead of the page. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Al Viro Acked-by: Al Viro --- fs/iomap/buffered-io.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index 49dccd9050f1..8ce8720093b9 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -435,18 +435,17 @@ bool iomap_is_partially_uptodate(struct folio *folio, size_t from, size_t count) { struct iomap_page *iop = to_iomap_page(folio); struct inode *inode = folio->mapping->host; - size_t len; unsigned first, last, i; if (!iop) return false; - /* Limit range to this folio */ - len = min(folio_size(folio) - from, count); + /* Caller's range may extend past the end of this folio */ + count = min(folio_size(folio) - from, count); - /* First and last blocks in range within page */ + /* First and last blocks in range within folio */ first = from >> inode->i_blkbits; - last = (from + len - 1) >> inode->i_blkbits; + last = (from + count - 1) >> inode->i_blkbits; for (i = first; i <= last; i++) if (!test_bit(i, iop->uptodate)) From a9fcd89d67bb8c4ad613b54ab691fc603c94a03a Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 14 Feb 2022 09:13:43 -0500 Subject: [PATCH 71/95] fs: Remove read_actor_t This typedef is not used any more. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Al Viro Acked-by: Al Viro --- include/linux/fs.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/linux/fs.h b/include/linux/fs.h index 7c81887cc7e8..7588d3a0ced8 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -357,9 +357,6 @@ typedef struct { int error; } read_descriptor_t; -typedef int (*read_actor_t)(read_descriptor_t *, struct page *, - unsigned long, unsigned long); - struct address_space_operations { int (*writepage)(struct page *page, struct writeback_control *wbc); int (*readpage)(struct file *, struct page *); From b2403a61308533c576c9dd783fcb73a9186e0b37 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 14 Feb 2022 09:15:34 -0500 Subject: [PATCH 72/95] fs, net: Move read_descriptor_t to net.h fs.h has no more need for this typedef; networking is now the sole user of the read_descriptor_t. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Al Viro Acked-by: Al Viro --- include/linux/fs.h | 19 ------------------- include/linux/net.h | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/include/linux/fs.h b/include/linux/fs.h index 7588d3a0ced8..8ff28939de60 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -338,25 +338,6 @@ static inline bool is_sync_kiocb(struct kiocb *kiocb) return kiocb->ki_complete == NULL; } -/* - * "descriptor" for what we're up to with a read. - * This allows us to use the same read code yet - * have multiple different users of the data that - * we read from a file. - * - * The simplest case just copies the data to user - * mode. - */ -typedef struct { - size_t written; - size_t count; - union { - char __user *buf; - void *data; - } arg; - int error; -} read_descriptor_t; - struct address_space_operations { int (*writepage)(struct page *page, struct writeback_control *wbc); int (*readpage)(struct file *, struct page *); diff --git a/include/linux/net.h b/include/linux/net.h index ba736b457a06..12093f4db50c 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -125,6 +125,25 @@ struct socket { struct socket_wq wq; }; +/* + * "descriptor" for what we're up to with a read. + * This allows us to use the same read code yet + * have multiple different users of the data that + * we read from a file. + * + * The simplest case just copies the data to user + * mode. + */ +typedef struct { + size_t written; + size_t count; + union { + char __user *buf; + void *data; + } arg; + int error; +} read_descriptor_t; + struct vm_area_struct; struct page; struct sockaddr; From 800ba29547e16d5fbe67ca764ba660e049e9f1bf Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sat, 19 Feb 2022 23:19:49 -0500 Subject: [PATCH 73/95] fs: Pass an iocb to generic_perform_write() We can extract both the file pointer and the pos from the iocb. This simplifies each caller as well as allowing generic_perform_write() to see more of the iocb contents in the future. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Christian Brauner Reviewed-by: Al Viro Acked-by: Al Viro --- fs/ceph/file.c | 2 +- fs/ext4/file.c | 2 +- fs/f2fs/file.c | 2 +- fs/nfs/file.c | 2 +- include/linux/fs.h | 2 +- mm/filemap.c | 10 ++++++---- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/fs/ceph/file.c b/fs/ceph/file.c index feb75eb1cd82..6c9e837aa1d3 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -1869,7 +1869,7 @@ retry_snap: * are pending vmtruncate. So write and vmtruncate * can not run at the same time */ - written = generic_perform_write(file, from, pos); + written = generic_perform_write(iocb, from); if (likely(written >= 0)) iocb->ki_pos = pos + written; ceph_end_io_write(inode); diff --git a/fs/ext4/file.c b/fs/ext4/file.c index 8bd66cdc41be..6feb07e3e1eb 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -267,7 +267,7 @@ static ssize_t ext4_buffered_write_iter(struct kiocb *iocb, goto out; current->backing_dev_info = inode_to_bdi(inode); - ret = generic_perform_write(iocb->ki_filp, from, iocb->ki_pos); + ret = generic_perform_write(iocb, from); current->backing_dev_info = NULL; out: diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index d3f39a704b8b..5b89af0f27f0 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -4448,7 +4448,7 @@ static ssize_t f2fs_buffered_write_iter(struct kiocb *iocb, return -EOPNOTSUPP; current->backing_dev_info = inode_to_bdi(inode); - ret = generic_perform_write(file, from, iocb->ki_pos); + ret = generic_perform_write(iocb, from); current->backing_dev_info = NULL; if (ret > 0) { diff --git a/fs/nfs/file.c b/fs/nfs/file.c index b0ca244c50d0..150b7fa8f0a7 100644 --- a/fs/nfs/file.c +++ b/fs/nfs/file.c @@ -646,7 +646,7 @@ ssize_t nfs_file_write(struct kiocb *iocb, struct iov_iter *from) result = generic_write_checks(iocb, from); if (result > 0) { current->backing_dev_info = inode_to_bdi(inode); - result = generic_perform_write(file, from, iocb->ki_pos); + result = generic_perform_write(iocb, from); current->backing_dev_info = NULL; } nfs_end_io_write(inode); diff --git a/include/linux/fs.h b/include/linux/fs.h index 8ff28939de60..468dc7ec821f 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2999,7 +2999,7 @@ extern ssize_t generic_file_read_iter(struct kiocb *, struct iov_iter *); extern ssize_t __generic_file_write_iter(struct kiocb *, struct iov_iter *); extern ssize_t generic_file_write_iter(struct kiocb *, struct iov_iter *); extern ssize_t generic_file_direct_write(struct kiocb *, struct iov_iter *); -extern ssize_t generic_perform_write(struct file *, struct iov_iter *, loff_t); +ssize_t generic_perform_write(struct kiocb *, struct iov_iter *); ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos, rwf_t flags); diff --git a/mm/filemap.c b/mm/filemap.c index d904cd7e4181..3a5ffb5587cd 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3752,9 +3752,10 @@ out: } EXPORT_SYMBOL(generic_file_direct_write); -ssize_t generic_perform_write(struct file *file, - struct iov_iter *i, loff_t pos) +ssize_t generic_perform_write(struct kiocb *iocb, struct iov_iter *i) { + struct file *file = iocb->ki_filp; + loff_t pos = iocb->ki_pos; struct address_space *mapping = file->f_mapping; const struct address_space_operations *a_ops = mapping->a_ops; long status = 0; @@ -3884,7 +3885,8 @@ ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) if (written < 0 || !iov_iter_count(from) || IS_DAX(inode)) goto out; - status = generic_perform_write(file, from, pos = iocb->ki_pos); + pos = iocb->ki_pos; + status = generic_perform_write(iocb, from); /* * If generic_perform_write() returned a synchronous error * then we want to return the number of bytes which were @@ -3916,7 +3918,7 @@ ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from) */ } } else { - written = generic_perform_write(file, from, iocb->ki_pos); + written = generic_perform_write(iocb, from); if (likely(written > 0)) iocb->ki_pos += written; } From d7414ba14a3a67f81321069219dc7dbc095022c3 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Sun, 20 Feb 2022 22:28:03 -0500 Subject: [PATCH 74/95] filemap: Remove AOP_FLAG_CONT_EXPAND This flag is no longer used, so remove it. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Al Viro Acked-by: Al Viro --- fs/buffer.c | 3 +-- include/linux/fs.h | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index d67fbe063a3a..2b5561ae5d0b 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2352,8 +2352,7 @@ int generic_cont_expand_simple(struct inode *inode, loff_t size) if (err) goto out; - err = pagecache_write_begin(NULL, mapping, size, 0, - AOP_FLAG_CONT_EXPAND, &page, &fsdata); + err = pagecache_write_begin(NULL, mapping, size, 0, 0, &page, &fsdata); if (err) goto out; diff --git a/include/linux/fs.h b/include/linux/fs.h index 468dc7ec821f..bbde95387a23 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -275,7 +275,6 @@ enum positive_aop_returns { AOP_TRUNCATED_PAGE = 0x80001, }; -#define AOP_FLAG_CONT_EXPAND 0x0001 /* called from cont_expand */ #define AOP_FLAG_NOFS 0x0002 /* used by filesystem to direct * helper code (eg buffer layer) * to clear GFP_FS from alloc */ From 0f2523366386f29d56040887555989a1de548625 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 29 Mar 2022 16:21:53 -0400 Subject: [PATCH 75/95] ext4: Correct ext4_journalled_dirty_folio() conversion This should use the new folio_buffers() instead of page_has_buffers(). Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Al Viro Acked-by: Al Viro --- fs/ext4/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index 1ce13f69fbec..13740f2d0e61 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -3589,7 +3589,7 @@ const struct iomap_ops ext4_iomap_report_ops = { static bool ext4_journalled_dirty_folio(struct address_space *mapping, struct folio *folio) { - WARN_ON_ONCE(!page_has_buffers(&folio->page)); + WARN_ON_ONCE(!folio_buffers(folio)); folio_set_checked(folio); return filemap_dirty_folio(mapping, folio); } From 0fb5b2ebc0422fccbb41b09ff38ff7f27616294d Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 29 Mar 2022 16:22:54 -0400 Subject: [PATCH 76/95] f2fs: Correct f2fs_dirty_data_folio() conversion I got the return value wrong. Very little checks the return value from set_page_dirty(), so nobody noticed during testing. Fixes: 4f5e34f71318 ("f2fs: Convert f2fs_set_data_page_dirty to f2fs_dirty_data_folio") Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Al Viro Acked-by: Al Viro --- fs/f2fs/data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index c92920c8661d..8e0c2e773c8d 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -3571,7 +3571,7 @@ static bool f2fs_dirty_data_folio(struct address_space *mapping, f2fs_update_dirty_folio(inode, folio); return true; } - return true; + return false; } From 29c87793eb3c6f78f215f85be863e5415e263e43 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 29 Mar 2022 16:27:21 -0400 Subject: [PATCH 77/95] f2fs: Get the superblock from the mapping instead of the page It's slightly more efficient to go directly from the mapping to the superblock than to go from the page. Now that these routines have the mapping passed to them, there's no reason not to use it. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Al Viro Acked-by: Al Viro --- fs/f2fs/checkpoint.c | 2 +- fs/f2fs/node.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c index a8fc4fa511a8..f5366feea82d 100644 --- a/fs/f2fs/checkpoint.c +++ b/fs/f2fs/checkpoint.c @@ -456,7 +456,7 @@ static bool f2fs_dirty_meta_folio(struct address_space *mapping, folio_mark_uptodate(folio); if (!folio_test_dirty(folio)) { filemap_dirty_folio(mapping, folio); - inc_page_count(F2FS_P_SB(&folio->page), F2FS_DIRTY_META); + inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_META); set_page_private_reference(&folio->page); return true; } diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 0b6e741e94a0..c45d341dcf6e 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -2146,11 +2146,11 @@ static bool f2fs_dirty_node_folio(struct address_space *mapping, folio_mark_uptodate(folio); #ifdef CONFIG_F2FS_CHECK_FS if (IS_INODE(&folio->page)) - f2fs_inode_chksum_set(F2FS_P_SB(&folio->page), &folio->page); + f2fs_inode_chksum_set(F2FS_M_SB(mapping), &folio->page); #endif if (!folio_test_dirty(folio)) { filemap_dirty_folio(mapping, folio); - inc_page_count(F2FS_P_SB(&folio->page), F2FS_DIRTY_NODES); + inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES); set_page_private_reference(&folio->page); return true; } From c37731301d66cce9c1a212c60e504f28e4342b39 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 29 Mar 2022 16:31:41 -0400 Subject: [PATCH 78/95] ntfs: Correct mark_ntfs_record_dirty() folio conversion We've already done the work of block_dirty_folio() here, leaving only the work that needs to be done by filemap_dirty_folio(). This was a misconversion where I misread __set_page_dirty_nobuffers() as __set_page_dirty_buffers(). Fixes: e621900ad28b ("fs: Convert __set_page_dirty_buffers to block_dirty_folio") Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Al Viro Acked-by: Al Viro --- fs/ntfs/aops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ntfs/aops.c b/fs/ntfs/aops.c index d154dcfe06af..90e3dad8ee45 100644 --- a/fs/ntfs/aops.c +++ b/fs/ntfs/aops.c @@ -1746,7 +1746,7 @@ void mark_ntfs_record_dirty(struct page *page, const unsigned int ofs) { set_buffer_dirty(bh); } while ((bh = bh->b_this_page) != head); spin_unlock(&mapping->private_lock); - block_dirty_folio(mapping, page_folio(page)); + filemap_dirty_folio(mapping, page_folio(page)); if (unlikely(buffers_to_free)) { do { bh = buffers_to_free->b_this_page; From 5a60542c61f3cce6e5dff2a38c8fb08a852a517b Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Tue, 29 Mar 2022 23:25:06 -0400 Subject: [PATCH 79/95] btrfs: Remove a use of PAGE_SIZE in btrfs_invalidate_folio() While btrfs doesn't use large folios yet, this should have been changed as part of the conversion from invalidatepage to invalidate_folio. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Christoph Hellwig Reviewed-by: Al Viro Acked-by: Al Viro --- fs/btrfs/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index aa0a60ee26cb..6bfc4343c98d 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -8296,7 +8296,7 @@ static void btrfs_invalidate_folio(struct folio *folio, size_t offset, * cover the full folio, like invalidating the last folio, we're * still safe to wait for ordered extent to finish. */ - if (!(offset == 0 && length == PAGE_SIZE)) { + if (!(offset == 0 && length == folio_size(folio))) { btrfs_releasepage(&folio->page, GFP_NOFS); return; } From e6b0a7b357659c332231621e4315658d062c23ee Mon Sep 17 00:00:00 2001 From: Charan Teja Kalla Date: Fri, 1 Apr 2022 11:28:12 -0700 Subject: [PATCH 80/95] Revert "mm: madvise: skip unmapped vma holes passed to process_madvise" This reverts commit 08095d6310a7 ("mm: madvise: skip unmapped vma holes passed to process_madvise") as process_madvise() fails to return the exact processed bytes in other cases too. As an example: if process_madvise() hits mlocked pages after processing some initial bytes passed in [start, end), it just returns EINVAL although some bytes are processed. Thus making an exception only for ENOMEM is partially fixing the problem of returning the proper advised bytes. Thus revert this patch and return proper bytes advised. Link: https://lkml.kernel.org/r/e73da1304a88b6a8a11907045117cccf4c2b8374.1648046642.git.quic_charante@quicinc.com Fixes: 08095d6310a7ce ("mm: madvise: skip unmapped vma holes passed to process_madvise") Signed-off-by: Charan Teja Kalla Acked-by: Michal Hocko Cc: Suren Baghdasaryan Cc: Vlastimil Babka Cc: David Rientjes Cc: Nadav Amit Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/madvise.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/mm/madvise.c b/mm/madvise.c index b41858ee937b..1873616a37d2 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -1464,16 +1464,9 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec, while (iov_iter_count(&iter)) { iovec = iov_iter_iovec(&iter); - /* - * do_madvise returns ENOMEM if unmapped holes are present - * in the passed VMA. process_madvise() is expected to skip - * unmapped holes passed to it in the 'struct iovec' list - * and not fail because of them. Thus treat -ENOMEM return - * from do_madvise as valid and continue processing. - */ ret = do_madvise(mm, (unsigned long)iovec.iov_base, iovec.iov_len, behavior); - if (ret < 0 && ret != -ENOMEM) + if (ret < 0) break; iov_iter_advance(&iter, iovec.iov_len); } From de19433423c7bedabbd4f9a25f7dbc62c5e78921 Mon Sep 17 00:00:00 2001 From: Joseph Qi Date: Fri, 1 Apr 2022 11:28:15 -0700 Subject: [PATCH 81/95] ocfs2: fix crash when mount with quota enabled There is a reported crash when mounting ocfs2 with quota enabled. RIP: 0010:ocfs2_qinfo_lock_res_init+0x44/0x50 [ocfs2] Call Trace: ocfs2_local_read_info+0xb9/0x6f0 [ocfs2] dquot_load_quota_sb+0x216/0x470 dquot_load_quota_inode+0x85/0x100 ocfs2_enable_quotas+0xa0/0x1c0 [ocfs2] ocfs2_fill_super.cold+0xc8/0x1bf [ocfs2] mount_bdev+0x185/0x1b0 legacy_get_tree+0x27/0x40 vfs_get_tree+0x25/0xb0 path_mount+0x465/0xac0 __x64_sys_mount+0x103/0x140 It is caused by when initializing dqi_gqlock, the corresponding dqi_type and dqi_sb are not properly initialized. This issue is introduced by commit 6c85c2c72819, which wants to avoid accessing uninitialized variables in error cases. So make global quota info properly initialized. Link: https://lkml.kernel.org/r/20220323023644.40084-1-joseph.qi@linux.alibaba.com Link: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1007141 Fixes: 6c85c2c72819 ("ocfs2: quota_local: fix possible uninitialized-variable access in ocfs2_local_read_info()") Signed-off-by: Joseph Qi Reported-by: Dayvison Tested-by: Valentin Vidic Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/ocfs2/quota_global.c | 23 ++++++++++++----------- fs/ocfs2/quota_local.c | 2 -- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/fs/ocfs2/quota_global.c b/fs/ocfs2/quota_global.c index 273f65e0aaba..0b6f551a342a 100644 --- a/fs/ocfs2/quota_global.c +++ b/fs/ocfs2/quota_global.c @@ -337,7 +337,6 @@ void ocfs2_unlock_global_qf(struct ocfs2_mem_dqinfo *oinfo, int ex) /* Read information header from global quota file */ int ocfs2_global_read_info(struct super_block *sb, int type) { - struct inode *gqinode = NULL; unsigned int ino[OCFS2_MAXQUOTAS] = { USER_QUOTA_SYSTEM_INODE, GROUP_QUOTA_SYSTEM_INODE }; struct ocfs2_global_disk_dqinfo dinfo; @@ -346,29 +345,31 @@ int ocfs2_global_read_info(struct super_block *sb, int type) u64 pcount; int status; + oinfo->dqi_gi.dqi_sb = sb; + oinfo->dqi_gi.dqi_type = type; + ocfs2_qinfo_lock_res_init(&oinfo->dqi_gqlock, oinfo); + oinfo->dqi_gi.dqi_entry_size = sizeof(struct ocfs2_global_disk_dqblk); + oinfo->dqi_gi.dqi_ops = &ocfs2_global_ops; + oinfo->dqi_gqi_bh = NULL; + oinfo->dqi_gqi_count = 0; + /* Read global header */ - gqinode = ocfs2_get_system_file_inode(OCFS2_SB(sb), ino[type], + oinfo->dqi_gqinode = ocfs2_get_system_file_inode(OCFS2_SB(sb), ino[type], OCFS2_INVALID_SLOT); - if (!gqinode) { + if (!oinfo->dqi_gqinode) { mlog(ML_ERROR, "failed to get global quota inode (type=%d)\n", type); status = -EINVAL; goto out_err; } - oinfo->dqi_gi.dqi_sb = sb; - oinfo->dqi_gi.dqi_type = type; - oinfo->dqi_gi.dqi_entry_size = sizeof(struct ocfs2_global_disk_dqblk); - oinfo->dqi_gi.dqi_ops = &ocfs2_global_ops; - oinfo->dqi_gqi_bh = NULL; - oinfo->dqi_gqi_count = 0; - oinfo->dqi_gqinode = gqinode; + status = ocfs2_lock_global_qf(oinfo, 0); if (status < 0) { mlog_errno(status); goto out_err; } - status = ocfs2_extent_map_get_blocks(gqinode, 0, &oinfo->dqi_giblk, + status = ocfs2_extent_map_get_blocks(oinfo->dqi_gqinode, 0, &oinfo->dqi_giblk, &pcount, NULL); if (status < 0) goto out_unlock; diff --git a/fs/ocfs2/quota_local.c b/fs/ocfs2/quota_local.c index 0e4b16d4c037..b1a8b046f4c2 100644 --- a/fs/ocfs2/quota_local.c +++ b/fs/ocfs2/quota_local.c @@ -702,8 +702,6 @@ static int ocfs2_local_read_info(struct super_block *sb, int type) info->dqi_priv = oinfo; oinfo->dqi_type = type; INIT_LIST_HEAD(&oinfo->dqi_chunk); - oinfo->dqi_gqinode = NULL; - ocfs2_qinfo_lock_res_init(&oinfo->dqi_gqlock, oinfo); oinfo->dqi_rec = NULL; oinfo->dqi_lqi_bh = NULL; oinfo->dqi_libh = NULL; From e897be17a441fa637cd166fc3de1445131e57692 Mon Sep 17 00:00:00 2001 From: Ryusuke Konishi Date: Fri, 1 Apr 2022 11:28:18 -0700 Subject: [PATCH 82/95] nilfs2: fix lockdep warnings in page operations for btree nodes Patch series "nilfs2 lockdep warning fixes". The first two are to resolve the lockdep warning issue, and the last one is the accompanying cleanup and low priority. Based on your comment, this series solves the issue by separating inode object as needed. Since I was worried about the impact of the object composition changes, I tested the series carefully not to cause regressions especially for delicate functions such like disk space reclamation and snapshots. This patch (of 3): If CONFIG_LOCKDEP is enabled, nilfs2 hits lockdep warnings at inode_to_wb() during page/folio operations for btree nodes: WARNING: CPU: 0 PID: 6575 at include/linux/backing-dev.h:269 inode_to_wb include/linux/backing-dev.h:269 [inline] WARNING: CPU: 0 PID: 6575 at include/linux/backing-dev.h:269 folio_account_dirtied mm/page-writeback.c:2460 [inline] WARNING: CPU: 0 PID: 6575 at include/linux/backing-dev.h:269 __folio_mark_dirty+0xa7c/0xe30 mm/page-writeback.c:2509 Modules linked in: ... RIP: 0010:inode_to_wb include/linux/backing-dev.h:269 [inline] RIP: 0010:folio_account_dirtied mm/page-writeback.c:2460 [inline] RIP: 0010:__folio_mark_dirty+0xa7c/0xe30 mm/page-writeback.c:2509 ... Call Trace: __set_page_dirty include/linux/pagemap.h:834 [inline] mark_buffer_dirty+0x4e6/0x650 fs/buffer.c:1145 nilfs_btree_propagate_p fs/nilfs2/btree.c:1889 [inline] nilfs_btree_propagate+0x4ae/0xea0 fs/nilfs2/btree.c:2085 nilfs_bmap_propagate+0x73/0x170 fs/nilfs2/bmap.c:337 nilfs_collect_dat_data+0x45/0xd0 fs/nilfs2/segment.c:625 nilfs_segctor_apply_buffers+0x14a/0x470 fs/nilfs2/segment.c:1009 nilfs_segctor_scan_file+0x47a/0x700 fs/nilfs2/segment.c:1048 nilfs_segctor_collect_blocks fs/nilfs2/segment.c:1224 [inline] nilfs_segctor_collect fs/nilfs2/segment.c:1494 [inline] nilfs_segctor_do_construct+0x14f3/0x6c60 fs/nilfs2/segment.c:2036 nilfs_segctor_construct+0x7a7/0xb30 fs/nilfs2/segment.c:2372 nilfs_segctor_thread_construct fs/nilfs2/segment.c:2480 [inline] nilfs_segctor_thread+0x3c3/0xf90 fs/nilfs2/segment.c:2563 kthread+0x405/0x4f0 kernel/kthread.c:327 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:295 This is because nilfs2 uses two page caches for each inode and inode->i_mapping never points to one of them, the btree node cache. This causes inode_to_wb(inode) to refer to a different page cache than the caller page/folio operations such like __folio_start_writeback(), __folio_end_writeback(), or __folio_mark_dirty() acquired the lock. This patch resolves the issue by allocating and using an additional inode to hold the page cache of btree nodes. The inode is attached one-to-one to the traditional nilfs2 inode if it requires a block mapping with b-tree. This setup change is in memory only and does not affect the disk format. Link: https://lkml.kernel.org/r/1647867427-30498-1-git-send-email-konishi.ryusuke@gmail.com Link: https://lkml.kernel.org/r/1647867427-30498-2-git-send-email-konishi.ryusuke@gmail.com Link: https://lore.kernel.org/r/YXrYvIo8YRnAOJCj@casper.infradead.org Link: https://lore.kernel.org/r/9a20b33d-b38f-b4a2-4742-c1eb5b8e4d6c@redhat.com Signed-off-by: Ryusuke Konishi Reported-by: syzbot+0d5b462a6f07447991b3@syzkaller.appspotmail.com Reported-by: syzbot+34ef28bb2aeb28724aa0@syzkaller.appspotmail.com Reported-by: Hao Sun Reported-by: David Hildenbrand Tested-by: Ryusuke Konishi Cc: Matthew Wilcox Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/nilfs2/btnode.c | 23 ++++++++-- fs/nilfs2/btnode.h | 1 + fs/nilfs2/btree.c | 27 ++++++++---- fs/nilfs2/gcinode.c | 7 +-- fs/nilfs2/inode.c | 104 ++++++++++++++++++++++++++++++++++++++------ fs/nilfs2/mdt.c | 7 +-- fs/nilfs2/nilfs.h | 14 +++--- fs/nilfs2/page.c | 7 ++- fs/nilfs2/segment.c | 9 ++-- fs/nilfs2/super.c | 5 +-- 10 files changed, 154 insertions(+), 50 deletions(-) diff --git a/fs/nilfs2/btnode.c b/fs/nilfs2/btnode.c index 66bdaa2cf496..ca611ac09f7c 100644 --- a/fs/nilfs2/btnode.c +++ b/fs/nilfs2/btnode.c @@ -20,6 +20,23 @@ #include "page.h" #include "btnode.h" + +/** + * nilfs_init_btnc_inode - initialize B-tree node cache inode + * @btnc_inode: inode to be initialized + * + * nilfs_init_btnc_inode() sets up an inode for B-tree node cache. + */ +void nilfs_init_btnc_inode(struct inode *btnc_inode) +{ + struct nilfs_inode_info *ii = NILFS_I(btnc_inode); + + btnc_inode->i_mode = S_IFREG; + ii->i_flags = 0; + memset(&ii->i_bmap_data, 0, sizeof(struct nilfs_bmap)); + mapping_set_gfp_mask(btnc_inode->i_mapping, GFP_NOFS); +} + void nilfs_btnode_cache_clear(struct address_space *btnc) { invalidate_mapping_pages(btnc, 0, -1); @@ -29,7 +46,7 @@ void nilfs_btnode_cache_clear(struct address_space *btnc) struct buffer_head * nilfs_btnode_create_block(struct address_space *btnc, __u64 blocknr) { - struct inode *inode = NILFS_BTNC_I(btnc); + struct inode *inode = btnc->host; struct buffer_head *bh; bh = nilfs_grab_buffer(inode, btnc, blocknr, BIT(BH_NILFS_Node)); @@ -57,7 +74,7 @@ int nilfs_btnode_submit_block(struct address_space *btnc, __u64 blocknr, struct buffer_head **pbh, sector_t *submit_ptr) { struct buffer_head *bh; - struct inode *inode = NILFS_BTNC_I(btnc); + struct inode *inode = btnc->host; struct page *page; int err; @@ -157,7 +174,7 @@ int nilfs_btnode_prepare_change_key(struct address_space *btnc, struct nilfs_btnode_chkey_ctxt *ctxt) { struct buffer_head *obh, *nbh; - struct inode *inode = NILFS_BTNC_I(btnc); + struct inode *inode = btnc->host; __u64 oldkey = ctxt->oldkey, newkey = ctxt->newkey; int err; diff --git a/fs/nilfs2/btnode.h b/fs/nilfs2/btnode.h index 11663650add7..bd5544e63a01 100644 --- a/fs/nilfs2/btnode.h +++ b/fs/nilfs2/btnode.h @@ -30,6 +30,7 @@ struct nilfs_btnode_chkey_ctxt { struct buffer_head *newbh; }; +void nilfs_init_btnc_inode(struct inode *btnc_inode); void nilfs_btnode_cache_clear(struct address_space *); struct buffer_head *nilfs_btnode_create_block(struct address_space *btnc, __u64 blocknr); diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c index 3594eabe1419..f544c22fff78 100644 --- a/fs/nilfs2/btree.c +++ b/fs/nilfs2/btree.c @@ -58,7 +58,8 @@ static void nilfs_btree_free_path(struct nilfs_btree_path *path) static int nilfs_btree_get_new_block(const struct nilfs_bmap *btree, __u64 ptr, struct buffer_head **bhp) { - struct address_space *btnc = &NILFS_BMAP_I(btree)->i_btnode_cache; + struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode; + struct address_space *btnc = btnc_inode->i_mapping; struct buffer_head *bh; bh = nilfs_btnode_create_block(btnc, ptr); @@ -470,7 +471,8 @@ static int __nilfs_btree_get_block(const struct nilfs_bmap *btree, __u64 ptr, struct buffer_head **bhp, const struct nilfs_btree_readahead_info *ra) { - struct address_space *btnc = &NILFS_BMAP_I(btree)->i_btnode_cache; + struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode; + struct address_space *btnc = btnc_inode->i_mapping; struct buffer_head *bh, *ra_bh; sector_t submit_ptr = 0; int ret; @@ -1741,6 +1743,10 @@ nilfs_btree_prepare_convert_and_insert(struct nilfs_bmap *btree, __u64 key, dat = nilfs_bmap_get_dat(btree); } + ret = nilfs_attach_btree_node_cache(&NILFS_BMAP_I(btree)->vfs_inode); + if (ret < 0) + return ret; + ret = nilfs_bmap_prepare_alloc_ptr(btree, dreq, dat); if (ret < 0) return ret; @@ -1913,7 +1919,7 @@ static int nilfs_btree_prepare_update_v(struct nilfs_bmap *btree, path[level].bp_ctxt.newkey = path[level].bp_newreq.bpr_ptr; path[level].bp_ctxt.bh = path[level].bp_bh; ret = nilfs_btnode_prepare_change_key( - &NILFS_BMAP_I(btree)->i_btnode_cache, + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, &path[level].bp_ctxt); if (ret < 0) { nilfs_dat_abort_update(dat, @@ -1939,7 +1945,7 @@ static void nilfs_btree_commit_update_v(struct nilfs_bmap *btree, if (buffer_nilfs_node(path[level].bp_bh)) { nilfs_btnode_commit_change_key( - &NILFS_BMAP_I(btree)->i_btnode_cache, + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, &path[level].bp_ctxt); path[level].bp_bh = path[level].bp_ctxt.bh; } @@ -1958,7 +1964,7 @@ static void nilfs_btree_abort_update_v(struct nilfs_bmap *btree, &path[level].bp_newreq.bpr_req); if (buffer_nilfs_node(path[level].bp_bh)) nilfs_btnode_abort_change_key( - &NILFS_BMAP_I(btree)->i_btnode_cache, + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, &path[level].bp_ctxt); } @@ -2134,7 +2140,8 @@ static void nilfs_btree_add_dirty_buffer(struct nilfs_bmap *btree, static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree, struct list_head *listp) { - struct address_space *btcache = &NILFS_BMAP_I(btree)->i_btnode_cache; + struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode; + struct address_space *btcache = btnc_inode->i_mapping; struct list_head lists[NILFS_BTREE_LEVEL_MAX]; struct pagevec pvec; struct buffer_head *bh, *head; @@ -2188,12 +2195,12 @@ static int nilfs_btree_assign_p(struct nilfs_bmap *btree, path[level].bp_ctxt.newkey = blocknr; path[level].bp_ctxt.bh = *bh; ret = nilfs_btnode_prepare_change_key( - &NILFS_BMAP_I(btree)->i_btnode_cache, + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, &path[level].bp_ctxt); if (ret < 0) return ret; nilfs_btnode_commit_change_key( - &NILFS_BMAP_I(btree)->i_btnode_cache, + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, &path[level].bp_ctxt); *bh = path[level].bp_ctxt.bh; } @@ -2398,6 +2405,10 @@ int nilfs_btree_init(struct nilfs_bmap *bmap) if (nilfs_btree_root_broken(nilfs_btree_get_root(bmap), bmap->b_inode)) ret = -EIO; + else + ret = nilfs_attach_btree_node_cache( + &NILFS_BMAP_I(bmap)->vfs_inode); + return ret; } diff --git a/fs/nilfs2/gcinode.c b/fs/nilfs2/gcinode.c index a8f5315f01e3..04fdd420eae7 100644 --- a/fs/nilfs2/gcinode.c +++ b/fs/nilfs2/gcinode.c @@ -126,9 +126,10 @@ int nilfs_gccache_submit_read_data(struct inode *inode, sector_t blkoff, int nilfs_gccache_submit_read_node(struct inode *inode, sector_t pbn, __u64 vbn, struct buffer_head **out_bh) { + struct inode *btnc_inode = NILFS_I(inode)->i_assoc_inode; int ret; - ret = nilfs_btnode_submit_block(&NILFS_I(inode)->i_btnode_cache, + ret = nilfs_btnode_submit_block(btnc_inode->i_mapping, vbn ? : pbn, pbn, REQ_OP_READ, 0, out_bh, &pbn); if (ret == -EEXIST) /* internal code (cache hit) */ @@ -170,7 +171,7 @@ int nilfs_init_gcinode(struct inode *inode) ii->i_flags = 0; nilfs_bmap_init_gc(ii->i_bmap); - return 0; + return nilfs_attach_btree_node_cache(inode); } /** @@ -185,7 +186,7 @@ void nilfs_remove_all_gcinodes(struct the_nilfs *nilfs) ii = list_first_entry(head, struct nilfs_inode_info, i_dirty); list_del_init(&ii->i_dirty); truncate_inode_pages(&ii->vfs_inode.i_data, 0); - nilfs_btnode_cache_clear(&ii->i_btnode_cache); + nilfs_btnode_cache_clear(ii->i_assoc_inode->i_mapping); iput(&ii->vfs_inode); } } diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c index 476a4a649f38..2557f5f5a935 100644 --- a/fs/nilfs2/inode.c +++ b/fs/nilfs2/inode.c @@ -29,12 +29,14 @@ * @cno: checkpoint number * @root: pointer on NILFS root object (mounted checkpoint) * @for_gc: inode for GC flag + * @for_btnc: inode for B-tree node cache flag */ struct nilfs_iget_args { u64 ino; __u64 cno; struct nilfs_root *root; - int for_gc; + bool for_gc; + bool for_btnc; }; static int nilfs_iget_test(struct inode *inode, void *opaque); @@ -312,7 +314,8 @@ static int nilfs_insert_inode_locked(struct inode *inode, unsigned long ino) { struct nilfs_iget_args args = { - .ino = ino, .root = root, .cno = 0, .for_gc = 0 + .ino = ino, .root = root, .cno = 0, .for_gc = false, + .for_btnc = false }; return insert_inode_locked4(inode, ino, nilfs_iget_test, &args); @@ -525,6 +528,13 @@ static int nilfs_iget_test(struct inode *inode, void *opaque) return 0; ii = NILFS_I(inode); + if (test_bit(NILFS_I_BTNC, &ii->i_state)) { + if (!args->for_btnc) + return 0; + } else if (args->for_btnc) { + return 0; + } + if (!test_bit(NILFS_I_GCINODE, &ii->i_state)) return !args->for_gc; @@ -536,15 +546,15 @@ static int nilfs_iget_set(struct inode *inode, void *opaque) struct nilfs_iget_args *args = opaque; inode->i_ino = args->ino; - if (args->for_gc) { + NILFS_I(inode)->i_cno = args->cno; + NILFS_I(inode)->i_root = args->root; + if (args->root && args->ino == NILFS_ROOT_INO) + nilfs_get_root(args->root); + + if (args->for_gc) NILFS_I(inode)->i_state = BIT(NILFS_I_GCINODE); - NILFS_I(inode)->i_cno = args->cno; - NILFS_I(inode)->i_root = NULL; - } else { - if (args->root && args->ino == NILFS_ROOT_INO) - nilfs_get_root(args->root); - NILFS_I(inode)->i_root = args->root; - } + if (args->for_btnc) + NILFS_I(inode)->i_state |= BIT(NILFS_I_BTNC); return 0; } @@ -552,7 +562,8 @@ struct inode *nilfs_ilookup(struct super_block *sb, struct nilfs_root *root, unsigned long ino) { struct nilfs_iget_args args = { - .ino = ino, .root = root, .cno = 0, .for_gc = 0 + .ino = ino, .root = root, .cno = 0, .for_gc = false, + .for_btnc = false }; return ilookup5(sb, ino, nilfs_iget_test, &args); @@ -562,7 +573,8 @@ struct inode *nilfs_iget_locked(struct super_block *sb, struct nilfs_root *root, unsigned long ino) { struct nilfs_iget_args args = { - .ino = ino, .root = root, .cno = 0, .for_gc = 0 + .ino = ino, .root = root, .cno = 0, .for_gc = false, + .for_btnc = false }; return iget5_locked(sb, ino, nilfs_iget_test, nilfs_iget_set, &args); @@ -593,7 +605,8 @@ struct inode *nilfs_iget_for_gc(struct super_block *sb, unsigned long ino, __u64 cno) { struct nilfs_iget_args args = { - .ino = ino, .root = NULL, .cno = cno, .for_gc = 1 + .ino = ino, .root = NULL, .cno = cno, .for_gc = true, + .for_btnc = false }; struct inode *inode; int err; @@ -613,6 +626,68 @@ struct inode *nilfs_iget_for_gc(struct super_block *sb, unsigned long ino, return inode; } +/** + * nilfs_attach_btree_node_cache - attach a B-tree node cache to the inode + * @inode: inode object + * + * nilfs_attach_btree_node_cache() attaches a B-tree node cache to @inode, + * or does nothing if the inode already has it. This function allocates + * an additional inode to maintain page cache of B-tree nodes one-on-one. + * + * Return Value: On success, 0 is returned. On errors, one of the following + * negative error code is returned. + * + * %-ENOMEM - Insufficient memory available. + */ +int nilfs_attach_btree_node_cache(struct inode *inode) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + struct inode *btnc_inode; + struct nilfs_iget_args args; + + if (ii->i_assoc_inode) + return 0; + + args.ino = inode->i_ino; + args.root = ii->i_root; + args.cno = ii->i_cno; + args.for_gc = test_bit(NILFS_I_GCINODE, &ii->i_state) != 0; + args.for_btnc = true; + + btnc_inode = iget5_locked(inode->i_sb, inode->i_ino, nilfs_iget_test, + nilfs_iget_set, &args); + if (unlikely(!btnc_inode)) + return -ENOMEM; + if (btnc_inode->i_state & I_NEW) { + nilfs_init_btnc_inode(btnc_inode); + unlock_new_inode(btnc_inode); + } + NILFS_I(btnc_inode)->i_assoc_inode = inode; + NILFS_I(btnc_inode)->i_bmap = ii->i_bmap; + ii->i_assoc_inode = btnc_inode; + + return 0; +} + +/** + * nilfs_detach_btree_node_cache - detach the B-tree node cache from the inode + * @inode: inode object + * + * nilfs_detach_btree_node_cache() detaches the B-tree node cache and its + * holder inode bound to @inode, or does nothing if @inode doesn't have it. + */ +void nilfs_detach_btree_node_cache(struct inode *inode) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + struct inode *btnc_inode = ii->i_assoc_inode; + + if (btnc_inode) { + NILFS_I(btnc_inode)->i_assoc_inode = NULL; + ii->i_assoc_inode = NULL; + iput(btnc_inode); + } +} + void nilfs_write_inode_common(struct inode *inode, struct nilfs_inode *raw_inode, int has_bmap) { @@ -760,7 +835,8 @@ static void nilfs_clear_inode(struct inode *inode) if (test_bit(NILFS_I_BMAP, &ii->i_state)) nilfs_bmap_clear(ii->i_bmap); - nilfs_btnode_cache_clear(&ii->i_btnode_cache); + if (!test_bit(NILFS_I_BTNC, &ii->i_state)) + nilfs_detach_btree_node_cache(inode); if (ii->i_root && inode->i_ino == NILFS_ROOT_INO) nilfs_put_root(ii->i_root); diff --git a/fs/nilfs2/mdt.c b/fs/nilfs2/mdt.c index 78db33decd72..a8274d0efa37 100644 --- a/fs/nilfs2/mdt.c +++ b/fs/nilfs2/mdt.c @@ -533,7 +533,7 @@ int nilfs_mdt_save_to_shadow_map(struct inode *inode) goto out; ret = nilfs_copy_dirty_pages(&shadow->frozen_btnodes, - &ii->i_btnode_cache); + ii->i_assoc_inode->i_mapping); if (ret) goto out; @@ -624,8 +624,9 @@ void nilfs_mdt_restore_from_shadow_map(struct inode *inode) nilfs_clear_dirty_pages(inode->i_mapping, true); nilfs_copy_back_pages(inode->i_mapping, &shadow->frozen_data); - nilfs_clear_dirty_pages(&ii->i_btnode_cache, true); - nilfs_copy_back_pages(&ii->i_btnode_cache, &shadow->frozen_btnodes); + nilfs_clear_dirty_pages(ii->i_assoc_inode->i_mapping, true); + nilfs_copy_back_pages(ii->i_assoc_inode->i_mapping, + &shadow->frozen_btnodes); nilfs_bmap_restore(ii->i_bmap, &shadow->bmap_store); diff --git a/fs/nilfs2/nilfs.h b/fs/nilfs2/nilfs.h index a7b81755c350..36b048db00b7 100644 --- a/fs/nilfs2/nilfs.h +++ b/fs/nilfs2/nilfs.h @@ -28,7 +28,7 @@ * @i_xattr: * @i_dir_start_lookup: page index of last successful search * @i_cno: checkpoint number for GC inode - * @i_btnode_cache: cached pages of b-tree nodes + * @i_assoc_inode: associated inode (B-tree node cache holder or back pointer) * @i_dirty: list for connecting dirty files * @xattr_sem: semaphore for extended attributes processing * @i_bh: buffer contains disk inode @@ -43,7 +43,7 @@ struct nilfs_inode_info { __u64 i_xattr; /* sector_t ??? */ __u32 i_dir_start_lookup; __u64 i_cno; /* check point number for GC inode */ - struct address_space i_btnode_cache; + struct inode *i_assoc_inode; struct list_head i_dirty; /* List for connecting dirty files */ #ifdef CONFIG_NILFS_XATTR @@ -75,13 +75,6 @@ NILFS_BMAP_I(const struct nilfs_bmap *bmap) return container_of(bmap, struct nilfs_inode_info, i_bmap_data); } -static inline struct inode *NILFS_BTNC_I(struct address_space *btnc) -{ - struct nilfs_inode_info *ii = - container_of(btnc, struct nilfs_inode_info, i_btnode_cache); - return &ii->vfs_inode; -} - /* * Dynamic state flags of NILFS on-memory inode (i_state) */ @@ -98,6 +91,7 @@ enum { NILFS_I_INODE_SYNC, /* dsync is not allowed for inode */ NILFS_I_BMAP, /* has bmap and btnode_cache */ NILFS_I_GCINODE, /* inode for GC, on memory only */ + NILFS_I_BTNC, /* inode for btree node cache */ }; /* @@ -267,6 +261,8 @@ struct inode *nilfs_iget(struct super_block *sb, struct nilfs_root *root, unsigned long ino); extern struct inode *nilfs_iget_for_gc(struct super_block *sb, unsigned long ino, __u64 cno); +int nilfs_attach_btree_node_cache(struct inode *inode); +void nilfs_detach_btree_node_cache(struct inode *inode); extern void nilfs_update_inode(struct inode *, struct buffer_head *, int); extern void nilfs_truncate(struct inode *); extern void nilfs_evict_inode(struct inode *); diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c index 063dd16d75b5..45e079295008 100644 --- a/fs/nilfs2/page.c +++ b/fs/nilfs2/page.c @@ -448,10 +448,9 @@ void nilfs_mapping_init(struct address_space *mapping, struct inode *inode) /* * NILFS2 needs clear_page_dirty() in the following two cases: * - * 1) For B-tree node pages and data pages of the dat/gcdat, NILFS2 clears - * page dirty flags when it copies back pages from the shadow cache - * (gcdat->{i_mapping,i_btnode_cache}) to its original cache - * (dat->{i_mapping,i_btnode_cache}). + * 1) For B-tree node pages and data pages of DAT file, NILFS2 clears dirty + * flag of pages when it copies back pages from shadow cache to the + * original cache. * * 2) Some B-tree operations like insertion or deletion may dispose buffers * in dirty state, and this needs to cancel the dirty state of their pages. diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c index 85a853334771..0afe0832c754 100644 --- a/fs/nilfs2/segment.c +++ b/fs/nilfs2/segment.c @@ -733,15 +733,18 @@ static void nilfs_lookup_dirty_node_buffers(struct inode *inode, struct list_head *listp) { struct nilfs_inode_info *ii = NILFS_I(inode); - struct address_space *mapping = &ii->i_btnode_cache; + struct inode *btnc_inode = ii->i_assoc_inode; struct pagevec pvec; struct buffer_head *bh, *head; unsigned int i; pgoff_t index = 0; + if (!btnc_inode) + return; + pagevec_init(&pvec); - while (pagevec_lookup_tag(&pvec, mapping, &index, + while (pagevec_lookup_tag(&pvec, btnc_inode->i_mapping, &index, PAGECACHE_TAG_DIRTY)) { for (i = 0; i < pagevec_count(&pvec); i++) { bh = head = page_buffers(pvec.pages[i]); @@ -2410,7 +2413,7 @@ nilfs_remove_written_gcinodes(struct the_nilfs *nilfs, struct list_head *head) continue; list_del_init(&ii->i_dirty); truncate_inode_pages(&ii->vfs_inode.i_data, 0); - nilfs_btnode_cache_clear(&ii->i_btnode_cache); + nilfs_btnode_cache_clear(ii->i_assoc_inode->i_mapping); iput(&ii->vfs_inode); } } diff --git a/fs/nilfs2/super.c b/fs/nilfs2/super.c index 3e05c98631ec..ba108f915391 100644 --- a/fs/nilfs2/super.c +++ b/fs/nilfs2/super.c @@ -157,7 +157,8 @@ struct inode *nilfs_alloc_inode(struct super_block *sb) ii->i_bh = NULL; ii->i_state = 0; ii->i_cno = 0; - nilfs_mapping_init(&ii->i_btnode_cache, &ii->vfs_inode); + ii->i_assoc_inode = NULL; + ii->i_bmap = &ii->i_bmap_data; return &ii->vfs_inode; } @@ -1377,8 +1378,6 @@ static void nilfs_inode_init_once(void *obj) #ifdef CONFIG_NILFS_XATTR init_rwsem(&ii->xattr_sem); #endif - address_space_init_once(&ii->i_btnode_cache); - ii->i_bmap = &ii->i_bmap_data; inode_init_once(&ii->vfs_inode); } From 6e211930f79aa45d422009a5f2e5467d2369ffe5 Mon Sep 17 00:00:00 2001 From: Ryusuke Konishi Date: Fri, 1 Apr 2022 11:28:21 -0700 Subject: [PATCH 83/95] nilfs2: fix lockdep warnings during disk space reclamation During disk space reclamation, nilfs2 still emits the following lockdep warning due to page/folio operations on shadowed page caches that nilfs2 uses to get a snapshot of DAT file in memory: WARNING: CPU: 0 PID: 2643 at include/linux/backing-dev.h:272 __folio_mark_dirty+0x645/0x670 ... RIP: 0010:__folio_mark_dirty+0x645/0x670 ... Call Trace: filemap_dirty_folio+0x74/0xd0 __set_page_dirty_nobuffers+0x85/0xb0 nilfs_copy_dirty_pages+0x288/0x510 [nilfs2] nilfs_mdt_save_to_shadow_map+0x50/0xe0 [nilfs2] nilfs_clean_segments+0xee/0x5d0 [nilfs2] nilfs_ioctl_clean_segments.isra.19+0xb08/0xf40 [nilfs2] nilfs_ioctl+0xc52/0xfb0 [nilfs2] __x64_sys_ioctl+0x11d/0x170 This fixes the remaining warning by using inode objects to hold those page caches. Link: https://lkml.kernel.org/r/1647867427-30498-3-git-send-email-konishi.ryusuke@gmail.com Signed-off-by: Ryusuke Konishi Tested-by: Ryusuke Konishi Cc: Matthew Wilcox Cc: David Hildenbrand Cc: Hao Sun Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/nilfs2/dat.c | 4 ++- fs/nilfs2/inode.c | 63 ++++++++++++++++++++++++++++++++++++++++++++--- fs/nilfs2/mdt.c | 38 +++++++++++++++++++--------- fs/nilfs2/mdt.h | 6 ++--- fs/nilfs2/nilfs.h | 2 ++ 5 files changed, 92 insertions(+), 21 deletions(-) diff --git a/fs/nilfs2/dat.c b/fs/nilfs2/dat.c index dc51d3b7a7bf..3b55e239705f 100644 --- a/fs/nilfs2/dat.c +++ b/fs/nilfs2/dat.c @@ -497,7 +497,9 @@ int nilfs_dat_read(struct super_block *sb, size_t entry_size, di = NILFS_DAT_I(dat); lockdep_set_class(&di->mi.mi_sem, &dat_lock_key); nilfs_palloc_setup_cache(dat, &di->palloc_cache); - nilfs_mdt_setup_shadow_map(dat, &di->shadow); + err = nilfs_mdt_setup_shadow_map(dat, &di->shadow); + if (err) + goto failed; err = nilfs_read_inode_common(dat, raw_inode); if (err) diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c index 2557f5f5a935..6045cea21f52 100644 --- a/fs/nilfs2/inode.c +++ b/fs/nilfs2/inode.c @@ -30,6 +30,7 @@ * @root: pointer on NILFS root object (mounted checkpoint) * @for_gc: inode for GC flag * @for_btnc: inode for B-tree node cache flag + * @for_shadow: inode for shadowed page cache flag */ struct nilfs_iget_args { u64 ino; @@ -37,6 +38,7 @@ struct nilfs_iget_args { struct nilfs_root *root; bool for_gc; bool for_btnc; + bool for_shadow; }; static int nilfs_iget_test(struct inode *inode, void *opaque); @@ -315,7 +317,7 @@ static int nilfs_insert_inode_locked(struct inode *inode, { struct nilfs_iget_args args = { .ino = ino, .root = root, .cno = 0, .for_gc = false, - .for_btnc = false + .for_btnc = false, .for_shadow = false }; return insert_inode_locked4(inode, ino, nilfs_iget_test, &args); @@ -534,6 +536,12 @@ static int nilfs_iget_test(struct inode *inode, void *opaque) } else if (args->for_btnc) { return 0; } + if (test_bit(NILFS_I_SHADOW, &ii->i_state)) { + if (!args->for_shadow) + return 0; + } else if (args->for_shadow) { + return 0; + } if (!test_bit(NILFS_I_GCINODE, &ii->i_state)) return !args->for_gc; @@ -555,6 +563,8 @@ static int nilfs_iget_set(struct inode *inode, void *opaque) NILFS_I(inode)->i_state = BIT(NILFS_I_GCINODE); if (args->for_btnc) NILFS_I(inode)->i_state |= BIT(NILFS_I_BTNC); + if (args->for_shadow) + NILFS_I(inode)->i_state |= BIT(NILFS_I_SHADOW); return 0; } @@ -563,7 +573,7 @@ struct inode *nilfs_ilookup(struct super_block *sb, struct nilfs_root *root, { struct nilfs_iget_args args = { .ino = ino, .root = root, .cno = 0, .for_gc = false, - .for_btnc = false + .for_btnc = false, .for_shadow = false }; return ilookup5(sb, ino, nilfs_iget_test, &args); @@ -574,7 +584,7 @@ struct inode *nilfs_iget_locked(struct super_block *sb, struct nilfs_root *root, { struct nilfs_iget_args args = { .ino = ino, .root = root, .cno = 0, .for_gc = false, - .for_btnc = false + .for_btnc = false, .for_shadow = false }; return iget5_locked(sb, ino, nilfs_iget_test, nilfs_iget_set, &args); @@ -606,7 +616,7 @@ struct inode *nilfs_iget_for_gc(struct super_block *sb, unsigned long ino, { struct nilfs_iget_args args = { .ino = ino, .root = NULL, .cno = cno, .for_gc = true, - .for_btnc = false + .for_btnc = false, .for_shadow = false }; struct inode *inode; int err; @@ -653,6 +663,7 @@ int nilfs_attach_btree_node_cache(struct inode *inode) args.cno = ii->i_cno; args.for_gc = test_bit(NILFS_I_GCINODE, &ii->i_state) != 0; args.for_btnc = true; + args.for_shadow = test_bit(NILFS_I_SHADOW, &ii->i_state) != 0; btnc_inode = iget5_locked(inode->i_sb, inode->i_ino, nilfs_iget_test, nilfs_iget_set, &args); @@ -688,6 +699,50 @@ void nilfs_detach_btree_node_cache(struct inode *inode) } } +/** + * nilfs_iget_for_shadow - obtain inode for shadow mapping + * @inode: inode object that uses shadow mapping + * + * nilfs_iget_for_shadow() allocates a pair of inodes that holds page + * caches for shadow mapping. The page cache for data pages is set up + * in one inode and the one for b-tree node pages is set up in the + * other inode, which is attached to the former inode. + * + * Return Value: On success, a pointer to the inode for data pages is + * returned. On errors, one of the following negative error code is returned + * in a pointer type. + * + * %-ENOMEM - Insufficient memory available. + */ +struct inode *nilfs_iget_for_shadow(struct inode *inode) +{ + struct nilfs_iget_args args = { + .ino = inode->i_ino, .root = NULL, .cno = 0, .for_gc = false, + .for_btnc = false, .for_shadow = true + }; + struct inode *s_inode; + int err; + + s_inode = iget5_locked(inode->i_sb, inode->i_ino, nilfs_iget_test, + nilfs_iget_set, &args); + if (unlikely(!s_inode)) + return ERR_PTR(-ENOMEM); + if (!(s_inode->i_state & I_NEW)) + return inode; + + NILFS_I(s_inode)->i_flags = 0; + memset(NILFS_I(s_inode)->i_bmap, 0, sizeof(struct nilfs_bmap)); + mapping_set_gfp_mask(s_inode->i_mapping, GFP_NOFS); + + err = nilfs_attach_btree_node_cache(s_inode); + if (unlikely(err)) { + iget_failed(s_inode); + return ERR_PTR(err); + } + unlock_new_inode(s_inode); + return s_inode; +} + void nilfs_write_inode_common(struct inode *inode, struct nilfs_inode *raw_inode, int has_bmap) { diff --git a/fs/nilfs2/mdt.c b/fs/nilfs2/mdt.c index a8274d0efa37..d29a0f2b9c16 100644 --- a/fs/nilfs2/mdt.c +++ b/fs/nilfs2/mdt.c @@ -471,9 +471,18 @@ int nilfs_mdt_init(struct inode *inode, gfp_t gfp_mask, size_t objsz) void nilfs_mdt_clear(struct inode *inode) { struct nilfs_mdt_info *mdi = NILFS_MDT(inode); + struct nilfs_shadow_map *shadow = mdi->mi_shadow; if (mdi->mi_palloc_cache) nilfs_palloc_destroy_cache(inode); + + if (shadow) { + struct inode *s_inode = shadow->inode; + + shadow->inode = NULL; + iput(s_inode); + mdi->mi_shadow = NULL; + } } /** @@ -507,12 +516,15 @@ int nilfs_mdt_setup_shadow_map(struct inode *inode, struct nilfs_shadow_map *shadow) { struct nilfs_mdt_info *mi = NILFS_MDT(inode); + struct inode *s_inode; INIT_LIST_HEAD(&shadow->frozen_buffers); - address_space_init_once(&shadow->frozen_data); - nilfs_mapping_init(&shadow->frozen_data, inode); - address_space_init_once(&shadow->frozen_btnodes); - nilfs_mapping_init(&shadow->frozen_btnodes, inode); + + s_inode = nilfs_iget_for_shadow(inode); + if (IS_ERR(s_inode)) + return PTR_ERR(s_inode); + + shadow->inode = s_inode; mi->mi_shadow = shadow; return 0; } @@ -526,13 +538,14 @@ int nilfs_mdt_save_to_shadow_map(struct inode *inode) struct nilfs_mdt_info *mi = NILFS_MDT(inode); struct nilfs_inode_info *ii = NILFS_I(inode); struct nilfs_shadow_map *shadow = mi->mi_shadow; + struct inode *s_inode = shadow->inode; int ret; - ret = nilfs_copy_dirty_pages(&shadow->frozen_data, inode->i_mapping); + ret = nilfs_copy_dirty_pages(s_inode->i_mapping, inode->i_mapping); if (ret) goto out; - ret = nilfs_copy_dirty_pages(&shadow->frozen_btnodes, + ret = nilfs_copy_dirty_pages(NILFS_I(s_inode)->i_assoc_inode->i_mapping, ii->i_assoc_inode->i_mapping); if (ret) goto out; @@ -549,7 +562,7 @@ int nilfs_mdt_freeze_buffer(struct inode *inode, struct buffer_head *bh) struct page *page; int blkbits = inode->i_blkbits; - page = grab_cache_page(&shadow->frozen_data, bh->b_page->index); + page = grab_cache_page(shadow->inode->i_mapping, bh->b_page->index); if (!page) return -ENOMEM; @@ -581,7 +594,7 @@ nilfs_mdt_get_frozen_buffer(struct inode *inode, struct buffer_head *bh) struct page *page; int n; - page = find_lock_page(&shadow->frozen_data, bh->b_page->index); + page = find_lock_page(shadow->inode->i_mapping, bh->b_page->index); if (page) { if (page_has_buffers(page)) { n = bh_offset(bh) >> inode->i_blkbits; @@ -622,11 +635,11 @@ void nilfs_mdt_restore_from_shadow_map(struct inode *inode) nilfs_palloc_clear_cache(inode); nilfs_clear_dirty_pages(inode->i_mapping, true); - nilfs_copy_back_pages(inode->i_mapping, &shadow->frozen_data); + nilfs_copy_back_pages(inode->i_mapping, shadow->inode->i_mapping); nilfs_clear_dirty_pages(ii->i_assoc_inode->i_mapping, true); nilfs_copy_back_pages(ii->i_assoc_inode->i_mapping, - &shadow->frozen_btnodes); + NILFS_I(shadow->inode)->i_assoc_inode->i_mapping); nilfs_bmap_restore(ii->i_bmap, &shadow->bmap_store); @@ -641,10 +654,11 @@ void nilfs_mdt_clear_shadow_map(struct inode *inode) { struct nilfs_mdt_info *mi = NILFS_MDT(inode); struct nilfs_shadow_map *shadow = mi->mi_shadow; + struct inode *shadow_btnc_inode = NILFS_I(shadow->inode)->i_assoc_inode; down_write(&mi->mi_sem); nilfs_release_frozen_buffers(shadow); - truncate_inode_pages(&shadow->frozen_data, 0); - truncate_inode_pages(&shadow->frozen_btnodes, 0); + truncate_inode_pages(shadow->inode->i_mapping, 0); + truncate_inode_pages(shadow_btnc_inode->i_mapping, 0); up_write(&mi->mi_sem); } diff --git a/fs/nilfs2/mdt.h b/fs/nilfs2/mdt.h index 8f86080a436d..9e23bab3ff12 100644 --- a/fs/nilfs2/mdt.h +++ b/fs/nilfs2/mdt.h @@ -18,14 +18,12 @@ /** * struct nilfs_shadow_map - shadow mapping of meta data file * @bmap_store: shadow copy of bmap state - * @frozen_data: shadowed dirty data pages - * @frozen_btnodes: shadowed dirty b-tree nodes' pages + * @inode: holder of page caches used in shadow mapping * @frozen_buffers: list of frozen buffers */ struct nilfs_shadow_map { struct nilfs_bmap_store bmap_store; - struct address_space frozen_data; - struct address_space frozen_btnodes; + struct inode *inode; struct list_head frozen_buffers; }; diff --git a/fs/nilfs2/nilfs.h b/fs/nilfs2/nilfs.h index 36b048db00b7..1344f7d475d3 100644 --- a/fs/nilfs2/nilfs.h +++ b/fs/nilfs2/nilfs.h @@ -92,6 +92,7 @@ enum { NILFS_I_BMAP, /* has bmap and btnode_cache */ NILFS_I_GCINODE, /* inode for GC, on memory only */ NILFS_I_BTNC, /* inode for btree node cache */ + NILFS_I_SHADOW, /* inode for shadowed page cache */ }; /* @@ -263,6 +264,7 @@ extern struct inode *nilfs_iget_for_gc(struct super_block *sb, unsigned long ino, __u64 cno); int nilfs_attach_btree_node_cache(struct inode *inode); void nilfs_detach_btree_node_cache(struct inode *inode); +struct inode *nilfs_iget_for_shadow(struct inode *inode); extern void nilfs_update_inode(struct inode *, struct buffer_head *, int); extern void nilfs_truncate(struct inode *); extern void nilfs_evict_inode(struct inode *); From cdd81b313d5e09e8a248c2dbd9c6b069f82e5d71 Mon Sep 17 00:00:00 2001 From: Ryusuke Konishi Date: Fri, 1 Apr 2022 11:28:24 -0700 Subject: [PATCH 84/95] nilfs2: get rid of nilfs_mapping_init() After applying the lockdep warning fixes, nilfs_mapping_init() is no longer used, so delete it. Link: https://lkml.kernel.org/r/1647867427-30498-4-git-send-email-konishi.ryusuke@gmail.com Signed-off-by: Ryusuke Konishi Cc: David Hildenbrand Cc: Hao Sun Cc: Matthew Wilcox Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- fs/nilfs2/page.c | 9 --------- fs/nilfs2/page.h | 1 - 2 files changed, 10 deletions(-) diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c index 45e079295008..a8e88cc38e16 100644 --- a/fs/nilfs2/page.c +++ b/fs/nilfs2/page.c @@ -436,15 +436,6 @@ unsigned int nilfs_page_count_clean_buffers(struct page *page, return nc; } -void nilfs_mapping_init(struct address_space *mapping, struct inode *inode) -{ - mapping->host = inode; - mapping->flags = 0; - mapping_set_gfp_mask(mapping, GFP_NOFS); - mapping->private_data = NULL; - mapping->a_ops = &empty_aops; -} - /* * NILFS2 needs clear_page_dirty() in the following two cases: * diff --git a/fs/nilfs2/page.h b/fs/nilfs2/page.h index 569263b23c0c..21ddcdd4d63e 100644 --- a/fs/nilfs2/page.h +++ b/fs/nilfs2/page.h @@ -43,7 +43,6 @@ int nilfs_copy_dirty_pages(struct address_space *, struct address_space *); void nilfs_copy_back_pages(struct address_space *, struct address_space *); void nilfs_clear_dirty_page(struct page *, bool); void nilfs_clear_dirty_pages(struct address_space *, bool); -void nilfs_mapping_init(struct address_space *mapping, struct inode *inode); unsigned int nilfs_page_count_clean_buffers(struct page *, unsigned int, unsigned int); unsigned long nilfs_find_uncommitted_extent(struct inode *inode, From ece369c7e1044a333dc002d3c3c228b8853fc5f7 Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Fri, 1 Apr 2022 11:28:27 -0700 Subject: [PATCH 85/95] mm/munlock: add lru_add_drain() to fix memcg_stat_test Mike reports that LTP memcg_stat_test usually leads to memcg_stat_test 3 TINFO: Test unevictable with MAP_LOCKED memcg_stat_test 3 TINFO: Running memcg_process --mmap-lock1 -s 135168 memcg_stat_test 3 TINFO: Warming up pid: 3460 memcg_stat_test 3 TINFO: Process is still here after warm up: 3460 memcg_stat_test 3 TFAIL: unevictable is 122880, 135168 expected but may also lead to memcg_stat_test 4 TINFO: Test unevictable with mlock memcg_stat_test 4 TINFO: Running memcg_process --mmap-lock2 -s 135168 memcg_stat_test 4 TINFO: Warming up pid: 4271 memcg_stat_test 4 TINFO: Process is still here after warm up: 4271 memcg_stat_test 4 TFAIL: unevictable is 122880, 135168 expected or both. A wee bit flaky. follow_page_pte() used to have an lru_add_drain() per each page mlocked, and the test came to rely on accurate stats. The pagevec to be drained is different now, but still covered by lru_add_drain(); and, never mind the test, I believe it's in everyone's interest that a bulk faulting interface like populate_vma_page_range() or faultin_vma_page_range() should drain its local pagevecs at the end, to save others sometimes needing the much more expensive lru_add_drain_all(). This does not absolutely guarantee exact stats - the mlocking task can be migrated between CPUs as it proceeds - but it's good enough and the tests pass. Link: https://lkml.kernel.org/r/47f6d39c-a075-50cb-1cfb-26dd957a48af@google.com Fixes: b67bf49ce7aa ("mm/munlock: delete FOLL_MLOCK and FOLL_POPULATE") Signed-off-by: Hugh Dickins Reported-by: Mike Galbraith Cc: Vlastimil Babka Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/gup.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index 271fbe8195d7..f598a037eb04 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1404,6 +1404,7 @@ long populate_vma_page_range(struct vm_area_struct *vma, struct mm_struct *mm = vma->vm_mm; unsigned long nr_pages = (end - start) / PAGE_SIZE; int gup_flags; + long ret; VM_BUG_ON(!PAGE_ALIGNED(start)); VM_BUG_ON(!PAGE_ALIGNED(end)); @@ -1438,8 +1439,10 @@ long populate_vma_page_range(struct vm_area_struct *vma, * We made sure addr is within a VMA, so the following will * not result in a stack expansion that recurses back here. */ - return __get_user_pages(mm, start, nr_pages, gup_flags, + ret = __get_user_pages(mm, start, nr_pages, gup_flags, NULL, NULL, locked); + lru_add_drain(); + return ret; } /* @@ -1471,6 +1474,7 @@ long faultin_vma_page_range(struct vm_area_struct *vma, unsigned long start, struct mm_struct *mm = vma->vm_mm; unsigned long nr_pages = (end - start) / PAGE_SIZE; int gup_flags; + long ret; VM_BUG_ON(!PAGE_ALIGNED(start)); VM_BUG_ON(!PAGE_ALIGNED(end)); @@ -1498,8 +1502,10 @@ long faultin_vma_page_range(struct vm_area_struct *vma, unsigned long start, if (check_vma_flags(vma, gup_flags)) return -EINVAL; - return __get_user_pages(mm, start, nr_pages, gup_flags, + ret = __get_user_pages(mm, start, nr_pages, gup_flags, NULL, NULL, locked); + lru_add_drain(); + return ret; } /* From 577e9846f8a9e7b09cd356ae0d59a66e19402e8b Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Fri, 1 Apr 2022 11:28:30 -0700 Subject: [PATCH 86/95] mm/munlock: update Documentation/vm/unevictable-lru.rst Update Documentation/vm/unevictable-lru.rst to reflect the changes made by the mm/munlock series: keeping an mlock_count instead of page_mlock() (formerly try_to_munlock()) and munlock_vma_pages_all() etc. Also make other little updates or cleanups wherever noticed. But, I apologize, this is already out of date, in that "folio" appears nowhere: 5.18 will be in a transitional state from "page" to "folio", and documenting its current mix of the two does not help to understand "the Unevictable LRU". Should be revisited when naming is more settled. Link: https://lkml.kernel.org/r/3753962-d491-bf60-f59f-51bfe84fd6a0@google.com Signed-off-by: Hugh Dickins Cc: Jonathan Corbet Cc: Randy Dunlap Cc: Matthew Wilcox Cc: Michal Hocko Cc: Vlastimil Babka Cc: "Kirill A. Shutemov" Cc: David Hildenbrand Cc: Alistair Popple Cc: Johannes Weiner Cc: Rik van Riel Cc: Suren Baghdasaryan Cc: Yu Zhao Cc: Greg Thelen Cc: Shakeel Butt Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/vm/unevictable-lru.rst | 449 ++++++++++++--------------- 1 file changed, 199 insertions(+), 250 deletions(-) diff --git a/Documentation/vm/unevictable-lru.rst b/Documentation/vm/unevictable-lru.rst index eae3af17f2d9..b280367d6a44 100644 --- a/Documentation/vm/unevictable-lru.rst +++ b/Documentation/vm/unevictable-lru.rst @@ -52,8 +52,13 @@ The infrastructure may also be able to handle other conditions that make pages unevictable, either by definition or by circumstance, in the future. -The Unevictable Page List -------------------------- +The Unevictable LRU Page List +----------------------------- + +The Unevictable LRU page list is a lie. It was never an LRU-ordered list, but a +companion to the LRU-ordered anonymous and file, active and inactive page lists; +and now it is not even a page list. But following familiar convention, here in +this document and in the source, we often imagine it as a fifth LRU page list. The Unevictable LRU infrastructure consists of an additional, per-node, LRU list called the "unevictable" list and an associated page flag, PG_unevictable, to @@ -63,8 +68,8 @@ The PG_unevictable flag is analogous to, and mutually exclusive with, the PG_active flag in that it indicates on which LRU list a page resides when PG_lru is set. -The Unevictable LRU infrastructure maintains unevictable pages on an additional -LRU list for a few reasons: +The Unevictable LRU infrastructure maintains unevictable pages as if they were +on an additional LRU list for a few reasons: (1) We get to "treat unevictable pages just like we treat other pages in the system - which means we get to use the same code to manipulate them, the @@ -72,13 +77,11 @@ LRU list for a few reasons: of the statistics, etc..." [Rik van Riel] (2) We want to be able to migrate unevictable pages between nodes for memory - defragmentation, workload management and memory hotplug. The linux kernel + defragmentation, workload management and memory hotplug. The Linux kernel can only migrate pages that it can successfully isolate from the LRU - lists. If we were to maintain pages elsewhere than on an LRU-like list, - where they can be found by isolate_lru_page(), we would prevent their - migration, unless we reworked migration code to find the unevictable pages - itself. - + lists (or "Movable" pages: outside of consideration here). If we were to + maintain pages elsewhere than on an LRU-like list, where they can be + detected by isolate_lru_page(), we would prevent their migration. The unevictable list does not differentiate between file-backed and anonymous, swap-backed pages. This differentiation is only important while the pages are, @@ -92,8 +95,8 @@ Memory Control Group Interaction -------------------------------- The unevictable LRU facility interacts with the memory control group [aka -memory controller; see Documentation/admin-guide/cgroup-v1/memory.rst] by extending the -lru_list enum. +memory controller; see Documentation/admin-guide/cgroup-v1/memory.rst] by +extending the lru_list enum. The memory controller data structure automatically gets a per-node unevictable list as a result of the "arrayification" of the per-node LRU lists (one per @@ -143,7 +146,6 @@ These are currently used in three places in the kernel: and this mark remains for the life of the inode. (2) By SYSV SHM to mark SHM_LOCK'd address spaces until SHM_UNLOCK is called. - Note that SHM_LOCK is not required to page in the locked pages if they're swapped out; the application must touch the pages manually if it wants to ensure they're in memory. @@ -156,19 +158,19 @@ These are currently used in three places in the kernel: Detecting Unevictable Pages --------------------------- -The function page_evictable() in vmscan.c determines whether a page is +The function page_evictable() in mm/internal.h determines whether a page is evictable or not using the query function outlined above [see section :ref:`Marking address spaces unevictable `] to check the AS_UNEVICTABLE flag. For address spaces that are so marked after being populated (as SHM regions -might be), the lock action (eg: SHM_LOCK) can be lazy, and need not populate +might be), the lock action (e.g. SHM_LOCK) can be lazy, and need not populate the page tables for the region as does, for example, mlock(), nor need it make any special effort to push any pages in the SHM_LOCK'd area to the unevictable list. Instead, vmscan will do this if and when it encounters the pages during a reclamation scan. -On an unlock action (such as SHM_UNLOCK), the unlocker (eg: shmctl()) must scan +On an unlock action (such as SHM_UNLOCK), the unlocker (e.g. shmctl()) must scan the pages in the region and "rescue" them from the unevictable list if no other condition is keeping them unevictable. If an unevictable region is destroyed, the pages are also "rescued" from the unevictable list in the process of @@ -176,7 +178,7 @@ freeing them. page_evictable() also checks for mlocked pages by testing an additional page flag, PG_mlocked (as wrapped by PageMlocked()), which is set when a page is -faulted into a VM_LOCKED vma, or found in a vma being VM_LOCKED. +faulted into a VM_LOCKED VMA, or found in a VMA being VM_LOCKED. Vmscan's Handling of Unevictable Pages @@ -186,28 +188,23 @@ If unevictable pages are culled in the fault path, or moved to the unevictable list at mlock() or mmap() time, vmscan will not encounter the pages until they have become evictable again (via munlock() for example) and have been "rescued" from the unevictable list. However, there may be situations where we decide, -for the sake of expediency, to leave a unevictable page on one of the regular +for the sake of expediency, to leave an unevictable page on one of the regular active/inactive LRU lists for vmscan to deal with. vmscan checks for such pages in all of the shrink_{active|inactive|page}_list() functions and will "cull" such pages that it encounters: that is, it diverts those pages to the -unevictable list for the node being scanned. +unevictable list for the memory cgroup and node being scanned. There may be situations where a page is mapped into a VM_LOCKED VMA, but the page is not marked as PG_mlocked. Such pages will make it all the way to -shrink_page_list() where they will be detected when vmscan walks the reverse -map in try_to_unmap(). If try_to_unmap() returns SWAP_MLOCK, -shrink_page_list() will cull the page at that point. +shrink_active_list() or shrink_page_list() where they will be detected when +vmscan walks the reverse map in page_referenced() or try_to_unmap(). The page +is culled to the unevictable list when it is released by the shrinker. To "cull" an unevictable page, vmscan simply puts the page back on the LRU list using putback_lru_page() - the inverse operation to isolate_lru_page() - after dropping the page lock. Because the condition which makes the page unevictable -may change once the page is unlocked, putback_lru_page() will recheck the -unevictable state of a page that it places on the unevictable list. If the -page has become unevictable, putback_lru_page() removes it from the list and -retries, including the page_unevictable() test. Because such a race is a rare -event and movement of pages onto the unevictable list should be rare, these -extra evictabilty checks should not occur in the majority of calls to -putback_lru_page(). +may change once the page is unlocked, __pagevec_lru_add_fn() will recheck the +unevictable state of a page before placing it on the unevictable list. MLOCKED Pages @@ -227,16 +224,25 @@ Nick posted his patch as an alternative to a patch posted by Christoph Lameter to achieve the same objective: hiding mlocked pages from vmscan. In Nick's patch, he used one of the struct page LRU list link fields as a count -of VM_LOCKED VMAs that map the page. This use of the link field for a count -prevented the management of the pages on an LRU list, and thus mlocked pages -were not migratable as isolate_lru_page() could not find them, and the LRU list -link field was not available to the migration subsystem. +of VM_LOCKED VMAs that map the page (Rik van Riel had the same idea three years +earlier). But this use of the link field for a count prevented the management +of the pages on an LRU list, and thus mlocked pages were not migratable as +isolate_lru_page() could not detect them, and the LRU list link field was not +available to the migration subsystem. -Nick resolved this by putting mlocked pages back on the lru list before +Nick resolved this by putting mlocked pages back on the LRU list before attempting to isolate them, thus abandoning the count of VM_LOCKED VMAs. When Nick's patch was integrated with the Unevictable LRU work, the count was -replaced by walking the reverse map to determine whether any VM_LOCKED VMAs -mapped the page. More on this below. +replaced by walking the reverse map when munlocking, to determine whether any +other VM_LOCKED VMAs still mapped the page. + +However, walking the reverse map for each page when munlocking was ugly and +inefficient, and could lead to catastrophic contention on a file's rmap lock, +when many processes which had it mlocked were trying to exit. In 5.18, the +idea of keeping mlock_count in Unevictable LRU list link field was revived and +put to work, without preventing the migration of mlocked pages. This is why +the "Unevictable LRU list" cannot be a linked list of pages now; but there was +no use for that linked list anyway - though its size is maintained for meminfo. Basic Management @@ -250,22 +256,18 @@ PageMlocked() functions. A PG_mlocked page will be placed on the unevictable list when it is added to the LRU. Such pages can be "noticed" by memory management in several places: - (1) in the mlock()/mlockall() system call handlers; + (1) in the mlock()/mlock2()/mlockall() system call handlers; (2) in the mmap() system call handler when mmapping a region with the MAP_LOCKED flag; (3) mmapping a region in a task that has called mlockall() with the MCL_FUTURE - flag + flag; - (4) in the fault path, if mlocked pages are "culled" in the fault path, - and when a VM_LOCKED stack segment is expanded; or + (4) in the fault path and when a VM_LOCKED stack segment is expanded; or (5) as mentioned above, in vmscan:shrink_page_list() when attempting to - reclaim a page in a VM_LOCKED VMA via try_to_unmap() - -all of which result in the VM_LOCKED flag being set for the VMA if it doesn't -already have it set. + reclaim a page in a VM_LOCKED VMA by page_referenced() or try_to_unmap(). mlocked pages become unlocked and rescued from the unevictable list when: @@ -280,51 +282,53 @@ mlocked pages become unlocked and rescued from the unevictable list when: (4) before a page is COW'd in a VM_LOCKED VMA. -mlock()/mlockall() System Call Handling ---------------------------------------- +mlock()/mlock2()/mlockall() System Call Handling +------------------------------------------------ -Both [do\_]mlock() and [do\_]mlockall() system call handlers call mlock_fixup() +mlock(), mlock2() and mlockall() system call handlers proceed to mlock_fixup() for each VMA in the range specified by the call. In the case of mlockall(), this is the entire active address space of the task. Note that mlock_fixup() is used for both mlocking and munlocking a range of memory. A call to mlock() -an already VM_LOCKED VMA, or to munlock() a VMA that is not VM_LOCKED is -treated as a no-op, and mlock_fixup() simply returns. +an already VM_LOCKED VMA, or to munlock() a VMA that is not VM_LOCKED, is +treated as a no-op and mlock_fixup() simply returns. -If the VMA passes some filtering as described in "Filtering Special Vmas" +If the VMA passes some filtering as described in "Filtering Special VMAs" below, mlock_fixup() will attempt to merge the VMA with its neighbors or split -off a subset of the VMA if the range does not cover the entire VMA. Once the -VMA has been merged or split or neither, mlock_fixup() will call -populate_vma_page_range() to fault in the pages via get_user_pages() and to -mark the pages as mlocked via mlock_vma_page(). +off a subset of the VMA if the range does not cover the entire VMA. Any pages +already present in the VMA are then marked as mlocked by mlock_page() via +mlock_pte_range() via walk_page_range() via mlock_vma_pages_range(). + +Before returning from the system call, do_mlock() or mlockall() will call +__mm_populate() to fault in the remaining pages via get_user_pages() and to +mark those pages as mlocked as they are faulted. Note that the VMA being mlocked might be mapped with PROT_NONE. In this case, get_user_pages() will be unable to fault in the pages. That's okay. If pages -do end up getting faulted into this VM_LOCKED VMA, we'll handle them in the -fault path or in vmscan. +do end up getting faulted into this VM_LOCKED VMA, they will be handled in the +fault path - which is also how mlock2()'s MLOCK_ONFAULT areas are handled. -Also note that a page returned by get_user_pages() could be truncated or -migrated out from under us, while we're trying to mlock it. To detect this, -populate_vma_page_range() checks page_mapping() after acquiring the page lock. -If the page is still associated with its mapping, we'll go ahead and call -mlock_vma_page(). If the mapping is gone, we just unlock the page and move on. -In the worst case, this will result in a page mapped in a VM_LOCKED VMA -remaining on a normal LRU list without being PageMlocked(). Again, vmscan will -detect and cull such pages. +For each PTE (or PMD) being faulted into a VMA, the page add rmap function +calls mlock_vma_page(), which calls mlock_page() when the VMA is VM_LOCKED +(unless it is a PTE mapping of a part of a transparent huge page). Or when +it is a newly allocated anonymous page, lru_cache_add_inactive_or_unevictable() +calls mlock_new_page() instead: similar to mlock_page(), but can make better +judgments, since this page is held exclusively and known not to be on LRU yet. -mlock_vma_page() will call TestSetPageMlocked() for each page returned by -get_user_pages(). We use TestSetPageMlocked() because the page might already -be mlocked by another task/VMA and we don't want to do extra work. We -especially do not want to count an mlocked page more than once in the -statistics. If the page was already mlocked, mlock_vma_page() need do nothing -more. +mlock_page() sets PageMlocked immediately, then places the page on the CPU's +mlock pagevec, to batch up the rest of the work to be done under lru_lock by +__mlock_page(). __mlock_page() sets PageUnevictable, initializes mlock_count +and moves the page to unevictable state ("the unevictable LRU", but with +mlock_count in place of LRU threading). Or if the page was already PageLRU +and PageUnevictable and PageMlocked, it simply increments the mlock_count. -If the page was NOT already mlocked, mlock_vma_page() attempts to isolate the -page from the LRU, as it is likely on the appropriate active or inactive list -at that time. If the isolate_lru_page() succeeds, mlock_vma_page() will put -back the page - by calling putback_lru_page() - which will notice that the page -is now mlocked and divert the page to the node's unevictable list. If -mlock_vma_page() is unable to isolate the page from the LRU, vmscan will handle -it later if and when it attempts to reclaim the page. +But in practice that may not work ideally: the page may not yet be on an LRU, or +it may have been temporarily isolated from LRU. In such cases the mlock_count +field cannot be touched, but will be set to 0 later when __pagevec_lru_add_fn() +returns the page to "LRU". Races prohibit mlock_count from being set to 1 then: +rather than risk stranding a page indefinitely as unevictable, always err with +mlock_count on the low side, so that when munlocked the page will be rescued to +an evictable LRU, then perhaps be mlocked again later if vmscan finds it in a +VM_LOCKED VMA. Filtering Special VMAs @@ -339,68 +343,48 @@ mlock_fixup() filters several classes of "special" VMAs: so there is no sense in attempting to visit them. 2) VMAs mapping hugetlbfs page are already effectively pinned into memory. We - neither need nor want to mlock() these pages. However, to preserve the - prior behavior of mlock() - before the unevictable/mlock changes - - mlock_fixup() will call make_pages_present() in the hugetlbfs VMA range to - allocate the huge pages and populate the ptes. + neither need nor want to mlock() these pages. But __mm_populate() includes + hugetlbfs ranges, allocating the huge pages and populating the PTEs. 3) VMAs with VM_DONTEXPAND are generally userspace mappings of kernel pages, - such as the VDSO page, relay channel pages, etc. These pages - are inherently unevictable and are not managed on the LRU lists. - mlock_fixup() treats these VMAs the same as hugetlbfs VMAs. It calls - make_pages_present() to populate the ptes. + such as the VDSO page, relay channel pages, etc. These pages are inherently + unevictable and are not managed on the LRU lists. __mm_populate() includes + these ranges, populating the PTEs if not already populated. + +4) VMAs with VM_MIXEDMAP set are not marked VM_LOCKED, but __mm_populate() + includes these ranges, populating the PTEs if not already populated. Note that for all of these special VMAs, mlock_fixup() does not set the VM_LOCKED flag. Therefore, we won't have to deal with them later during munlock(), munmap() or task exit. Neither does mlock_fixup() account these VMAs against the task's "locked_vm". -.. _munlock_munlockall_handling: munlock()/munlockall() System Call Handling ------------------------------------------- -The munlock() and munlockall() system calls are handled by the same functions - -do_mlock[all]() - as the mlock() and mlockall() system calls with the unlock vs -lock operation indicated by an argument. So, these system calls are also -handled by mlock_fixup(). Again, if called for an already munlocked VMA, -mlock_fixup() simply returns. Because of the VMA filtering discussed above, -VM_LOCKED will not be set in any "special" VMAs. So, these VMAs will be -ignored for munlock. +The munlock() and munlockall() system calls are handled by the same +mlock_fixup() function as mlock(), mlock2() and mlockall() system calls are. +If called to munlock an already munlocked VMA, mlock_fixup() simply returns. +Because of the VMA filtering discussed above, VM_LOCKED will not be set in +any "special" VMAs. So, those VMAs will be ignored for munlock. If the VMA is VM_LOCKED, mlock_fixup() again attempts to merge or split off the -specified range. The range is then munlocked via the function -populate_vma_page_range() - the same function used to mlock a VMA range - -passing a flag to indicate that munlock() is being performed. +specified range. All pages in the VMA are then munlocked by munlock_page() via +mlock_pte_range() via walk_page_range() via mlock_vma_pages_range() - the same +function used when mlocking a VMA range, with new flags for the VMA indicating +that it is munlock() being performed. -Because the VMA access protections could have been changed to PROT_NONE after -faulting in and mlocking pages, get_user_pages() was unreliable for visiting -these pages for munlocking. Because we don't want to leave pages mlocked, -get_user_pages() was enhanced to accept a flag to ignore the permissions when -fetching the pages - all of which should be resident as a result of previous -mlocking. +munlock_page() uses the mlock pagevec to batch up work to be done under +lru_lock by __munlock_page(). __munlock_page() decrements the page's +mlock_count, and when that reaches 0 it clears PageMlocked and clears +PageUnevictable, moving the page from unevictable state to inactive LRU. -For munlock(), populate_vma_page_range() unlocks individual pages by calling -munlock_vma_page(). munlock_vma_page() unconditionally clears the PG_mlocked -flag using TestClearPageMlocked(). As with mlock_vma_page(), -munlock_vma_page() use the Test*PageMlocked() function to handle the case where -the page might have already been unlocked by another task. If the page was -mlocked, munlock_vma_page() updates that zone statistics for the number of -mlocked pages. Note, however, that at this point we haven't checked whether -the page is mapped by other VM_LOCKED VMAs. - -We can't call page_mlock(), the function that walks the reverse map to -check for other VM_LOCKED VMAs, without first isolating the page from the LRU. -page_mlock() is a variant of try_to_unmap() and thus requires that the page -not be on an LRU list [more on these below]. However, the call to -isolate_lru_page() could fail, in which case we can't call page_mlock(). So, -we go ahead and clear PG_mlocked up front, as this might be the only chance we -have. If we can successfully isolate the page, we go ahead and call -page_mlock(), which will restore the PG_mlocked flag and update the zone -page statistics if it finds another VMA holding the page mlocked. If we fail -to isolate the page, we'll have left a potentially mlocked page on the LRU. -This is fine, because we'll catch it later if and if vmscan tries to reclaim -the page. This should be relatively rare. +But in practice that may not work ideally: the page may not yet have reached +"the unevictable LRU", or it may have been temporarily isolated from it. In +those cases its mlock_count field is unusable and must be assumed to be 0: so +that the page will be rescued to an evictable LRU, then perhaps be mlocked +again later if vmscan finds it in a VM_LOCKED VMA. Migrating MLOCKED Pages @@ -410,33 +394,38 @@ A page that is being migrated has been isolated from the LRU lists and is held locked across unmapping of the page, updating the page's address space entry and copying the contents and state, until the page table entry has been replaced with an entry that refers to the new page. Linux supports migration -of mlocked pages and other unevictable pages. This involves simply moving the -PG_mlocked and PG_unevictable states from the old page to the new page. +of mlocked pages and other unevictable pages. PG_mlocked is cleared from the +the old page when it is unmapped from the last VM_LOCKED VMA, and set when the +new page is mapped in place of migration entry in a VM_LOCKED VMA. If the page +was unevictable because mlocked, PG_unevictable follows PG_mlocked; but if the +page was unevictable for other reasons, PG_unevictable is copied explicitly. Note that page migration can race with mlocking or munlocking of the same page. -This has been discussed from the mlock/munlock perspective in the respective -sections above. Both processes (migration and m[un]locking) hold the page -locked. This provides the first level of synchronization. Page migration -zeros out the page_mapping of the old page before unlocking it, so m[un]lock -can skip these pages by testing the page mapping under page lock. +There is mostly no problem since page migration requires unmapping all PTEs of +the old page (including munlock where VM_LOCKED), then mapping in the new page +(including mlock where VM_LOCKED). The page table locks provide sufficient +synchronization. -To complete page migration, we place the new and old pages back onto the LRU -after dropping the page lock. The "unneeded" page - old page on success, new -page on failure - will be freed when the reference count held by the migration -process is released. To ensure that we don't strand pages on the unevictable -list because of a race between munlock and migration, page migration uses the -putback_lru_page() function to add migrated pages back to the LRU. +However, since mlock_vma_pages_range() starts by setting VM_LOCKED on a VMA, +before mlocking any pages already present, if one of those pages were migrated +before mlock_pte_range() reached it, it would get counted twice in mlock_count. +To prevent that, mlock_vma_pages_range() temporarily marks the VMA as VM_IO, +so that mlock_vma_page() will skip it. + +To complete page migration, we place the old and new pages back onto the LRU +afterwards. The "unneeded" page - old page on success, new page on failure - +is freed when the reference count held by the migration process is released. Compacting MLOCKED Pages ------------------------ -The unevictable LRU can be scanned for compactable regions and the default -behavior is to do so. /proc/sys/vm/compact_unevictable_allowed controls -this behavior (see Documentation/admin-guide/sysctl/vm.rst). Once scanning of the -unevictable LRU is enabled, the work of compaction is mostly handled by -the page migration code and the same work flow as described in MIGRATING -MLOCKED PAGES will apply. +The memory map can be scanned for compactable regions and the default behavior +is to let unevictable pages be moved. /proc/sys/vm/compact_unevictable_allowed +controls this behavior (see Documentation/admin-guide/sysctl/vm.rst). The work +of compaction is mostly handled by the page migration code and the same work +flow as described in Migrating MLOCKED Pages will apply. + MLOCKING Transparent Huge Pages ------------------------------- @@ -445,51 +434,44 @@ A transparent huge page is represented by a single entry on an LRU list. Therefore, we can only make unevictable an entire compound page, not individual subpages. -If a user tries to mlock() part of a huge page, we want the rest of the -page to be reclaimable. +If a user tries to mlock() part of a huge page, and no user mlock()s the +whole of the huge page, we want the rest of the page to be reclaimable. We cannot just split the page on partial mlock() as split_huge_page() can -fail and new intermittent failure mode for the syscall is undesirable. +fail and a new intermittent failure mode for the syscall is undesirable. -We handle this by keeping PTE-mapped huge pages on normal LRU lists: the -PMD on border of VM_LOCKED VMA will be split into PTE table. +We handle this by keeping PTE-mlocked huge pages on evictable LRU lists: +the PMD on the border of a VM_LOCKED VMA will be split into a PTE table. -This way the huge page is accessible for vmscan. Under memory pressure the +This way the huge page is accessible for vmscan. Under memory pressure the page will be split, subpages which belong to VM_LOCKED VMAs will be moved -to unevictable LRU and the rest can be reclaimed. +to the unevictable LRU and the rest can be reclaimed. + +/proc/meminfo's Unevictable and Mlocked amounts do not include those parts +of a transparent huge page which are mapped only by PTEs in VM_LOCKED VMAs. -See also comment in follow_trans_huge_pmd(). mmap(MAP_LOCKED) System Call Handling ------------------------------------- -In addition the mlock()/mlockall() system calls, an application can request -that a region of memory be mlocked supplying the MAP_LOCKED flag to the mmap() -call. There is one important and subtle difference here, though. mmap() + mlock() -will fail if the range cannot be faulted in (e.g. because mm_populate fails) -and returns with ENOMEM while mmap(MAP_LOCKED) will not fail. The mmaped -area will still have properties of the locked area - aka. pages will not get -swapped out - but major page faults to fault memory in might still happen. +In addition to the mlock(), mlock2() and mlockall() system calls, an application +can request that a region of memory be mlocked by supplying the MAP_LOCKED flag +to the mmap() call. There is one important and subtle difference here, though. +mmap() + mlock() will fail if the range cannot be faulted in (e.g. because +mm_populate fails) and returns with ENOMEM while mmap(MAP_LOCKED) will not fail. +The mmaped area will still have properties of the locked area - pages will not +get swapped out - but major page faults to fault memory in might still happen. -Furthermore, any mmap() call or brk() call that expands the heap by a -task that has previously called mlockall() with the MCL_FUTURE flag will result +Furthermore, any mmap() call or brk() call that expands the heap by a task +that has previously called mlockall() with the MCL_FUTURE flag will result in the newly mapped memory being mlocked. Before the unevictable/mlock -changes, the kernel simply called make_pages_present() to allocate pages and -populate the page table. +changes, the kernel simply called make_pages_present() to allocate pages +and populate the page table. -To mlock a range of memory under the unevictable/mlock infrastructure, the -mmap() handler and task address space expansion functions call +To mlock a range of memory under the unevictable/mlock infrastructure, +the mmap() handler and task address space expansion functions call populate_vma_page_range() specifying the vma and the address range to mlock. -The callers of populate_vma_page_range() will have already added the memory range -to be mlocked to the task's "locked_vm". To account for filtered VMAs, -populate_vma_page_range() returns the number of pages NOT mlocked. All of the -callers then subtract a non-negative return value from the task's locked_vm. A -negative return value represent an error - for example, from get_user_pages() -attempting to fault in a VMA with PROT_NONE access. In this case, we leave the -memory range accounted as locked_vm, as the protections could be changed later -and pages allocated into that region. - munmap()/exit()/exec() System Call Handling ------------------------------------------- @@ -500,81 +482,53 @@ munlock the pages if we're removing the last VM_LOCKED VMA that maps the pages. Before the unevictable/mlock changes, mlocking did not mark the pages in any way, so unmapping them required no processing. -To munlock a range of memory under the unevictable/mlock infrastructure, the -munmap() handler and task address space call tear down function -munlock_vma_pages_all(). The name reflects the observation that one always -specifies the entire VMA range when munlock()ing during unmap of a region. -Because of the VMA filtering when mlocking() regions, only "normal" VMAs that -actually contain mlocked pages will be passed to munlock_vma_pages_all(). +For each PTE (or PMD) being unmapped from a VMA, page_remove_rmap() calls +munlock_vma_page(), which calls munlock_page() when the VMA is VM_LOCKED +(unless it was a PTE mapping of a part of a transparent huge page). -munlock_vma_pages_all() clears the VM_LOCKED VMA flag and, like mlock_fixup() -for the munlock case, calls __munlock_vma_pages_range() to walk the page table -for the VMA's memory range and munlock_vma_page() each resident page mapped by -the VMA. This effectively munlocks the page, only if this is the last -VM_LOCKED VMA that maps the page. +munlock_page() uses the mlock pagevec to batch up work to be done under +lru_lock by __munlock_page(). __munlock_page() decrements the page's +mlock_count, and when that reaches 0 it clears PageMlocked and clears +PageUnevictable, moving the page from unevictable state to inactive LRU. + +But in practice that may not work ideally: the page may not yet have reached +"the unevictable LRU", or it may have been temporarily isolated from it. In +those cases its mlock_count field is unusable and must be assumed to be 0: so +that the page will be rescued to an evictable LRU, then perhaps be mlocked +again later if vmscan finds it in a VM_LOCKED VMA. -try_to_unmap() --------------- +Truncating MLOCKED Pages +------------------------ -Pages can, of course, be mapped into multiple VMAs. Some of these VMAs may -have VM_LOCKED flag set. It is possible for a page mapped into one or more -VM_LOCKED VMAs not to have the PG_mlocked flag set and therefore reside on one -of the active or inactive LRU lists. This could happen if, for example, a task -in the process of munlocking the page could not isolate the page from the LRU. -As a result, vmscan/shrink_page_list() might encounter such a page as described -in section "vmscan's handling of unevictable pages". To handle this situation, -try_to_unmap() checks for VM_LOCKED VMAs while it is walking a page's reverse -map. +File truncation or hole punching forcibly unmaps the deleted pages from +userspace; truncation even unmaps and deletes any private anonymous pages +which had been Copied-On-Write from the file pages now being truncated. -try_to_unmap() is always called, by either vmscan for reclaim or for page -migration, with the argument page locked and isolated from the LRU. Separate -functions handle anonymous and mapped file and KSM pages, as these types of -pages have different reverse map lookup mechanisms, with different locking. -In each case, whether rmap_walk_anon() or rmap_walk_file() or rmap_walk_ksm(), -it will call try_to_unmap_one() for every VMA which might contain the page. +Mlocked pages can be munlocked and deleted in this way: like with munmap(), +for each PTE (or PMD) being unmapped from a VMA, page_remove_rmap() calls +munlock_vma_page(), which calls munlock_page() when the VMA is VM_LOCKED +(unless it was a PTE mapping of a part of a transparent huge page). -When trying to reclaim, if try_to_unmap_one() finds the page in a VM_LOCKED -VMA, it will then mlock the page via mlock_vma_page() instead of unmapping it, -and return SWAP_MLOCK to indicate that the page is unevictable: and the scan -stops there. - -mlock_vma_page() is called while holding the page table's lock (in addition -to the page lock, and the rmap lock): to serialize against concurrent mlock or -munlock or munmap system calls, mm teardown (munlock_vma_pages_all), reclaim, -holepunching, and truncation of file pages and their anonymous COWed pages. - - -page_mlock() Reverse Map Scan ---------------------------------- - -When munlock_vma_page() [see section :ref:`munlock()/munlockall() System Call -Handling ` above] tries to munlock a -page, it needs to determine whether or not the page is mapped by any -VM_LOCKED VMA without actually attempting to unmap all PTEs from the -page. For this purpose, the unevictable/mlock infrastructure -introduced a variant of try_to_unmap() called page_mlock(). - -page_mlock() walks the respective reverse maps looking for VM_LOCKED VMAs. When -such a VMA is found the page is mlocked via mlock_vma_page(). This undoes the -pre-clearing of the page's PG_mlocked done by munlock_vma_page. - -Note that page_mlock()'s reverse map walk must visit every VMA in a page's -reverse map to determine that a page is NOT mapped into any VM_LOCKED VMA. -However, the scan can terminate when it encounters a VM_LOCKED VMA. -Although page_mlock() might be called a great many times when munlocking a -large region or tearing down a large address space that has been mlocked via -mlockall(), overall this is a fairly rare event. +However, if there is a racing munlock(), since mlock_vma_pages_range() starts +munlocking by clearing VM_LOCKED from a VMA, before munlocking all the pages +present, if one of those pages were unmapped by truncation or hole punch before +mlock_pte_range() reached it, it would not be recognized as mlocked by this VMA, +and would not be counted out of mlock_count. In this rare case, a page may +still appear as PageMlocked after it has been fully unmapped: and it is left to +release_pages() (or __page_cache_release()) to clear it and update statistics +before freeing (this event is counted in /proc/vmstat unevictable_pgs_cleared, +which is usually 0). Page Reclaim in shrink_*_list() ------------------------------- -shrink_active_list() culls any obviously unevictable pages - i.e. -!page_evictable(page) - diverting these to the unevictable list. +vmscan's shrink_active_list() culls any obviously unevictable pages - +i.e. !page_evictable(page) pages - diverting those to the unevictable list. However, shrink_active_list() only sees unevictable pages that made it onto the -active/inactive lru lists. Note that these pages do not have PageUnevictable -set - otherwise they would be on the unevictable list and shrink_active_list +active/inactive LRU lists. Note that these pages do not have PageUnevictable +set - otherwise they would be on the unevictable list and shrink_active_list() would never see them. Some examples of these unevictable pages on the LRU lists are: @@ -586,20 +540,15 @@ Some examples of these unevictable pages on the LRU lists are: when an application accesses the page the first time after SHM_LOCK'ing the segment. - (3) mlocked pages that could not be isolated from the LRU and moved to the - unevictable list in mlock_vma_page(). + (3) pages still mapped into VM_LOCKED VMAs, which should be marked mlocked, + but events left mlock_count too low, so they were munlocked too early. -shrink_inactive_list() also diverts any unevictable pages that it finds on the -inactive lists to the appropriate node's unevictable list. +vmscan's shrink_inactive_list() and shrink_page_list() also divert obviously +unevictable pages found on the inactive lists to the appropriate memory cgroup +and node unevictable list. -shrink_inactive_list() should only see SHM_LOCK'd pages that became SHM_LOCK'd -after shrink_active_list() had moved them to the inactive list, or pages mapped -into VM_LOCKED VMAs that munlock_vma_page() couldn't isolate from the LRU to -recheck via page_mlock(). shrink_inactive_list() won't notice the latter, -but will pass on to shrink_page_list(). - -shrink_page_list() again culls obviously unevictable pages that it could -encounter for similar reason to shrink_inactive_list(). Pages mapped into -VM_LOCKED VMAs but without PG_mlocked set will make it all the way to -try_to_unmap(). shrink_page_list() will divert them to the unevictable list -when try_to_unmap() returns SWAP_MLOCK, as discussed above. +rmap's page_referenced_one(), called via vmscan's shrink_active_list() or +shrink_page_list(), and rmap's try_to_unmap_one() called via shrink_page_list(), +check for (3) pages still mapped into VM_LOCKED VMAs, and call mlock_vma_page() +to correct them. Such pages are culled to the unevictable list when released +by the shrinker. From adb11e78c5dc5e26774acb05f983da36447f7911 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 1 Apr 2022 11:28:33 -0700 Subject: [PATCH 87/95] mm/munlock: protect the per-CPU pagevec by a local_lock_t The access to mlock_pvec is protected by disabling preemption via get_cpu_var() or implicit by having preemption disabled by the caller (in mlock_page_drain() case). This breaks on PREEMPT_RT since folio_lruvec_lock_irq() acquires a sleeping lock in this section. Create struct mlock_pvec which consits of the local_lock_t and the pagevec. Acquire the local_lock() before accessing the per-CPU pagevec. Replace mlock_page_drain() with a _local() version which is invoked on the local CPU and acquires the local_lock_t and a _remote() version which uses the pagevec from a remote CPU which offline. Link: https://lkml.kernel.org/r/YjizWi9IY0mpvIfb@linutronix.de Signed-off-by: Sebastian Andrzej Siewior Acked-by: Hugh Dickins Cc: Vlastimil Babka Cc: Matthew Wilcox Cc: Thomas Gleixner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/internal.h | 6 ++++-- mm/migrate.c | 2 +- mm/mlock.c | 46 ++++++++++++++++++++++++++++++++++++---------- mm/page_alloc.c | 1 + mm/rmap.c | 4 ++-- mm/swap.c | 4 +++- 6 files changed, 47 insertions(+), 16 deletions(-) diff --git a/mm/internal.h b/mm/internal.h index 58dc6adc19c5..cf16280ce132 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -456,7 +456,8 @@ static inline void munlock_vma_page(struct page *page, } void mlock_new_page(struct page *page); bool need_mlock_page_drain(int cpu); -void mlock_page_drain(int cpu); +void mlock_page_drain_local(void); +void mlock_page_drain_remote(int cpu); extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma); @@ -539,7 +540,8 @@ static inline void munlock_vma_page(struct page *page, struct vm_area_struct *vma, bool compound) { } static inline void mlock_new_page(struct page *page) { } static inline bool need_mlock_page_drain(int cpu) { return false; } -static inline void mlock_page_drain(int cpu) { } +static inline void mlock_page_drain_local(void) { } +static inline void mlock_page_drain_remote(int cpu) { } static inline void vunmap_range_noflush(unsigned long start, unsigned long end) { } diff --git a/mm/migrate.c b/mm/migrate.c index 3d60823afd2d..de175e2fdba5 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -246,7 +246,7 @@ static bool remove_migration_pte(struct folio *folio, set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte); } if (vma->vm_flags & VM_LOCKED) - mlock_page_drain(smp_processor_id()); + mlock_page_drain_local(); trace_remove_migration_pte(pvmw.address, pte_val(pte), compound_order(new)); diff --git a/mm/mlock.c b/mm/mlock.c index 529fbc1f27c8..716caf851043 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -28,7 +28,14 @@ #include "internal.h" -static DEFINE_PER_CPU(struct pagevec, mlock_pvec); +struct mlock_pvec { + local_lock_t lock; + struct pagevec vec; +}; + +static DEFINE_PER_CPU(struct mlock_pvec, mlock_pvec) = { + .lock = INIT_LOCAL_LOCK(lock), +}; bool can_do_mlock(void) { @@ -203,18 +210,30 @@ static void mlock_pagevec(struct pagevec *pvec) pagevec_reinit(pvec); } -void mlock_page_drain(int cpu) +void mlock_page_drain_local(void) { struct pagevec *pvec; - pvec = &per_cpu(mlock_pvec, cpu); + local_lock(&mlock_pvec.lock); + pvec = this_cpu_ptr(&mlock_pvec.vec); + if (pagevec_count(pvec)) + mlock_pagevec(pvec); + local_unlock(&mlock_pvec.lock); +} + +void mlock_page_drain_remote(int cpu) +{ + struct pagevec *pvec; + + WARN_ON_ONCE(cpu_online(cpu)); + pvec = &per_cpu(mlock_pvec.vec, cpu); if (pagevec_count(pvec)) mlock_pagevec(pvec); } bool need_mlock_page_drain(int cpu) { - return pagevec_count(&per_cpu(mlock_pvec, cpu)); + return pagevec_count(&per_cpu(mlock_pvec.vec, cpu)); } /** @@ -223,7 +242,10 @@ bool need_mlock_page_drain(int cpu) */ void mlock_folio(struct folio *folio) { - struct pagevec *pvec = &get_cpu_var(mlock_pvec); + struct pagevec *pvec; + + local_lock(&mlock_pvec.lock); + pvec = this_cpu_ptr(&mlock_pvec.vec); if (!folio_test_set_mlocked(folio)) { int nr_pages = folio_nr_pages(folio); @@ -236,7 +258,7 @@ void mlock_folio(struct folio *folio) if (!pagevec_add(pvec, mlock_lru(&folio->page)) || folio_test_large(folio) || lru_cache_disabled()) mlock_pagevec(pvec); - put_cpu_var(mlock_pvec); + local_unlock(&mlock_pvec.lock); } /** @@ -245,9 +267,11 @@ void mlock_folio(struct folio *folio) */ void mlock_new_page(struct page *page) { - struct pagevec *pvec = &get_cpu_var(mlock_pvec); + struct pagevec *pvec; int nr_pages = thp_nr_pages(page); + local_lock(&mlock_pvec.lock); + pvec = this_cpu_ptr(&mlock_pvec.vec); SetPageMlocked(page); mod_zone_page_state(page_zone(page), NR_MLOCK, nr_pages); __count_vm_events(UNEVICTABLE_PGMLOCKED, nr_pages); @@ -256,7 +280,7 @@ void mlock_new_page(struct page *page) if (!pagevec_add(pvec, mlock_new(page)) || PageHead(page) || lru_cache_disabled()) mlock_pagevec(pvec); - put_cpu_var(mlock_pvec); + local_unlock(&mlock_pvec.lock); } /** @@ -265,8 +289,10 @@ void mlock_new_page(struct page *page) */ void munlock_page(struct page *page) { - struct pagevec *pvec = &get_cpu_var(mlock_pvec); + struct pagevec *pvec; + local_lock(&mlock_pvec.lock); + pvec = this_cpu_ptr(&mlock_pvec.vec); /* * TestClearPageMlocked(page) must be left to __munlock_page(), * which will check whether the page is multiply mlocked. @@ -276,7 +302,7 @@ void munlock_page(struct page *page) if (!pagevec_add(pvec, page) || PageHead(page) || lru_cache_disabled()) mlock_pagevec(pvec); - put_cpu_var(mlock_pvec); + local_unlock(&mlock_pvec.lock); } static int mlock_pte_range(pmd_t *pmd, unsigned long addr, diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 6c6af8658775..2db95780e003 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -8367,6 +8367,7 @@ static int page_alloc_cpu_dead(unsigned int cpu) struct zone *zone; lru_add_drain_cpu(cpu); + mlock_page_drain_remote(cpu); drain_pages(cpu); /* diff --git a/mm/rmap.c b/mm/rmap.c index 5cb970d51f0a..fedb82371efe 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1683,7 +1683,7 @@ discard: */ page_remove_rmap(subpage, vma, folio_test_hugetlb(folio)); if (vma->vm_flags & VM_LOCKED) - mlock_page_drain(smp_processor_id()); + mlock_page_drain_local(); folio_put(folio); } @@ -1961,7 +1961,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, */ page_remove_rmap(subpage, vma, folio_test_hugetlb(folio)); if (vma->vm_flags & VM_LOCKED) - mlock_page_drain(smp_processor_id()); + mlock_page_drain_local(); folio_put(folio); } diff --git a/mm/swap.c b/mm/swap.c index bceff0cb559c..7e320ec08c6a 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -624,7 +624,6 @@ void lru_add_drain_cpu(int cpu) pagevec_lru_move_fn(pvec, lru_lazyfree_fn); activate_page_drain(cpu); - mlock_page_drain(cpu); } /** @@ -706,6 +705,7 @@ void lru_add_drain(void) local_lock(&lru_pvecs.lock); lru_add_drain_cpu(smp_processor_id()); local_unlock(&lru_pvecs.lock); + mlock_page_drain_local(); } /* @@ -720,6 +720,7 @@ static void lru_add_and_bh_lrus_drain(void) lru_add_drain_cpu(smp_processor_id()); local_unlock(&lru_pvecs.lock); invalidate_bh_lrus_cpu(); + mlock_page_drain_local(); } void lru_add_drain_cpu_zone(struct zone *zone) @@ -728,6 +729,7 @@ void lru_add_drain_cpu_zone(struct zone *zone) lru_add_drain_cpu(smp_processor_id()); drain_local_pages(zone); local_unlock(&lru_pvecs.lock); + mlock_page_drain_local(); } #ifdef CONFIG_SMP From 8f0b36497303487d5a32c75789c77859cc2ee895 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Fri, 1 Apr 2022 11:28:36 -0700 Subject: [PATCH 88/95] mm: kfence: fix objcgs vector allocation If the kfence object is allocated to be used for objects vector, then this slot of the pool eventually being occupied permanently since the vector is never freed. The solutions could be (1) freeing vector when the kfence object is freed or (2) allocating all vectors statically. Since the memory consumption of object vectors is low, it is better to chose (2) to fix the issue and it is also can reduce overhead of vectors allocating in the future. Link: https://lkml.kernel.org/r/20220328132843.16624-1-songmuchun@bytedance.com Fixes: d3fb45f370d9 ("mm, kfence: insert KFENCE hooks for SLAB") Signed-off-by: Muchun Song Reviewed-by: Marco Elver Reviewed-by: Roman Gushchin Cc: Alexander Potapenko Cc: Dmitry Vyukov Cc: Xiongchun Duan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/kfence/core.c | 11 ++++++++++- mm/kfence/kfence.h | 3 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 2f9fdfde1941..a203747ad2c0 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -566,6 +566,8 @@ static unsigned long kfence_init_pool(void) * enters __slab_free() slow-path. */ for (i = 0; i < KFENCE_POOL_SIZE / PAGE_SIZE; i++) { + struct slab *slab = page_slab(&pages[i]); + if (!i || (i % 2)) continue; @@ -573,7 +575,11 @@ static unsigned long kfence_init_pool(void) if (WARN_ON(compound_head(&pages[i]) != &pages[i])) return addr; - __SetPageSlab(&pages[i]); + __folio_set_slab(slab_folio(slab)); +#ifdef CONFIG_MEMCG + slab->memcg_data = (unsigned long)&kfence_metadata[i / 2 - 1].objcg | + MEMCG_DATA_OBJCGS; +#endif } /* @@ -1033,6 +1039,9 @@ void __kfence_free(void *addr) { struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr); +#ifdef CONFIG_MEMCG + KFENCE_WARN_ON(meta->objcg); +#endif /* * If the objects of the cache are SLAB_TYPESAFE_BY_RCU, defer freeing * the object, as the object page may be recycled for other-typed diff --git a/mm/kfence/kfence.h b/mm/kfence/kfence.h index 2a2d5de9d379..9a6c4b1b12a8 100644 --- a/mm/kfence/kfence.h +++ b/mm/kfence/kfence.h @@ -89,6 +89,9 @@ struct kfence_metadata { struct kfence_track free_track; /* For updating alloc_covered on frees. */ u32 alloc_stack_hash; +#ifdef CONFIG_MEMCG + struct obj_cgroup *objcg; +#endif }; extern struct kfence_metadata kfence_metadata[CONFIG_KFENCE_NUM_OBJECTS]; From 4f1f9698d77734a90cae1bb9a50188f44f15e305 Mon Sep 17 00:00:00 2001 From: Kirill Tkhai Date: Fri, 1 Apr 2022 11:28:39 -0700 Subject: [PATCH 89/95] mailmap: update Kirill's email My new email address is kirill.tkhai@openvz.org. Link: https://lkml.kernel.org/r/164846762354.278960.13129571556274098855.stgit@pro Signed-off-by: Kirill Tkhai Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- .mailmap | 1 + 1 file changed, 1 insertion(+) diff --git a/.mailmap b/.mailmap index 8fd9b3c7a42b..b9d358217586 100644 --- a/.mailmap +++ b/.mailmap @@ -213,6 +213,7 @@ Kees Cook Keith Busch Keith Busch Kenneth W Chen +Kirill Tkhai Konstantin Khlebnikov Konstantin Khlebnikov Koushik From 3149c79f3cb0e2e3bafb7cfadacec090cbd250d3 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Fri, 1 Apr 2022 11:28:42 -0700 Subject: [PATCH 90/95] mm,hwpoison: unmap poisoned page before invalidation In some cases it appears the invalidation of a hwpoisoned page fails because the page is still mapped in another process. This can cause a program to be continuously restarted and die when it page faults on the page that was not invalidated. Avoid that problem by unmapping the hwpoisoned page when we find it. Another issue is that sometimes we end up oopsing in finish_fault, if the code tries to do something with the now-NULL vmf->page. I did not hit this error when submitting the previous patch because there are several opportunities for alloc_set_pte to bail out before accessing vmf->page, and that apparently happened on those systems, and most of the time on other systems, too. However, across several million systems that error does occur a handful of times a day. It can be avoided by returning VM_FAULT_NOPAGE which will cause do_read_fault to return before calling finish_fault. Link: https://lkml.kernel.org/r/20220325161428.5068d97e@imladris.surriel.com Fixes: e53ac7374e64 ("mm: invalidate hwpoison page cache page in fault path") Signed-off-by: Rik van Riel Reviewed-by: Miaohe Lin Tested-by: Naoya Horiguchi Reviewed-by: Oscar Salvador Cc: Mel Gorman Cc: Johannes Weiner Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memory.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mm/memory.c b/mm/memory.c index be44d0b36b18..76e3af9639d9 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3918,14 +3918,18 @@ static vm_fault_t __do_fault(struct vm_fault *vmf) return ret; if (unlikely(PageHWPoison(vmf->page))) { + struct page *page = vmf->page; vm_fault_t poisonret = VM_FAULT_HWPOISON; if (ret & VM_FAULT_LOCKED) { + if (page_mapped(page)) + unmap_mapping_pages(page_mapping(page), + page->index, 1, false); /* Retry if a clean page was removed from the cache. */ - if (invalidate_inode_page(vmf->page)) - poisonret = 0; - unlock_page(vmf->page); + if (invalidate_inode_page(page)) + poisonret = VM_FAULT_NOPAGE; + unlock_page(page); } - put_page(vmf->page); + put_page(page); vmf->page = NULL; return poisonret; } From ada543af3bfe3d953986eca118601b9612382c13 Mon Sep 17 00:00:00 2001 From: Andrey Konovalov Date: Fri, 1 Apr 2022 11:28:45 -0700 Subject: [PATCH 91/95] mm, kasan: fix __GFP_BITS_SHIFT definition breaking LOCKDEP KASAN changes that added new GFP flags mistakenly updated __GFP_BITS_SHIFT as the total number of GFP bits instead of as a shift used to define __GFP_BITS_MASK. This broke LOCKDEP, as __GFP_BITS_MASK now gets the 25th bit enabled instead of the 28th for __GFP_NOLOCKDEP. Update __GFP_BITS_SHIFT to always count KASAN GFP bits. In the future, we could handle all combinations of KASAN and LOCKDEP to occupy as few bits as possible. For now, we have enough GFP bits to be inefficient in this quick fix. Link: https://lkml.kernel.org/r/462ff52742a1fcc95a69778685737f723ee4dfb3.1648400273.git.andreyknvl@google.com Fixes: 9353ffa6e9e9 ("kasan, page_alloc: allow skipping memory init for HW_TAGS") Fixes: 53ae233c30a6 ("kasan, page_alloc: allow skipping unpoisoning for HW_TAGS") Fixes: f49d9c5bb15c ("kasan, mm: only define ___GFP_SKIP_KASAN_POISON with HW_TAGS") Signed-off-by: Andrey Konovalov Reported-by: Sebastian Andrzej Siewior Tested-by: Sebastian Andrzej Siewior Acked-by: Vlastimil Babka Cc: Marco Elver Cc: Alexander Potapenko Cc: Dmitry Vyukov Cc: Andrey Ryabinin Cc: Matthew Wilcox Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/gfp.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 0fa17fb85de5..761f8f1885c7 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -264,9 +264,7 @@ struct vm_area_struct; #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) /* Room for N __GFP_FOO bits */ -#define __GFP_BITS_SHIFT (24 + \ - 3 * IS_ENABLED(CONFIG_KASAN_HW_TAGS) + \ - IS_ENABLED(CONFIG_LOCKDEP)) +#define __GFP_BITS_SHIFT (27 + IS_ENABLED(CONFIG_LOCKDEP)) #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) /** From d8b7b3fa9f9b2dc67fa1df29c4ce98eb10d62824 Mon Sep 17 00:00:00 2001 From: Yinan Zhang Date: Fri, 1 Apr 2022 11:28:48 -0700 Subject: [PATCH 92/95] tools/vm/page_owner_sort.c: remove -c option The -c option is used to cull by stacktrace. Now, --cull option has been Added in page_owner_sort.c. Culling by stacktrace is one of the function of "--cull". No need to set an extra parameter. So remove -c option. Remove parsing of -c when parse parameter and remove "-c" from usage. This work is coauthored by Shenghong Han Yixuan Cao Chongxi Zhao Jiajian Ye Yuhong Feng Yongqiang Liu Link: https://lkml.kernel.org/r/20220326085920.1470081-1-zhangyinan2019@email.szu.edu.cn Signed-off-by: Yinan Zhang Cc: Chongxi Zhao Cc: Georgi Djakov Cc: Jiajian Ye Cc: Jonathan Corbet Cc: Sean Anderson Cc: Stephen Rothwell Cc: Tang Bin Cc: Yixuan Cao Cc: Yongqiang Liu Cc: Yuhong Feng Cc: Zhenliang Wei Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- tools/vm/page_owner_sort.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tools/vm/page_owner_sort.c b/tools/vm/page_owner_sort.c index 7679335fce5b..7d98e76c2291 100644 --- a/tools/vm/page_owner_sort.c +++ b/tools/vm/page_owner_sort.c @@ -441,7 +441,6 @@ static void usage(void) "-n\t\tSort by task command name.\n" "-a\t\tSort by memory allocate time.\n" "-r\t\tSort by memory release time.\n" - "-c\t\tCull by comparing stacktrace instead of total block.\n" "-f\t\tFilter out the information of blocks whose memory has been released.\n" "--pid \tSelect by pid. This selects the information of blocks whose process ID number equals to .\n" "--tgid \tSelect by tgid. This selects the information of blocks whose Thread Group ID number equals to .\n" @@ -466,14 +465,11 @@ int main(int argc, char **argv) { 0, 0, 0, 0}, }; - while ((opt = getopt_long(argc, argv, "acfmnprstP", longopts, NULL)) != -1) + while ((opt = getopt_long(argc, argv, "afmnprstP", longopts, NULL)) != -1) switch (opt) { case 'a': cmp = compare_ts; break; - case 'c': - cull = cull | CULL_STACKTRACE; - break; case 'f': filter = filter | FILTER_UNRELEASE; break; From c89b3ad2dea254ad17ae2585b17c2cf9f78e64d9 Mon Sep 17 00:00:00 2001 From: Yinan Zhang Date: Fri, 1 Apr 2022 11:28:51 -0700 Subject: [PATCH 93/95] doc/vm/page_owner.rst: remove content related to -c option -c option has been removed from page_owner_sort.c. Remove the usage of -c option from Documentation. This work is coauthored by Shenghong Han Yixuan Cao Chongxi Zhao Jiajian Ye Yuhong Feng Yongqiang Liu Link: https://lkml.kernel.org/r/20220326085920.1470081-2-zhangyinan2019@email.szu.edu.cn Signed-off-by: Yinan Zhang Cc: Jonathan Corbet Cc: Stephen Rothwell Cc: Sean Anderson Cc: Tang Bin Cc: Zhenliang Wei Cc: Georgi Djakov Cc: Chongxi Zhao Cc: Jiajian Ye Cc: Yixuan Cao Cc: Yuhong Feng Cc: Yongqiang Liu Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- Documentation/vm/page_owner.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/Documentation/vm/page_owner.rst b/Documentation/vm/page_owner.rst index c4de6f8dabe9..65204d7f004f 100644 --- a/Documentation/vm/page_owner.rst +++ b/Documentation/vm/page_owner.rst @@ -125,7 +125,6 @@ Usage additional function: Cull: - -c Cull by comparing stacktrace instead of total block. --cull Specify culling rules.Culling syntax is key[,key[,...]].Choose a multi-letter key from the **STANDARD FORMAT SPECIFIERS** section. From bfc8089f00fa526dea983844c880fa8106c33ac4 Mon Sep 17 00:00:00 2001 From: Kuan-Ying Lee Date: Fri, 1 Apr 2022 11:28:54 -0700 Subject: [PATCH 94/95] mm/kmemleak: reset tag when compare object pointer When we use HW-tag based kasan and enable vmalloc support, we hit the following bug. It is due to comparison between tagged object and non-tagged pointer. We need to reset the kasan tag when we need to compare tagged object and non-tagged pointer. kmemleak: [name:kmemleak&]Scan area larger than object 0xffffffe77076f440 CPU: 4 PID: 1 Comm: init Tainted: G S W 5.15.25-android13-0-g5cacf919c2bc #1 Hardware name: MT6983(ENG) (DT) Call trace: add_scan_area+0xc4/0x244 kmemleak_scan_area+0x40/0x9c layout_and_allocate+0x1e8/0x288 load_module+0x2c8/0xf00 __se_sys_finit_module+0x190/0x1d0 __arm64_sys_finit_module+0x20/0x30 invoke_syscall+0x60/0x170 el0_svc_common+0xc8/0x114 do_el0_svc+0x28/0xa0 el0_svc+0x60/0xf8 el0t_64_sync_handler+0x88/0xec el0t_64_sync+0x1b4/0x1b8 kmemleak: [name:kmemleak&]Object 0xf5ffffe77076b000 (size 32768): kmemleak: [name:kmemleak&] comm "init", pid 1, jiffies 4294894197 kmemleak: [name:kmemleak&] min_count = 0 kmemleak: [name:kmemleak&] count = 0 kmemleak: [name:kmemleak&] flags = 0x1 kmemleak: [name:kmemleak&] checksum = 0 kmemleak: [name:kmemleak&] backtrace: module_alloc+0x9c/0x120 move_module+0x34/0x19c layout_and_allocate+0x1c4/0x288 load_module+0x2c8/0xf00 __se_sys_finit_module+0x190/0x1d0 __arm64_sys_finit_module+0x20/0x30 invoke_syscall+0x60/0x170 el0_svc_common+0xc8/0x114 do_el0_svc+0x28/0xa0 el0_svc+0x60/0xf8 el0t_64_sync_handler+0x88/0xec el0t_64_sync+0x1b4/0x1b8 Link: https://lkml.kernel.org/r/20220318034051.30687-1-Kuan-Ying.Lee@mediatek.com Signed-off-by: Kuan-Ying Lee Reviewed-by: Catalin Marinas Cc: Matthias Brugger Cc: Chinwen Chang Cc: Nicholas Tang Cc: Yee Lee Cc: Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/kmemleak.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 7580baa76af1..acd7cbb82e16 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -796,6 +796,8 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp) unsigned long flags; struct kmemleak_object *object; struct kmemleak_scan_area *area = NULL; + unsigned long untagged_ptr; + unsigned long untagged_objp; object = find_and_get_object(ptr, 1); if (!object) { @@ -804,6 +806,9 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp) return; } + untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr); + untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); + if (scan_area_cache) area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp)); @@ -815,8 +820,8 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp) goto out_unlock; } if (size == SIZE_MAX) { - size = object->pointer + object->size - ptr; - } else if (ptr + size > object->pointer + object->size) { + size = untagged_objp + object->size - untagged_ptr; + } else if (untagged_ptr + size > untagged_objp + object->size) { kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr); dump_object_info(object); kmem_cache_free(scan_area_cache, area); From 78049e94a171837e5a882814ca5bc4f14f265603 Mon Sep 17 00:00:00 2001 From: Jonghyeon Kim Date: Fri, 1 Apr 2022 11:28:57 -0700 Subject: [PATCH 95/95] mm/damon: prevent activated scheme from sleeping by deactivated schemes In the DAMON, the minimum wait time of the schemes decides whether the kernel wakes up 'kdamon_fn()'. But since the minimum wait time is initialized to zero, there are corner cases against the original objective. For example, if we have several schemes for one target, and if the wait time of the first scheme is zero, the minimum wait time will set zero, which means 'kdamond_fn()' should wake up to apply this scheme. However, in the following scheme, wait time can be set to non-zero. Thus, the mininum wait time will be set to non-zero, which can cause sleeping this interval for 'kdamon_fn()' due to one deactivated last scheme. This commit prevents making DAMON monitoring inactive state due to other deactivated schemes. Link: https://lkml.kernel.org/r/20220330105302.32114-1-tome01@ajou.ac.kr Signed-off-by: Jonghyeon Kim Reviewed-by: SeongJae Park Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/damon/core.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mm/damon/core.c b/mm/damon/core.c index c1e0fed4e877..5ce8d7c867f0 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -1019,12 +1019,15 @@ static int kdamond_wait_activation(struct damon_ctx *ctx) struct damos *s; unsigned long wait_time; unsigned long min_wait_time = 0; + bool init_wait_time = false; while (!kdamond_need_stop(ctx)) { damon_for_each_scheme(s, ctx) { wait_time = damos_wmark_wait_us(s); - if (!min_wait_time || wait_time < min_wait_time) + if (!init_wait_time || wait_time < min_wait_time) { + init_wait_time = true; min_wait_time = wait_time; + } } if (!min_wait_time) return 0;