From 7aba5dcc234635b44b2781dbc268048cfba388ad Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Mon, 9 Mar 2020 13:23:08 -0500 Subject: [PATCH 0001/1043] jfs: Replace zero-length array with flexible-array member The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Dave Kleikamp --- fs/jfs/jfs_dtree.c | 2 +- fs/jfs/jfs_xattr.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/jfs/jfs_dtree.c b/fs/jfs/jfs_dtree.c index 3acc954f7c04..837d42f61464 100644 --- a/fs/jfs/jfs_dtree.c +++ b/fs/jfs/jfs_dtree.c @@ -2964,7 +2964,7 @@ struct jfs_dirent { loff_t position; int ino; u16 name_len; - char name[0]; + char name[]; }; /* diff --git a/fs/jfs/jfs_xattr.h b/fs/jfs/jfs_xattr.h index f0558b3348da..c50167a7bc50 100644 --- a/fs/jfs/jfs_xattr.h +++ b/fs/jfs/jfs_xattr.h @@ -17,12 +17,12 @@ struct jfs_ea { u8 flag; /* Unused? */ u8 namelen; /* Length of name */ __le16 valuelen; /* Length of value */ - char name[0]; /* Attribute name (includes null-terminator) */ + char name[]; /* Attribute name (includes null-terminator) */ }; /* Value immediately follows name */ struct jfs_ea_list { __le32 size; /* overall size */ - struct jfs_ea ea[0]; /* Variable length list */ + struct jfs_ea ea[]; /* Variable length list */ }; /* Macros for defining maxiumum number of bytes supported for EAs */ From bdf8783c0dae9d3d8fc1c4078fe849ab8aa8b583 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Thu, 9 Apr 2020 11:25:43 +0200 Subject: [PATCH 0002/1043] clocksource/drivers/davinci: Avoid trailing '\n' hidden in pr_fmt() pr_xxx() functions usually have '\n' at the end of the logging message. Here, this '\n' is added via the 'pr_fmt' macro. In order to be more consistent with other files, use a more standard convention and put these '\n' back in the messages themselves and remove it from the pr_fmt macro. While at it, remove a useless message in case of 'kzalloc' failure, especially with a __GFP_NOFAIL flag. Signed-off-by: Christophe JAILLET Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200409092543.14727-1-christophe.jaillet@wanadoo.fr --- drivers/clocksource/timer-davinci.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/drivers/clocksource/timer-davinci.c b/drivers/clocksource/timer-davinci.c index e421946a91c5..aae938368230 100644 --- a/drivers/clocksource/timer-davinci.c +++ b/drivers/clocksource/timer-davinci.c @@ -18,7 +18,7 @@ #include #undef pr_fmt -#define pr_fmt(fmt) "%s: " fmt "\n", __func__ +#define pr_fmt(fmt) "%s: " fmt, __func__ #define DAVINCI_TIMER_REG_TIM12 0x10 #define DAVINCI_TIMER_REG_TIM34 0x14 @@ -250,20 +250,20 @@ int __init davinci_timer_register(struct clk *clk, rv = clk_prepare_enable(clk); if (rv) { - pr_err("Unable to prepare and enable the timer clock"); + pr_err("Unable to prepare and enable the timer clock\n"); return rv; } if (!request_mem_region(timer_cfg->reg.start, resource_size(&timer_cfg->reg), "davinci-timer")) { - pr_err("Unable to request memory region"); + pr_err("Unable to request memory region\n"); return -EBUSY; } base = ioremap(timer_cfg->reg.start, resource_size(&timer_cfg->reg)); if (!base) { - pr_err("Unable to map the register range"); + pr_err("Unable to map the register range\n"); return -ENOMEM; } @@ -271,10 +271,8 @@ int __init davinci_timer_register(struct clk *clk, tick_rate = clk_get_rate(clk); clockevent = kzalloc(sizeof(*clockevent), GFP_KERNEL | __GFP_NOFAIL); - if (!clockevent) { - pr_err("Error allocating memory for clockevent data"); + if (!clockevent) return -ENOMEM; - } clockevent->dev.name = "tim12"; clockevent->dev.features = CLOCK_EVT_FEAT_ONESHOT; @@ -298,7 +296,7 @@ int __init davinci_timer_register(struct clk *clk, davinci_timer_irq_timer, IRQF_TIMER, "clockevent/tim12", clockevent); if (rv) { - pr_err("Unable to request the clockevent interrupt"); + pr_err("Unable to request the clockevent interrupt\n"); return rv; } @@ -325,7 +323,7 @@ int __init davinci_timer_register(struct clk *clk, rv = clocksource_register_hz(&davinci_clocksource.dev, tick_rate); if (rv) { - pr_err("Unable to register clocksource"); + pr_err("Unable to register clocksource\n"); return rv; } @@ -343,20 +341,20 @@ static int __init of_davinci_timer_register(struct device_node *np) rv = of_address_to_resource(np, 0, &timer_cfg.reg); if (rv) { - pr_err("Unable to get the register range for timer"); + pr_err("Unable to get the register range for timer\n"); return rv; } rv = of_irq_to_resource_table(np, timer_cfg.irq, DAVINCI_TIMER_NUM_IRQS); if (rv != DAVINCI_TIMER_NUM_IRQS) { - pr_err("Unable to get the interrupts for timer"); + pr_err("Unable to get the interrupts for timer\n"); return rv; } clk = of_clk_get(np, 0); if (IS_ERR(clk)) { - pr_err("Unable to get the timer clock"); + pr_err("Unable to get the timer clock\n"); return PTR_ERR(clk); } From 4855f2bd91b6e3461af7d795bfe9a40420122131 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Thu, 9 Apr 2020 12:12:26 +0200 Subject: [PATCH 0003/1043] clocksource: davinci: axe a pointless __GFP_NOFAIL There is no need to specify __GFP_NOFAIL when allocating memory here, so axe it. Signed-off-by: Christophe JAILLET Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200409101226.15432-1-christophe.jaillet@wanadoo.fr --- drivers/clocksource/timer-davinci.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clocksource/timer-davinci.c b/drivers/clocksource/timer-davinci.c index aae938368230..bb4eee31ae08 100644 --- a/drivers/clocksource/timer-davinci.c +++ b/drivers/clocksource/timer-davinci.c @@ -270,7 +270,7 @@ int __init davinci_timer_register(struct clk *clk, davinci_timer_init(base); tick_rate = clk_get_rate(clk); - clockevent = kzalloc(sizeof(*clockevent), GFP_KERNEL | __GFP_NOFAIL); + clockevent = kzalloc(sizeof(*clockevent), GFP_KERNEL); if (!clockevent) return -ENOMEM; From ac161f57b66dcf14b3339b1c5857c08a9ad4d833 Mon Sep 17 00:00:00 2001 From: Anson Huang Date: Wed, 1 Apr 2020 12:27:02 +0800 Subject: [PATCH 0004/1043] clocksource/drivers/imx-tpm: Add support for ARM64 Allows building and compile-testing the i.MX TPM driver also for ARM64. The delay_timer is only supported on ARMv7. Signed-off-by: Anson Huang Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/1585715222-24489-1-git-send-email-Anson.Huang@nxp.com --- drivers/clocksource/timer-imx-tpm.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/clocksource/timer-imx-tpm.c b/drivers/clocksource/timer-imx-tpm.c index 6334a35fdc2f..2cdc077a39f5 100644 --- a/drivers/clocksource/timer-imx-tpm.c +++ b/drivers/clocksource/timer-imx-tpm.c @@ -61,17 +61,19 @@ static inline void tpm_irq_acknowledge(void) writel(TPM_STATUS_CH0F, timer_base + TPM_STATUS); } -static struct delay_timer tpm_delay_timer; - static inline unsigned long tpm_read_counter(void) { return readl(timer_base + TPM_CNT); } +#if defined(CONFIG_ARM) +static struct delay_timer tpm_delay_timer; + static unsigned long tpm_read_current_timer(void) { return tpm_read_counter(); } +#endif static u64 notrace tpm_read_sched_clock(void) { @@ -144,9 +146,11 @@ static struct timer_of to_tpm = { static int __init tpm_clocksource_init(void) { +#if defined(CONFIG_ARM) tpm_delay_timer.read_current_timer = &tpm_read_current_timer; tpm_delay_timer.freq = timer_of_rate(&to_tpm) >> 3; register_current_timer_delay(&tpm_delay_timer); +#endif sched_clock_register(tpm_read_sched_clock, counter_width, timer_of_rate(&to_tpm) >> 3); From 5125bfeeb6e4ffc47a5cb2715c342c938854cf20 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 31 Mar 2020 15:00:06 +0800 Subject: [PATCH 0005/1043] MIPS: Loongson: Use CONFIG_NR_CPUS_DEFAULT_64 to support more CPUs When I update the mainline kernel on the Loongson 2-way platform which has 8 CPUs, it only shows 4 CPUs due to NR_CPUS is 4, this is obviously wrong. In order to support more CPUs on the Loongson platform, it is better to use CONFIG_NR_CPUS_DEFAULT_64 instead of CONFIG_NR_CPUS_DEFAULT_4 to specify the maximum number of CPUs which the kernel will support. Signed-off-by: Tiezhu Yang Acked-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 690718b3701a..4f9303f6a687 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -475,7 +475,7 @@ config MACH_LOONGSON64 select ISA select I8259 select IRQ_MIPS_CPU - select NR_CPUS_DEFAULT_4 + select NR_CPUS_DEFAULT_64 select USE_GENERIC_EARLY_PRINTK_8250 select SYS_HAS_CPU_LOONGSON64 select SYS_HAS_EARLY_PRINTK From de0c4eb44c39cbb2b3b11bac699b618b0f8d4142 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Fri, 3 Apr 2020 17:49:04 +0800 Subject: [PATCH 0006/1043] MIPS: TXx9: Fix Kconfig warnings If TTY and SND is not n, we got this warnings: WARNING: unmet direct dependencies detected for HAS_TXX9_SERIAL Depends on [n]: TTY [=n] && HAS_IOMEM [=y] Selected by [y]: - SOC_TX3927 [=y] WARNING: unmet direct dependencies detected for HAS_TXX9_SERIAL Depends on [n]: TTY [=n] && HAS_IOMEM [=y] Selected by [y]: - SOC_TX4938 [=y] Only dependencies is enabled, they can be enabled, so use 'imply' instead of 'select' to fix this. Signed-off-by: YueHaibing Signed-off-by: Thomas Bogendoerfer --- arch/mips/txx9/Kconfig | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/mips/txx9/Kconfig b/arch/mips/txx9/Kconfig index 9a22a182b7a4..85c4c121c71f 100644 --- a/arch/mips/txx9/Kconfig +++ b/arch/mips/txx9/Kconfig @@ -58,7 +58,7 @@ config TOSHIBA_RBTX4939 config SOC_TX3927 bool select CEVT_TXX9 - select HAS_TXX9_SERIAL + imply HAS_TXX9_SERIAL select HAVE_PCI select IRQ_TXX9 select GPIO_TXX9 @@ -66,30 +66,30 @@ config SOC_TX3927 config SOC_TX4927 bool select CEVT_TXX9 - select HAS_TXX9_SERIAL + imply HAS_TXX9_SERIAL select HAVE_PCI select IRQ_TXX9 select PCI_TX4927 select GPIO_TXX9 - select HAS_TXX9_ACLC + imply HAS_TXX9_ACLC config SOC_TX4938 bool select CEVT_TXX9 - select HAS_TXX9_SERIAL + imply HAS_TXX9_SERIAL select HAVE_PCI select IRQ_TXX9 select PCI_TX4927 select GPIO_TXX9 - select HAS_TXX9_ACLC + imply HAS_TXX9_ACLC config SOC_TX4939 bool select CEVT_TXX9 - select HAS_TXX9_SERIAL + imply HAS_TXX9_SERIAL select HAVE_PCI select PCI_TX4927 - select HAS_TXX9_ACLC + imply HAS_TXX9_ACLC config TXX9_7SEGLED bool From 3fbfb4585bfd4ff34e9d3b4edd5b3e49e8f5a541 Mon Sep 17 00:00:00 2001 From: Mike Rapoport Date: Thu, 2 Apr 2020 11:16:14 +0300 Subject: [PATCH 0007/1043] mips: define pud_index() regardless of page table folding Commit 31168f033e37 ("mips: drop __pXd_offset() macros that duplicate pXd_index() ones") is correct that pud_index() & __pud_offset() are the same when pud_index() is actually provided, however it does not take into account the __PAGETABLE_PUD_FOLDED case. This has broken MIPS KVM compilation because it relied on availability of pud_index(). Define pud_index() regardless of page table folded. It will evaluate to actual index for 4-level pagetables and to 0 for folded PUD level. Link: https://lore.kernel.org/lkml/20200331154749.5457-1-pbonzini@redhat.com Reported-by: Paolo Bonzini Signed-off-by: Mike Rapoport Tested-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/pgtable-64.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/include/asm/pgtable-64.h b/arch/mips/include/asm/pgtable-64.h index f92716cfa4f4..ee5dc0c145b9 100644 --- a/arch/mips/include/asm/pgtable-64.h +++ b/arch/mips/include/asm/pgtable-64.h @@ -172,6 +172,8 @@ extern pte_t invalid_pte_table[PTRS_PER_PTE]; +#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) + #ifndef __PAGETABLE_PUD_FOLDED /* * For 4-level pagetables we defines these ourselves, for 3-level the @@ -210,8 +212,6 @@ static inline void p4d_clear(p4d_t *p4dp) p4d_val(*p4dp) = (unsigned long)invalid_pud_table; } -#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) - static inline unsigned long p4d_page_vaddr(p4d_t p4d) { return p4d_val(p4d); From 408b722d029b14d87bd5039c04f59a282d6c1f74 Mon Sep 17 00:00:00 2001 From: bibo mao Date: Thu, 26 Mar 2020 03:42:47 -0400 Subject: [PATCH 0008/1043] MIPS: xilfpga: Removed unused header files Header in directory asm/mach-xilfpga is not used any more. Remove it here, and it passes to compile with xilfpga_defconfig Signed-off-by: bibo mao Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mach-xilfpga/irq.h | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 arch/mips/include/asm/mach-xilfpga/irq.h diff --git a/arch/mips/include/asm/mach-xilfpga/irq.h b/arch/mips/include/asm/mach-xilfpga/irq.h deleted file mode 100644 index 15ad29ec1dee..000000000000 --- a/arch/mips/include/asm/mach-xilfpga/irq.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2015 Imagination Technologies - * Author: Zubair Lutfullah Kakakhel - */ - -#ifndef __MIPS_ASM_MACH_XILFPGA_IRQ_H__ -#define __MIPS_ASM_MACH_XILFPGA_IRQ_H__ - -#define NR_IRQS 32 - -#include - -#endif /* __MIPS_ASM_MACH_XILFPGA_IRQ_H__ */ From e82c878d49bf58f3f2199cba00400530856ad11e Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Thu, 26 Mar 2020 14:16:58 +0800 Subject: [PATCH 0009/1043] MIPS: Kill MIPS_GIC_IRQ_BASE It never got used by any driver. Signed-off-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mach-generic/irq.h | 6 ------ arch/mips/include/asm/mach-ralink/mt7621.h | 2 -- 2 files changed, 8 deletions(-) diff --git a/arch/mips/include/asm/mach-generic/irq.h b/arch/mips/include/asm/mach-generic/irq.h index be546a0f65fa..72ac2c202c55 100644 --- a/arch/mips/include/asm/mach-generic/irq.h +++ b/arch/mips/include/asm/mach-generic/irq.h @@ -36,10 +36,4 @@ #endif /* CONFIG_IRQ_MIPS_CPU */ -#ifdef CONFIG_MIPS_GIC -#ifndef MIPS_GIC_IRQ_BASE -#define MIPS_GIC_IRQ_BASE (MIPS_CPU_IRQ_BASE + 8) -#endif -#endif /* CONFIG_MIPS_GIC */ - #endif /* __ASM_MACH_GENERIC_IRQ_H */ diff --git a/arch/mips/include/asm/mach-ralink/mt7621.h b/arch/mips/include/asm/mach-ralink/mt7621.h index 65483a4681ab..e1af1ba50bd8 100644 --- a/arch/mips/include/asm/mach-ralink/mt7621.h +++ b/arch/mips/include/asm/mach-ralink/mt7621.h @@ -31,6 +31,4 @@ #define MT7621_CHIP_NAME0 0x3637544D #define MT7621_CHIP_NAME1 0x20203132 -#define MIPS_GIC_IRQ_BASE (MIPS_CPU_IRQ_BASE + 8) - #endif From 4b8503967ef5d1123d6e0a87d5723bdaeddf8b3f Mon Sep 17 00:00:00 2001 From: Zou Wei Date: Tue, 14 Apr 2020 16:18:07 +0800 Subject: [PATCH 0010/1043] selinux: fix warning Comparison to bool fix below warnings reported by coccicheck security/selinux/ss/mls.c:539:39-43: WARNING: Comparison to bool security/selinux/ss/services.c:1815:46-50: WARNING: Comparison to bool security/selinux/ss/services.c:1827:46-50: WARNING: Comparison to bool Reported-by: Hulk Robot Signed-off-by: Zou Wei Signed-off-by: Paul Moore --- security/selinux/ss/mls.c | 2 +- security/selinux/ss/services.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/security/selinux/ss/mls.c b/security/selinux/ss/mls.c index ec5e3d1da9ac..6a5d7d08933d 100644 --- a/security/selinux/ss/mls.c +++ b/security/selinux/ss/mls.c @@ -536,7 +536,7 @@ int mls_compute_sid(struct policydb *p, /* Fallthrough */ case AVTAB_CHANGE: - if ((tclass == p->process_class) || (sock == true)) + if ((tclass == p->process_class) || sock) /* Use the process MLS attributes. */ return mls_context_cpy(newcontext, scontext); else diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index 8ad34fd031d1..3b592d17d2d3 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -1812,7 +1812,7 @@ static int security_compute_sid(struct selinux_state *state, } else if (cladatum && cladatum->default_role == DEFAULT_TARGET) { newcontext.role = tcontext->role; } else { - if ((tclass == policydb->process_class) || (sock == true)) + if ((tclass == policydb->process_class) || sock) newcontext.role = scontext->role; else newcontext.role = OBJECT_R_VAL; @@ -1824,7 +1824,7 @@ static int security_compute_sid(struct selinux_state *state, } else if (cladatum && cladatum->default_type == DEFAULT_TARGET) { newcontext.type = tcontext->type; } else { - if ((tclass == policydb->process_class) || (sock == true)) { + if ((tclass == policydb->process_class) || sock) { /* Use the type of process. */ newcontext.type = scontext->type; } else { From 8c42c0f72d7c4f295646d3eba73f62e5579b1732 Mon Sep 17 00:00:00 2001 From: Jason Yan Date: Tue, 14 Apr 2020 20:02:38 +0800 Subject: [PATCH 0011/1043] clocksource/drivers/atmel-st: Remove useless 'status' Fix the following coccicheck warning: drivers/clocksource/timer-atmel-st.c:142:6-12: Unneeded variable: "status". Return "0" on line 166 Signed-off-by: Jason Yan Acked-by: Alexandre Belloni Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200414120238.35704-1-yanaijie@huawei.com --- drivers/clocksource/timer-atmel-st.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/clocksource/timer-atmel-st.c b/drivers/clocksource/timer-atmel-st.c index ab0aabfae5f0..73e8aee445da 100644 --- a/drivers/clocksource/timer-atmel-st.c +++ b/drivers/clocksource/timer-atmel-st.c @@ -139,7 +139,6 @@ static int clkevt32k_next_event(unsigned long delta, struct clock_event_device *dev) { u32 alm; - int status = 0; unsigned int val; BUG_ON(delta < 2); @@ -163,7 +162,7 @@ clkevt32k_next_event(unsigned long delta, struct clock_event_device *dev) alm += delta; regmap_write(regmap_st, AT91_ST_RTAR, alm); - return status; + return 0; } static struct clock_event_device clkevt = { From 788109c1ccf70f29d2d15de94aabf100710d0069 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 10 Apr 2020 12:35:26 +0100 Subject: [PATCH 0012/1043] KVM: remove redundant assignment to variable r The variable r is being assigned with a value that is never read and it is being updated later with a new value. The initialization is redundant and can be removed. Addresses-Coverity: ("Unused value") Signed-off-by: Colin Ian King Message-Id: <20200410113526.13822-1-colin.king@canonical.com> Signed-off-by: Paolo Bonzini --- virt/kvm/kvm_main.c | 1 - 1 file changed, 1 deletion(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 74bdb7bf3295..03571f6acaa8 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -3160,7 +3160,6 @@ out_free1: case KVM_SET_REGS: { struct kvm_regs *kvm_regs; - r = -ENOMEM; kvm_regs = memdup_user(argp, sizeof(*kvm_regs)); if (IS_ERR(kvm_regs)) { r = PTR_ERR(kvm_regs); From 24647e0a39b6c5ef007fa2f9a91ac8b581e093b9 Mon Sep 17 00:00:00 2001 From: Peter Shier Date: Wed, 10 Oct 2018 15:56:53 -0700 Subject: [PATCH 0013/1043] KVM: x86: Return updated timer current count register from KVM_GET_LAPIC kvm_vcpu_ioctl_get_lapic (implements KVM_GET_LAPIC ioctl) does a bulk copy of the LAPIC registers but must take into account that the one-shot and periodic timer current count register is computed upon reads and is not present in register state. When restoring LAPIC state (e.g. after migration), restart timers from their their current count values at time of save. Note: When a one-shot timer expires, the code in arch/x86/kvm/lapic.c does not zero the value of the LAPIC initial count register (emulating HW behavior). If no other timer is run and pending prior to a subsequent KVM_GET_LAPIC call, the returned register set will include the expired one-shot initial count. On a subsequent KVM_SET_LAPIC call the code will see a non-zero initial count and start a new one-shot timer using the expired timer's count. This is a prior existing bug and will be addressed in a separate patch. Thanks to jmattson@google.com for this find. Signed-off-by: Peter Shier Reviewed-by: Jim Mattson Reviewed-by: Wanpeng Li Message-Id: <20181010225653.238911-1-pshier@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/lapic.c | 61 ++++++++++++++++++++++++++++++++++++-------- arch/x86/kvm/lapic.h | 7 ++++- 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 9af25c97612a..38f7dc9c16ee 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1648,13 +1648,18 @@ static void start_sw_tscdeadline(struct kvm_lapic *apic) local_irq_restore(flags); } +static inline u64 tmict_to_ns(struct kvm_lapic *apic, u32 tmict) +{ + return (u64)tmict * APIC_BUS_CYCLE_NS * (u64)apic->divide_count; +} + static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor) { ktime_t now, remaining; u64 ns_remaining_old, ns_remaining_new; - apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) - * APIC_BUS_CYCLE_NS * apic->divide_count; + apic->lapic_timer.period = + tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT)); limit_periodic_timer_frequency(apic); now = ktime_get(); @@ -1672,14 +1677,15 @@ static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_diviso apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new); } -static bool set_target_expiration(struct kvm_lapic *apic) +static bool set_target_expiration(struct kvm_lapic *apic, u32 count_reg) { ktime_t now; u64 tscl = rdtsc(); + s64 deadline; now = ktime_get(); - apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT) - * APIC_BUS_CYCLE_NS * apic->divide_count; + apic->lapic_timer.period = + tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT)); if (!apic->lapic_timer.period) { apic->lapic_timer.tscdeadline = 0; @@ -1687,10 +1693,32 @@ static bool set_target_expiration(struct kvm_lapic *apic) } limit_periodic_timer_frequency(apic); + deadline = apic->lapic_timer.period; + + if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) { + if (unlikely(count_reg != APIC_TMICT)) { + deadline = tmict_to_ns(apic, + kvm_lapic_get_reg(apic, count_reg)); + if (unlikely(deadline <= 0)) + deadline = apic->lapic_timer.period; + else if (unlikely(deadline > apic->lapic_timer.period)) { + pr_info_ratelimited( + "kvm: vcpu %i: requested lapic timer restore with " + "starting count register %#x=%u (%lld ns) > initial count (%lld ns). " + "Using initial count to start timer.\n", + apic->vcpu->vcpu_id, + count_reg, + kvm_lapic_get_reg(apic, count_reg), + deadline, apic->lapic_timer.period); + kvm_lapic_set_reg(apic, count_reg, 0); + deadline = apic->lapic_timer.period; + } + } + } apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) + - nsec_to_cycles(apic->vcpu, apic->lapic_timer.period); - apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period); + nsec_to_cycles(apic->vcpu, deadline); + apic->lapic_timer.target_expiration = ktime_add_ns(now, deadline); return true; } @@ -1872,17 +1900,22 @@ void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu) restart_apic_timer(apic); } -static void start_apic_timer(struct kvm_lapic *apic) +static void __start_apic_timer(struct kvm_lapic *apic, u32 count_reg) { atomic_set(&apic->lapic_timer.pending, 0); if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) - && !set_target_expiration(apic)) + && !set_target_expiration(apic, count_reg)) return; restart_apic_timer(apic); } +static void start_apic_timer(struct kvm_lapic *apic) +{ + __start_apic_timer(apic, APIC_TMICT); +} + static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) { bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val); @@ -2493,6 +2526,14 @@ static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu, int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) { memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s)); + + /* + * Get calculated timer current count for remaining timer period (if + * any) and store it in the returned register set. + */ + __kvm_lapic_set_reg(s->regs, APIC_TMCCT, + __apic_read(vcpu->arch.apic, APIC_TMCCT)); + return kvm_apic_state_fixup(vcpu, s, false); } @@ -2520,7 +2561,7 @@ int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) apic_update_lvtt(apic); apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); update_divide_count(apic); - start_apic_timer(apic); + __start_apic_timer(apic, APIC_TMCCT); kvm_apic_update_apicv(vcpu); apic->highest_isr_cache = -1; if (vcpu->arch.apicv_active) { diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index a0ffb4331418..7f15f9e69efe 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -161,9 +161,14 @@ static inline u32 kvm_lapic_get_reg(struct kvm_lapic *apic, int reg_off) return *((u32 *) (apic->regs + reg_off)); } +static inline void __kvm_lapic_set_reg(char *regs, int reg_off, u32 val) +{ + *((u32 *) (regs + reg_off)) = val; +} + static inline void kvm_lapic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) { - *((u32 *) (apic->regs + reg_off)) = val; + __kvm_lapic_set_reg(apic->regs, reg_off, val); } extern struct static_key kvm_no_apic_vcpu; From 43d05de2bee7561b7958628b7c27c89b779de990 Mon Sep 17 00:00:00 2001 From: Eric Northup Date: Tue, 14 Apr 2020 18:23:20 -0700 Subject: [PATCH 0014/1043] KVM: pass through CPUID(0x80000006) Return the host's L2 cache and TLB information for CPUID.0x80000006 instead of zeroing out the entry as part of KVM_GET_SUPPORTED_CPUID. This allows a userspace VMM to feed KVM_GET_SUPPORTED_CPUID's output directly into KVM_SET_CPUID2 (without breaking the guest). Signed-off-by: Eric Northup (Google) Signed-off-by: Jim Mattson Signed-off-by: Jon Cargille Message-Id: <20200415012320.236065-1-jcargill@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/cpuid.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 901cd1fdecd9..6828be99b908 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -728,6 +728,9 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) cpuid_entry_override(entry, CPUID_8000_0001_EDX); cpuid_entry_override(entry, CPUID_8000_0001_ECX); break; + case 0x80000006: + /* L2 cache and TLB: pass through host info. */ + break; case 0x80000007: /* Advanced power management */ /* invariant TSC is CPUID.80000007H:EDX[8] */ entry->edx &= (1 << 8); From 238022ff5d05f6cdceac67c03a66cc28a6cb30c9 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 10 Apr 2020 16:16:58 -0700 Subject: [PATCH 0015/1043] KVM: selftests: Take vcpu pointer instead of id in vm_vcpu_rm() The sole caller of vm_vcpu_rm() already has the vcpu pointer, take it directly instead of doing an extra lookup. Signed-off-by: Sean Christopherson Reviewed-by: Wainer dos Santos Moschetta Reviewed-by: Andrew Jones Message-Id: <20200410231707.7128-2-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/lib/kvm_util.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 8a3523d4434f..9a783c20dd26 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -393,7 +393,7 @@ struct vcpu *vcpu_find(struct kvm_vm *vm, uint32_t vcpuid) * * Input Args: * vm - Virtual Machine - * vcpuid - VCPU ID + * vcpu - VCPU to remove * * Output Args: None * @@ -401,9 +401,8 @@ struct vcpu *vcpu_find(struct kvm_vm *vm, uint32_t vcpuid) * * Within the VM specified by vm, removes the VCPU given by vcpuid. */ -static void vm_vcpu_rm(struct kvm_vm *vm, uint32_t vcpuid) +static void vm_vcpu_rm(struct kvm_vm *vm, struct vcpu *vcpu) { - struct vcpu *vcpu = vcpu_find(vm, vcpuid); int ret; ret = munmap(vcpu->state, sizeof(*vcpu->state)); @@ -427,7 +426,7 @@ void kvm_vm_release(struct kvm_vm *vmp) int ret; while (vmp->vcpu_head) - vm_vcpu_rm(vmp, vmp->vcpu_head->id); + vm_vcpu_rm(vmp, vmp->vcpu_head); ret = close(vmp->fd); TEST_ASSERT(ret == 0, "Close of vm fd failed,\n" From 4d9bba9007167074504187d27fd80352eb3a04e7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 10 Apr 2020 16:16:59 -0700 Subject: [PATCH 0016/1043] KVM: selftests: Use kernel's list instead of homebrewed replacement Replace the KVM selftests' homebrewed linked lists for vCPUs and memory regions with the kernel's 'struct list_head'. Signed-off-by: Sean Christopherson Reviewed-by: Andrew Jones Message-Id: <20200410231707.7128-3-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- .../testing/selftests/kvm/include/kvm_util.h | 1 + tools/testing/selftests/kvm/lib/kvm_util.c | 94 ++++++++----------- .../selftests/kvm/lib/kvm_util_internal.h | 8 +- .../selftests/kvm/lib/s390x/processor.c | 5 +- 4 files changed, 48 insertions(+), 60 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index a99b875f50d2..2f329e785c58 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -10,6 +10,7 @@ #include "test_util.h" #include "asm/kvm.h" +#include "linux/list.h" #include "linux/kvm.h" #include diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 9a783c20dd26..105ee9bc09f0 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -161,6 +161,9 @@ struct kvm_vm *_vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm) vm = calloc(1, sizeof(*vm)); TEST_ASSERT(vm != NULL, "Insufficient Memory"); + INIT_LIST_HEAD(&vm->vcpus); + INIT_LIST_HEAD(&vm->userspace_mem_regions); + vm->mode = mode; vm->type = 0; @@ -258,8 +261,7 @@ void kvm_vm_restart(struct kvm_vm *vmp, int perm) if (vmp->has_irqchip) vm_create_irqchip(vmp); - for (region = vmp->userspace_mem_region_head; region; - region = region->next) { + list_for_each_entry(region, &vmp->userspace_mem_regions, list) { int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region); TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n" " rc: %i errno: %i\n" @@ -319,8 +321,7 @@ userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end) { struct userspace_mem_region *region; - for (region = vm->userspace_mem_region_head; region; - region = region->next) { + list_for_each_entry(region, &vm->userspace_mem_regions, list) { uint64_t existing_start = region->region.guest_phys_addr; uint64_t existing_end = region->region.guest_phys_addr + region->region.memory_size - 1; @@ -378,11 +379,11 @@ kvm_userspace_memory_region_find(struct kvm_vm *vm, uint64_t start, */ struct vcpu *vcpu_find(struct kvm_vm *vm, uint32_t vcpuid) { - struct vcpu *vcpup; + struct vcpu *vcpu; - for (vcpup = vm->vcpu_head; vcpup; vcpup = vcpup->next) { - if (vcpup->id == vcpuid) - return vcpup; + list_for_each_entry(vcpu, &vm->vcpus, list) { + if (vcpu->id == vcpuid) + return vcpu; } return NULL; @@ -392,16 +393,15 @@ struct vcpu *vcpu_find(struct kvm_vm *vm, uint32_t vcpuid) * VM VCPU Remove * * Input Args: - * vm - Virtual Machine * vcpu - VCPU to remove * * Output Args: None * * Return: None, TEST_ASSERT failures for all error conditions * - * Within the VM specified by vm, removes the VCPU given by vcpuid. + * Removes a vCPU from a VM and frees its resources. */ -static void vm_vcpu_rm(struct kvm_vm *vm, struct vcpu *vcpu) +static void vm_vcpu_rm(struct vcpu *vcpu) { int ret; @@ -412,21 +412,17 @@ static void vm_vcpu_rm(struct kvm_vm *vm, struct vcpu *vcpu) TEST_ASSERT(ret == 0, "Close of VCPU fd failed, rc: %i " "errno: %i", ret, errno); - if (vcpu->next) - vcpu->next->prev = vcpu->prev; - if (vcpu->prev) - vcpu->prev->next = vcpu->next; - else - vm->vcpu_head = vcpu->next; + list_del(&vcpu->list); free(vcpu); } void kvm_vm_release(struct kvm_vm *vmp) { + struct vcpu *vcpu, *tmp; int ret; - while (vmp->vcpu_head) - vm_vcpu_rm(vmp, vmp->vcpu_head); + list_for_each_entry_safe(vcpu, tmp, &vmp->vcpus, list) + vm_vcpu_rm(vcpu); ret = close(vmp->fd); TEST_ASSERT(ret == 0, "Close of vm fd failed,\n" @@ -442,15 +438,15 @@ void kvm_vm_release(struct kvm_vm *vmp) */ void kvm_vm_free(struct kvm_vm *vmp) { + struct userspace_mem_region *region, *tmp; int ret; if (vmp == NULL) return; /* Free userspace_mem_regions. */ - while (vmp->userspace_mem_region_head) { - struct userspace_mem_region *region - = vmp->userspace_mem_region_head; + list_for_each_entry_safe(region, tmp, &vmp->userspace_mem_regions, list) { + list_del(®ion->list); region->region.memory_size = 0; ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION, @@ -458,7 +454,6 @@ void kvm_vm_free(struct kvm_vm *vmp) TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed, " "rc: %i errno: %i", ret, errno); - vmp->userspace_mem_region_head = region->next; sparsebit_free(®ion->unused_phy_pages); ret = munmap(region->mmap_start, region->mmap_size); TEST_ASSERT(ret == 0, "munmap failed, rc: %i errno: %i", @@ -611,12 +606,10 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, (uint64_t) region->region.memory_size); /* Confirm no region with the requested slot already exists. */ - for (region = vm->userspace_mem_region_head; region; - region = region->next) { - if (region->region.slot == slot) - break; - } - if (region != NULL) + list_for_each_entry(region, &vm->userspace_mem_regions, list) { + if (region->region.slot != slot) + continue; + TEST_FAIL("A mem region with the requested slot " "already exists.\n" " requested slot: %u paddr: 0x%lx npages: 0x%lx\n" @@ -625,6 +618,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, region->region.slot, (uint64_t) region->region.guest_phys_addr, (uint64_t) region->region.memory_size); + } /* Allocate and initialize new mem region structure. */ region = calloc(1, sizeof(*region)); @@ -685,10 +679,7 @@ void vm_userspace_mem_region_add(struct kvm_vm *vm, guest_paddr, (uint64_t) region->region.memory_size); /* Add to linked-list of memory regions. */ - if (vm->userspace_mem_region_head) - vm->userspace_mem_region_head->prev = region; - region->next = vm->userspace_mem_region_head; - vm->userspace_mem_region_head = region; + list_add(®ion->list, &vm->userspace_mem_regions); } /* @@ -711,20 +702,17 @@ memslot2region(struct kvm_vm *vm, uint32_t memslot) { struct userspace_mem_region *region; - for (region = vm->userspace_mem_region_head; region; - region = region->next) { + list_for_each_entry(region, &vm->userspace_mem_regions, list) { if (region->region.slot == memslot) - break; - } - if (region == NULL) { - fprintf(stderr, "No mem region with the requested slot found,\n" - " requested slot: %u\n", memslot); - fputs("---- vm dump ----\n", stderr); - vm_dump(stderr, vm, 2); - TEST_FAIL("Mem region not found"); + return region; } - return region; + fprintf(stderr, "No mem region with the requested slot found,\n" + " requested slot: %u\n", memslot); + fputs("---- vm dump ----\n", stderr); + vm_dump(stderr, vm, 2); + TEST_FAIL("Mem region not found"); + return NULL; } /* @@ -862,10 +850,7 @@ void vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpuid) "vcpu id: %u errno: %i", vcpuid, errno); /* Add to linked-list of VCPUs. */ - if (vm->vcpu_head) - vm->vcpu_head->prev = vcpu; - vcpu->next = vm->vcpu_head; - vm->vcpu_head = vcpu; + list_add(&vcpu->list, &vm->vcpus); } /* @@ -1058,8 +1043,8 @@ void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa) { struct userspace_mem_region *region; - for (region = vm->userspace_mem_region_head; region; - region = region->next) { + + list_for_each_entry(region, &vm->userspace_mem_regions, list) { if ((gpa >= region->region.guest_phys_addr) && (gpa <= (region->region.guest_phys_addr + region->region.memory_size - 1))) @@ -1091,8 +1076,8 @@ void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa) vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva) { struct userspace_mem_region *region; - for (region = vm->userspace_mem_region_head; region; - region = region->next) { + + list_for_each_entry(region, &vm->userspace_mem_regions, list) { if ((hva >= region->host_mem) && (hva <= (region->host_mem + region->region.memory_size - 1))) @@ -1519,8 +1504,7 @@ void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) fprintf(stream, "%*sfd: %i\n", indent, "", vm->fd); fprintf(stream, "%*spage_size: 0x%x\n", indent, "", vm->page_size); fprintf(stream, "%*sMem Regions:\n", indent, ""); - for (region = vm->userspace_mem_region_head; region; - region = region->next) { + list_for_each_entry(region, &vm->userspace_mem_regions, list) { fprintf(stream, "%*sguest_phys: 0x%lx size: 0x%lx " "host_virt: %p\n", indent + 2, "", (uint64_t) region->region.guest_phys_addr, @@ -1539,7 +1523,7 @@ void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) virt_dump(stream, vm, indent + 4); } fprintf(stream, "%*sVCPUs:\n", indent, ""); - for (vcpu = vm->vcpu_head; vcpu; vcpu = vcpu->next) + list_for_each_entry(vcpu, &vm->vcpus, list) vcpu_dump(stream, vm, vcpu->id, indent + 2); } diff --git a/tools/testing/selftests/kvm/lib/kvm_util_internal.h b/tools/testing/selftests/kvm/lib/kvm_util_internal.h index ca56a0133127..2ef446520748 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util_internal.h +++ b/tools/testing/selftests/kvm/lib/kvm_util_internal.h @@ -13,7 +13,6 @@ #define KVM_DEV_PATH "/dev/kvm" struct userspace_mem_region { - struct userspace_mem_region *next, *prev; struct kvm_userspace_memory_region region; struct sparsebit *unused_phy_pages; int fd; @@ -21,10 +20,11 @@ struct userspace_mem_region { void *host_mem; void *mmap_start; size_t mmap_size; + struct list_head list; }; struct vcpu { - struct vcpu *next, *prev; + struct list_head list; uint32_t id; int fd; struct kvm_run *state; @@ -41,8 +41,8 @@ struct kvm_vm { unsigned int pa_bits; unsigned int va_bits; uint64_t max_gfn; - struct vcpu *vcpu_head; - struct userspace_mem_region *userspace_mem_region_head; + struct list_head vcpus; + struct list_head userspace_mem_regions; struct sparsebit *vpages_valid; struct sparsebit *vpages_mapped; bool has_irqchip; diff --git a/tools/testing/selftests/kvm/lib/s390x/processor.c b/tools/testing/selftests/kvm/lib/s390x/processor.c index 8d94961bd046..a88c5d665725 100644 --- a/tools/testing/selftests/kvm/lib/s390x/processor.c +++ b/tools/testing/selftests/kvm/lib/s390x/processor.c @@ -233,7 +233,10 @@ void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...) void vcpu_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent) { - struct vcpu *vcpu = vm->vcpu_head; + struct vcpu *vcpu = vcpu_find(vm, vcpuid); + + if (!vcpu) + return; fprintf(stream, "%*spstate: psw: 0x%.16llx:0x%.16llx\n", indent, "", vcpu->state->psw_mask, vcpu->state->psw_addr); From 8c996e4dae177e6921fe61d20b1f30fc06a3d275 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 10 Apr 2020 16:17:00 -0700 Subject: [PATCH 0017/1043] KVM: selftests: Add util to delete memory region Add a utility to delete a memory region, it will be used by x86's set_memory_region_test. Signed-off-by: Sean Christopherson Reviewed-by: Wainer dos Santos Moschetta Reviewed-by: Andrew Jones Message-Id: <20200410231707.7128-4-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- .../testing/selftests/kvm/include/kvm_util.h | 1 + tools/testing/selftests/kvm/lib/kvm_util.c | 56 +++++++++++++------ 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 2f329e785c58..d4c3e4d9cd92 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -114,6 +114,7 @@ int _vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid, unsigned long ioctl, void vm_ioctl(struct kvm_vm *vm, unsigned long ioctl, void *arg); void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags); void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa); +void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot); void vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpuid); vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min, uint32_t data_memslot, uint32_t pgd_memslot); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 105ee9bc09f0..ab5b7ea60f4b 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -433,34 +433,38 @@ void kvm_vm_release(struct kvm_vm *vmp) " vmp->kvm_fd: %i rc: %i errno: %i", vmp->kvm_fd, ret, errno); } +static void __vm_mem_region_delete(struct kvm_vm *vm, + struct userspace_mem_region *region) +{ + int ret; + + list_del(®ion->list); + + region->region.memory_size = 0; + ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region); + TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed, " + "rc: %i errno: %i", ret, errno); + + sparsebit_free(®ion->unused_phy_pages); + ret = munmap(region->mmap_start, region->mmap_size); + TEST_ASSERT(ret == 0, "munmap failed, rc: %i errno: %i", ret, errno); + + free(region); +} + /* * Destroys and frees the VM pointed to by vmp. */ void kvm_vm_free(struct kvm_vm *vmp) { struct userspace_mem_region *region, *tmp; - int ret; if (vmp == NULL) return; /* Free userspace_mem_regions. */ - list_for_each_entry_safe(region, tmp, &vmp->userspace_mem_regions, list) { - list_del(®ion->list); - - region->region.memory_size = 0; - ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION, - ®ion->region); - TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed, " - "rc: %i errno: %i", ret, errno); - - sparsebit_free(®ion->unused_phy_pages); - ret = munmap(region->mmap_start, region->mmap_size); - TEST_ASSERT(ret == 0, "munmap failed, rc: %i errno: %i", - ret, errno); - - free(region); - } + list_for_each_entry_safe(region, tmp, &vmp->userspace_mem_regions, list) + __vm_mem_region_delete(vmp, region); /* Free sparsebit arrays. */ sparsebit_free(&vmp->vpages_valid); @@ -775,6 +779,24 @@ void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa) ret, errno, slot, new_gpa); } +/* + * VM Memory Region Delete + * + * Input Args: + * vm - Virtual Machine + * slot - Slot of the memory region to delete + * + * Output Args: None + * + * Return: None + * + * Delete a memory region. + */ +void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot) +{ + __vm_mem_region_delete(vm, memslot2region(vm, slot)); +} + /* * VCPU mmap Size * From 3e6b94126784fd5b0e781f2e8e980948ee945195 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 10 Apr 2020 16:17:01 -0700 Subject: [PATCH 0018/1043] KVM: selftests: Add GUEST_ASSERT variants to pass values to host Add variants of GUEST_ASSERT to pass values back to the host, e.g. to help debug/understand a failure when the the cause of the assert isn't necessarily binary. It'd probably be possible to auto-calculate the number of arguments and just have a single GUEST_ASSERT, but there are a limited number of variants and silently eating arguments could lead to subtle code bugs. Signed-off-by: Sean Christopherson Message-Id: <20200410231707.7128-5-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- .../testing/selftests/kvm/include/kvm_util.h | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index d4c3e4d9cd92..e38d91bd8ec1 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -313,11 +313,26 @@ uint64_t get_ucall(struct kvm_vm *vm, uint32_t vcpu_id, struct ucall *uc); #define GUEST_SYNC(stage) ucall(UCALL_SYNC, 2, "hello", stage) #define GUEST_DONE() ucall(UCALL_DONE, 0) -#define GUEST_ASSERT(_condition) do { \ - if (!(_condition)) \ - ucall(UCALL_ABORT, 2, \ - "Failed guest assert: " \ - #_condition, __LINE__); \ +#define __GUEST_ASSERT(_condition, _nargs, _args...) do { \ + if (!(_condition)) \ + ucall(UCALL_ABORT, 2 + _nargs, \ + "Failed guest assert: " \ + #_condition, __LINE__, _args); \ } while (0) +#define GUEST_ASSERT(_condition) \ + __GUEST_ASSERT((_condition), 0, 0) + +#define GUEST_ASSERT_1(_condition, arg1) \ + __GUEST_ASSERT((_condition), 1, (arg1)) + +#define GUEST_ASSERT_2(_condition, arg1, arg2) \ + __GUEST_ASSERT((_condition), 2, (arg1), (arg2)) + +#define GUEST_ASSERT_3(_condition, arg1, arg2, arg3) \ + __GUEST_ASSERT((_condition), 3, (arg1), (arg2), (arg3)) + +#define GUEST_ASSERT_4(_condition, arg1, arg2, arg3, arg4) \ + __GUEST_ASSERT((_condition), 4, (arg1), (arg2), (arg3), (arg4)) + #endif /* SELFTEST_KVM_UTIL_H */ From 8a0639fe9201ed7af73dd40fc4b6f9a4f7b16a42 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 10 Apr 2020 16:17:02 -0700 Subject: [PATCH 0019/1043] KVM: sefltests: Add explicit synchronization to move mem region test Use sem_post() and sem_timedwait() to synchronize test stages between the vCPU thread and the main thread instead of using usleep() to wait for the vCPU thread and hoping for the best. Opportunistically refactor the code to make it suck less in general, and to prepare for adding more testcases. Suggested-by: Peter Xu Signed-off-by: Sean Christopherson Message-Id: <20200410231707.7128-6-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- .../kvm/x86_64/set_memory_region_test.c | 117 +++++++++++++++--- 1 file changed, 99 insertions(+), 18 deletions(-) diff --git a/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c b/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c index c6691cff4e19..629dd8579b73 100644 --- a/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -26,18 +27,20 @@ #define MEM_REGION_SIZE 0x200000 #define MEM_REGION_SLOT 10 -static void guest_code(void) +static const uint64_t MMIO_VAL = 0xbeefull; + +static sem_t vcpu_ready; + +static inline uint64_t guest_spin_on_val(uint64_t spin_val) { uint64_t val; do { val = READ_ONCE(*((uint64_t *)MEM_REGION_GPA)); - } while (!val); + } while (val == spin_val); - if (val != 1) - ucall(UCALL_ABORT, 1, val); - - GUEST_DONE(); + GUEST_SYNC(0); + return val; } static void *vcpu_worker(void *data) @@ -49,25 +52,60 @@ static void *vcpu_worker(void *data) /* * Loop until the guest is done. Re-enter the guest on all MMIO exits, - * which will occur if the guest attempts to access a memslot while it - * is being moved. + * which will occur if the guest attempts to access a memslot after it + * has been deleted or while it is being moved . */ run = vcpu_state(vm, VCPU_ID); - do { + + while (1) { vcpu_run(vm, VCPU_ID); - } while (run->exit_reason == KVM_EXIT_MMIO); - TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, - "Unexpected exit reason = %d", run->exit_reason); + if (run->exit_reason == KVM_EXIT_IO) { + cmd = get_ucall(vm, VCPU_ID, &uc); + if (cmd != UCALL_SYNC) + break; + + sem_post(&vcpu_ready); + continue; + } + + if (run->exit_reason != KVM_EXIT_MMIO) + break; + + TEST_ASSERT(!run->mmio.is_write, "Unexpected exit mmio write"); + TEST_ASSERT(run->mmio.len == 8, + "Unexpected exit mmio size = %u", run->mmio.len); + + TEST_ASSERT(run->mmio.phys_addr == MEM_REGION_GPA, + "Unexpected exit mmio address = 0x%llx", + run->mmio.phys_addr); + memcpy(run->mmio.data, &MMIO_VAL, 8); + } + + if (run->exit_reason == KVM_EXIT_IO && cmd == UCALL_ABORT) + TEST_FAIL("%s at %s:%ld, val = %lu", (const char *)uc.args[0], + __FILE__, uc.args[1], uc.args[2]); - cmd = get_ucall(vm, VCPU_ID, &uc); - TEST_ASSERT(cmd == UCALL_DONE, "Unexpected val in guest = %lu", uc.args[0]); return NULL; } -static void test_move_memory_region(void) +static void wait_for_vcpu(void) +{ + struct timespec ts; + + TEST_ASSERT(!clock_gettime(CLOCK_REALTIME, &ts), + "clock_gettime() failed: %d\n", errno); + + ts.tv_sec += 2; + TEST_ASSERT(!sem_timedwait(&vcpu_ready, &ts), + "sem_timedwait() failed: %d\n", errno); + + /* Wait for the vCPU thread to reenter the guest. */ + usleep(100000); +} + +static struct kvm_vm *spawn_vm(pthread_t *vcpu_thread, void *guest_code) { - pthread_t vcpu_thread; struct kvm_vm *vm; uint64_t *hva; uint64_t gpa; @@ -93,10 +131,45 @@ static void test_move_memory_region(void) hva = addr_gpa2hva(vm, MEM_REGION_GPA); memset(hva, 0, 2 * 4096); - pthread_create(&vcpu_thread, NULL, vcpu_worker, vm); + pthread_create(vcpu_thread, NULL, vcpu_worker, vm); /* Ensure the guest thread is spun up. */ - usleep(100000); + wait_for_vcpu(); + + return vm; +} + + +static void guest_code_move_memory_region(void) +{ + uint64_t val; + + GUEST_SYNC(0); + + /* + * Spin until the memory region is moved to a misaligned address. This + * may or may not trigger MMIO, as the window where the memslot is + * invalid is quite small. + */ + val = guest_spin_on_val(0); + GUEST_ASSERT_1(val == 1 || val == MMIO_VAL, val); + + /* Spin until the memory region is realigned. */ + val = guest_spin_on_val(MMIO_VAL); + GUEST_ASSERT_1(val == 1, val); + + GUEST_DONE(); +} + +static void test_move_memory_region(void) +{ + pthread_t vcpu_thread; + struct kvm_vm *vm; + uint64_t *hva; + + vm = spawn_vm(&vcpu_thread, guest_code_move_memory_region); + + hva = addr_gpa2hva(vm, MEM_REGION_GPA); /* * Shift the region's base GPA. The guest should not see "2" as the @@ -106,6 +179,11 @@ static void test_move_memory_region(void) vm_mem_region_move(vm, MEM_REGION_SLOT, MEM_REGION_GPA - 4096); WRITE_ONCE(*hva, 2); + /* + * The guest _might_ see an invalid memslot and trigger MMIO, but it's + * a tiny window. Spin and defer the sync until the memslot is + * restored and guest behavior is once again deterministic. + */ usleep(100000); /* @@ -116,6 +194,9 @@ static void test_move_memory_region(void) /* Restore the original base, the guest should see "1". */ vm_mem_region_move(vm, MEM_REGION_SLOT, MEM_REGION_GPA); + wait_for_vcpu(); + /* Defered sync from when the memslot was misaligned (above). */ + wait_for_vcpu(); pthread_join(vcpu_thread, NULL); From 8fb38f05ca9ffcca895d545a68a02f9a312c30d5 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 10 Apr 2020 16:17:03 -0700 Subject: [PATCH 0020/1043] KVM: selftests: Add "delete" testcase to set_memory_region_test Add a testcase for deleting memslots while the guest is running. Like the "move" testcase, this is x86_64-only as it relies on MMIO happening when a non-existent memslot is encountered. Signed-off-by: Sean Christopherson Message-Id: <20200410231707.7128-7-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- .../kvm/x86_64/set_memory_region_test.c | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c b/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c index 629dd8579b73..b556024af683 100644 --- a/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c @@ -29,6 +29,9 @@ static const uint64_t MMIO_VAL = 0xbeefull; +extern const uint64_t final_rip_start; +extern const uint64_t final_rip_end; + static sem_t vcpu_ready; static inline uint64_t guest_spin_on_val(uint64_t spin_val) @@ -203,6 +206,89 @@ static void test_move_memory_region(void) kvm_vm_free(vm); } +static void guest_code_delete_memory_region(void) +{ + uint64_t val; + + GUEST_SYNC(0); + + /* Spin until the memory region is deleted. */ + val = guest_spin_on_val(0); + GUEST_ASSERT_1(val == MMIO_VAL, val); + + /* Spin until the memory region is recreated. */ + val = guest_spin_on_val(MMIO_VAL); + GUEST_ASSERT_1(val == 0, val); + + /* Spin until the memory region is deleted. */ + val = guest_spin_on_val(0); + GUEST_ASSERT_1(val == MMIO_VAL, val); + + asm("1:\n\t" + ".pushsection .rodata\n\t" + ".global final_rip_start\n\t" + "final_rip_start: .quad 1b\n\t" + ".popsection"); + + /* Spin indefinitely (until the code memslot is deleted). */ + guest_spin_on_val(MMIO_VAL); + + asm("1:\n\t" + ".pushsection .rodata\n\t" + ".global final_rip_end\n\t" + "final_rip_end: .quad 1b\n\t" + ".popsection"); + + GUEST_ASSERT_1(0, 0); +} + +static void test_delete_memory_region(void) +{ + pthread_t vcpu_thread; + struct kvm_regs regs; + struct kvm_run *run; + struct kvm_vm *vm; + + vm = spawn_vm(&vcpu_thread, guest_code_delete_memory_region); + + /* Delete the memory region, the guest should not die. */ + vm_mem_region_delete(vm, MEM_REGION_SLOT); + wait_for_vcpu(); + + /* Recreate the memory region. The guest should see "0". */ + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP, + MEM_REGION_GPA, MEM_REGION_SLOT, + MEM_REGION_SIZE / getpagesize(), 0); + wait_for_vcpu(); + + /* Delete the region again so that there's only one memslot left. */ + vm_mem_region_delete(vm, MEM_REGION_SLOT); + wait_for_vcpu(); + + /* + * Delete the primary memslot. This should cause an emulation error or + * shutdown due to the page tables getting nuked. + */ + vm_mem_region_delete(vm, 0); + + pthread_join(vcpu_thread, NULL); + + run = vcpu_state(vm, VCPU_ID); + + TEST_ASSERT(run->exit_reason == KVM_EXIT_SHUTDOWN || + run->exit_reason == KVM_EXIT_INTERNAL_ERROR, + "Unexpected exit reason = %d", run->exit_reason); + + vcpu_regs_get(vm, VCPU_ID, ®s); + + TEST_ASSERT(regs.rip >= final_rip_start && + regs.rip < final_rip_end, + "Bad rip, expected 0x%lx - 0x%lx, got 0x%llx\n", + final_rip_start, final_rip_end, regs.rip); + + kvm_vm_free(vm); +} + int main(int argc, char *argv[]) { int i, loops; @@ -215,8 +301,13 @@ int main(int argc, char *argv[]) else loops = 10; + pr_info("Testing MOVE of in-use region, %d loops\n", loops); for (i = 0; i < loops; i++) test_move_memory_region(); + pr_info("Testing DELETE of in-use region, %d loops\n", loops); + for (i = 0; i < loops; i++) + test_delete_memory_region(); + return 0; } From 4cd94d125df531f9f569478eb3e025f08bc180f2 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 10 Apr 2020 16:17:04 -0700 Subject: [PATCH 0021/1043] selftests: kvm: Add vm_get_fd() in kvm_util Introduces the vm_get_fd() function in kvm_util which returns the VM file descriptor. Reviewed-by: Andrew Jones Signed-off-by: Wainer dos Santos Moschetta Signed-off-by: Sean Christopherson Message-Id: <20200410231707.7128-8-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/include/kvm_util.h | 1 + tools/testing/selftests/kvm/lib/kvm_util.c | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index e38d91bd8ec1..53b11d725d81 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -256,6 +256,7 @@ bool vm_is_unrestricted_guest(struct kvm_vm *vm); unsigned int vm_get_page_size(struct kvm_vm *vm); unsigned int vm_get_page_shift(struct kvm_vm *vm); unsigned int vm_get_max_gfn(struct kvm_vm *vm); +int vm_get_fd(struct kvm_vm *vm); unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size); unsigned int vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index ab5b7ea60f4b..33ab0a36d230 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -1739,6 +1739,11 @@ unsigned int vm_get_max_gfn(struct kvm_vm *vm) return vm->max_gfn; } +int vm_get_fd(struct kvm_vm *vm) +{ + return vm->fd; +} + static unsigned int vm_calc_num_pages(unsigned int num_pages, unsigned int page_shift, unsigned int new_page_shift, From 8cc2dd637b890d75613387daf57af7e8f3a32e33 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 10 Apr 2020 16:17:05 -0700 Subject: [PATCH 0022/1043] KVM: selftests: Add "zero" testcase to set_memory_region_test Add a testcase for running a guest with no memslots to the memory region test. The expected result on x86_64 is that the guest will trigger an internal KVM error due to the initial code fetch encountering a non-existent memslot and resulting in an emulation failure. Signed-off-by: Sean Christopherson Message-Id: <20200410231707.7128-9-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- .../kvm/x86_64/set_memory_region_test.c | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c b/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c index b556024af683..c274ce6b4ba2 100644 --- a/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c @@ -289,6 +289,28 @@ static void test_delete_memory_region(void) kvm_vm_free(vm); } +static void test_zero_memory_regions(void) +{ + struct kvm_run *run; + struct kvm_vm *vm; + + pr_info("Testing KVM_RUN with zero added memory regions\n"); + + vm = vm_create(VM_MODE_DEFAULT, 0, O_RDWR); + vm_vcpu_add(vm, VCPU_ID); + + TEST_ASSERT(!ioctl(vm_get_fd(vm), KVM_SET_NR_MMU_PAGES, 64), + "KVM_SET_NR_MMU_PAGES failed, errno = %d\n", errno); + + vcpu_run(vm, VCPU_ID); + + run = vcpu_state(vm, VCPU_ID); + TEST_ASSERT(run->exit_reason == KVM_EXIT_INTERNAL_ERROR, + "Unexpected exit_reason = %u\n", run->exit_reason); + + kvm_vm_free(vm); +} + int main(int argc, char *argv[]) { int i, loops; @@ -296,6 +318,8 @@ int main(int argc, char *argv[]) /* Tell stdout not to buffer its content */ setbuf(stdout, NULL); + test_zero_memory_regions(); + if (argc > 1) loops = atoi(argv[1]); else From 5b4f758f454b6e3d9c7182031631c745105ec24b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 10 Apr 2020 16:17:06 -0700 Subject: [PATCH 0023/1043] KVM: selftests: Make set_memory_region_test common to all architectures Make set_memory_region_test available on all architectures by wrapping the bits that are x86-specific in ifdefs. A future testcase to create the maximum number of memslots will be architecture agnostic. Signed-off-by: Sean Christopherson Message-Id: <20200410231707.7128-10-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/.gitignore | 2 +- tools/testing/selftests/kvm/Makefile | 4 +++- .../kvm/{x86_64 => }/set_memory_region_test.c | 11 ++++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) rename tools/testing/selftests/kvm/{x86_64 => }/set_memory_region_test.c (97%) diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 16877c3daabf..5947cc119abc 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -6,7 +6,6 @@ /x86_64/hyperv_cpuid /x86_64/mmio_warning_test /x86_64/platform_info_test -/x86_64/set_memory_region_test /x86_64/set_sregs_test /x86_64/smm_test /x86_64/state_test @@ -21,4 +20,5 @@ /demand_paging_test /dirty_log_test /kvm_create_max_vcpus +/set_memory_region_test /steal_time diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 712a2ddd2a27..7af62030c12f 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -17,7 +17,6 @@ TEST_GEN_PROGS_x86_64 += x86_64/evmcs_test TEST_GEN_PROGS_x86_64 += x86_64/hyperv_cpuid TEST_GEN_PROGS_x86_64 += x86_64/mmio_warning_test TEST_GEN_PROGS_x86_64 += x86_64/platform_info_test -TEST_GEN_PROGS_x86_64 += x86_64/set_memory_region_test TEST_GEN_PROGS_x86_64 += x86_64/set_sregs_test TEST_GEN_PROGS_x86_64 += x86_64/smm_test TEST_GEN_PROGS_x86_64 += x86_64/state_test @@ -32,12 +31,14 @@ TEST_GEN_PROGS_x86_64 += clear_dirty_log_test TEST_GEN_PROGS_x86_64 += demand_paging_test TEST_GEN_PROGS_x86_64 += dirty_log_test TEST_GEN_PROGS_x86_64 += kvm_create_max_vcpus +TEST_GEN_PROGS_x86_64 += set_memory_region_test TEST_GEN_PROGS_x86_64 += steal_time TEST_GEN_PROGS_aarch64 += clear_dirty_log_test TEST_GEN_PROGS_aarch64 += demand_paging_test TEST_GEN_PROGS_aarch64 += dirty_log_test TEST_GEN_PROGS_aarch64 += kvm_create_max_vcpus +TEST_GEN_PROGS_aarch64 += set_memory_region_test TEST_GEN_PROGS_aarch64 += steal_time TEST_GEN_PROGS_s390x = s390x/memop @@ -46,6 +47,7 @@ TEST_GEN_PROGS_s390x += s390x/sync_regs_test TEST_GEN_PROGS_s390x += demand_paging_test TEST_GEN_PROGS_s390x += dirty_log_test TEST_GEN_PROGS_s390x += kvm_create_max_vcpus +TEST_GEN_PROGS_s390x += set_memory_region_test TEST_GEN_PROGS += $(TEST_GEN_PROGS_$(UNAME_M)) LIBKVM += $(LIBKVM_$(UNAME_M)) diff --git a/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c similarity index 97% rename from tools/testing/selftests/kvm/x86_64/set_memory_region_test.c rename to tools/testing/selftests/kvm/set_memory_region_test.c index c274ce6b4ba2..ac4945fa1c89 100644 --- a/tools/testing/selftests/kvm/x86_64/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -18,6 +18,7 @@ #define VCPU_ID 0 +#ifdef __x86_64__ /* * Somewhat arbitrary location and slot, intended to not overlap anything. The * location and size are specifically 2mb sized/aligned so that the initial @@ -301,7 +302,6 @@ static void test_zero_memory_regions(void) TEST_ASSERT(!ioctl(vm_get_fd(vm), KVM_SET_NR_MMU_PAGES, 64), "KVM_SET_NR_MMU_PAGES failed, errno = %d\n", errno); - vcpu_run(vm, VCPU_ID); run = vcpu_state(vm, VCPU_ID); @@ -310,14 +310,22 @@ static void test_zero_memory_regions(void) kvm_vm_free(vm); } +#endif /* __x86_64__ */ int main(int argc, char *argv[]) { +#ifdef __x86_64__ int i, loops; +#endif /* Tell stdout not to buffer its content */ setbuf(stdout, NULL); +#ifdef __x86_64__ + /* + * FIXME: the zero-memslot test fails on aarch64 and s390x because + * KVM_RUN fails with ENOEXEC or EFAULT. + */ test_zero_memory_regions(); if (argc > 1) @@ -332,6 +340,7 @@ int main(int argc, char *argv[]) pr_info("Testing DELETE of in-use region, %d loops\n", loops); for (i = 0; i < loops; i++) test_delete_memory_region(); +#endif return 0; } From 909e0abaac0f3331e2016d4b9b19001a390d6019 Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Fri, 10 Apr 2020 16:17:07 -0700 Subject: [PATCH 0024/1043] selftests: kvm: Add testcase for creating max number of memslots This patch introduces test_add_max_memory_regions(), which checks that a VM can have added memory slots up to the limit defined in KVM_CAP_NR_MEMSLOTS. Then attempt to add one more slot to verify it fails as expected. Signed-off-by: Wainer dos Santos Moschetta Reviewed-by: Andrew Jones Signed-off-by: Sean Christopherson Message-Id: <20200410231707.7128-11-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- .../selftests/kvm/set_memory_region_test.c | 65 +++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index ac4945fa1c89..260e638826dc 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -18,14 +19,18 @@ #define VCPU_ID 0 +/* + * s390x needs at least 1MB alignment, and the x86_64 MOVE/DELETE tests need a + * 2MB sized and aligned region so that the initial region corresponds to + * exactly one large page. + */ +#define MEM_REGION_SIZE 0x200000 + #ifdef __x86_64__ /* - * Somewhat arbitrary location and slot, intended to not overlap anything. The - * location and size are specifically 2mb sized/aligned so that the initial - * region corresponds to exactly one large page. + * Somewhat arbitrary location and slot, intended to not overlap anything. */ #define MEM_REGION_GPA 0xc0000000 -#define MEM_REGION_SIZE 0x200000 #define MEM_REGION_SLOT 10 static const uint64_t MMIO_VAL = 0xbeefull; @@ -312,6 +317,54 @@ static void test_zero_memory_regions(void) } #endif /* __x86_64__ */ +/* + * Test it can be added memory slots up to KVM_CAP_NR_MEMSLOTS, then any + * tentative to add further slots should fail. + */ +static void test_add_max_memory_regions(void) +{ + int ret; + struct kvm_vm *vm; + uint32_t max_mem_slots; + uint32_t slot; + uint64_t guest_addr = 0x0; + uint64_t mem_reg_npages; + void *mem; + + max_mem_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS); + TEST_ASSERT(max_mem_slots > 0, + "KVM_CAP_NR_MEMSLOTS should be greater than 0"); + pr_info("Allowed number of memory slots: %i\n", max_mem_slots); + + vm = vm_create(VM_MODE_DEFAULT, 0, O_RDWR); + + mem_reg_npages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, MEM_REGION_SIZE); + + /* Check it can be added memory slots up to the maximum allowed */ + pr_info("Adding slots 0..%i, each memory region with %dK size\n", + (max_mem_slots - 1), MEM_REGION_SIZE >> 10); + for (slot = 0; slot < max_mem_slots; slot++) { + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, + guest_addr, slot, mem_reg_npages, + 0); + guest_addr += MEM_REGION_SIZE; + } + + /* Check it cannot be added memory slots beyond the limit */ + mem = mmap(NULL, MEM_REGION_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + TEST_ASSERT(mem != MAP_FAILED, "Failed to mmap() host"); + + ret = ioctl(vm_get_fd(vm), KVM_SET_USER_MEMORY_REGION, + &(struct kvm_userspace_memory_region) {slot, 0, guest_addr, + MEM_REGION_SIZE, (uint64_t) mem}); + TEST_ASSERT(ret == -1 && errno == EINVAL, + "Adding one more memory slot should fail with EINVAL"); + + munmap(mem, MEM_REGION_SIZE); + kvm_vm_free(vm); +} + int main(int argc, char *argv[]) { #ifdef __x86_64__ @@ -327,7 +380,11 @@ int main(int argc, char *argv[]) * KVM_RUN fails with ENOEXEC or EFAULT. */ test_zero_memory_regions(); +#endif + test_add_max_memory_regions(); + +#ifdef __x86_64__ if (argc > 1) loops = atoi(argv[1]); else From e8eff282154fc392dadf6a779009c7ecaa7e169b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:27:57 -0700 Subject: [PATCH 0025/1043] KVM: VMX: Flush all EPTP/VPID contexts on remote TLB flush Flush all EPTP/VPID contexts if a TLB flush _may_ have been triggered by a remote or deferred TLB flush, i.e. by KVM_REQ_TLB_FLUSH. Remote TLB flushes require all contexts to be invalidated, not just the active contexts, e.g. all mappings in all contexts for a given HVA need to be invalidated on a mmu_notifier invalidation. Similarly, the instigator of the deferred TLB flush may be expecting all contexts to be flushed, e.g. vmx_vcpu_load_vmcs(). Without nested VMX, flushing only the current EPTP/VPID context isn't problematic because KVM uses a constant VPID for each vCPU, and mmu_alloc_direct_roots() all but guarantees KVM will use a single EPTP for L1. In the rare case where a different EPTP is created or reused, KVM (currently) unconditionally flushes the new EPTP context prior to entering the guest. With nested VMX, KVM conditionally uses a different VPID for L2, and unconditionally uses a different EPTP for L2. Because KVM doesn't _intentionally_ guarantee L2's EPTP/VPID context is flushed on nested VM-Enter, it'd be possible for a malicious L1 to attack the host and/or different VMs by exploiting the lack of flushing for L2. 1) Launch nested guest from malicious L1. 2) Nested VM-Enter to L2. 3) Access target GPA 'g'. CPU inserts TLB entry tagged with L2's ASID mapping 'g' to host PFN 'x'. 2) Nested VM-Exit to L1. 3) L1 triggers kernel same-page merging (ksm) by duplicating/zeroing the page for PFN 'x'. 4) Host kernel merges PFN 'x' with PFN 'y', i.e. unmaps PFN 'x' and remaps the page to PFN 'y'. mmu_notifier sends invalidate command, KVM flushes TLB only for L1's ASID. 4) Host kernel reallocates PFN 'x' to some other task/guest. 5) Nested VM-Enter to L2. KVM does not invalidate L2's EPTP or VPID. 6) L2 accesses GPA 'g' and gains read/write access to PFN 'x' via its stale TLB entry. However, current KVM unconditionally flushes L1's EPTP/VPID context on nested VM-Exit. But, that behavior is mostly unintentional, KVM doesn't go out of its way to flush EPTP/VPID on nested VM-Enter/VM-Exit, rather a TLB flush is guaranteed to occur prior to re-entering L1 due to __kvm_mmu_new_cr3() always being called with skip_tlb_flush=false. On nested VM-Enter, this happens via kvm_init_shadow_ept_mmu() (nested EPT enabled) or in nested_vmx_load_cr3() (nested EPT disabled). On nested VM-Exit it occurs via nested_vmx_load_cr3(). This also fixes a bug where a deferred TLB flush in the context of L2, with EPT disabled, would flush L1's VPID instead of L2's VPID, as vmx_flush_tlb() flushes L1's VPID regardless of is_guest_mode(). Cc: Vitaly Kuznetsov Cc: Ben Gardon Cc: Jim Mattson Cc: Junaid Shahid Cc: Liran Alon Cc: Boris Ostrovsky Cc: John Haxby Reviewed-by: Liran Alon Fixes: efebf0aaec3d ("KVM: nVMX: Do not flush TLB on L1<->L2 transitions if L1 uses VPID and EPT") Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-2-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.h | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index aab9df55336e..fa40319f1698 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -515,7 +515,33 @@ static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid, static inline void vmx_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa) { - __vmx_flush_tlb(vcpu, to_vmx(vcpu)->vpid, invalidate_gpa); + struct vcpu_vmx *vmx = to_vmx(vcpu); + + /* + * Flush all EPTP/VPID contexts if the TLB flush _may_ have been + * invoked via kvm_flush_remote_tlbs(), which always passes %true for + * @invalidate_gpa. Flushing remote TLBs requires all contexts to be + * flushed, not just the active context. + * + * Note, this also ensures a deferred TLB flush with VPID enabled and + * EPT disabled invalidates the "correct" VPID, by nuking both L1 and + * L2's VPIDs. + */ + if (invalidate_gpa) { + if (enable_ept) { + ept_sync_global(); + } else if (enable_vpid) { + if (cpu_has_vmx_invvpid_global()) { + vpid_sync_vcpu_global(); + } else { + WARN_ON_ONCE(!cpu_has_vmx_invvpid_single()); + vpid_sync_vcpu_single(vmx->vpid); + vpid_sync_vcpu_single(vmx->nested.vpid02); + } + } + } else { + __vmx_flush_tlb(vcpu, vmx->vpid, false); + } } static inline void decache_tsc_multiplier(struct vcpu_vmx *vmx) From eed0030e4caa941cfbdfca00981395d85e6b3c3e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:27:58 -0700 Subject: [PATCH 0026/1043] KVM: nVMX: Validate the EPTP when emulating INVEPT(EXTENT_CONTEXT) Signal VM-Fail for the single-context variant of INVEPT if the specified EPTP is invalid. Per the INEVPT pseudocode in Intel's SDM, it's subject to the standard EPT checks: If VM entry with the "enable EPT" VM execution control set to 1 would fail due to the EPTP value then VMfail(Invalid operand to INVEPT/INVVPID); Fixes: bfd0a56b90005 ("nEPT: Nested INVEPT") Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-3-sean.j.christopherson@intel.com> Reviewed-by: Vitaly Kuznetsov Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index fd78ffbde644..dc7d8104b58e 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5157,8 +5157,12 @@ static int handle_invept(struct kvm_vcpu *vcpu) } switch (type) { - case VMX_EPT_EXTENT_GLOBAL: case VMX_EPT_EXTENT_CONTEXT: + if (!nested_vmx_check_eptp(vcpu, operand.eptp)) + return nested_vmx_failValid(vcpu, + VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); + fallthrough; + case VMX_EPT_EXTENT_GLOBAL: /* * TODO: Sync the necessary shadow EPT roots here, rather than * at the next emulated VM-entry. From f8aa7e3958bc433087ae7b9d7f24a92036c41141 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:27:59 -0700 Subject: [PATCH 0027/1043] KVM: nVMX: Invalidate all EPTP contexts when emulating INVEPT for L1 Free all L2 (guest_mmu) roots when emulating INVEPT for L1. Outstanding changes to the EPT tables managed by L1 need to be recognized, and relying on KVM to always flush L2's EPTP context on nested VM-Enter is dangerous. Similar to handle_invpcid(), rely on kvm_mmu_free_roots() to do a remote TLB flush if necessary, e.g. if L1 has never entered L2 then there is nothing to be done. Nuking all L2 roots is overkill for the single-context variant, but it's the safe and easy bet. A more precise zap mechanism will be added in the future. Add a TODO to call out that KVM only needs to invalidate affected contexts. Fixes: 14c07ad89f4d ("x86/kvm/mmu: introduce guest_mmu") Reported-by: Jim Mattson Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-4-sean.j.christopherson@intel.com> Reviewed-by: Vitaly Kuznetsov Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index dc7d8104b58e..7ecb555e385a 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5161,12 +5161,16 @@ static int handle_invept(struct kvm_vcpu *vcpu) if (!nested_vmx_check_eptp(vcpu, operand.eptp)) return nested_vmx_failValid(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); + + /* TODO: sync only the target EPTP context. */ fallthrough; case VMX_EPT_EXTENT_GLOBAL: - /* - * TODO: Sync the necessary shadow EPT roots here, rather than - * at the next emulated VM-entry. - */ + /* + * Nested EPT roots are always held through guest_mmu, + * not root_mmu. + */ + kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, + KVM_MMU_ROOTS_ALL); break; default: BUG_ON(1); From d6e3f8385d27e6357221e01158fdd369f762cd0f Mon Sep 17 00:00:00 2001 From: Junaid Shahid Date: Fri, 20 Mar 2020 14:28:00 -0700 Subject: [PATCH 0028/1043] KVM: nVMX: Invalidate all roots when emulating INVVPID without EPT Free all roots when emulating INVVPID for L1 and EPT is disabled, as outstanding changes to the page tables managed by L1 need to be recognized. Because L1 and L2 share an MMU when EPT is disabled, and because VPID is not tracked by the MMU role, all roots in the current MMU (root_mmu) need to be freed, otherwise a future nested VM-Enter or VM-Exit could do a fast CR3 switch (without a flush/sync) and consume stale SPTEs. Fixes: 5c614b3583e7b ("KVM: nVMX: nested VPID emulation") Signed-off-by: Junaid Shahid [sean: ported to upstream KVM, reworded the comment and changelog] Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-5-sean.j.christopherson@intel.com> Reviewed-by: Vitaly Kuznetsov Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 7ecb555e385a..8f3ff07c7e66 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5255,6 +5255,20 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) return kvm_skip_emulated_instruction(vcpu); } + /* + * Sync the shadow page tables if EPT is disabled, L1 is invalidating + * linear mappings for L2 (tagged with L2's VPID). Free all roots as + * VPIDs are not tracked in the MMU role. + * + * Note, this operates on root_mmu, not guest_mmu, as L1 and L2 share + * an MMU when EPT is disabled. + * + * TODO: sync only the affected SPTEs for INVDIVIDUAL_ADDR. + */ + if (!enable_ept) + kvm_mmu_free_roots(vcpu, &vcpu->arch.root_mmu, + KVM_MMU_ROOTS_ALL); + return nested_vmx_succeed(vcpu); } From 53b3d8e9d57753295b33065f80b1e2fb4fcb946d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:01 -0700 Subject: [PATCH 0029/1043] KVM: x86: Export kvm_propagate_fault() (as kvm_inject_emulated_page_fault) Export the page fault propagation helper so that VMX can use it to correctly emulate TLB invalidation on page faults in an upcoming patch. In the (hopefully) not-too-distant future, SGX virtualization will also want access to the helper for injecting page faults to the correct level (L1 vs. L2) when emulating ENCLS instructions. Rename the function to kvm_inject_emulated_page_fault() to clarify that it is (a) injecting a fault and (b) only for page faults. WARN if it's invoked with an exception other than PF_VECTOR. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-6-sean.j.christopherson@intel.com> Reviewed-by: Vitaly Kuznetsov Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/x86.c | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 42a2d0d3984a..34ad7297b06b 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1452,6 +1452,8 @@ void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr); void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault); +bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, + struct x86_exception *fault); int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, gfn_t gfn, void *data, int offset, int len, u32 access); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 027dfd278a97..003e625367b7 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -611,8 +611,11 @@ void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) } EXPORT_SYMBOL_GPL(kvm_inject_page_fault); -static bool kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) +bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, + struct x86_exception *fault) { + WARN_ON_ONCE(fault->vector != PF_VECTOR); + if (mmu_is_nested(vcpu) && !fault->nested_page_fault) vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault); else @@ -620,6 +623,7 @@ static bool kvm_propagate_fault(struct kvm_vcpu *vcpu, struct x86_exception *fau return fault->nested_page_fault; } +EXPORT_SYMBOL_GPL(kvm_inject_emulated_page_fault); void kvm_inject_nmi(struct kvm_vcpu *vcpu) { @@ -6381,7 +6385,7 @@ static bool inject_emulated_exception(struct kvm_vcpu *vcpu) { struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; if (ctxt->exception.vector == PF_VECTOR) - return kvm_propagate_fault(vcpu, &ctxt->exception); + return kvm_inject_emulated_page_fault(vcpu, &ctxt->exception); if (ctxt->exception.error_code_valid) kvm_queue_exception_e(vcpu, ctxt->exception.vector, From 433e3aa37773e8a36858b9417c3e345eff79a079 Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Wed, 8 Apr 2020 11:08:08 +0200 Subject: [PATCH 0030/1043] selinux: drop unnecessary smp_load_acquire() call In commit 66f8e2f03c02 ("selinux: sidtab reverse lookup hash table") the corresponding load is moved under the spin lock, so there is no race possible and we can read the count directly. The smp_store_release() is still needed to avoid racing with the lock-free readers. Signed-off-by: Ondrej Mosnacek Signed-off-by: Paul Moore --- security/selinux/ss/sidtab.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/security/selinux/ss/sidtab.c b/security/selinux/ss/sidtab.c index f511ffccb131..98d5ea3fcde4 100644 --- a/security/selinux/ss/sidtab.c +++ b/security/selinux/ss/sidtab.c @@ -276,8 +276,7 @@ int sidtab_context_to_sid(struct sidtab *s, struct context *context, if (*sid) goto out_unlock; - /* read entries only after reading count */ - count = smp_load_acquire(&s->count); + count = s->count; convert = s->convert; /* bail out if we already reached max entries */ From 5ceb89f8a301b2d5c2ffa20c9b41eb2501360113 Mon Sep 17 00:00:00 2001 From: bibo mao Date: Wed, 15 Apr 2020 17:56:43 +0800 Subject: [PATCH 0031/1043] MIPS: Fix typo for user_ld macro definition There is typo for macro user_ld if __ASSEMBLY__ is declared, this patch fixes this issue. Signed-off-by: bibo mao Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/asm-eva.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/include/asm/asm-eva.h b/arch/mips/include/asm/asm-eva.h index d80be38c4144..e327ebc76753 100644 --- a/arch/mips/include/asm/asm-eva.h +++ b/arch/mips/include/asm/asm-eva.h @@ -180,7 +180,7 @@ #define user_ld(reg, addr) kernel_lw(reg, addr) #else #define user_sd(reg, addr) kernel_sd(reg, addr) -#define user_ld(reg, addr) kernel_sd(reg, addr) +#define user_ld(reg, addr) kernel_ld(reg, addr) #endif /* CONFIG_32BIT */ #endif /* CONFIG_EVA */ From c02e96304451ad3e22c58235061a8363c30116b6 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 9 Apr 2020 11:02:28 +0200 Subject: [PATCH 0032/1043] mips: loongsoon2ef: remove private clk api MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As platforms are moving to COMMON_CLK in general, loongson2ef stuck out as something that has a private implementation but does not actually use it except for setting the frequency of the CPU itself from the loongson2_cpufreq driver. Change that driver to call the register setting function directly and remove the rest of the stub implementation. Signed-off-by: Arnd Bergmann Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/clock.h | 49 ---------- .../include/asm/mach-loongson2ef/loongson.h | 1 + arch/mips/loongson2ef/Kconfig | 1 - arch/mips/loongson2ef/lemote-2f/clock.c | 98 +------------------ arch/mips/loongson64/smp.c | 1 - drivers/cpufreq/loongson2_cpufreq.c | 22 +---- 6 files changed, 8 insertions(+), 164 deletions(-) delete mode 100644 arch/mips/include/asm/clock.h diff --git a/arch/mips/include/asm/clock.h b/arch/mips/include/asm/clock.h deleted file mode 100644 index 5a8f96ebe5fa..000000000000 --- a/arch/mips/include/asm/clock.h +++ /dev/null @@ -1,49 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ASM_MIPS_CLOCK_H -#define __ASM_MIPS_CLOCK_H - -#include -#include -#include -#include - -struct clk; - -struct clk_ops { - void (*init) (struct clk *clk); - void (*enable) (struct clk *clk); - void (*disable) (struct clk *clk); - void (*recalc) (struct clk *clk); - int (*set_rate) (struct clk *clk, unsigned long rate, int algo_id); - long (*round_rate) (struct clk *clk, unsigned long rate); -}; - -struct clk { - struct list_head node; - const char *name; - int id; - struct module *owner; - - struct clk *parent; - struct clk_ops *ops; - - struct kref kref; - - unsigned long rate; - unsigned long flags; -}; - -#define CLK_ALWAYS_ENABLED (1 << 0) -#define CLK_RATE_PROPAGATES (1 << 1) - -int clk_init(void); - -int __clk_enable(struct clk *); -void __clk_disable(struct clk *); - -void clk_recalc_rate(struct clk *); - -int clk_register(struct clk *); -void clk_unregister(struct clk *); - -#endif /* __ASM_MIPS_CLOCK_H */ diff --git a/arch/mips/include/asm/mach-loongson2ef/loongson.h b/arch/mips/include/asm/mach-loongson2ef/loongson.h index 5008af0a1a19..57e571128489 100644 --- a/arch/mips/include/asm/mach-loongson2ef/loongson.h +++ b/arch/mips/include/asm/mach-loongson2ef/loongson.h @@ -244,6 +244,7 @@ static inline void do_perfcnt_IRQ(void) #ifdef CONFIG_CPU_SUPPORTS_CPUFREQ #include extern struct cpufreq_frequency_table loongson2_clockmod_table[]; +extern int loongson2_cpu_set_rate(unsigned long rate_khz); #endif /* diff --git a/arch/mips/loongson2ef/Kconfig b/arch/mips/loongson2ef/Kconfig index 595dd48e1e4d..96dc6eba4310 100644 --- a/arch/mips/loongson2ef/Kconfig +++ b/arch/mips/loongson2ef/Kconfig @@ -46,7 +46,6 @@ config LEMOTE_MACH2F select CSRC_R4K if ! MIPS_EXTERNAL_TIMER select DMA_NONCOHERENT select GENERIC_ISA_DMA_SUPPORT_BROKEN - select HAVE_CLK select FORCE_PCI select I8259 select IRQ_MIPS_CPU diff --git a/arch/mips/loongson2ef/lemote-2f/clock.c b/arch/mips/loongson2ef/lemote-2f/clock.c index 414f282c8ab5..850b6b9f8f15 100644 --- a/arch/mips/loongson2ef/lemote-2f/clock.c +++ b/arch/mips/loongson2ef/lemote-2f/clock.c @@ -6,22 +6,12 @@ * License. See the file "COPYING" in the main directory of this archive * for more details. */ -#include #include #include #include -#include -#include -#include -#include #include -static LIST_HEAD(clock_list); -static DEFINE_SPINLOCK(clock_lock); -static DEFINE_MUTEX(clock_list_sem); - -/* Minimum CLK support */ enum { DC_ZERO, DC_25PT = 2, DC_37PT, DC_50PT, DC_62PT, DC_75PT, DC_87PT, DC_DISABLE, DC_RESV @@ -41,103 +31,21 @@ struct cpufreq_frequency_table loongson2_clockmod_table[] = { }; EXPORT_SYMBOL_GPL(loongson2_clockmod_table); -static struct clk cpu_clk = { - .name = "cpu_clk", - .flags = CLK_ALWAYS_ENABLED | CLK_RATE_PROPAGATES, - .rate = 800000000, -}; - -struct clk *clk_get(struct device *dev, const char *id) +int loongson2_cpu_set_rate(unsigned long rate_khz) { - return &cpu_clk; -} -EXPORT_SYMBOL(clk_get); - -static void propagate_rate(struct clk *clk) -{ - struct clk *clkp; - - list_for_each_entry(clkp, &clock_list, node) { - if (likely(clkp->parent != clk)) - continue; - if (likely(clkp->ops && clkp->ops->recalc)) - clkp->ops->recalc(clkp); - if (unlikely(clkp->flags & CLK_RATE_PROPAGATES)) - propagate_rate(clkp); - } -} - -int clk_enable(struct clk *clk) -{ - return 0; -} -EXPORT_SYMBOL(clk_enable); - -void clk_disable(struct clk *clk) -{ -} -EXPORT_SYMBOL(clk_disable); - -unsigned long clk_get_rate(struct clk *clk) -{ - if (!clk) - return 0; - - return (unsigned long)clk->rate; -} -EXPORT_SYMBOL(clk_get_rate); - -void clk_put(struct clk *clk) -{ -} -EXPORT_SYMBOL(clk_put); - -int clk_set_rate(struct clk *clk, unsigned long rate) -{ - unsigned int rate_khz = rate / 1000; struct cpufreq_frequency_table *pos; - int ret = 0; int regval; - if (likely(clk->ops && clk->ops->set_rate)) { - unsigned long flags; - - spin_lock_irqsave(&clock_lock, flags); - ret = clk->ops->set_rate(clk, rate, 0); - spin_unlock_irqrestore(&clock_lock, flags); - } - - if (unlikely(clk->flags & CLK_RATE_PROPAGATES)) - propagate_rate(clk); - cpufreq_for_each_valid_entry(pos, loongson2_clockmod_table) if (rate_khz == pos->frequency) break; if (rate_khz != pos->frequency) return -ENOTSUPP; - clk->rate = rate; - regval = readl(LOONGSON_CHIPCFG); regval = (regval & ~0x7) | (pos->driver_data - 1); writel(regval, LOONGSON_CHIPCFG); - return ret; + return 0; } -EXPORT_SYMBOL_GPL(clk_set_rate); - -long clk_round_rate(struct clk *clk, unsigned long rate) -{ - if (likely(clk->ops && clk->ops->round_rate)) { - unsigned long flags, rounded; - - spin_lock_irqsave(&clock_lock, flags); - rounded = clk->ops->round_rate(clk, rate); - spin_unlock_irqrestore(&clock_lock, flags); - - return rounded; - } - - return rate; -} -EXPORT_SYMBOL_GPL(clk_round_rate); +EXPORT_SYMBOL_GPL(loongson2_cpu_set_rate); diff --git a/arch/mips/loongson64/smp.c b/arch/mips/loongson64/smp.c index e1fe8bbb377d..e744e1bee49e 100644 --- a/arch/mips/loongson64/smp.c +++ b/arch/mips/loongson64/smp.c @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/cpufreq/loongson2_cpufreq.c b/drivers/cpufreq/loongson2_cpufreq.c index 909f40fbcde2..d05e761d9572 100644 --- a/drivers/cpufreq/loongson2_cpufreq.c +++ b/drivers/cpufreq/loongson2_cpufreq.c @@ -20,7 +20,6 @@ #include #include -#include #include #include @@ -58,29 +57,20 @@ static int loongson2_cpufreq_target(struct cpufreq_policy *policy, loongson2_clockmod_table[index].driver_data) / 8; /* setting the cpu frequency */ - clk_set_rate(policy->clk, freq * 1000); + loongson2_cpu_set_rate(freq); return 0; } static int loongson2_cpufreq_cpu_init(struct cpufreq_policy *policy) { - struct clk *cpuclk; int i; unsigned long rate; int ret; - cpuclk = clk_get(NULL, "cpu_clk"); - if (IS_ERR(cpuclk)) { - pr_err("couldn't get CPU clk\n"); - return PTR_ERR(cpuclk); - } - rate = cpu_clock_freq / 1000; - if (!rate) { - clk_put(cpuclk); + if (!rate) return -EINVAL; - } /* clock table init */ for (i = 2; @@ -88,20 +78,16 @@ static int loongson2_cpufreq_cpu_init(struct cpufreq_policy *policy) i++) loongson2_clockmod_table[i].frequency = (rate * i) / 8; - ret = clk_set_rate(cpuclk, rate * 1000); - if (ret) { - clk_put(cpuclk); + ret = loongson2_cpu_set_rate(rate); + if (ret) return ret; - } - policy->clk = cpuclk; cpufreq_generic_init(policy, &loongson2_clockmod_table[0], 0); return 0; } static int loongson2_cpufreq_exit(struct cpufreq_policy *policy) { - clk_put(policy->clk); return 0; } From e67b2ec9f6171895e774f6543626913960e019df Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Tue, 7 Apr 2020 20:28:58 +0200 Subject: [PATCH 0033/1043] selinux: store role transitions in a hash table Currently, they are stored in a linked list, which adds significant overhead to security_transition_sid(). On Fedora, with 428 role transitions in policy, converting this list to a hash table cuts down its run time by about 50%. This was measured by running 'stress-ng --msg 1 --msg-ops 100000' under perf with and without this patch. Signed-off-by: Ondrej Mosnacek Signed-off-by: Paul Moore --- security/selinux/ss/policydb.c | 142 ++++++++++++++++++++++----------- security/selinux/ss/policydb.h | 8 +- security/selinux/ss/services.c | 21 +++-- 3 files changed, 109 insertions(+), 62 deletions(-) diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index 70ecdc78efbd..4f0cfffd008d 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -352,6 +352,13 @@ static int range_tr_destroy(void *key, void *datum, void *p) return 0; } +static int role_tr_destroy(void *key, void *datum, void *p) +{ + kfree(key); + kfree(datum); + return 0; +} + static void ocontext_destroy(struct ocontext *c, int i) { if (!c) @@ -458,6 +465,30 @@ static int rangetr_cmp(struct hashtab *h, const void *k1, const void *k2) return v; } +static u32 role_trans_hash(struct hashtab *h, const void *k) +{ + const struct role_trans_key *key = k; + + return (key->role + (key->type << 3) + (key->tclass << 5)) & + (h->size - 1); +} + +static int role_trans_cmp(struct hashtab *h, const void *k1, const void *k2) +{ + const struct role_trans_key *key1 = k1, *key2 = k2; + int v; + + v = key1->role - key2->role; + if (v) + return v; + + v = key1->type - key2->type; + if (v) + return v; + + return key1->tclass - key2->tclass; +} + /* * Initialize a policy database structure. */ @@ -728,7 +759,6 @@ void policydb_destroy(struct policydb *p) struct genfs *g, *gtmp; int i; struct role_allow *ra, *lra = NULL; - struct role_trans *tr, *ltr = NULL; for (i = 0; i < SYM_NUM; i++) { cond_resched(); @@ -775,12 +805,8 @@ void policydb_destroy(struct policydb *p) cond_policydb_destroy(p); - for (tr = p->role_tr; tr; tr = tr->next) { - cond_resched(); - kfree(ltr); - ltr = tr; - } - kfree(ltr); + hashtab_map(p->role_tr, role_tr_destroy, NULL); + hashtab_destroy(p->role_tr); for (ra = p->role_allow; ra; ra = ra->next) { cond_resched(); @@ -2251,7 +2277,8 @@ out: int policydb_read(struct policydb *p, void *fp) { struct role_allow *ra, *lra; - struct role_trans *tr, *ltr; + struct role_trans_key *rtk = NULL; + struct role_trans_datum *rtd = NULL; int i, j, rc; __le32 buf[4]; u32 len, nprim, nel; @@ -2416,39 +2443,50 @@ int policydb_read(struct policydb *p, void *fp) if (rc) goto bad; nel = le32_to_cpu(buf[0]); - ltr = NULL; + + p->role_tr = hashtab_create(role_trans_hash, role_trans_cmp, nel); + if (!p->role_tr) + goto bad; for (i = 0; i < nel; i++) { rc = -ENOMEM; - tr = kzalloc(sizeof(*tr), GFP_KERNEL); - if (!tr) + rtk = kmalloc(sizeof(*rtk), GFP_KERNEL); + if (!rtk) goto bad; - if (ltr) - ltr->next = tr; - else - p->role_tr = tr; + + rc = -ENOMEM; + rtd = kmalloc(sizeof(*rtd), GFP_KERNEL); + if (!rtd) + goto bad; + rc = next_entry(buf, fp, sizeof(u32)*3); if (rc) goto bad; rc = -EINVAL; - tr->role = le32_to_cpu(buf[0]); - tr->type = le32_to_cpu(buf[1]); - tr->new_role = le32_to_cpu(buf[2]); + rtk->role = le32_to_cpu(buf[0]); + rtk->type = le32_to_cpu(buf[1]); + rtd->new_role = le32_to_cpu(buf[2]); if (p->policyvers >= POLICYDB_VERSION_ROLETRANS) { rc = next_entry(buf, fp, sizeof(u32)); if (rc) goto bad; - tr->tclass = le32_to_cpu(buf[0]); + rtk->tclass = le32_to_cpu(buf[0]); } else - tr->tclass = p->process_class; + rtk->tclass = p->process_class; rc = -EINVAL; - if (!policydb_role_isvalid(p, tr->role) || - !policydb_type_isvalid(p, tr->type) || - !policydb_class_isvalid(p, tr->tclass) || - !policydb_role_isvalid(p, tr->new_role)) + if (!policydb_role_isvalid(p, rtk->role) || + !policydb_type_isvalid(p, rtk->type) || + !policydb_class_isvalid(p, rtk->tclass) || + !policydb_role_isvalid(p, rtd->new_role)) goto bad; - ltr = tr; + + rc = hashtab_insert(p->role_tr, rtk, rtd); + if (rc) + goto bad; + + rtk = NULL; + rtd = NULL; } rc = next_entry(buf, fp, sizeof(u32)); @@ -2536,6 +2574,8 @@ int policydb_read(struct policydb *p, void *fp) out: return rc; bad: + kfree(rtk); + kfree(rtd); policydb_destroy(p); goto out; } @@ -2653,37 +2693,43 @@ static int cat_write(void *vkey, void *datum, void *ptr) return 0; } -static int role_trans_write(struct policydb *p, void *fp) +static int role_trans_write_one(void *key, void *datum, void *ptr) { - struct role_trans *r = p->role_tr; - struct role_trans *tr; + struct role_trans_key *rtk = key; + struct role_trans_datum *rtd = datum; + struct policy_data *pd = ptr; + void *fp = pd->fp; + struct policydb *p = pd->p; __le32 buf[3]; - size_t nel; int rc; - nel = 0; - for (tr = r; tr; tr = tr->next) - nel++; - buf[0] = cpu_to_le32(nel); + buf[0] = cpu_to_le32(rtk->role); + buf[1] = cpu_to_le32(rtk->type); + buf[2] = cpu_to_le32(rtd->new_role); + rc = put_entry(buf, sizeof(u32), 3, fp); + if (rc) + return rc; + if (p->policyvers >= POLICYDB_VERSION_ROLETRANS) { + buf[0] = cpu_to_le32(rtk->tclass); + rc = put_entry(buf, sizeof(u32), 1, fp); + if (rc) + return rc; + } + return 0; +} + +static int role_trans_write(struct policydb *p, void *fp) +{ + struct policy_data pd = { .p = p, .fp = fp }; + __le32 buf[1]; + int rc; + + buf[0] = cpu_to_le32(p->role_tr->nel); rc = put_entry(buf, sizeof(u32), 1, fp); if (rc) return rc; - for (tr = r; tr; tr = tr->next) { - buf[0] = cpu_to_le32(tr->role); - buf[1] = cpu_to_le32(tr->type); - buf[2] = cpu_to_le32(tr->new_role); - rc = put_entry(buf, sizeof(u32), 3, fp); - if (rc) - return rc; - if (p->policyvers >= POLICYDB_VERSION_ROLETRANS) { - buf[0] = cpu_to_le32(tr->tclass); - rc = put_entry(buf, sizeof(u32), 1, fp); - if (rc) - return rc; - } - } - return 0; + return hashtab_map(p->role_tr, role_trans_write_one, &pd); } static int role_allow_write(struct role_allow *r, void *fp) diff --git a/security/selinux/ss/policydb.h b/security/selinux/ss/policydb.h index 72e2932fb12d..d3adb522d3f3 100644 --- a/security/selinux/ss/policydb.h +++ b/security/selinux/ss/policydb.h @@ -81,12 +81,14 @@ struct role_datum { struct ebitmap types; /* set of authorized types for role */ }; -struct role_trans { +struct role_trans_key { u32 role; /* current role */ u32 type; /* program executable type, or new object type */ u32 tclass; /* process class, or new object class */ +}; + +struct role_trans_datum { u32 new_role; /* new role */ - struct role_trans *next; }; struct filename_trans_key { @@ -261,7 +263,7 @@ struct policydb { struct avtab te_avtab; /* role transitions */ - struct role_trans *role_tr; + struct hashtab *role_tr; /* file transitions with the last path component */ /* quickly exclude lookups when parent ttype has no rules */ diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index 3b592d17d2d3..07cdda2ff49c 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -1731,7 +1731,6 @@ static int security_compute_sid(struct selinux_state *state, struct class_datum *cladatum = NULL; struct context *scontext, *tcontext, newcontext; struct sidtab_entry *sentry, *tentry; - struct role_trans *roletr = NULL; struct avtab_key avkey; struct avtab_datum *avdatum; struct avtab_node *node; @@ -1864,16 +1863,16 @@ static int security_compute_sid(struct selinux_state *state, /* Check for class-specific changes. */ if (specified & AVTAB_TRANSITION) { /* Look for a role transition rule. */ - for (roletr = policydb->role_tr; roletr; - roletr = roletr->next) { - if ((roletr->role == scontext->role) && - (roletr->type == tcontext->type) && - (roletr->tclass == tclass)) { - /* Use the role transition rule. */ - newcontext.role = roletr->new_role; - break; - } - } + struct role_trans_datum *rtd; + struct role_trans_key rtk = { + .role = scontext->role, + .type = tcontext->type, + .tclass = tclass, + }; + + rtd = hashtab_search(policydb->role_tr, &rtk); + if (rtd) + newcontext.role = rtd->new_role; } /* Set the MLS attributes. From 50077289804c9bd4e6cfd5b3a10d4da0487f7e42 Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Fri, 17 Apr 2020 10:11:56 +0200 Subject: [PATCH 0034/1043] selinux: hash context structure directly Always hashing the string representation is inefficient. Just hash the contents of the structure directly (using jhash). If the context is invalid (str & len are set), then hash the string as before, otherwise hash the structured data. Since the context hashing function is now faster (about 10 times), this patch decreases the overhead of security_transition_sid(), which is called from many hooks. The jhash function seemed as a good choice, since it is used as the default hashing algorithm in rhashtable. Signed-off-by: Ondrej Mosnacek Reviewed-by: Jeff Vander Stoep Tested-by: Jeff Vander Stoep [PM: fixed some spelling errors in the comments pointed out by JVS] Signed-off-by: Paul Moore --- security/selinux/Makefile | 2 +- security/selinux/ss/context.c | 32 +++++++++++++++++++++++++++++++ security/selinux/ss/context.h | 6 ++++-- security/selinux/ss/ebitmap.c | 14 ++++++++++++++ security/selinux/ss/ebitmap.h | 1 + security/selinux/ss/mls.h | 11 +++++++++++ security/selinux/ss/policydb.c | 7 ++----- security/selinux/ss/services.c | 35 ++++------------------------------ security/selinux/ss/services.h | 3 --- 9 files changed, 69 insertions(+), 42 deletions(-) create mode 100644 security/selinux/ss/context.c diff --git a/security/selinux/Makefile b/security/selinux/Makefile index 0c77ede1cc11..4d8e0e8adf0b 100644 --- a/security/selinux/Makefile +++ b/security/selinux/Makefile @@ -8,7 +8,7 @@ obj-$(CONFIG_SECURITY_SELINUX) := selinux.o selinux-y := avc.o hooks.o selinuxfs.o netlink.o nlmsgtab.o netif.o \ netnode.o netport.o status.o \ ss/ebitmap.o ss/hashtab.o ss/symtab.o ss/sidtab.o ss/avtab.o \ - ss/policydb.o ss/services.o ss/conditional.o ss/mls.o + ss/policydb.o ss/services.o ss/conditional.o ss/mls.o ss/context.o selinux-$(CONFIG_SECURITY_NETWORK_XFRM) += xfrm.o diff --git a/security/selinux/ss/context.c b/security/selinux/ss/context.c new file mode 100644 index 000000000000..38bc0aa524a6 --- /dev/null +++ b/security/selinux/ss/context.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Implementations of the security context functions. + * + * Author: Ondrej Mosnacek + * Copyright (C) 2020 Red Hat, Inc. + */ + +#include + +#include "context.h" +#include "mls.h" + +u32 context_compute_hash(const struct context *c) +{ + u32 hash = 0; + + /* + * If a context is invalid, it will always be represented by a + * context struct with only the len & str set (and vice versa) + * under a given policy. Since context structs from different + * policies should never meet, it is safe to hash valid and + * invalid contexts differently. The context_cmp() function + * already operates under the same assumption. + */ + if (c->len) + return full_name_hash(NULL, c->str, c->len); + + hash = jhash_3words(c->user, c->role, c->type, hash); + hash = mls_range_hash(&c->range, hash); + return hash; +} diff --git a/security/selinux/ss/context.h b/security/selinux/ss/context.h index 3ba044fe02ed..e7ae7e21449b 100644 --- a/security/selinux/ss/context.h +++ b/security/selinux/ss/context.h @@ -196,9 +196,11 @@ static inline int context_cmp(struct context *c1, struct context *c2) mls_context_cmp(c1, c2)); } -static inline unsigned int context_compute_hash(const char *s) +u32 context_compute_hash(const struct context *c); + +static inline void context_add_hash(struct context *context) { - return full_name_hash(NULL, s, strlen(s)); + context->hash = context_compute_hash(context); } #endif /* _SS_CONTEXT_H_ */ diff --git a/security/selinux/ss/ebitmap.c b/security/selinux/ss/ebitmap.c index c8c3663111e2..14bedc95c6dc 100644 --- a/security/selinux/ss/ebitmap.c +++ b/security/selinux/ss/ebitmap.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "ebitmap.h" #include "policydb.h" @@ -542,6 +543,19 @@ int ebitmap_write(struct ebitmap *e, void *fp) return 0; } +u32 ebitmap_hash(const struct ebitmap *e, u32 hash) +{ + struct ebitmap_node *node; + + /* need to change hash even if ebitmap is empty */ + hash = jhash_1word(e->highbit, hash); + for (node = e->node; node; node = node->next) { + hash = jhash_1word(node->startbit, hash); + hash = jhash(node->maps, sizeof(node->maps), hash); + } + return hash; +} + void __init ebitmap_cache_init(void) { ebitmap_node_cachep = kmem_cache_create("ebitmap_node", diff --git a/security/selinux/ss/ebitmap.h b/security/selinux/ss/ebitmap.h index 9a23b81b8832..9eb2d0af2805 100644 --- a/security/selinux/ss/ebitmap.h +++ b/security/selinux/ss/ebitmap.h @@ -131,6 +131,7 @@ int ebitmap_set_bit(struct ebitmap *e, unsigned long bit, int value); void ebitmap_destroy(struct ebitmap *e); int ebitmap_read(struct ebitmap *e, void *fp); int ebitmap_write(struct ebitmap *e, void *fp); +u32 ebitmap_hash(const struct ebitmap *e, u32 hash); #ifdef CONFIG_NETLABEL int ebitmap_netlbl_export(struct ebitmap *ebmap, diff --git a/security/selinux/ss/mls.h b/security/selinux/ss/mls.h index 7954b1e60b64..15cacde0ff61 100644 --- a/security/selinux/ss/mls.h +++ b/security/selinux/ss/mls.h @@ -22,7 +22,10 @@ #ifndef _SS_MLS_H_ #define _SS_MLS_H_ +#include + #include "context.h" +#include "ebitmap.h" #include "policydb.h" int mls_compute_context_len(struct policydb *p, struct context *context); @@ -101,5 +104,13 @@ static inline int mls_import_netlbl_cat(struct policydb *p, } #endif +static inline u32 mls_range_hash(const struct mls_range *r, u32 hash) +{ + hash = jhash_2words(r->level[0].sens, r->level[1].sens, hash); + hash = ebitmap_hash(&r->level[0].cat, hash); + hash = ebitmap_hash(&r->level[1].cat, hash); + return hash; +} + #endif /* _SS_MLS_H */ diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index 4f0cfffd008d..2849bc362828 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -862,11 +862,8 @@ int policydb_load_isids(struct policydb *p, struct sidtab *s) if (!name) continue; - rc = context_add_hash(p, &c->context[0]); - if (rc) { - sidtab_destroy(s); - goto out; - } + context_add_hash(&c->context[0]); + rc = sidtab_set_initial(s, sid, &c->context[0]); if (rc) { pr_err("SELinux: unable to load initial SID %s.\n", diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index 07cdda2ff49c..ed3306914309 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -1490,38 +1490,13 @@ out: return rc; } -int context_add_hash(struct policydb *policydb, - struct context *context) -{ - int rc; - char *str; - int len; - - if (context->str) { - context->hash = context_compute_hash(context->str); - } else { - rc = context_struct_to_string(policydb, context, - &str, &len); - if (rc) - return rc; - context->hash = context_compute_hash(str); - kfree(str); - } - return 0; -} - static int context_struct_to_sid(struct selinux_state *state, struct context *context, u32 *sid) { - int rc; struct sidtab *sidtab = state->ss->sidtab; - struct policydb *policydb = &state->ss->policydb; - if (!context->hash) { - rc = context_add_hash(policydb, context); - if (rc) - return rc; - } + if (!context->hash) + context_add_hash(context); return sidtab_context_to_sid(sidtab, context, sid); } @@ -2119,9 +2094,7 @@ static int convert_context(struct context *oldc, struct context *newc, void *p) goto bad; } - rc = context_add_hash(args->newp, newc); - if (rc) - goto bad; + context_add_hash(newc); return 0; bad: @@ -2132,7 +2105,7 @@ bad: context_destroy(newc); newc->str = s; newc->len = len; - newc->hash = context_compute_hash(s); + context_add_hash(newc); pr_info("SELinux: Context %s became invalid (unmapped).\n", newc->str); return 0; diff --git a/security/selinux/ss/services.h b/security/selinux/ss/services.h index e9bddf33e53d..a06f3d835216 100644 --- a/security/selinux/ss/services.h +++ b/security/selinux/ss/services.h @@ -8,7 +8,6 @@ #define _SS_SERVICES_H_ #include "policydb.h" -#include "context.h" /* Mapping for a single class */ struct selinux_mapping { @@ -37,6 +36,4 @@ void services_compute_xperms_drivers(struct extended_perms *xperms, void services_compute_xperms_decision(struct extended_perms_decision *xpermd, struct avtab_node *node); -int context_add_hash(struct policydb *policydb, struct context *context); - #endif /* _SS_SERVICES_H_ */ From 225621c9348d2a759db141024d5986d48e8c50dc Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Fri, 17 Apr 2020 10:11:57 +0200 Subject: [PATCH 0035/1043] selinux: move context hashing under sidtab Now that context hash computation no longer depends on policydb, we can simplify things by moving the context hashing completely under sidtab. The hash is still cached in sidtab entries, but not for the in-flight context structures. Signed-off-by: Ondrej Mosnacek Signed-off-by: Paul Moore --- security/selinux/ss/context.h | 11 +------ security/selinux/ss/policydb.c | 2 -- security/selinux/ss/services.c | 59 +++++++++++++++------------------- security/selinux/ss/sidtab.c | 32 ++++++++++-------- security/selinux/ss/sidtab.h | 1 + 5 files changed, 47 insertions(+), 58 deletions(-) diff --git a/security/selinux/ss/context.h b/security/selinux/ss/context.h index e7ae7e21449b..62990aa1ec9e 100644 --- a/security/selinux/ss/context.h +++ b/security/selinux/ss/context.h @@ -31,7 +31,6 @@ struct context { u32 len; /* length of string in bytes */ struct mls_range range; char *str; /* string representation if context cannot be mapped. */ - u32 hash; /* a hash of the string representation */ }; static inline void mls_context_init(struct context *c) @@ -169,13 +168,12 @@ static inline int context_cpy(struct context *dst, struct context *src) kfree(dst->str); return rc; } - dst->hash = src->hash; return 0; } static inline void context_destroy(struct context *c) { - c->user = c->role = c->type = c->hash = 0; + c->user = c->role = c->type = 0; kfree(c->str); c->str = NULL; c->len = 0; @@ -184,8 +182,6 @@ static inline void context_destroy(struct context *c) static inline int context_cmp(struct context *c1, struct context *c2) { - if (c1->hash && c2->hash && (c1->hash != c2->hash)) - return 0; if (c1->len && c2->len) return (c1->len == c2->len && !strcmp(c1->str, c2->str)); if (c1->len || c2->len) @@ -198,10 +194,5 @@ static inline int context_cmp(struct context *c1, struct context *c2) u32 context_compute_hash(const struct context *c); -static inline void context_add_hash(struct context *context) -{ - context->hash = context_compute_hash(context); -} - #endif /* _SS_CONTEXT_H_ */ diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index 2849bc362828..dc6729860bd6 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -862,8 +862,6 @@ int policydb_load_isids(struct policydb *p, struct sidtab *s) if (!name) continue; - context_add_hash(&c->context[0]); - rc = sidtab_set_initial(s, sid, &c->context[0]); if (rc) { pr_err("SELinux: unable to load initial SID %s.\n", diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index ed3306914309..b49a336b1e6e 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -1490,17 +1490,6 @@ out: return rc; } -static int context_struct_to_sid(struct selinux_state *state, - struct context *context, u32 *sid) -{ - struct sidtab *sidtab = state->ss->sidtab; - - if (!context->hash) - context_add_hash(context); - - return sidtab_context_to_sid(sidtab, context, sid); -} - static int security_context_to_sid_core(struct selinux_state *state, const char *scontext, u32 scontext_len, u32 *sid, u32 def_sid, gfp_t gfp_flags, @@ -1555,7 +1544,7 @@ static int security_context_to_sid_core(struct selinux_state *state, str = NULL; } else if (rc) goto out_unlock; - rc = context_struct_to_sid(state, &context, sid); + rc = sidtab_context_to_sid(sidtab, &context, sid); context_destroy(&context); out_unlock: read_unlock(&state->ss->policy_rwlock); @@ -1865,7 +1854,7 @@ static int security_compute_sid(struct selinux_state *state, goto out_unlock; } /* Obtain the sid for the context. */ - rc = context_struct_to_sid(state, &newcontext, out_sid); + rc = sidtab_context_to_sid(sidtab, &newcontext, out_sid); out_unlock: read_unlock(&state->ss->policy_rwlock); context_destroy(&newcontext); @@ -2017,7 +2006,6 @@ static int convert_context(struct context *oldc, struct context *newc, void *p) context_init(newc); newc->str = s; newc->len = oldc->len; - newc->hash = oldc->hash; return 0; } kfree(s); @@ -2094,8 +2082,6 @@ static int convert_context(struct context *oldc, struct context *newc, void *p) goto bad; } - context_add_hash(newc); - return 0; bad: /* Map old representation to string and save it. */ @@ -2105,7 +2091,6 @@ bad: context_destroy(newc); newc->str = s; newc->len = len; - context_add_hash(newc); pr_info("SELinux: Context %s became invalid (unmapped).\n", newc->str); return 0; @@ -2322,12 +2307,14 @@ int security_port_sid(struct selinux_state *state, u8 protocol, u16 port, u32 *out_sid) { struct policydb *policydb; + struct sidtab *sidtab; struct ocontext *c; int rc = 0; read_lock(&state->ss->policy_rwlock); policydb = &state->ss->policydb; + sidtab = state->ss->sidtab; c = policydb->ocontexts[OCON_PORT]; while (c) { @@ -2340,7 +2327,7 @@ int security_port_sid(struct selinux_state *state, if (c) { if (!c->sid[0]) { - rc = context_struct_to_sid(state, &c->context[0], + rc = sidtab_context_to_sid(sidtab, &c->context[0], &c->sid[0]); if (rc) goto out; @@ -2365,12 +2352,14 @@ int security_ib_pkey_sid(struct selinux_state *state, u64 subnet_prefix, u16 pkey_num, u32 *out_sid) { struct policydb *policydb; + struct sidtab *sidtab; struct ocontext *c; int rc = 0; read_lock(&state->ss->policy_rwlock); policydb = &state->ss->policydb; + sidtab = state->ss->sidtab; c = policydb->ocontexts[OCON_IBPKEY]; while (c) { @@ -2384,7 +2373,7 @@ int security_ib_pkey_sid(struct selinux_state *state, if (c) { if (!c->sid[0]) { - rc = context_struct_to_sid(state, + rc = sidtab_context_to_sid(sidtab, &c->context[0], &c->sid[0]); if (rc) @@ -2409,12 +2398,14 @@ int security_ib_endport_sid(struct selinux_state *state, const char *dev_name, u8 port_num, u32 *out_sid) { struct policydb *policydb; + struct sidtab *sidtab; struct ocontext *c; int rc = 0; read_lock(&state->ss->policy_rwlock); policydb = &state->ss->policydb; + sidtab = state->ss->sidtab; c = policydb->ocontexts[OCON_IBENDPORT]; while (c) { @@ -2429,7 +2420,7 @@ int security_ib_endport_sid(struct selinux_state *state, if (c) { if (!c->sid[0]) { - rc = context_struct_to_sid(state, &c->context[0], + rc = sidtab_context_to_sid(sidtab, &c->context[0], &c->sid[0]); if (rc) goto out; @@ -2452,12 +2443,14 @@ int security_netif_sid(struct selinux_state *state, char *name, u32 *if_sid) { struct policydb *policydb; + struct sidtab *sidtab; int rc = 0; struct ocontext *c; read_lock(&state->ss->policy_rwlock); policydb = &state->ss->policydb; + sidtab = state->ss->sidtab; c = policydb->ocontexts[OCON_NETIF]; while (c) { @@ -2468,11 +2461,11 @@ int security_netif_sid(struct selinux_state *state, if (c) { if (!c->sid[0] || !c->sid[1]) { - rc = context_struct_to_sid(state, &c->context[0], + rc = sidtab_context_to_sid(sidtab, &c->context[0], &c->sid[0]); if (rc) goto out; - rc = context_struct_to_sid(state, &c->context[1], + rc = sidtab_context_to_sid(sidtab, &c->context[1], &c->sid[1]); if (rc) goto out; @@ -2513,12 +2506,14 @@ int security_node_sid(struct selinux_state *state, u32 *out_sid) { struct policydb *policydb; + struct sidtab *sidtab; int rc; struct ocontext *c; read_lock(&state->ss->policy_rwlock); policydb = &state->ss->policydb; + sidtab = state->ss->sidtab; switch (domain) { case AF_INET: { @@ -2560,7 +2555,7 @@ int security_node_sid(struct selinux_state *state, if (c) { if (!c->sid[0]) { - rc = context_struct_to_sid(state, + rc = sidtab_context_to_sid(sidtab, &c->context[0], &c->sid[0]); if (rc) @@ -2644,17 +2639,12 @@ int security_get_user_sids(struct selinux_state *state, usercon.role = i + 1; ebitmap_for_each_positive_bit(&role->types, tnode, j) { usercon.type = j + 1; - /* - * The same context struct is reused here so the hash - * must be reset. - */ - usercon.hash = 0; if (mls_setup_user_range(policydb, fromcon, user, &usercon)) continue; - rc = context_struct_to_sid(state, &usercon, &sid); + rc = sidtab_context_to_sid(sidtab, &usercon, &sid); if (rc) goto out_unlock; if (mynel < maxnel) { @@ -2725,6 +2715,7 @@ static inline int __security_genfs_sid(struct selinux_state *state, u32 *sid) { struct policydb *policydb = &state->ss->policydb; + struct sidtab *sidtab = state->ss->sidtab; int len; u16 sclass; struct genfs *genfs; @@ -2759,7 +2750,7 @@ static inline int __security_genfs_sid(struct selinux_state *state, goto out; if (!c->sid[0]) { - rc = context_struct_to_sid(state, &c->context[0], &c->sid[0]); + rc = sidtab_context_to_sid(sidtab, &c->context[0], &c->sid[0]); if (rc) goto out; } @@ -2801,6 +2792,7 @@ int security_genfs_sid(struct selinux_state *state, int security_fs_use(struct selinux_state *state, struct super_block *sb) { struct policydb *policydb; + struct sidtab *sidtab; int rc = 0; struct ocontext *c; struct superblock_security_struct *sbsec = sb->s_security; @@ -2809,6 +2801,7 @@ int security_fs_use(struct selinux_state *state, struct super_block *sb) read_lock(&state->ss->policy_rwlock); policydb = &state->ss->policydb; + sidtab = state->ss->sidtab; c = policydb->ocontexts[OCON_FSUSE]; while (c) { @@ -2820,7 +2813,7 @@ int security_fs_use(struct selinux_state *state, struct super_block *sb) if (c) { sbsec->behavior = c->v.behavior; if (!c->sid[0]) { - rc = context_struct_to_sid(state, &c->context[0], + rc = sidtab_context_to_sid(sidtab, &c->context[0], &c->sid[0]); if (rc) goto out; @@ -3068,7 +3061,7 @@ int security_sid_mls_copy(struct selinux_state *state, goto out_unlock; } } - rc = context_struct_to_sid(state, &newcon, new_sid); + rc = sidtab_context_to_sid(sidtab, &newcon, new_sid); out_unlock: read_unlock(&state->ss->policy_rwlock); context_destroy(&newcon); @@ -3661,7 +3654,7 @@ int security_netlbl_secattr_to_sid(struct selinux_state *state, if (!mls_context_isvalid(policydb, &ctx_new)) goto out_free; - rc = context_struct_to_sid(state, &ctx_new, sid); + rc = sidtab_context_to_sid(sidtab, &ctx_new, sid); if (rc) goto out_free; diff --git a/security/selinux/ss/sidtab.c b/security/selinux/ss/sidtab.c index 98d5ea3fcde4..eb6d27b5aeb4 100644 --- a/security/selinux/ss/sidtab.c +++ b/security/selinux/ss/sidtab.c @@ -54,14 +54,15 @@ int sidtab_init(struct sidtab *s) return 0; } -static u32 context_to_sid(struct sidtab *s, struct context *context) +static u32 context_to_sid(struct sidtab *s, struct context *context, u32 hash) { struct sidtab_entry *entry; u32 sid = 0; rcu_read_lock(); - hash_for_each_possible_rcu(s->context_to_sid, entry, list, - context->hash) { + hash_for_each_possible_rcu(s->context_to_sid, entry, list, hash) { + if (entry->hash != hash) + continue; if (context_cmp(&entry->context, context)) { sid = entry->sid; break; @@ -74,6 +75,7 @@ static u32 context_to_sid(struct sidtab *s, struct context *context) int sidtab_set_initial(struct sidtab *s, u32 sid, struct context *context) { struct sidtab_isid_entry *isid; + u32 hash; int rc; if (sid == 0 || sid > SECINITSID_NUM) @@ -90,15 +92,18 @@ int sidtab_set_initial(struct sidtab *s, u32 sid, struct context *context) #endif isid->set = 1; + hash = context_compute_hash(context); + /* * Multiple initial sids may map to the same context. Check that this * context is not already represented in the context_to_sid hashtable * to avoid duplicate entries and long linked lists upon hash * collision. */ - if (!context_to_sid(s, context)) { + if (!context_to_sid(s, context, hash)) { isid->entry.sid = sid; - hash_add(s->context_to_sid, &isid->entry.list, context->hash); + isid->entry.hash = hash; + hash_add(s->context_to_sid, &isid->entry.list, hash); } return 0; @@ -259,12 +264,12 @@ int sidtab_context_to_sid(struct sidtab *s, struct context *context, u32 *sid) { unsigned long flags; - u32 count; + u32 count, hash = context_compute_hash(context); struct sidtab_convert_params *convert; struct sidtab_entry *dst, *dst_convert; int rc; - *sid = context_to_sid(s, context); + *sid = context_to_sid(s, context, hash); if (*sid) return 0; @@ -272,7 +277,7 @@ int sidtab_context_to_sid(struct sidtab *s, struct context *context, spin_lock_irqsave(&s->lock, flags); rc = 0; - *sid = context_to_sid(s, context); + *sid = context_to_sid(s, context, hash); if (*sid) goto out_unlock; @@ -291,6 +296,7 @@ int sidtab_context_to_sid(struct sidtab *s, struct context *context, goto out_unlock; dst->sid = index_to_sid(count); + dst->hash = hash; rc = context_cpy(&dst->context, context); if (rc) @@ -315,10 +321,11 @@ int sidtab_context_to_sid(struct sidtab *s, struct context *context, goto out_unlock; } dst_convert->sid = index_to_sid(count); + dst_convert->hash = context_compute_hash(&dst_convert->context); convert->target->count = count + 1; hash_add_rcu(convert->target->context_to_sid, - &dst_convert->list, dst_convert->context.hash); + &dst_convert->list, dst_convert->hash); } if (context->len) @@ -329,7 +336,7 @@ int sidtab_context_to_sid(struct sidtab *s, struct context *context, /* write entries before updating count */ smp_store_release(&s->count, count + 1); - hash_add_rcu(s->context_to_sid, &dst->list, dst->context.hash); + hash_add_rcu(s->context_to_sid, &dst->list, dst->hash); rc = 0; out_unlock: @@ -345,10 +352,9 @@ static void sidtab_convert_hashtable(struct sidtab *s, u32 count) for (i = 0; i < count; i++) { entry = sidtab_do_lookup(s, i, 0); entry->sid = index_to_sid(i); + entry->hash = context_compute_hash(&entry->context); - hash_add_rcu(s->context_to_sid, &entry->list, - entry->context.hash); - + hash_add_rcu(s->context_to_sid, &entry->list, entry->hash); } } diff --git a/security/selinux/ss/sidtab.h b/security/selinux/ss/sidtab.h index 3311d9f236c0..f2a84560b8b3 100644 --- a/security/selinux/ss/sidtab.h +++ b/security/selinux/ss/sidtab.h @@ -19,6 +19,7 @@ struct sidtab_entry { u32 sid; + u32 hash; struct context context; #if CONFIG_SECURITY_SELINUX_SID2STR_CACHE_SIZE > 0 struct sidtab_str_cache __rcu *cache; From 4300590243895ac39e8c97a2f5acd004dad8a42f Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Thu, 16 Apr 2020 19:13:55 +0200 Subject: [PATCH 0036/1043] selinux: implement new format of filename transitions Implement a new, more space-efficient way of storing filename transitions in the binary policy. The internal structures have already been converted to this new representation; this patch just implements reading/writing an equivalent represntation from/to the binary policy. This new format reduces the size of Fedora policy from 7.6 MB to only 3.3 MB (with policy optimization enabled in both cases). With the unconfined module disabled, the size is reduced from 3.3 MB to 2.4 MB. The time to load policy into kernel is also shorter with the new format. On Fedora Rawhide x86_64 it dropped from 157 ms to 106 ms; without the unconfined module from 115 ms to 105 ms. Signed-off-by: Ondrej Mosnacek Signed-off-by: Paul Moore --- security/selinux/include/security.h | 3 +- security/selinux/ss/policydb.c | 214 ++++++++++++++++++++++++---- 2 files changed, 190 insertions(+), 27 deletions(-) diff --git a/security/selinux/include/security.h b/security/selinux/include/security.h index d6036c018cf2..b0e02cfe3ce1 100644 --- a/security/selinux/include/security.h +++ b/security/selinux/include/security.h @@ -41,10 +41,11 @@ #define POLICYDB_VERSION_XPERMS_IOCTL 30 #define POLICYDB_VERSION_INFINIBAND 31 #define POLICYDB_VERSION_GLBLUB 32 +#define POLICYDB_VERSION_COMP_FTRANS 33 /* compressed filename transitions */ /* Range of policy versions we understand*/ #define POLICYDB_VERSION_MIN POLICYDB_VERSION_BASE -#define POLICYDB_VERSION_MAX POLICYDB_VERSION_GLBLUB +#define POLICYDB_VERSION_MAX POLICYDB_VERSION_COMP_FTRANS /* Mask for just the mount related flags */ #define SE_MNTMASK 0x0f diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index dc6729860bd6..ef8d5b46b05b 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -154,6 +154,11 @@ static struct policydb_compat_info policydb_compat[] = { .sym_num = SYM_NUM, .ocon_num = OCON_NUM, }, + { + .version = POLICYDB_VERSION_COMP_FTRANS, + .sym_num = SYM_NUM, + .ocon_num = OCON_NUM, + }, }; static struct policydb_compat_info *policydb_lookup_compat(int version) @@ -492,23 +497,16 @@ static int role_trans_cmp(struct hashtab *h, const void *k1, const void *k2) /* * Initialize a policy database structure. */ -static int policydb_init(struct policydb *p) +static void policydb_init(struct policydb *p) { memset(p, 0, sizeof(*p)); avtab_init(&p->te_avtab); cond_policydb_init(p); - p->filename_trans = hashtab_create(filenametr_hash, filenametr_cmp, - (1 << 11)); - if (!p->filename_trans) - return -ENOMEM; - ebitmap_init(&p->filename_trans_ttypes); ebitmap_init(&p->policycaps); ebitmap_init(&p->permissive_map); - - return 0; } /* @@ -1862,7 +1860,7 @@ out: return rc; } -static int filename_trans_read_one(struct policydb *p, void *fp) +static int filename_trans_read_helper_compat(struct policydb *p, void *fp) { struct filename_trans_key key, *ft = NULL; struct filename_trans_datum *last, *datum = NULL; @@ -1945,6 +1943,99 @@ out: return rc; } +static int filename_trans_read_helper(struct policydb *p, void *fp) +{ + struct filename_trans_key *ft = NULL; + struct filename_trans_datum **dst, *datum, *first = NULL; + char *name = NULL; + u32 len, ttype, tclass, ndatum, i; + __le32 buf[3]; + int rc; + + /* length of the path component string */ + rc = next_entry(buf, fp, sizeof(u32)); + if (rc) + return rc; + len = le32_to_cpu(buf[0]); + + /* path component string */ + rc = str_read(&name, GFP_KERNEL, fp, len); + if (rc) + return rc; + + rc = next_entry(buf, fp, sizeof(u32) * 3); + if (rc) + goto out; + + ttype = le32_to_cpu(buf[0]); + tclass = le32_to_cpu(buf[1]); + + ndatum = le32_to_cpu(buf[2]); + if (ndatum == 0) { + pr_err("SELinux: Filename transition key with no datum\n"); + rc = -ENOENT; + goto out; + } + + dst = &first; + for (i = 0; i < ndatum; i++) { + rc = -ENOMEM; + datum = kmalloc(sizeof(*datum), GFP_KERNEL); + if (!datum) + goto out; + + *dst = datum; + + /* ebitmap_read() will at least init the bitmap */ + rc = ebitmap_read(&datum->stypes, fp); + if (rc) + goto out; + + rc = next_entry(buf, fp, sizeof(u32)); + if (rc) + goto out; + + datum->otype = le32_to_cpu(buf[0]); + datum->next = NULL; + + dst = &datum->next; + } + + rc = -ENOMEM; + ft = kmalloc(sizeof(*ft), GFP_KERNEL); + if (!ft) + goto out; + + ft->ttype = ttype; + ft->tclass = tclass; + ft->name = name; + + rc = hashtab_insert(p->filename_trans, ft, first); + if (rc == -EEXIST) + pr_err("SELinux: Duplicate filename transition key\n"); + if (rc) + goto out; + + rc = ebitmap_set_bit(&p->filename_trans_ttypes, ttype, 1); + if (rc) + return rc; + + p->filename_trans_count += ndatum; + return 0; + +out: + kfree(ft); + kfree(name); + while (first) { + datum = first; + first = first->next; + + ebitmap_destroy(&datum->stypes); + kfree(datum); + } + return rc; +} + static int filename_trans_read(struct policydb *p, void *fp) { u32 nel; @@ -1959,12 +2050,29 @@ static int filename_trans_read(struct policydb *p, void *fp) return rc; nel = le32_to_cpu(buf[0]); - p->filename_trans_count = nel; + if (p->policyvers < POLICYDB_VERSION_COMP_FTRANS) { + p->filename_trans_count = nel; + p->filename_trans = hashtab_create(filenametr_hash, + filenametr_cmp, (1 << 11)); + if (!p->filename_trans) + return -ENOMEM; - for (i = 0; i < nel; i++) { - rc = filename_trans_read_one(p, fp); - if (rc) - return rc; + for (i = 0; i < nel; i++) { + rc = filename_trans_read_helper_compat(p, fp); + if (rc) + return rc; + } + } else { + p->filename_trans = hashtab_create(filenametr_hash, + filenametr_cmp, nel); + if (!p->filename_trans) + return -ENOMEM; + + for (i = 0; i < nel; i++) { + rc = filename_trans_read_helper(p, fp); + if (rc) + return rc; + } } hash_eval(p->filename_trans, "filenametr"); return 0; @@ -2281,9 +2389,7 @@ int policydb_read(struct policydb *p, void *fp) char *policydb_str; struct policydb_compat_info *info; - rc = policydb_init(p); - if (rc) - return rc; + policydb_init(p); /* Read the magic number and string length. */ rc = next_entry(buf, fp, sizeof(u32) * 2); @@ -3367,7 +3473,7 @@ static int range_write(struct policydb *p, void *fp) return 0; } -static int filename_write_helper(void *key, void *data, void *ptr) +static int filename_write_helper_compat(void *key, void *data, void *ptr) { struct filename_trans_key *ft = key; struct filename_trans_datum *datum = data; @@ -3404,6 +3510,55 @@ static int filename_write_helper(void *key, void *data, void *ptr) return 0; } +static int filename_write_helper(void *key, void *data, void *ptr) +{ + struct filename_trans_key *ft = key; + struct filename_trans_datum *datum; + void *fp = ptr; + __le32 buf[3]; + int rc; + u32 ndatum, len = strlen(ft->name); + + buf[0] = cpu_to_le32(len); + rc = put_entry(buf, sizeof(u32), 1, fp); + if (rc) + return rc; + + rc = put_entry(ft->name, sizeof(char), len, fp); + if (rc) + return rc; + + ndatum = 0; + datum = data; + do { + ndatum++; + datum = datum->next; + } while (unlikely(datum)); + + buf[0] = cpu_to_le32(ft->ttype); + buf[1] = cpu_to_le32(ft->tclass); + buf[2] = cpu_to_le32(ndatum); + rc = put_entry(buf, sizeof(u32), 3, fp); + if (rc) + return rc; + + datum = data; + do { + rc = ebitmap_write(&datum->stypes, fp); + if (rc) + return rc; + + buf[0] = cpu_to_le32(datum->otype); + rc = put_entry(buf, sizeof(u32), 1, fp); + if (rc) + return rc; + + datum = datum->next; + } while (unlikely(datum)); + + return 0; +} + static int filename_trans_write(struct policydb *p, void *fp) { __le32 buf[1]; @@ -3412,16 +3567,23 @@ static int filename_trans_write(struct policydb *p, void *fp) if (p->policyvers < POLICYDB_VERSION_FILENAME_TRANS) return 0; - buf[0] = cpu_to_le32(p->filename_trans_count); - rc = put_entry(buf, sizeof(u32), 1, fp); - if (rc) - return rc; + if (p->policyvers < POLICYDB_VERSION_COMP_FTRANS) { + buf[0] = cpu_to_le32(p->filename_trans_count); + rc = put_entry(buf, sizeof(u32), 1, fp); + if (rc) + return rc; - rc = hashtab_map(p->filename_trans, filename_write_helper, fp); - if (rc) - return rc; + rc = hashtab_map(p->filename_trans, + filename_write_helper_compat, fp); + } else { + buf[0] = cpu_to_le32(p->filename_trans->nel); + rc = put_entry(buf, sizeof(u32), 1, fp); + if (rc) + return rc; - return 0; + rc = hashtab_map(p->filename_trans, filename_write_helper, fp); + } + return rc; } /* From 9446aa5062e972265427368314039fe74635bec2 Mon Sep 17 00:00:00 2001 From: Jason Yan Date: Sat, 18 Apr 2020 16:18:34 +0800 Subject: [PATCH 0037/1043] MIPS: Netlogic: remove unneeded semicolon in fmn_message_handler() Fix the following coccicheck warning: arch/mips/netlogic/xlr/fmn.c:106:2-3: Unneeded semicolon Reported-by: Hulk Robot Signed-off-by: Jason Yan Signed-off-by: Thomas Bogendoerfer --- arch/mips/netlogic/xlr/fmn.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/netlogic/xlr/fmn.c b/arch/mips/netlogic/xlr/fmn.c index d7db1533889a..f90303f31967 100644 --- a/arch/mips/netlogic/xlr/fmn.c +++ b/arch/mips/netlogic/xlr/fmn.c @@ -103,7 +103,7 @@ static irqreturn_t fmn_message_handler(int irq, void *data) mflags = nlm_cop2_enable_irqsave(); } } - }; + } /* Enable message ring intr, to any thread in core */ nlm_fmn_setup_intr(irq, (1 << nlm_threads_per_core) - 1); nlm_cop2_disable_irqrestore(mflags); From acfaaf52ebfd02de509bc212fc3e8c8bac187110 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 16 Apr 2020 17:00:05 +0200 Subject: [PATCH 0038/1043] ASoC: txx9: don't work around too small resource_size_t The txx9 sound driver deends on HAS_TXX9_ACLC, which is only set for three tx49xx SOCs, and thus always has a 64-bit phys_addr_t and resource_size_t. Instead of poking into ioremap internals to work around a potentially too small resource_size_t just add a BUILD_BUG_ON to catch such a case. Signed-off-by: Christoph Hellwig Acked-by: Mark Brown Signed-off-by: Thomas Bogendoerfer --- sound/soc/txx9/txx9aclc-ac97.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sound/soc/txx9/txx9aclc-ac97.c b/sound/soc/txx9/txx9aclc-ac97.c index b1d9615f2375..7402448bdb09 100644 --- a/sound/soc/txx9/txx9aclc-ac97.c +++ b/sound/soc/txx9/txx9aclc-ac97.c @@ -175,6 +175,8 @@ static int txx9aclc_ac97_dev_probe(struct platform_device *pdev) int err; int irq; + BUILD_BUG_ON(sizeof(drvdata->physbase) > sizeof(r->start)); + irq = platform_get_irq(pdev, 0); if (irq < 0) return irq; @@ -190,10 +192,6 @@ static int txx9aclc_ac97_dev_probe(struct platform_device *pdev) platform_set_drvdata(pdev, drvdata); drvdata->physbase = r->start; - if (sizeof(drvdata->physbase) > sizeof(r->start) && - r->start >= TXX9_DIRECTMAP_BASE && - r->start < TXX9_DIRECTMAP_BASE + 0x400000) - drvdata->physbase |= 0xf00000000ull; err = devm_request_irq(&pdev->dev, irq, txx9aclc_ac97_irq, 0, dev_name(&pdev->dev), drvdata); if (err < 0) From b604d4973af7df4bbe970b2c380f5a0477fa53ec Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 16 Apr 2020 17:00:06 +0200 Subject: [PATCH 0039/1043] MIPS: remove cpu_has_64bit_addresses This macro is identical to CONFIG_64BIT, and using a Kconfig variable for the only places that checks them (the ioremap implementation) will simplify later patches in this series. Signed-off-by: Christoph Hellwig Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/cpu-features.h | 6 ------ arch/mips/include/asm/io.h | 4 ++-- arch/mips/include/asm/mach-ath25/cpu-feature-overrides.h | 1 - arch/mips/include/asm/mach-ath79/cpu-feature-overrides.h | 1 - .../include/asm/mach-lantiq/falcon/cpu-feature-overrides.h | 1 - .../include/asm/mach-ralink/mt7620/cpu-feature-overrides.h | 1 - .../include/asm/mach-ralink/mt7621/cpu-feature-overrides.h | 1 - .../include/asm/mach-ralink/rt288x/cpu-feature-overrides.h | 1 - .../include/asm/mach-ralink/rt305x/cpu-feature-overrides.h | 1 - .../include/asm/mach-ralink/rt3883/cpu-feature-overrides.h | 1 - arch/mips/include/asm/mach-rc32434/cpu-feature-overrides.h | 1 - 11 files changed, 2 insertions(+), 17 deletions(-) diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h index de44c92b1c1f..400b123cb6da 100644 --- a/arch/mips/include/asm/cpu-features.h +++ b/arch/mips/include/asm/cpu-features.h @@ -435,9 +435,6 @@ # ifndef cpu_has_64bit_gp_regs # define cpu_has_64bit_gp_regs 0 # endif -# ifndef cpu_has_64bit_addresses -# define cpu_has_64bit_addresses 0 -# endif # ifndef cpu_vmbits # define cpu_vmbits 31 # endif @@ -456,9 +453,6 @@ # ifndef cpu_has_64bit_gp_regs # define cpu_has_64bit_gp_regs 1 # endif -# ifndef cpu_has_64bit_addresses -# define cpu_has_64bit_addresses 1 -# endif # ifndef cpu_vmbits # define cpu_vmbits cpu_data[0].vmbits # define __NEED_VMBITS_PROBE diff --git a/arch/mips/include/asm/io.h b/arch/mips/include/asm/io.h index cf1f2a4a2418..7be323ed2bfd 100644 --- a/arch/mips/include/asm/io.h +++ b/arch/mips/include/asm/io.h @@ -166,7 +166,7 @@ static inline void __iomem * __ioremap_mode(phys_addr_t offset, unsigned long si #define __IS_LOW512(addr) (!((phys_addr_t)(addr) & (phys_addr_t) ~0x1fffffffULL)) - if (cpu_has_64bit_addresses) { + if (IS_ENABLED(CONFIG_64BIT)) { u64 base = UNCAC_BASE; /* @@ -275,7 +275,7 @@ static inline void iounmap(const volatile void __iomem *addr) #define __IS_KSEG1(addr) (((unsigned long)(addr) & ~0x1fffffffUL) == CKSEG1) - if (cpu_has_64bit_addresses || + if (IS_ENABLED(CONFIG_64BIT) || (__builtin_constant_p(addr) && __IS_KSEG1(addr))) return; diff --git a/arch/mips/include/asm/mach-ath25/cpu-feature-overrides.h b/arch/mips/include/asm/mach-ath25/cpu-feature-overrides.h index 95a0b580909d..a54f20d956a2 100644 --- a/arch/mips/include/asm/mach-ath25/cpu-feature-overrides.h +++ b/arch/mips/include/asm/mach-ath25/cpu-feature-overrides.h @@ -56,6 +56,5 @@ #define cpu_has_64bits 0 #define cpu_has_64bit_zero_reg 0 #define cpu_has_64bit_gp_regs 0 -#define cpu_has_64bit_addresses 0 #endif /* __ASM_MACH_ATH25_CPU_FEATURE_OVERRIDES_H */ diff --git a/arch/mips/include/asm/mach-ath79/cpu-feature-overrides.h b/arch/mips/include/asm/mach-ath79/cpu-feature-overrides.h index e7c972fccd9f..79ab3ad9fee8 100644 --- a/arch/mips/include/asm/mach-ath79/cpu-feature-overrides.h +++ b/arch/mips/include/asm/mach-ath79/cpu-feature-overrides.h @@ -45,7 +45,6 @@ #define cpu_has_64bits 0 #define cpu_has_64bit_zero_reg 0 #define cpu_has_64bit_gp_regs 0 -#define cpu_has_64bit_addresses 0 #define cpu_dcache_line_size() 32 #define cpu_icache_line_size() 32 diff --git a/arch/mips/include/asm/mach-lantiq/falcon/cpu-feature-overrides.h b/arch/mips/include/asm/mach-lantiq/falcon/cpu-feature-overrides.h index f03c1c42dd90..10226976f7b7 100644 --- a/arch/mips/include/asm/mach-lantiq/falcon/cpu-feature-overrides.h +++ b/arch/mips/include/asm/mach-lantiq/falcon/cpu-feature-overrides.h @@ -46,7 +46,6 @@ #define cpu_has_64bits 0 #define cpu_has_64bit_zero_reg 0 #define cpu_has_64bit_gp_regs 0 -#define cpu_has_64bit_addresses 0 #define cpu_dcache_line_size() 32 #define cpu_icache_line_size() 32 diff --git a/arch/mips/include/asm/mach-ralink/mt7620/cpu-feature-overrides.h b/arch/mips/include/asm/mach-ralink/mt7620/cpu-feature-overrides.h index 6ea5908f0c11..c4579f1705c2 100644 --- a/arch/mips/include/asm/mach-ralink/mt7620/cpu-feature-overrides.h +++ b/arch/mips/include/asm/mach-ralink/mt7620/cpu-feature-overrides.h @@ -45,7 +45,6 @@ #define cpu_has_64bits 0 #define cpu_has_64bit_zero_reg 0 #define cpu_has_64bit_gp_regs 0 -#define cpu_has_64bit_addresses 0 #define cpu_dcache_line_size() 32 #define cpu_icache_line_size() 32 diff --git a/arch/mips/include/asm/mach-ralink/mt7621/cpu-feature-overrides.h b/arch/mips/include/asm/mach-ralink/mt7621/cpu-feature-overrides.h index e06f517b2588..168359a0a58d 100644 --- a/arch/mips/include/asm/mach-ralink/mt7621/cpu-feature-overrides.h +++ b/arch/mips/include/asm/mach-ralink/mt7621/cpu-feature-overrides.h @@ -46,7 +46,6 @@ #define cpu_has_64bits 0 #define cpu_has_64bit_zero_reg 0 #define cpu_has_64bit_gp_regs 0 -#define cpu_has_64bit_addresses 0 #define cpu_dcache_line_size() 32 #define cpu_icache_line_size() 32 diff --git a/arch/mips/include/asm/mach-ralink/rt288x/cpu-feature-overrides.h b/arch/mips/include/asm/mach-ralink/rt288x/cpu-feature-overrides.h index 9c069646d0bd..fdaf8c9182bc 100644 --- a/arch/mips/include/asm/mach-ralink/rt288x/cpu-feature-overrides.h +++ b/arch/mips/include/asm/mach-ralink/rt288x/cpu-feature-overrides.h @@ -44,7 +44,6 @@ #define cpu_has_64bits 0 #define cpu_has_64bit_zero_reg 0 #define cpu_has_64bit_gp_regs 0 -#define cpu_has_64bit_addresses 0 #define cpu_dcache_line_size() 16 #define cpu_icache_line_size() 16 diff --git a/arch/mips/include/asm/mach-ralink/rt305x/cpu-feature-overrides.h b/arch/mips/include/asm/mach-ralink/rt305x/cpu-feature-overrides.h index 2e423fd15384..7a385fe784a6 100644 --- a/arch/mips/include/asm/mach-ralink/rt305x/cpu-feature-overrides.h +++ b/arch/mips/include/asm/mach-ralink/rt305x/cpu-feature-overrides.h @@ -44,7 +44,6 @@ #define cpu_has_64bits 0 #define cpu_has_64bit_zero_reg 0 #define cpu_has_64bit_gp_regs 0 -#define cpu_has_64bit_addresses 0 #define cpu_dcache_line_size() 32 #define cpu_icache_line_size() 32 diff --git a/arch/mips/include/asm/mach-ralink/rt3883/cpu-feature-overrides.h b/arch/mips/include/asm/mach-ralink/rt3883/cpu-feature-overrides.h index 7cee0e232580..0a61910f6521 100644 --- a/arch/mips/include/asm/mach-ralink/rt3883/cpu-feature-overrides.h +++ b/arch/mips/include/asm/mach-ralink/rt3883/cpu-feature-overrides.h @@ -43,7 +43,6 @@ #define cpu_has_64bits 0 #define cpu_has_64bit_zero_reg 0 #define cpu_has_64bit_gp_regs 0 -#define cpu_has_64bit_addresses 0 #define cpu_dcache_line_size() 32 #define cpu_icache_line_size() 32 diff --git a/arch/mips/include/asm/mach-rc32434/cpu-feature-overrides.h b/arch/mips/include/asm/mach-rc32434/cpu-feature-overrides.h index bc46179fdf40..8539ccfb69b7 100644 --- a/arch/mips/include/asm/mach-rc32434/cpu-feature-overrides.h +++ b/arch/mips/include/asm/mach-rc32434/cpu-feature-overrides.h @@ -54,7 +54,6 @@ #define cpu_has_64bits 0 #define cpu_has_64bit_zero_reg 0 #define cpu_has_64bit_gp_regs 0 -#define cpu_has_64bit_addresses 0 #define cpu_has_inclusive_pcaches 0 From d399157283fb457cdc3ae03f2897070f0bd5d7c6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 16 Apr 2020 17:00:07 +0200 Subject: [PATCH 0040/1043] MIPS: cleanup fixup_bigphys_addr handling fixup_bigphys_addr is only provided by the alchemy platform. Remove all the stubs, and ensure we only call it if it is actually implemented. Also don't bother implementing io_remap_pfn_range if we don't have to, and move the remaining implementation to alchemy platform code. Signed-off-by: Christoph Hellwig Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kconfig | 4 +++ arch/mips/alchemy/common/setup.c | 16 ++++++--- arch/mips/include/asm/mach-au1x00/ioremap.h | 38 -------------------- arch/mips/include/asm/mach-bcm63xx/ioremap.h | 5 --- arch/mips/include/asm/mach-bmips/ioremap.h | 5 --- arch/mips/include/asm/mach-generic/ioremap.h | 9 ----- arch/mips/include/asm/mach-tx39xx/ioremap.h | 9 ----- arch/mips/include/asm/mach-tx49xx/ioremap.h | 9 ----- arch/mips/include/asm/pgtable.h | 23 ++++++------ arch/mips/pci/pci-alchemy.c | 2 +- 10 files changed, 27 insertions(+), 93 deletions(-) delete mode 100644 arch/mips/include/asm/mach-au1x00/ioremap.h diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 4f9303f6a687..9f15539a6342 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -92,6 +92,9 @@ config MIPS select SYSCTL_EXCEPTION_TRACE select VIRT_TO_BUS +config MIPS_FIXUP_BIGPHYS_ADDR + bool + menu "Machine selection" choice @@ -157,6 +160,7 @@ config MIPS_ALCHEMY select CSRC_R4K select IRQ_MIPS_CPU select DMA_MAYBE_COHERENT # Au1000,1500,1100 aren't, rest is + select MIPS_FIXUP_BIGPHYS_ADDR if PCI select SYS_HAS_CPU_MIPS32_R1 select SYS_SUPPORTS_32BIT_KERNEL select SYS_SUPPORTS_APM_EMULATION diff --git a/arch/mips/alchemy/common/setup.c b/arch/mips/alchemy/common/setup.c index 7faaa6d593a7..a8cbc552bd64 100644 --- a/arch/mips/alchemy/common/setup.c +++ b/arch/mips/alchemy/common/setup.c @@ -72,9 +72,9 @@ void __init plat_mem_setup(void) iomem_resource.end = IOMEM_RESOURCE_END; } -#if defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_PCI) +#ifdef CONFIG_MIPS_FIXUP_BIGPHYS_ADDR /* This routine should be valid for all Au1x based boards */ -phys_addr_t __fixup_bigphys_addr(phys_addr_t phys_addr, phys_addr_t size) +phys_addr_t fixup_bigphys_addr(phys_addr_t phys_addr, phys_addr_t size) { unsigned long start = ALCHEMY_PCI_MEMWIN_START; unsigned long end = ALCHEMY_PCI_MEMWIN_END; @@ -90,5 +90,13 @@ phys_addr_t __fixup_bigphys_addr(phys_addr_t phys_addr, phys_addr_t size) /* default nop */ return phys_addr; } -EXPORT_SYMBOL(__fixup_bigphys_addr); -#endif + +int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long vaddr, + unsigned long pfn, unsigned long size, pgprot_t prot) +{ + phys_addr_t phys_addr = fixup_bigphys_addr(pfn << PAGE_SHIFT, size); + + return remap_pfn_range(vma, vaddr, phys_addr >> PAGE_SHIFT, size, prot); +} +EXPORT_SYMBOL(io_remap_pfn_range); +#endif /* CONFIG_MIPS_FIXUP_BIGPHYS_ADDR */ diff --git a/arch/mips/include/asm/mach-au1x00/ioremap.h b/arch/mips/include/asm/mach-au1x00/ioremap.h deleted file mode 100644 index f6877ed8b8d0..000000000000 --- a/arch/mips/include/asm/mach-au1x00/ioremap.h +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * include/asm-mips/mach-au1x00/ioremap.h - */ -#ifndef __ASM_MACH_AU1X00_IOREMAP_H -#define __ASM_MACH_AU1X00_IOREMAP_H - -#include - -#if defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_PCI) -extern phys_addr_t __fixup_bigphys_addr(phys_addr_t, phys_addr_t); -#else -static inline phys_addr_t __fixup_bigphys_addr(phys_addr_t phys_addr, phys_addr_t size) -{ - return phys_addr; -} -#endif - -/* - * Allow physical addresses to be fixed up to help 36-bit peripherals. - */ -static inline phys_addr_t fixup_bigphys_addr(phys_addr_t phys_addr, phys_addr_t size) -{ - return __fixup_bigphys_addr(phys_addr, size); -} - -static inline void __iomem *plat_ioremap(phys_addr_t offset, unsigned long size, - unsigned long flags) -{ - return NULL; -} - -static inline int plat_iounmap(const volatile void __iomem *addr) -{ - return 0; -} - -#endif /* __ASM_MACH_AU1X00_IOREMAP_H */ diff --git a/arch/mips/include/asm/mach-bcm63xx/ioremap.h b/arch/mips/include/asm/mach-bcm63xx/ioremap.h index 8cd261ec0a75..73f31825bbf3 100644 --- a/arch/mips/include/asm/mach-bcm63xx/ioremap.h +++ b/arch/mips/include/asm/mach-bcm63xx/ioremap.h @@ -4,11 +4,6 @@ #include -static inline phys_addr_t fixup_bigphys_addr(phys_addr_t phys_addr, phys_addr_t size) -{ - return phys_addr; -} - static inline int is_bcm63xx_internal_registers(phys_addr_t offset) { switch (bcm63xx_get_cpu_id()) { diff --git a/arch/mips/include/asm/mach-bmips/ioremap.h b/arch/mips/include/asm/mach-bmips/ioremap.h index 52632ebc705f..63b4af9916b6 100644 --- a/arch/mips/include/asm/mach-bmips/ioremap.h +++ b/arch/mips/include/asm/mach-bmips/ioremap.h @@ -4,11 +4,6 @@ #include -static inline phys_addr_t fixup_bigphys_addr(phys_addr_t phys_addr, phys_addr_t size) -{ - return phys_addr; -} - static inline int is_bmips_internal_registers(phys_addr_t offset) { if (offset >= 0xfff80000) diff --git a/arch/mips/include/asm/mach-generic/ioremap.h b/arch/mips/include/asm/mach-generic/ioremap.h index 4e36ea25ed33..f2442b84545c 100644 --- a/arch/mips/include/asm/mach-generic/ioremap.h +++ b/arch/mips/include/asm/mach-generic/ioremap.h @@ -7,15 +7,6 @@ #include -/* - * Allow physical addresses to be fixed up to help peripherals located - * outside the low 32-bit range -- generic pass-through version. - */ -static inline phys_addr_t fixup_bigphys_addr(phys_addr_t phys_addr, phys_addr_t size) -{ - return phys_addr; -} - static inline void __iomem *plat_ioremap(phys_addr_t offset, unsigned long size, unsigned long flags) { diff --git a/arch/mips/include/asm/mach-tx39xx/ioremap.h b/arch/mips/include/asm/mach-tx39xx/ioremap.h index 077b3c9971f7..157a7292397e 100644 --- a/arch/mips/include/asm/mach-tx39xx/ioremap.h +++ b/arch/mips/include/asm/mach-tx39xx/ioremap.h @@ -7,15 +7,6 @@ #include -/* - * Allow physical addresses to be fixed up to help peripherals located - * outside the low 32-bit range -- generic pass-through version. - */ -static inline phys_addr_t fixup_bigphys_addr(phys_addr_t phys_addr, phys_addr_t size) -{ - return phys_addr; -} - static inline void __iomem *plat_ioremap(phys_addr_t offset, unsigned long size, unsigned long flags) { diff --git a/arch/mips/include/asm/mach-tx49xx/ioremap.h b/arch/mips/include/asm/mach-tx49xx/ioremap.h index c6b9e05f44c4..b1f3710acf8e 100644 --- a/arch/mips/include/asm/mach-tx49xx/ioremap.h +++ b/arch/mips/include/asm/mach-tx49xx/ioremap.h @@ -7,15 +7,6 @@ #include -/* - * Allow physical addresses to be fixed up to help peripherals located - * outside the low 32-bit range -- generic pass-through version. - */ -static inline phys_addr_t fixup_bigphys_addr(phys_addr_t phys_addr, phys_addr_t size) -{ - return phys_addr; -} - static inline void __iomem *plat_ioremap(phys_addr_t offset, unsigned long size, unsigned long flags) { diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h index f1801e7a4b15..aab0ec174f68 100644 --- a/arch/mips/include/asm/pgtable.h +++ b/arch/mips/include/asm/pgtable.h @@ -507,20 +507,17 @@ static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, #define kern_addr_valid(addr) (1) -#ifdef CONFIG_PHYS_ADDR_T_64BIT -extern int remap_pfn_range(struct vm_area_struct *vma, unsigned long from, unsigned long pfn, unsigned long size, pgprot_t prot); - -static inline int io_remap_pfn_range(struct vm_area_struct *vma, - unsigned long vaddr, - unsigned long pfn, - unsigned long size, - pgprot_t prot) -{ - phys_addr_t phys_addr_high = fixup_bigphys_addr(pfn << PAGE_SHIFT, size); - return remap_pfn_range(vma, vaddr, phys_addr_high >> PAGE_SHIFT, size, prot); -} +/* + * Allow physical addresses to be fixed up to help 36-bit peripherals. + */ +#ifdef CONFIG_MIPS_FIXUP_BIGPHYS_ADDR +phys_addr_t fixup_bigphys_addr(phys_addr_t addr, phys_addr_t size); +int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long vaddr, + unsigned long pfn, unsigned long size, pgprot_t prot); #define io_remap_pfn_range io_remap_pfn_range -#endif +#else +#define fixup_bigphys_addr(addr, size) (addr) +#endif /* CONFIG_MIPS_FIXUP_BIGPHYS_ADDR */ #ifdef CONFIG_TRANSPARENT_HUGEPAGE diff --git a/arch/mips/pci/pci-alchemy.c b/arch/mips/pci/pci-alchemy.c index 01a2af8215c8..7285b5667568 100644 --- a/arch/mips/pci/pci-alchemy.c +++ b/arch/mips/pci/pci-alchemy.c @@ -52,7 +52,7 @@ struct alchemy_pci_context { static struct alchemy_pci_context *__alchemy_pci_ctx; -/* IO/MEM resources for PCI. Keep the memres in sync with __fixup_bigphys_addr +/* IO/MEM resources for PCI. Keep the memres in sync with fixup_bigphys_addr * in arch/mips/alchemy/common/setup.c */ static struct resource alchemy_pci_def_memres = { From 5c9ff5709dcf82bfd9a6d0d99a13b15ca522f510 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 16 Apr 2020 17:00:08 +0200 Subject: [PATCH 0041/1043] MIPS: merge __ioremap_mode into ioremap_prot There is no reason to have two ioremap with flags interfaces. Merge the historic mips __ioremap_mode into ioremap_prot which is a generic kernel interface. Signed-off-by: Christoph Hellwig Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/io.h | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/arch/mips/include/asm/io.h b/arch/mips/include/asm/io.h index 7be323ed2bfd..60513250f8f8 100644 --- a/arch/mips/include/asm/io.h +++ b/arch/mips/include/asm/io.h @@ -156,9 +156,17 @@ static inline void *isa_bus_to_virt(unsigned long address) extern void __iomem * __ioremap(phys_addr_t offset, phys_addr_t size, unsigned long flags); extern void __iounmap(const volatile void __iomem *addr); -static inline void __iomem * __ioremap_mode(phys_addr_t offset, unsigned long size, - unsigned long flags) +/* + * ioremap_prot - map bus memory into CPU space + * @offset: bus address of the memory + * @size: size of the resource to map + + * ioremap_prot gives the caller control over cache coherency attributes (CCA) + */ +static inline void __iomem *ioremap_prot(phys_addr_t offset, + unsigned long size, unsigned long prot_val) { + unsigned long flags = prot_val & _CACHE_MASK; void __iomem *addr = plat_ioremap(offset, size, flags); if (addr) @@ -202,18 +210,6 @@ static inline void __iomem * __ioremap_mode(phys_addr_t offset, unsigned long si #undef __IS_LOW512 } -/* - * ioremap_prot - map bus memory into CPU space - * @offset: bus address of the memory - * @size: size of the resource to map - - * ioremap_prot gives the caller control over cache coherency attributes (CCA) - */ -static inline void __iomem *ioremap_prot(phys_addr_t offset, - unsigned long size, unsigned long prot_val) { - return __ioremap_mode(offset, size, prot_val & _CACHE_MASK); -} - /* * ioremap - map bus memory into CPU space * @offset: bus address of the memory @@ -226,7 +222,7 @@ static inline void __iomem *ioremap_prot(phys_addr_t offset, * address. */ #define ioremap(offset, size) \ - __ioremap_mode((offset), (size), _CACHE_UNCACHED) + ioremap_prot((offset), (size), _CACHE_UNCACHED) #define ioremap_uc ioremap /* @@ -245,7 +241,7 @@ static inline void __iomem *ioremap_prot(phys_addr_t offset, * memory-like regions on I/O busses. */ #define ioremap_cache(offset, size) \ - __ioremap_mode((offset), (size), _page_cachable_default) + ioremap_prot((offset), (size), _page_cachable_default) /* * ioremap_wc - map bus memory into CPU space @@ -266,7 +262,7 @@ static inline void __iomem *ioremap_prot(phys_addr_t offset, * _CACHE_UNCACHED option (see cpu_probe() method). */ #define ioremap_wc(offset, size) \ - __ioremap_mode((offset), (size), boot_cpu_data.writecombine) + ioremap_prot((offset), (size), boot_cpu_data.writecombine) static inline void iounmap(const volatile void __iomem *addr) { From 8e487c153c302a4cf274c1645c705ad6f551f138 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 16 Apr 2020 17:00:09 +0200 Subject: [PATCH 0042/1043] MIPS: split out the 64-bit ioremap implementation Split out the mips64 ioremap implementation entirely, as it will never use page table based remapping. Signed-off-by: Christoph Hellwig Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/io.h | 65 ++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/arch/mips/include/asm/io.h b/arch/mips/include/asm/io.h index 60513250f8f8..f007571e036d 100644 --- a/arch/mips/include/asm/io.h +++ b/arch/mips/include/asm/io.h @@ -153,6 +153,25 @@ static inline void *isa_bus_to_virt(unsigned long address) */ #define page_to_phys(page) ((dma_addr_t)page_to_pfn(page) << PAGE_SHIFT) +#ifdef CONFIG_64BIT +static inline void __iomem *ioremap_prot(phys_addr_t offset, + unsigned long size, unsigned long prot_val) +{ + unsigned long flags = prot_val & _CACHE_MASK; + u64 base = (flags == _CACHE_UNCACHED ? IO_BASE : UNCAC_BASE); + void __iomem *addr; + + addr = plat_ioremap(offset, size, flags); + if (!addr) + addr = (void __iomem *)(unsigned long)(base + offset); + return addr; +} + +static inline void iounmap(const volatile void __iomem *addr) +{ + plat_iounmap(addr); +} +#else /* CONFIG_64BIT */ extern void __iomem * __ioremap(phys_addr_t offset, phys_addr_t size, unsigned long flags); extern void __iounmap(const volatile void __iomem *addr); @@ -174,18 +193,8 @@ static inline void __iomem *ioremap_prot(phys_addr_t offset, #define __IS_LOW512(addr) (!((phys_addr_t)(addr) & (phys_addr_t) ~0x1fffffffULL)) - if (IS_ENABLED(CONFIG_64BIT)) { - u64 base = UNCAC_BASE; - - /* - * R10000 supports a 2 bit uncached attribute therefore - * UNCAC_BASE may not equal IO_BASE. - */ - if (flags == _CACHE_UNCACHED) - base = (u64) IO_BASE; - return (void __iomem *) (unsigned long) (base + offset); - } else if (__builtin_constant_p(offset) && - __builtin_constant_p(size) && __builtin_constant_p(flags)) { + if (__builtin_constant_p(offset) && + __builtin_constant_p(size) && __builtin_constant_p(flags)) { phys_addr_t phys_addr, last_addr; phys_addr = fixup_bigphys_addr(offset, size); @@ -210,6 +219,22 @@ static inline void __iomem *ioremap_prot(phys_addr_t offset, #undef __IS_LOW512 } +static inline void iounmap(const volatile void __iomem *addr) +{ + if (plat_iounmap(addr)) + return; + +#define __IS_KSEG1(addr) (((unsigned long)(addr) & ~0x1fffffffUL) == CKSEG1) + + if (__builtin_constant_p(addr) && __IS_KSEG1(addr)) + return; + + __iounmap(addr); + +#undef __IS_KSEG1 +} +#endif /* !CONFIG_64BIT */ + /* * ioremap - map bus memory into CPU space * @offset: bus address of the memory @@ -264,22 +289,6 @@ static inline void __iomem *ioremap_prot(phys_addr_t offset, #define ioremap_wc(offset, size) \ ioremap_prot((offset), (size), boot_cpu_data.writecombine) -static inline void iounmap(const volatile void __iomem *addr) -{ - if (plat_iounmap(addr)) - return; - -#define __IS_KSEG1(addr) (((unsigned long)(addr) & ~0x1fffffffUL) == CKSEG1) - - if (IS_ENABLED(CONFIG_64BIT) || - (__builtin_constant_p(addr) && __IS_KSEG1(addr))) - return; - - __iounmap(addr); - -#undef __IS_KSEG1 -} - #if defined(CONFIG_CPU_CAVIUM_OCTEON) || defined(CONFIG_CPU_LOONGSON64) #define war_io_reorder_wmb() wmb() #else From d257b8fe173a4b22ca32780a6e65b075a3b88301 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 16 Apr 2020 17:00:10 +0200 Subject: [PATCH 0043/1043] MIPS: move ioremap_prot und iounmap out of line Neither of these interfaces is anywhere near the fast path. Move them out of line and avoid exposing implementation details to the drivers. Signed-off-by: Christoph Hellwig Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/io.h | 86 ++------------------------------------ arch/mips/mm/Makefile | 2 +- arch/mips/mm/ioremap.c | 45 ++++++++++---------- arch/mips/mm/ioremap64.c | 23 ++++++++++ 4 files changed, 49 insertions(+), 107 deletions(-) create mode 100644 arch/mips/mm/ioremap64.c diff --git a/arch/mips/include/asm/io.h b/arch/mips/include/asm/io.h index f007571e036d..346fffd9e972 100644 --- a/arch/mips/include/asm/io.h +++ b/arch/mips/include/asm/io.h @@ -30,8 +30,6 @@ #include #include #include - -#include #include /* @@ -153,87 +151,9 @@ static inline void *isa_bus_to_virt(unsigned long address) */ #define page_to_phys(page) ((dma_addr_t)page_to_pfn(page) << PAGE_SHIFT) -#ifdef CONFIG_64BIT -static inline void __iomem *ioremap_prot(phys_addr_t offset, - unsigned long size, unsigned long prot_val) -{ - unsigned long flags = prot_val & _CACHE_MASK; - u64 base = (flags == _CACHE_UNCACHED ? IO_BASE : UNCAC_BASE); - void __iomem *addr; - - addr = plat_ioremap(offset, size, flags); - if (!addr) - addr = (void __iomem *)(unsigned long)(base + offset); - return addr; -} - -static inline void iounmap(const volatile void __iomem *addr) -{ - plat_iounmap(addr); -} -#else /* CONFIG_64BIT */ -extern void __iomem * __ioremap(phys_addr_t offset, phys_addr_t size, unsigned long flags); -extern void __iounmap(const volatile void __iomem *addr); - -/* - * ioremap_prot - map bus memory into CPU space - * @offset: bus address of the memory - * @size: size of the resource to map - - * ioremap_prot gives the caller control over cache coherency attributes (CCA) - */ -static inline void __iomem *ioremap_prot(phys_addr_t offset, - unsigned long size, unsigned long prot_val) -{ - unsigned long flags = prot_val & _CACHE_MASK; - void __iomem *addr = plat_ioremap(offset, size, flags); - - if (addr) - return addr; - -#define __IS_LOW512(addr) (!((phys_addr_t)(addr) & (phys_addr_t) ~0x1fffffffULL)) - - if (__builtin_constant_p(offset) && - __builtin_constant_p(size) && __builtin_constant_p(flags)) { - phys_addr_t phys_addr, last_addr; - - phys_addr = fixup_bigphys_addr(offset, size); - - /* Don't allow wraparound or zero size. */ - last_addr = phys_addr + size - 1; - if (!size || last_addr < phys_addr) - return NULL; - - /* - * Map uncached objects in the low 512MB of address - * space using KSEG1. - */ - if (__IS_LOW512(phys_addr) && __IS_LOW512(last_addr) && - flags == _CACHE_UNCACHED) - return (void __iomem *) - (unsigned long)CKSEG1ADDR(phys_addr); - } - - return __ioremap(offset, size, flags); - -#undef __IS_LOW512 -} - -static inline void iounmap(const volatile void __iomem *addr) -{ - if (plat_iounmap(addr)) - return; - -#define __IS_KSEG1(addr) (((unsigned long)(addr) & ~0x1fffffffUL) == CKSEG1) - - if (__builtin_constant_p(addr) && __IS_KSEG1(addr)) - return; - - __iounmap(addr); - -#undef __IS_KSEG1 -} -#endif /* !CONFIG_64BIT */ +void __iomem *ioremap_prot(phys_addr_t offset, unsigned long size, + unsigned long prot_val); +void iounmap(const volatile void __iomem *addr); /* * ioremap - map bus memory into CPU space diff --git a/arch/mips/mm/Makefile b/arch/mips/mm/Makefile index 46f483e952c8..865926a37775 100644 --- a/arch/mips/mm/Makefile +++ b/arch/mips/mm/Makefile @@ -23,7 +23,7 @@ obj-y += uasm-mips.o endif obj-$(CONFIG_32BIT) += ioremap.o pgtable-32.o -obj-$(CONFIG_64BIT) += pgtable-64.o +obj-$(CONFIG_64BIT) += ioremap64.o pgtable-64.o obj-$(CONFIG_HIGHMEM) += highmem.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_DMA_NONCOHERENT) += dma-noncoherent.o diff --git a/arch/mips/mm/ioremap.c b/arch/mips/mm/ioremap.c index 8317f337a86e..c5b5181c7cd0 100644 --- a/arch/mips/mm/ioremap.c +++ b/arch/mips/mm/ioremap.c @@ -17,6 +17,10 @@ #include #include #include +#include + +#define IS_LOW512(addr) (!((phys_addr_t)(addr) & (phys_addr_t) ~0x1fffffffULL)) +#define IS_KSEG1(addr) (((unsigned long)(addr) & ~0x1fffffffUL) == CKSEG1) static inline void remap_area_pte(pte_t * pte, unsigned long address, phys_addr_t size, phys_addr_t phys_addr, unsigned long flags) @@ -118,27 +122,25 @@ static int __ioremap_check_ram(unsigned long start_pfn, unsigned long nr_pages, } /* - * Generic mapping function (not visible outside): - */ - -/* - * Remap an arbitrary physical address space into the kernel virtual - * address space. Needed when the kernel wants to access high addresses - * directly. + * ioremap_prot - map bus memory into CPU space + * @phys_addr: bus address of the memory + * @size: size of the resource to map * - * NOTE! We need to allow non-page-aligned mappings too: we will obviously - * have to convert them into an offset in a page-aligned mapping, but the - * caller shouldn't need to know that small detail. + * ioremap_prot gives the caller control over cache coherency attributes (CCA) */ - -#define IS_LOW512(addr) (!((phys_addr_t)(addr) & (phys_addr_t) ~0x1fffffffULL)) - -void __iomem * __ioremap(phys_addr_t phys_addr, phys_addr_t size, unsigned long flags) +void __iomem *ioremap_prot(phys_addr_t phys_addr, unsigned long size, + unsigned long prot_val) { + unsigned long flags = prot_val & _CACHE_MASK; unsigned long offset, pfn, last_pfn; - struct vm_struct * area; + struct vm_struct *area; phys_addr_t last_addr; - void * addr; + void *addr; + void __iomem *cpu_addr; + + cpu_addr = plat_ioremap(phys_addr, size, flags); + if (cpu_addr) + return cpu_addr; phys_addr = fixup_bigphys_addr(phys_addr, size); @@ -189,14 +191,13 @@ void __iomem * __ioremap(phys_addr_t phys_addr, phys_addr_t size, unsigned long return (void __iomem *) (offset + (char *)addr); } +EXPORT_SYMBOL(ioremap_prot); -#define IS_KSEG1(addr) (((unsigned long)(addr) & ~0x1fffffffUL) == CKSEG1) - -void __iounmap(const volatile void __iomem *addr) +void iounmap(const volatile void __iomem *addr) { struct vm_struct *p; - if (IS_KSEG1(addr)) + if (plat_iounmap(addr) || IS_KSEG1(addr)) return; p = remove_vm_area((void *) (PAGE_MASK & (unsigned long __force) addr)); @@ -205,6 +206,4 @@ void __iounmap(const volatile void __iomem *addr) kfree(p); } - -EXPORT_SYMBOL(__ioremap); -EXPORT_SYMBOL(__iounmap); +EXPORT_SYMBOL(iounmap); diff --git a/arch/mips/mm/ioremap64.c b/arch/mips/mm/ioremap64.c new file mode 100644 index 000000000000..15e7820d6a5f --- /dev/null +++ b/arch/mips/mm/ioremap64.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +void __iomem *ioremap_prot(phys_addr_t offset, unsigned long size, + unsigned long prot_val) +{ + unsigned long flags = prot_val & _CACHE_MASK; + u64 base = (flags == _CACHE_UNCACHED ? IO_BASE : UNCAC_BASE); + void __iomem *addr; + + addr = plat_ioremap(offset, size, flags); + if (!addr) + addr = (void __iomem *)(unsigned long)(base + offset); + return addr; +} +EXPORT_SYMBOL(ioremap_prot); + +void iounmap(const volatile void __iomem *addr) +{ + plat_iounmap(addr); +} +EXPORT_SYMBOL(iounmap); From c2591eb5da521f34bd6023ece98b657719831752 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 16 Apr 2020 17:00:11 +0200 Subject: [PATCH 0044/1043] MIPS: use ioremap_page_range Use the generic ioremap_page_range helper instead of reimplementing it. Signed-off-by: Christoph Hellwig Signed-off-by: Thomas Bogendoerfer --- arch/mips/mm/ioremap.c | 112 ++++------------------------------------- 1 file changed, 11 insertions(+), 101 deletions(-) diff --git a/arch/mips/mm/ioremap.c b/arch/mips/mm/ioremap.c index c5b5181c7cd0..b6dad2fd5575 100644 --- a/arch/mips/mm/ioremap.c +++ b/arch/mips/mm/ioremap.c @@ -14,99 +14,14 @@ #include #include #include +#include #include -#include #include #include #define IS_LOW512(addr) (!((phys_addr_t)(addr) & (phys_addr_t) ~0x1fffffffULL)) #define IS_KSEG1(addr) (((unsigned long)(addr) & ~0x1fffffffUL) == CKSEG1) -static inline void remap_area_pte(pte_t * pte, unsigned long address, - phys_addr_t size, phys_addr_t phys_addr, unsigned long flags) -{ - phys_addr_t end; - unsigned long pfn; - pgprot_t pgprot = __pgprot(_PAGE_GLOBAL | _PAGE_PRESENT | __READABLE - | __WRITEABLE | flags); - - address &= ~PMD_MASK; - end = address + size; - if (end > PMD_SIZE) - end = PMD_SIZE; - BUG_ON(address >= end); - pfn = phys_addr >> PAGE_SHIFT; - do { - if (!pte_none(*pte)) { - printk("remap_area_pte: page already exists\n"); - BUG(); - } - set_pte(pte, pfn_pte(pfn, pgprot)); - address += PAGE_SIZE; - pfn++; - pte++; - } while (address && (address < end)); -} - -static inline int remap_area_pmd(pmd_t * pmd, unsigned long address, - phys_addr_t size, phys_addr_t phys_addr, unsigned long flags) -{ - phys_addr_t end; - - address &= ~PGDIR_MASK; - end = address + size; - if (end > PGDIR_SIZE) - end = PGDIR_SIZE; - phys_addr -= address; - BUG_ON(address >= end); - do { - pte_t * pte = pte_alloc_kernel(pmd, address); - if (!pte) - return -ENOMEM; - remap_area_pte(pte, address, end - address, address + phys_addr, flags); - address = (address + PMD_SIZE) & PMD_MASK; - pmd++; - } while (address && (address < end)); - return 0; -} - -static int remap_area_pages(unsigned long address, phys_addr_t phys_addr, - phys_addr_t size, unsigned long flags) -{ - int error; - pgd_t * dir; - unsigned long end = address + size; - - phys_addr -= address; - dir = pgd_offset(&init_mm, address); - flush_cache_all(); - BUG_ON(address >= end); - do { - p4d_t *p4d; - pud_t *pud; - pmd_t *pmd; - - error = -ENOMEM; - p4d = p4d_alloc(&init_mm, dir, address); - if (!p4d) - break; - pud = pud_alloc(&init_mm, p4d, address); - if (!pud) - break; - pmd = pmd_alloc(&init_mm, pud, address); - if (!pmd) - break; - if (remap_area_pmd(pmd, address, end - address, - phys_addr + address, flags)) - break; - error = 0; - address = (address + PGDIR_SIZE) & PGDIR_MASK; - dir++; - } while (address && (address < end)); - flush_tlb_all(); - return error; -} - static int __ioremap_check_ram(unsigned long start_pfn, unsigned long nr_pages, void *arg) { @@ -135,7 +50,7 @@ void __iomem *ioremap_prot(phys_addr_t phys_addr, unsigned long size, unsigned long offset, pfn, last_pfn; struct vm_struct *area; phys_addr_t last_addr; - void *addr; + unsigned long vaddr; void __iomem *cpu_addr; cpu_addr = plat_ioremap(phys_addr, size, flags); @@ -183,27 +98,22 @@ void __iomem *ioremap_prot(phys_addr_t phys_addr, unsigned long size, area = get_vm_area(size, VM_IOREMAP); if (!area) return NULL; - addr = area->addr; - if (remap_area_pages((unsigned long) addr, phys_addr, size, flags)) { - vunmap(addr); + vaddr = (unsigned long)area->addr; + + flags |= _PAGE_GLOBAL | _PAGE_PRESENT | __READABLE | __WRITEABLE; + if (ioremap_page_range(vaddr, vaddr + size, phys_addr, + __pgprot(flags))) { + free_vm_area(area); return NULL; } - return (void __iomem *) (offset + (char *)addr); + return (void __iomem *)(vaddr + offset); } EXPORT_SYMBOL(ioremap_prot); void iounmap(const volatile void __iomem *addr) { - struct vm_struct *p; - - if (plat_iounmap(addr) || IS_KSEG1(addr)) - return; - - p = remove_vm_area((void *) (PAGE_MASK & (unsigned long __force) addr)); - if (!p) - printk(KERN_ERR "iounmap: bad address %p\n", addr); - - kfree(p); + if (!plat_iounmap(addr) && !IS_KSEG1(addr)) + vunmap((void *)((unsigned long)addr & PAGE_MASK)); } EXPORT_SYMBOL(iounmap); From 190607f2d59e174bcf8415efb1bb390737f8d428 Mon Sep 17 00:00:00 2001 From: PrasannaKumar Muralidharan Date: Fri, 28 Feb 2020 17:00:52 +0100 Subject: [PATCH 0045/1043] MIPS: DTS: JZ4780: define node for JZ4780 efuse This patch brings support for the JZ4780 efuse. Currently it only exposes a read only access to the entire 8K bits efuse memory and the ethernet mac address for the davicom dm9000 chip on the CI20 board. It also changes the nemc ranges definition to give the driver access to the efuse registers, which are in the middle of the nemc reg range. Tested-by: Mathieu Malaterre Signed-off-by: PrasannaKumar Muralidharan Signed-off-by: Mathieu Malaterre Signed-off-by: H. Nikolaus Schaller Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ingenic/jz4780.dtsi | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/arch/mips/boot/dts/ingenic/jz4780.dtsi b/arch/mips/boot/dts/ingenic/jz4780.dtsi index bb89653d16a3..1c94f6791127 100644 --- a/arch/mips/boot/dts/ingenic/jz4780.dtsi +++ b/arch/mips/boot/dts/ingenic/jz4780.dtsi @@ -358,11 +358,12 @@ }; nemc: nemc@13410000 { - compatible = "ingenic,jz4780-nemc"; + compatible = "ingenic,jz4780-nemc", "simple-mfd"; reg = <0x13410000 0x10000>; #address-cells = <2>; #size-cells = <1>; - ranges = <1 0 0x1b000000 0x1000000 + ranges = <0 0 0x13410000 0x10000 + 1 0 0x1b000000 0x1000000 2 0 0x1a000000 0x1000000 3 0 0x19000000 0x1000000 4 0 0x18000000 0x1000000 @@ -372,6 +373,20 @@ clocks = <&cgu JZ4780_CLK_NEMC>; status = "disabled"; + + efuse: efuse@d0 { + reg = <0 0xd0 0x30>; + compatible = "ingenic,jz4780-efuse"; + + clocks = <&cgu JZ4780_CLK_AHB2>; + + #address-cells = <1>; + #size-cells = <1>; + + eth0_addr: eth-mac-addr@0x22 { + reg = <0x22 0x6>; + }; + }; }; dma: dma@13420000 { From 19c968222934835058e9ed3d515de744d8275213 Mon Sep 17 00:00:00 2001 From: "H. Nikolaus Schaller" Date: Fri, 28 Feb 2020 17:00:53 +0100 Subject: [PATCH 0046/1043] MIPS: DTS: CI20: make DM9000 Ethernet controller use NVMEM to find the default MAC address There is a unique MAC address programmed into the eFuses of the JZ4780 chip in the CI20 factory. By using this for initializing the DM9000 Ethernet controller, every CI20 board has an individual - but stable - MAC address and DHCP can assign stable IP addresses. Signed-off-by: H. Nikolaus Schaller Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ingenic/ci20.dts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/mips/boot/dts/ingenic/ci20.dts b/arch/mips/boot/dts/ingenic/ci20.dts index db0ca250bd1a..75f5bfbf2c37 100644 --- a/arch/mips/boot/dts/ingenic/ci20.dts +++ b/arch/mips/boot/dts/ingenic/ci20.dts @@ -386,6 +386,9 @@ interrupt-parent = <&gpe>; interrupts = <19 4>; + + nvmem-cells = <ð0_addr>; + nvmem-cell-names = "mac-address"; }; }; From d075fc3154be9de3b00b7bba2fb6009fe8ab611a Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Fri, 3 Apr 2020 17:30:49 +0200 Subject: [PATCH 0047/1043] KVM: s390: vsie: Move conditional reschedule Let's move it to the outer loop, in case we ever run again into long loops, trying to map the prefix. While at it, convert it to cond_resched(). Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200403153050.20569-5-david@redhat.com Reviewed-by: Claudio Imbrenda Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/kvm/vsie.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index 4f6c22d72072..ef05b4e167fb 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -1000,8 +1000,6 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) handle_last_fault(vcpu, vsie_page); - if (need_resched()) - schedule(); if (test_cpu_flag(CIF_MCCK_PENDING)) s390_handle_mcck(); @@ -1185,6 +1183,7 @@ static int vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page) kvm_s390_vcpu_has_irq(vcpu, 0) || kvm_s390_vcpu_sie_inhibited(vcpu)) break; + cond_resched(); } if (rc == -EFAULT) { From 62cf666e4eb8c00dee69bed505db60bf8c3fecf0 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Fri, 3 Apr 2020 17:30:50 +0200 Subject: [PATCH 0048/1043] KVM: s390: vsie: gmap_table_walk() simplifications Let's use asce_type where applicable. Also, simplify our sanity check for valid table levels and convert it into a WARN_ON_ONCE(). Check if we even have a valid gmap shadow as the very first step. Signed-off-by: David Hildenbrand Link: https://lore.kernel.org/r/20200403153050.20569-6-david@redhat.com Reviewed-by: Christian Borntraeger Signed-off-by: Christian Borntraeger --- arch/s390/mm/gmap.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c index 1a95d8809cc3..4b6903fbba4a 100644 --- a/arch/s390/mm/gmap.c +++ b/arch/s390/mm/gmap.c @@ -788,19 +788,19 @@ static inline unsigned long *gmap_table_walk(struct gmap *gmap, unsigned long gaddr, int level) { const int asce_type = gmap->asce & _ASCE_TYPE_MASK; - unsigned long *table; + unsigned long *table = gmap->table; - if ((gmap->asce & _ASCE_TYPE_MASK) + 4 < (level * 4)) - return NULL; if (gmap_is_shadow(gmap) && gmap->removed) return NULL; + if (WARN_ON_ONCE(level > (asce_type >> 2) + 1)) + return NULL; + if (asce_type != _ASCE_TYPE_REGION1 && gaddr & (-1UL << (31 + (asce_type >> 2) * 11))) return NULL; - table = gmap->table; - switch (gmap->asce & _ASCE_TYPE_MASK) { + switch (asce_type) { case _ASCE_TYPE_REGION1: table += (gaddr & _REGION1_INDEX) >> _REGION1_SHIFT; if (level == 4) From 0b545fd17f84184f9536bde68c3521e36c488448 Mon Sep 17 00:00:00 2001 From: Jason Yan Date: Sat, 18 Apr 2020 16:19:26 +0800 Subject: [PATCH 0049/1043] KVM: s390: remove unneeded semicolon in gisa_vcpu_kicker() Fix the following coccicheck warning: arch/s390/kvm/interrupt.c:3085:2-3: Unneeded semicolon Reported-by: Hulk Robot Signed-off-by: Jason Yan Signed-off-by: Christian Borntraeger Reviewed-by: Cornelia Huck Reviewed-by: David Hildenbrand Link: https://lore.kernel.org/r/20200418081926.41666-1-yanaijie@huawei.com Signed-off-by: Christian Borntraeger --- arch/s390/kvm/interrupt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index bfb481134994..a4d4ca2769bd 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -3082,7 +3082,7 @@ static enum hrtimer_restart gisa_vcpu_kicker(struct hrtimer *timer) __airqs_kick_single_vcpu(kvm, pending_mask); hrtimer_forward_now(timer, ns_to_ktime(gi->expires)); return HRTIMER_RESTART; - }; + } return HRTIMER_NORESTART; } From 7a6659a59198acb5e9be050f42dbb1060efe7434 Mon Sep 17 00:00:00 2001 From: Thomas Bogendoerfer Date: Mon, 20 Apr 2020 13:28:54 +0200 Subject: [PATCH 0050/1043] MIPS: alchemy: Fix build error after ioremap cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit IOremap changes caused following build error: arch/mips/alchemy/common/setup.c:99:9: error: implicit declaration of function +‘remap_pfn_range’; did you mean ‘io_remap_pfn_range’? +[-Werror=implicit-function-declaration] Fixed my including linux/mm.h Fixes: d399157283fb ("MIPS: cleanup fixup_bigphys_addr handling") Signed-off-by: Thomas Bogendoerfer --- arch/mips/alchemy/common/setup.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/mips/alchemy/common/setup.c b/arch/mips/alchemy/common/setup.c index a8cbc552bd64..0f60efe0481e 100644 --- a/arch/mips/alchemy/common/setup.c +++ b/arch/mips/alchemy/common/setup.c @@ -27,6 +27,7 @@ #include #include +#include #include #include From a746f50d69bf1cb13b17c1d0855471dc4c9fbd4f Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Mon, 20 Apr 2020 21:45:25 +0800 Subject: [PATCH 0051/1043] MIPS: Loongson64: Remove dead RTC code RTC is now enabled by devicetree. So platform code is no longer needed. Signed-off-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- .../include/asm/mach-loongson64/mc146818rtc.h | 36 ----------------- arch/mips/loongson64/Kconfig | 4 -- arch/mips/loongson64/Makefile | 1 - arch/mips/loongson64/rtc.c | 39 ------------------- arch/mips/loongson64/time.c | 8 +--- 5 files changed, 1 insertion(+), 87 deletions(-) delete mode 100644 arch/mips/include/asm/mach-loongson64/mc146818rtc.h delete mode 100644 arch/mips/loongson64/rtc.c diff --git a/arch/mips/include/asm/mach-loongson64/mc146818rtc.h b/arch/mips/include/asm/mach-loongson64/mc146818rtc.h deleted file mode 100644 index ebdccfee50be..000000000000 --- a/arch/mips/include/asm/mach-loongson64/mc146818rtc.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 1998, 2001, 03, 07 by Ralf Baechle (ralf@linux-mips.org) - * - * RTC routines for PC style attached Dallas chip. - */ -#ifndef __ASM_MACH_LOONGSON64_MC146818RTC_H -#define __ASM_MACH_LOONGSON64_MC146818RTC_H - -#include - -#define RTC_PORT(x) (0x70 + (x)) -#define RTC_IRQ 8 - -static inline unsigned char CMOS_READ(unsigned long addr) -{ - outb_p(addr, RTC_PORT(0)); - return inb_p(RTC_PORT(1)); -} - -static inline void CMOS_WRITE(unsigned char data, unsigned long addr) -{ - outb_p(addr, RTC_PORT(0)); - outb_p(data, RTC_PORT(1)); -} - -#define RTC_ALWAYS_BCD 0 - -#ifndef mc146818_decode_year -#define mc146818_decode_year(year) ((year) < 70 ? (year) + 2000 : (year) + 1970) -#endif - -#endif /* __ASM_MACH_LOONGSON64_MC146818RTC_H */ diff --git a/arch/mips/loongson64/Kconfig b/arch/mips/loongson64/Kconfig index 48b29c198acf..c386b8a3c753 100644 --- a/arch/mips/loongson64/Kconfig +++ b/arch/mips/loongson64/Kconfig @@ -14,8 +14,4 @@ config RS780_HPET If unsure, say Yes. -config LOONGSON_MC146818 - bool - default n - endif # MACH_LOONGSON64 diff --git a/arch/mips/loongson64/Makefile b/arch/mips/loongson64/Makefile index b7f40b179c71..32b8c224852f 100644 --- a/arch/mips/loongson64/Makefile +++ b/arch/mips/loongson64/Makefile @@ -9,5 +9,4 @@ obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_RS780_HPET) += hpet.o obj-$(CONFIG_PCI) += pci.o -obj-$(CONFIG_LOONGSON_MC146818) += rtc.o obj-$(CONFIG_SUSPEND) += pm.o diff --git a/arch/mips/loongson64/rtc.c b/arch/mips/loongson64/rtc.c deleted file mode 100644 index 8d7628c0f513..000000000000 --- a/arch/mips/loongson64/rtc.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Lemote Fuloong platform support - * - * Copyright(c) 2010 Arnaud Patard - */ - -#include -#include -#include -#include - -static struct resource loongson_rtc_resources[] = { - { - .start = RTC_PORT(0), - .end = RTC_PORT(1), - .flags = IORESOURCE_IO, - }, { - .start = RTC_IRQ, - .end = RTC_IRQ, - .flags = IORESOURCE_IRQ, - } -}; - -static struct platform_device loongson_rtc_device = { - .name = "rtc_cmos", - .id = -1, - .resource = loongson_rtc_resources, - .num_resources = ARRAY_SIZE(loongson_rtc_resources), -}; - - -static int __init loongson_rtc_platform_init(void) -{ - platform_device_register(&loongson_rtc_device); - return 0; -} - -device_initcall(loongson_rtc_platform_init); diff --git a/arch/mips/loongson64/time.c b/arch/mips/loongson64/time.c index 1245f22cec84..91e842b58365 100644 --- a/arch/mips/loongson64/time.c +++ b/arch/mips/loongson64/time.c @@ -6,7 +6,7 @@ * Copyright (C) 2009 Lemote Inc. * Author: Wu Zhangjin, wuzhangjin@gmail.com */ -#include + #include #include @@ -21,9 +21,3 @@ void __init plat_time_init(void) setup_hpet_timer(); #endif } - -void read_persistent_clock64(struct timespec64 *ts) -{ - ts->tv_sec = mc146818_get_cmos_time(); - ts->tv_nsec = 0; -} From 0cfd2440aa03ea3d4b04cc2565561cecadcb1257 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Mon, 20 Apr 2020 21:45:26 +0800 Subject: [PATCH 0052/1043] MIPS: Loongson64: Make RS780E ACPI as a platform driver Make RS780E ACPI as a platform driver so we can enable it by DeviceTree selectively. Signed-off-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/loongson64/Makefile | 2 +- arch/mips/loongson64/pci.c | 2 - drivers/platform/mips/Kconfig | 6 ++ drivers/platform/mips/Makefile | 1 + .../platform/mips/rs780e-acpi.c | 58 ++++++++++++------- 5 files changed, 46 insertions(+), 23 deletions(-) rename arch/mips/loongson64/acpi_init.c => drivers/platform/mips/rs780e-acpi.c (70%) diff --git a/arch/mips/loongson64/Makefile b/arch/mips/loongson64/Makefile index 32b8c224852f..6f3c2b47f66f 100644 --- a/arch/mips/loongson64/Makefile +++ b/arch/mips/loongson64/Makefile @@ -2,7 +2,7 @@ # # Makefile for Loongson-3 family machines # -obj-$(CONFIG_MACH_LOONGSON64) += cop2-ex.o platform.o acpi_init.o dma.o \ +obj-$(CONFIG_MACH_LOONGSON64) += cop2-ex.o platform.o dma.o \ setup.o init.o env.o time.o reset.o \ obj-$(CONFIG_SMP) += smp.o diff --git a/arch/mips/loongson64/pci.c b/arch/mips/loongson64/pci.c index e84ae20c3290..a440a2725a20 100644 --- a/arch/mips/loongson64/pci.c +++ b/arch/mips/loongson64/pci.c @@ -43,8 +43,6 @@ static int __init pcibios_init(void) register_pci_controller(&loongson_pci_controller); - sbx00_acpi_init(); - return 0; } diff --git a/drivers/platform/mips/Kconfig b/drivers/platform/mips/Kconfig index 5e77b0dc5fd6..8ac149173c64 100644 --- a/drivers/platform/mips/Kconfig +++ b/drivers/platform/mips/Kconfig @@ -24,4 +24,10 @@ config CPU_HWMON help Loongson-3A/3B CPU Hwmon (temperature sensor) driver. +config RS780E_ACPI + bool "Loongson RS780E ACPI Controller" + depends on MACH_LOONGSON64 || COMPILE_TEST + help + Loongson RS780E PCH ACPI Controller driver. + endif # MIPS_PLATFORM_DEVICES diff --git a/drivers/platform/mips/Makefile b/drivers/platform/mips/Makefile index be8146c20dc8..178149098777 100644 --- a/drivers/platform/mips/Makefile +++ b/drivers/platform/mips/Makefile @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_CPU_HWMON) += cpu_hwmon.o +obj-$(CONFIG_RS780E_ACPI) += rs780e-acpi.o diff --git a/arch/mips/loongson64/acpi_init.c b/drivers/platform/mips/rs780e-acpi.c similarity index 70% rename from arch/mips/loongson64/acpi_init.c rename to drivers/platform/mips/rs780e-acpi.c index 8d7c119ddf91..e5a643b78ac9 100644 --- a/arch/mips/loongson64/acpi_init.c +++ b/drivers/platform/mips/rs780e-acpi.c @@ -3,32 +3,23 @@ #include #include #include +#include +#include -#define SBX00_ACPI_IO_BASE 0x800 -#define SBX00_ACPI_IO_SIZE 0x100 +static unsigned long acpi_iobase; -#define ACPI_PM_EVT_BLK (SBX00_ACPI_IO_BASE + 0x00) /* 4 bytes */ -#define ACPI_PM_CNT_BLK (SBX00_ACPI_IO_BASE + 0x04) /* 2 bytes */ -#define ACPI_PMA_CNT_BLK (SBX00_ACPI_IO_BASE + 0x0F) /* 1 byte */ -#define ACPI_PM_TMR_BLK (SBX00_ACPI_IO_BASE + 0x18) /* 4 bytes */ -#define ACPI_GPE0_BLK (SBX00_ACPI_IO_BASE + 0x10) /* 8 bytes */ -#define ACPI_END (SBX00_ACPI_IO_BASE + 0x80) +#define ACPI_PM_EVT_BLK (acpi_iobase + 0x00) /* 4 bytes */ +#define ACPI_PM_CNT_BLK (acpi_iobase + 0x04) /* 2 bytes */ +#define ACPI_PMA_CNT_BLK (acpi_iobase + 0x0F) /* 1 byte */ +#define ACPI_PM_TMR_BLK (acpi_iobase + 0x18) /* 4 bytes */ +#define ACPI_GPE0_BLK (acpi_iobase + 0x10) /* 8 bytes */ +#define ACPI_END (acpi_iobase + 0x80) #define PM_INDEX 0xCD6 #define PM_DATA 0xCD7 #define PM2_INDEX 0xCD0 #define PM2_DATA 0xCD1 -/* - * SCI interrupt need acpi space, allocate here - */ - -static int __init register_acpi_resource(void) -{ - request_region(SBX00_ACPI_IO_BASE, SBX00_ACPI_IO_SIZE, "acpi"); - return 0; -} - static void pmio_write_index(u16 index, u8 reg, u8 value) { outb(reg, index); @@ -141,11 +132,38 @@ void acpi_registers_setup(void) pm2_iowrite(0xf8, value); } -int __init sbx00_acpi_init(void) +static int rs780e_acpi_probe(struct platform_device *pdev) { - register_acpi_resource(); + struct resource *res; + + res = platform_get_resource(pdev, IORESOURCE_IO, 0); + if (!res) + return -ENODEV; + + /* SCI interrupt need acpi space, allocate here */ + if (!request_region(res->start, resource_size(res), "acpi")) { + pr_err("RS780E-ACPI: Failed to request IO Region\n"); + return -EBUSY; + } + + acpi_iobase = res->start; + acpi_registers_setup(); acpi_hw_clear_status(); return 0; } + +static const struct of_device_id rs780e_acpi_match[] = { + { .compatible = "loongson,rs780e-acpi" }, + {}, +}; + +static struct platform_driver rs780e_acpi_driver = { + .probe = rs780e_acpi_probe, + .driver = { + .name = "RS780E-ACPI", + .of_match_table = rs780e_acpi_match, + }, +}; +builtin_platform_driver(rs780e_acpi_driver); From dbfd92fe57b125ec72ee6f9c75c40275826579b6 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Mon, 20 Apr 2020 21:45:27 +0800 Subject: [PATCH 0053/1043] dt-bindings: Document Loongson RS780E PCH ACPI Controller This controller is attached under ISA Bus and can be found in Loongson-3 systems with RS780E PCH. Signed-off-by: Jiaxun Yang Reviewed-by: Rob Herring Signed-off-by: Thomas Bogendoerfer --- .../bindings/mips/loongson/rs780e-acpi.yaml | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 Documentation/devicetree/bindings/mips/loongson/rs780e-acpi.yaml diff --git a/Documentation/devicetree/bindings/mips/loongson/rs780e-acpi.yaml b/Documentation/devicetree/bindings/mips/loongson/rs780e-acpi.yaml new file mode 100644 index 000000000000..d317897e1115 --- /dev/null +++ b/Documentation/devicetree/bindings/mips/loongson/rs780e-acpi.yaml @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/mips/loongson/rs780e-acpi.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: Loongson RS780E PCH ACPI Controller + +maintainers: + - Jiaxun Yang + +description: | + This controller can be found in Loongson-3 systems with RS780E PCH. + +properties: + compatible: + const: loongson,rs780e-acpi + + reg: + maxItems: 1 + +required: + - compatible + - reg + +examples: + - | + isa@0 { + compatible = "isa"; + #address-cells = <2>; + #size-cells = <1>; + ranges = <1 0 0 0x1000>; + + acpi@800 { + compatible = "loongson,rs780e-acpi"; + reg = <1 0x800 0x100>; + }; + }; + +... From a89aa749ece9c6fee7932163472d2ee0efd6ddd3 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Mon, 20 Apr 2020 21:45:28 +0800 Subject: [PATCH 0054/1043] MIPS: DTS: Loongson64: Add ACPI Controller Node Add ACPI Controller Node for RS780E PCH to fit newly added driver. Signed-off-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/loongson/rs780e-pch.dtsi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/mips/boot/dts/loongson/rs780e-pch.dtsi b/arch/mips/boot/dts/loongson/rs780e-pch.dtsi index 45c54d555fa4..8687c4f7370a 100644 --- a/arch/mips/boot/dts/loongson/rs780e-pch.dtsi +++ b/arch/mips/boot/dts/loongson/rs780e-pch.dtsi @@ -21,6 +21,11 @@ interrupts = <8>; interrupt-parent = <&htpic>; }; + + acpi@800 { + compatible = "loongson,rs780e-acpi"; + reg = <1 0x800 0x100>; + }; }; }; }; From f46e1dc982a04c1d2b46694a0c1350e76d4b0817 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Mon, 20 Apr 2020 21:45:29 +0800 Subject: [PATCH 0055/1043] MIPS: Loongson64: Mark RS780 HPET as broken This driver is using some dangerous hack to set MMIO address for HPET, which might break systems with other kinds of PCH. Also, as Loongson-3 cpufreq driver never appeared in mainline, this driver rarely got used. So we temporarily mark it as broken until we find a better solution. Signed-off-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/loongson64/Kconfig | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/arch/mips/loongson64/Kconfig b/arch/mips/loongson64/Kconfig index c386b8a3c753..517f1f8e81fb 100644 --- a/arch/mips/loongson64/Kconfig +++ b/arch/mips/loongson64/Kconfig @@ -4,14 +4,12 @@ if MACH_LOONGSON64 config RS780_HPET bool "RS780/SBX00 HPET Timer" depends on MACH_LOONGSON64 + depends on BROKEN select MIPS_EXTERNAL_TIMER help This option enables the hpet timer of AMD RS780/SBX00. - If you want to enable the Loongson3 CPUFreq Driver, Please enable - this option at first, otherwise, You will get wrong system time. - - If unsure, say Yes. - + Note: This driver is doing some dangerous hack. Please only enable + it on RS780E systems. endif # MACH_LOONGSON64 From 5efac0741ce238e0844d3f7af00198f81e84926a Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 23 Mar 2020 20:42:57 -0400 Subject: [PATCH 0056/1043] KVM: x86: introduce kvm_mmu_invalidate_gva Wrap the combination of mmu->invlpg and kvm_x86_ops->tlb_flush_gva into a new function. This function also lets us specify the host PGD to invalidate and also the MMU, both of which will be useful in fixing and simplifying kvm_inject_emulated_page_fault. A nested guest's MMU however has g_context->invlpg == NULL. Instead of setting it to nonpaging_invlpg, make kvm_mmu_invalidate_gva the only entry point to mmu->invlpg and make a NULL invlpg pointer equivalent to nonpaging_invlpg, saving a retpoline. Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 + arch/x86/kvm/mmu/mmu.c | 71 +++++++++++++++++++++------------ 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 34ad7297b06b..8c1e094a639f 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1511,6 +1511,8 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu); int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code, void *insn, int insn_len); void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva); +void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, + gva_t gva, hpa_t root_hpa); void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid); void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 8071952e9cf2..84eeaa4ea149 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2153,10 +2153,6 @@ static int nonpaging_sync_page(struct kvm_vcpu *vcpu, return 0; } -static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva, hpa_t root) -{ -} - static void nonpaging_update_pte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, u64 *spte, const void *pte) @@ -4237,7 +4233,7 @@ static void nonpaging_init_context(struct kvm_vcpu *vcpu, context->page_fault = nonpaging_page_fault; context->gva_to_gpa = nonpaging_gva_to_gpa; context->sync_page = nonpaging_sync_page; - context->invlpg = nonpaging_invlpg; + context->invlpg = NULL; context->update_pte = nonpaging_update_pte; context->root_level = 0; context->shadow_root_level = PT32E_ROOT_LEVEL; @@ -4928,7 +4924,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) context->mmu_role.as_u64 = new_role.as_u64; context->page_fault = kvm_tdp_page_fault; context->sync_page = nonpaging_sync_page; - context->invlpg = nonpaging_invlpg; + context->invlpg = NULL; context->update_pte = nonpaging_update_pte; context->shadow_root_level = kvm_x86_ops.get_tdp_level(vcpu); context->direct_map = true; @@ -5096,6 +5092,12 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu) g_context->get_pdptr = kvm_pdptr_read; g_context->inject_page_fault = kvm_inject_page_fault; + /* + * L2 page tables are never shadowed, so there is no need to sync + * SPTEs. + */ + g_context->invlpg = NULL; + /* * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using * L1's nested page tables (e.g. EPT12). The nested translation @@ -5497,37 +5499,54 @@ emulate: } EXPORT_SYMBOL_GPL(kvm_mmu_page_fault); -void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva) +void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, + gva_t gva, hpa_t root_hpa) { - struct kvm_mmu *mmu = vcpu->arch.mmu; int i; - /* INVLPG on a * non-canonical address is a NOP according to the SDM. */ - if (is_noncanonical_address(gva, vcpu)) + /* It's actually a GPA for vcpu->arch.guest_mmu. */ + if (mmu != &vcpu->arch.guest_mmu) { + /* INVLPG on a non-canonical address is a NOP according to the SDM. */ + if (is_noncanonical_address(gva, vcpu)) + return; + + kvm_x86_ops.tlb_flush_gva(vcpu, gva); + } + + if (!mmu->invlpg) return; - mmu->invlpg(vcpu, gva, mmu->root_hpa); + if (root_hpa == INVALID_PAGE) { + mmu->invlpg(vcpu, gva, mmu->root_hpa); - /* - * INVLPG is required to invalidate any global mappings for the VA, - * irrespective of PCID. Since it would take us roughly similar amount - * of work to determine whether any of the prev_root mappings of the VA - * is marked global, or to just sync it blindly, so we might as well - * just always sync it. - * - * Mappings not reachable via the current cr3 or the prev_roots will be - * synced when switching to that cr3, so nothing needs to be done here - * for them. - */ - for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) - if (VALID_PAGE(mmu->prev_roots[i].hpa)) - mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa); + /* + * INVLPG is required to invalidate any global mappings for the VA, + * irrespective of PCID. Since it would take us roughly similar amount + * of work to determine whether any of the prev_root mappings of the VA + * is marked global, or to just sync it blindly, so we might as well + * just always sync it. + * + * Mappings not reachable via the current cr3 or the prev_roots will be + * synced when switching to that cr3, so nothing needs to be done here + * for them. + */ + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) + if (VALID_PAGE(mmu->prev_roots[i].hpa)) + mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa); + } else { + mmu->invlpg(vcpu, gva, root_hpa); + } +} +EXPORT_SYMBOL_GPL(kvm_mmu_invalidate_gva); - kvm_x86_ops.tlb_flush_gva(vcpu, gva); +void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva) +{ + kvm_mmu_invalidate_gva(vcpu, vcpu->arch.mmu, gva, INVALID_PAGE); ++vcpu->stat.invlpg; } EXPORT_SYMBOL_GPL(kvm_mmu_invlpg); + void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) { struct kvm_mmu *mmu = vcpu->arch.mmu; From 0cd665bd20f9088d363158b4ac75592af18ecf4f Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 25 Mar 2020 12:50:03 -0400 Subject: [PATCH 0057/1043] KVM: x86: cleanup kvm_inject_emulated_page_fault To reconstruct the kvm_mmu to be used for page fault injection, we can simply use fault->nested_page_fault. This matches how fault->nested_page_fault is assigned in the first place by FNAME(walk_addr_generic). Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 6 ------ arch/x86/kvm/mmu/paging_tmpl.h | 2 +- arch/x86/kvm/x86.c | 8 ++++---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 84eeaa4ea149..32c9f4b2a281 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4353,12 +4353,6 @@ static unsigned long get_cr3(struct kvm_vcpu *vcpu) return kvm_read_cr3(vcpu); } -static void inject_page_fault(struct kvm_vcpu *vcpu, - struct x86_exception *fault) -{ - vcpu->arch.mmu->inject_page_fault(vcpu, fault); -} - static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, unsigned int access, int *nr_present) { diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 9bdf9b7d9a96..efec7d27b8c5 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -812,7 +812,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, if (!r) { pgprintk("%s: guest page fault\n", __func__); if (!prefault) - inject_page_fault(vcpu, &walker.fault); + kvm_inject_emulated_page_fault(vcpu, &walker.fault); return RET_PF_RETRY; } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 003e625367b7..2ab821f6281f 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -614,12 +614,12 @@ EXPORT_SYMBOL_GPL(kvm_inject_page_fault); bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault) { + struct kvm_mmu *fault_mmu; WARN_ON_ONCE(fault->vector != PF_VECTOR); - if (mmu_is_nested(vcpu) && !fault->nested_page_fault) - vcpu->arch.nested_mmu.inject_page_fault(vcpu, fault); - else - vcpu->arch.mmu->inject_page_fault(vcpu, fault); + fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu : + vcpu->arch.walk_mmu; + fault_mmu->inject_page_fault(vcpu, fault); return fault->nested_page_fault; } From ee1fa209f5e5ca5c1e76c7aa1c2aab292f371f4a Mon Sep 17 00:00:00 2001 From: Junaid Shahid Date: Fri, 20 Mar 2020 14:28:03 -0700 Subject: [PATCH 0058/1043] KVM: x86: Sync SPTEs when injecting page/EPT fault into L1 When injecting a page fault or EPT violation/misconfiguration, KVM is not syncing any shadow PTEs associated with the faulting address, including those in previous MMUs that are associated with L1's current EPTP (in a nested EPT scenario), nor is it flushing any hardware TLB entries. All this is done by kvm_mmu_invalidate_gva. Page faults that are either !PRESENT or RSVD are exempt from the flushing, as the CPU is not allowed to cache such translations. Signed-off-by: Junaid Shahid Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-8-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 12 ++++++------ arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/kvm/x86.c | 11 ++++++++++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 8f3ff07c7e66..d0857bfcb568 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -4560,7 +4560,7 @@ static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer) return 1; if (kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e)) { - kvm_inject_page_fault(vcpu, &e); + kvm_inject_emulated_page_fault(vcpu, &e); return 1; } @@ -4869,7 +4869,7 @@ static int handle_vmread(struct kvm_vcpu *vcpu) return 1; /* _system ok, nested_vmx_check_permission has verified cpl=0 */ if (kvm_write_guest_virt_system(vcpu, gva, &value, len, &e)) { - kvm_inject_page_fault(vcpu, &e); + kvm_inject_emulated_page_fault(vcpu, &e); return 1; } } @@ -4943,7 +4943,7 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu) instr_info, false, len, &gva)) return 1; if (kvm_read_guest_virt(vcpu, gva, &value, len, &e)) { - kvm_inject_page_fault(vcpu, &e); + kvm_inject_emulated_page_fault(vcpu, &e); return 1; } } @@ -5108,7 +5108,7 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu) /* *_system ok, nested_vmx_check_permission has verified cpl=0 */ if (kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, sizeof(gpa_t), &e)) { - kvm_inject_page_fault(vcpu, &e); + kvm_inject_emulated_page_fault(vcpu, &e); return 1; } return nested_vmx_succeed(vcpu); @@ -5152,7 +5152,7 @@ static int handle_invept(struct kvm_vcpu *vcpu) vmx_instruction_info, false, sizeof(operand), &gva)) return 1; if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { - kvm_inject_page_fault(vcpu, &e); + kvm_inject_emulated_page_fault(vcpu, &e); return 1; } @@ -5220,7 +5220,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) vmx_instruction_info, false, sizeof(operand), &gva)) return 1; if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { - kvm_inject_page_fault(vcpu, &e); + kvm_inject_emulated_page_fault(vcpu, &e); return 1; } if (operand.vpid >> 16) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 01330096ff3e..e671bcfa29c4 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5393,7 +5393,7 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) return 1; if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { - kvm_inject_page_fault(vcpu, &e); + kvm_inject_emulated_page_fault(vcpu, &e); return 1; } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 2ab821f6281f..3984574e09bf 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -619,8 +619,17 @@ bool kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu, fault_mmu = fault->nested_page_fault ? vcpu->arch.mmu : vcpu->arch.walk_mmu; - fault_mmu->inject_page_fault(vcpu, fault); + /* + * Invalidate the TLB entry for the faulting address, if it exists, + * else the access will fault indefinitely (and to emulate hardware). + */ + if ((fault->error_code & PFERR_PRESENT_MASK) && + !(fault->error_code & PFERR_RSVD_MASK)) + kvm_mmu_invalidate_gva(vcpu, fault_mmu, fault->address, + fault_mmu->root_hpa); + + fault_mmu->inject_page_fault(vcpu, fault); return fault->nested_page_fault; } EXPORT_SYMBOL_GPL(kvm_inject_emulated_page_fault); From c746b3a4b84cc6eb2f8a45e89210f57d252d98fd Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:04 -0700 Subject: [PATCH 0059/1043] KVM: VMX: Skip global INVVPID fallback if vpid==0 in vpid_sync_context() Skip the global INVVPID in the unlikely scenario that vpid==0 and the SINGLE_CONTEXT variant of INVVPID is unsupported. If vpid==0, there's no need to INVVPID as it's impossible to do VM-Enter with VPID enabled and vmcs.VPID==0, i.e. there can't be any TLB entries for the vCPU with vpid==0. The fact that the SINGLE_CONTEXT variant isn't supported is irrelevant. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-9-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/ops.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/ops.h b/arch/x86/kvm/vmx/ops.h index 19717d0a1100..a1cb6d4114dc 100644 --- a/arch/x86/kvm/vmx/ops.h +++ b/arch/x86/kvm/vmx/ops.h @@ -300,7 +300,7 @@ static inline void vpid_sync_context(int vpid) { if (cpu_has_vmx_invvpid_single()) vpid_sync_vcpu_single(vpid); - else + else if (vpid != 0) vpid_sync_vcpu_global(); } From 446ace4bca423297d62342801799ba4952d6b910 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:05 -0700 Subject: [PATCH 0060/1043] KVM: VMX: Use vpid_sync_context() directly when possible Use vpid_sync_context() directly for flows that run if and only if enable_vpid=1, or more specifically, nested VMX flows that are gated by vmx->nested.msrs.secondary_ctls_high.SECONDARY_EXEC_ENABLE_VPID being set, which is allowed if and only if enable_vpid=1. Because these flows call __vmx_flush_tlb() with @invalidate_gpa=false, the if-statement that decides between INVEPT and INVVPID will always go down the INVVPID path, i.e. call vpid_sync_context() because "enable_ept && (invalidate_gpa || !enable_vpid)" always evaluates false. This helps pave the way toward removing @invalidate_gpa and @vpid from __vmx_flush_tlb() and its callers. Opportunstically drop unnecessary brackets in handle_invvpid() around an affected __vmx_flush_tlb()->vpid_sync_context() conversion. No functional change intended. Reviewed-by: Miaohe Lin Reviewed-by: Vitaly Kuznetsov Reviewed-by: Paolo Bonzini Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-10-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index d0857bfcb568..919fb7409535 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -2459,7 +2459,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, if (nested_cpu_has_vpid(vmcs12) && nested_has_guest_tlb_tag(vcpu)) { if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) { vmx->nested.last_vpid = vmcs12->virtual_processor_id; - __vmx_flush_tlb(vcpu, nested_get_vpid02(vcpu), false); + vpid_sync_context(nested_get_vpid02(vcpu)); } } else { /* @@ -5234,21 +5234,21 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) is_noncanonical_address(operand.gla, vcpu)) return nested_vmx_failValid(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); - if (cpu_has_vmx_invvpid_individual_addr()) { + if (cpu_has_vmx_invvpid_individual_addr()) __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid02, operand.gla); - } else - __vmx_flush_tlb(vcpu, vpid02, false); + else + vpid_sync_context(vpid02); break; case VMX_VPID_EXTENT_SINGLE_CONTEXT: case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: if (!operand.vpid) return nested_vmx_failValid(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); - __vmx_flush_tlb(vcpu, vpid02, false); + vpid_sync_context(vpid02); break; case VMX_VPID_EXTENT_ALL_CONTEXT: - __vmx_flush_tlb(vcpu, vpid02, false); + vpid_sync_context(vpid02); break; default: WARN_ON_ONCE(1); From 8a8b097c6cd0041da6ba3a0701f911bfee05c652 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:06 -0700 Subject: [PATCH 0061/1043] KVM: VMX: Move vpid_sync_vcpu_addr() down a few lines Move vpid_sync_vcpu_addr() below vpid_sync_context() so that it can be refactored in a future patch to call vpid_sync_context() directly when the "individual address" INVVPID variant isn't supported. No functional change intended. Reviewed-by: Miaohe Lin Reviewed-by: Vitaly Kuznetsov Reviewed-by: Paolo Bonzini Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-11-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/ops.h | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/arch/x86/kvm/vmx/ops.h b/arch/x86/kvm/vmx/ops.h index a1cb6d4114dc..6e7ceb0a1c0f 100644 --- a/arch/x86/kvm/vmx/ops.h +++ b/arch/x86/kvm/vmx/ops.h @@ -268,19 +268,6 @@ static inline void __invept(unsigned long ext, u64 eptp, gpa_t gpa) vmx_asm2(invept, "r"(ext), "m"(operand), ext, eptp, gpa); } -static inline bool vpid_sync_vcpu_addr(int vpid, gva_t addr) -{ - if (vpid == 0) - return true; - - if (cpu_has_vmx_invvpid_individual_addr()) { - __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid, addr); - return true; - } - - return false; -} - static inline void vpid_sync_vcpu_single(int vpid) { if (vpid == 0) @@ -304,6 +291,19 @@ static inline void vpid_sync_context(int vpid) vpid_sync_vcpu_global(); } +static inline bool vpid_sync_vcpu_addr(int vpid, gva_t addr) +{ + if (vpid == 0) + return true; + + if (cpu_has_vmx_invvpid_individual_addr()) { + __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid, addr); + return true; + } + + return false; +} + static inline void ept_sync_global(void) { __invept(VMX_EPT_EXTENT_GLOBAL, 0, 0); From ab4b3597ffce01e9080ce5760e5a407a134701ae Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:07 -0700 Subject: [PATCH 0062/1043] KVM: VMX: Handle INVVPID fallback logic in vpid_sync_vcpu_addr() Directly invoke vpid_sync_context() to do a global INVVPID when the individual address variant is not supported instead of deferring such behavior to the caller. This allows for additional consolidation of code as the logic is basically identical to the emulation of the individual address variant in handle_invvpid(). No functional change intended. Reviewed-by: Miaohe Lin Reviewed-by: Vitaly Kuznetsov Reviewed-by: Paolo Bonzini Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-12-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/ops.h | 12 +++++------- arch/x86/kvm/vmx/vmx.c | 3 +-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/vmx/ops.h b/arch/x86/kvm/vmx/ops.h index 6e7ceb0a1c0f..c43853d2cdcc 100644 --- a/arch/x86/kvm/vmx/ops.h +++ b/arch/x86/kvm/vmx/ops.h @@ -291,17 +291,15 @@ static inline void vpid_sync_context(int vpid) vpid_sync_vcpu_global(); } -static inline bool vpid_sync_vcpu_addr(int vpid, gva_t addr) +static inline void vpid_sync_vcpu_addr(int vpid, gva_t addr) { if (vpid == 0) - return true; + return; - if (cpu_has_vmx_invvpid_individual_addr()) { + if (cpu_has_vmx_invvpid_individual_addr()) __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, vpid, addr); - return true; - } - - return false; + else + vpid_sync_context(vpid); } static inline void ept_sync_global(void) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index e671bcfa29c4..d973ff095066 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2842,8 +2842,7 @@ static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) { int vpid = to_vmx(vcpu)->vpid; - if (!vpid_sync_vcpu_addr(vpid, addr)) - vpid_sync_context(vpid); + vpid_sync_vcpu_addr(vpid, addr); /* * If VPIDs are not supported or enabled, then the above is a no-op. From ca431c0cc3317e36c7011df2d2319385465db442 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:08 -0700 Subject: [PATCH 0063/1043] KVM: VMX: Drop redundant capability checks in low level INVVPID helpers Remove the INVVPID capabilities checks from vpid_sync_vcpu_single() and vpid_sync_vcpu_global() now that all callers ensure the INVVPID variant is supported. Note, in some cases the guarantee is provided in concert with hardware_setup(), which enables VPID if and only if at least of invvpid_single() or invvpid_global() is supported. Drop the WARN_ON_ONCE() from vmx_flush_tlb() as vpid_sync_vcpu_single() will trigger a WARN() on INVVPID failure, i.e. if SINGLE_CONTEXT isn't supported. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-13-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/ops.h | 6 ++---- arch/x86/kvm/vmx/vmx.h | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/vmx/ops.h b/arch/x86/kvm/vmx/ops.h index c43853d2cdcc..5f1ac002b4b6 100644 --- a/arch/x86/kvm/vmx/ops.h +++ b/arch/x86/kvm/vmx/ops.h @@ -273,14 +273,12 @@ static inline void vpid_sync_vcpu_single(int vpid) if (vpid == 0) return; - if (cpu_has_vmx_invvpid_single()) - __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0); + __invvpid(VMX_VPID_EXTENT_SINGLE_CONTEXT, vpid, 0); } static inline void vpid_sync_vcpu_global(void) { - if (cpu_has_vmx_invvpid_global()) - __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0); + __invvpid(VMX_VPID_EXTENT_ALL_CONTEXT, 0, 0); } static inline void vpid_sync_context(int vpid) diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index fa40319f1698..1560296dde25 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -534,7 +534,6 @@ static inline void vmx_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa) if (cpu_has_vmx_invvpid_global()) { vpid_sync_vcpu_global(); } else { - WARN_ON_ONCE(!cpu_has_vmx_invvpid_single()); vpid_sync_vcpu_single(vmx->vpid); vpid_sync_vcpu_single(vmx->nested.vpid02); } From bc41d0c40ec2a09b775e2c65f425f903733f5b22 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:09 -0700 Subject: [PATCH 0064/1043] KVM: nVMX: Use vpid_sync_vcpu_addr() to emulate INVVPID with address Use vpid_sync_vcpu_addr() to emulate the "individual address" variant of INVVPID now that said function handles the fallback case of the (host) CPU not supporting "individual address". Note, the "vpid == 0" checks in the vpid_sync_*() helpers aren't actually redundant with the "!operand.vpid" check in handle_invvpid(), as the vpid passed to vpid_sync_vcpu_addr() is a KVM (host) controlled value, i.e. vpid02 can be zero even if operand.vpid is non-zero. No functional change intended. Reviewed-by: Vitaly Kuznetsov Reviewed-by: Paolo Bonzini Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-14-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 919fb7409535..cbc687e9209d 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5234,11 +5234,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) is_noncanonical_address(operand.gla, vcpu)) return nested_vmx_failValid(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); - if (cpu_has_vmx_invvpid_individual_addr()) - __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR, - vpid02, operand.gla); - else - vpid_sync_context(vpid02); + vpid_sync_vcpu_addr(vpid02, operand.gla); break; case VMX_VPID_EXTENT_SINGLE_CONTEXT: case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: From e64419d991ea212af087d3c57fcabb4d27db03fc Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:10 -0700 Subject: [PATCH 0065/1043] KVM: x86: Move "flush guest's TLB" logic to separate kvm_x86_ops hook Add a dedicated hook to handle flushing TLB entries on behalf of the guest, i.e. for a paravirtualized TLB flush, and use it directly instead of bouncing through kvm_vcpu_flush_tlb(). For VMX, change the effective implementation implementation to never do INVEPT and flush only the current context, i.e. to always flush via INVVPID(SINGLE_CONTEXT). The INVEPT performed by __vmx_flush_tlb() when @invalidate_gpa=false and enable_vpid=0 is unnecessary, as it will only flush guest-physical mappings; linear and combined mappings are flushed by VM-Enter when VPID is disabled, and changes in the guest pages tables do not affect guest-physical mappings. When EPT and VPID are enabled, doing INVVPID is not required (by Intel's architecture) to invalidate guest-physical mappings, i.e. TLB entries that cache guest-physical mappings can live across INVVPID as the mappings are associated with an EPTP, not a VPID. The intent of @invalidate_gpa is to inform vmx_flush_tlb() that it must "invalidate gpa mappings", i.e. do INVEPT and not simply INVVPID. Other than nested VPID handling, which now calls vpid_sync_context() directly, the only scenario where KVM can safely do INVVPID instead of INVEPT (when EPT is enabled) is if KVM is flushing TLB entries from the guest's perspective, i.e. is only required to invalidate linear mappings. For SVM, flushing TLB entries from the guest's perspective can be done by flushing the current ASID, as changes to the guest's page tables are associated only with the current ASID. Adding a dedicated ->tlb_flush_guest() paves the way toward removing @invalidate_gpa, which is a potentially dangerous control flag as its meaning is not exactly crystal clear, even for those who are familiar with the subtleties of what mappings Intel CPUs are/aren't allowed to keep across various invalidation scenarios. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-15-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 6 ++++++ arch/x86/kvm/svm/svm.c | 6 ++++++ arch/x86/kvm/vmx/vmx.c | 13 +++++++++++++ arch/x86/kvm/x86.c | 2 +- 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 8c1e094a639f..a7c1ea136c8c 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1114,6 +1114,12 @@ struct kvm_x86_ops { */ void (*tlb_flush_gva)(struct kvm_vcpu *vcpu, gva_t addr); + /* + * Flush any TLB entries created by the guest. Like tlb_flush_gva(), + * does not need to flush GPA->HPA mappings. + */ + void (*tlb_flush_guest)(struct kvm_vcpu *vcpu); + void (*run)(struct kvm_vcpu *vcpu); int (*handle_exit)(struct kvm_vcpu *vcpu, enum exit_fastpath_completion exit_fastpath); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 2f379bacbb26..2c0fcb87a054 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3170,6 +3170,11 @@ static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) invlpga(gva, svm->vmcb->control.asid); } +static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu) +{ + svm_flush_tlb(vcpu, false); +} + static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu) { } @@ -3939,6 +3944,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .tlb_flush = svm_flush_tlb, .tlb_flush_gva = svm_flush_tlb_gva, + .tlb_flush_guest = svm_flush_tlb_guest, .run = svm_vcpu_run, .handle_exit = handle_exit, diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d973ff095066..094acb53829b 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2851,6 +2851,18 @@ static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) */ } +static void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu) +{ + /* + * vpid_sync_context() is a nop if vmx->vpid==0, e.g. if enable_vpid==0 + * or a vpid couldn't be allocated for this vCPU. VM-Enter and VM-Exit + * are required to flush GVA->{G,H}PA mappings from the TLB if vpid is + * disabled (VM-Enter with vpid enabled and vpid==0 is disallowed), + * i.e. no explicit INVVPID is necessary. + */ + vpid_sync_context(to_vmx(vcpu)->vpid); +} + static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu) { ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits; @@ -7718,6 +7730,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .tlb_flush = vmx_flush_tlb, .tlb_flush_gva = vmx_flush_tlb_gva, + .tlb_flush_guest = vmx_flush_tlb_guest, .run = vmx_vcpu_run, .handle_exit = vmx_handle_exit, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 3984574e09bf..c7ad142b511f 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2719,7 +2719,7 @@ static void record_steal_time(struct kvm_vcpu *vcpu) trace_kvm_pv_tlb_flush(vcpu->vcpu_id, st->preempted & KVM_VCPU_FLUSH_TLB); if (xchg(&st->preempted, 0) & KVM_VCPU_FLUSH_TLB) - kvm_vcpu_flush_tlb(vcpu, false); + kvm_x86_ops.tlb_flush_guest(vcpu); vcpu->arch.st.preempted = 0; From a48b284b403a4a073d8beb72d2bb33e54df67fb6 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Mon, 20 Apr 2020 10:09:29 -0400 Subject: [PATCH 0066/1043] audit: fix a net reference leak in audit_send_reply() If audit_send_reply() fails when trying to create a new thread to send the reply it also fails to cleanup properly, leaking a reference to a net structure. This patch fixes the error path and makes a handful of other cleanups that came up while fixing the code. Reported-by: teroincn@gmail.com Reviewed-by: Richard Guy Briggs Signed-off-by: Paul Moore --- kernel/audit.c | 50 +++++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/kernel/audit.c b/kernel/audit.c index b69c8b460341..66b81358b64f 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -924,19 +924,30 @@ out_kfree_skb: return NULL; } +static void audit_free_reply(struct audit_reply *reply) +{ + if (!reply) + return; + + if (reply->skb) + kfree_skb(reply->skb); + if (reply->net) + put_net(reply->net); + kfree(reply); +} + static int audit_send_reply_thread(void *arg) { struct audit_reply *reply = (struct audit_reply *)arg; - struct sock *sk = audit_get_sk(reply->net); audit_ctl_lock(); audit_ctl_unlock(); /* Ignore failure. It'll only happen if the sender goes away, because our timeout is set to infinite. */ - netlink_unicast(sk, reply->skb, reply->portid, 0); - put_net(reply->net); - kfree(reply); + netlink_unicast(audit_get_sk(reply->net), reply->skb, reply->portid, 0); + reply->skb = NULL; + audit_free_reply(reply); return 0; } @@ -950,35 +961,32 @@ static int audit_send_reply_thread(void *arg) * @payload: payload data * @size: payload size * - * Allocates an skb, builds the netlink message, and sends it to the port id. - * No failure notifications. + * Allocates a skb, builds the netlink message, and sends it to the port id. */ static void audit_send_reply(struct sk_buff *request_skb, int seq, int type, int done, int multi, const void *payload, int size) { - struct net *net = sock_net(NETLINK_CB(request_skb).sk); - struct sk_buff *skb; struct task_struct *tsk; - struct audit_reply *reply = kmalloc(sizeof(struct audit_reply), - GFP_KERNEL); + struct audit_reply *reply; + reply = kzalloc(sizeof(*reply), GFP_KERNEL); if (!reply) return; - skb = audit_make_reply(seq, type, done, multi, payload, size); - if (!skb) - goto out; - - reply->net = get_net(net); + reply->skb = audit_make_reply(seq, type, done, multi, payload, size); + if (!reply->skb) + goto err; + reply->net = get_net(sock_net(NETLINK_CB(request_skb).sk)); reply->portid = NETLINK_CB(request_skb).portid; - reply->skb = skb; tsk = kthread_run(audit_send_reply_thread, reply, "audit_send_reply"); - if (!IS_ERR(tsk)) - return; - kfree_skb(skb); -out: - kfree(reply); + if (IS_ERR(tsk)) + goto err; + + return; + +err: + audit_free_reply(reply); } /* From 0baedd792713063213f1e2060dc6a5d536638f0a Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Wed, 25 Mar 2020 12:28:24 -0400 Subject: [PATCH 0067/1043] KVM: x86: make Hyper-V PV TLB flush use tlb_flush_guest() Hyper-V PV TLB flush mechanism does TLB flush on behalf of the guest so doing tlb_flush_all() is an overkill, switch to using tlb_flush_guest() (just like KVM PV TLB flush mechanism) instead. Introduce KVM_REQ_HV_TLB_FLUSH to support the change. Signed-off-by: Vitaly Kuznetsov Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/hyperv.c | 3 +-- arch/x86/kvm/x86.c | 10 +++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index a7c1ea136c8c..9951b01df57c 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -83,6 +83,8 @@ #define KVM_REQ_GET_VMCS12_PAGES KVM_ARCH_REQ(24) #define KVM_REQ_APICV_UPDATE \ KVM_ARCH_REQ_FLAGS(25, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) +#define KVM_REQ_HV_TLB_FLUSH \ + KVM_ARCH_REQ_FLAGS(26, KVM_REQUEST_NO_WAKEUP) #define CR0_RESERVED_BITS \ (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index bcefa9d4e57e..b850f676abe4 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1425,8 +1425,7 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *current_vcpu, u64 ingpa, * vcpu->arch.cr3 may not be up-to-date for running vCPUs so we can't * analyze it here, flush TLB regardless of the specified address space. */ - kvm_make_vcpus_request_mask(kvm, - KVM_REQ_TLB_FLUSH | KVM_REQUEST_NO_WAKEUP, + kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask, &hv_vcpu->tlb_flush); ret_success: diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c7ad142b511f..a7df68af65e5 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2696,6 +2696,12 @@ static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa) kvm_x86_ops.tlb_flush(vcpu, invalidate_gpa); } +static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu) +{ + ++vcpu->stat.tlb_flush; + kvm_x86_ops.tlb_flush_guest(vcpu); +} + static void record_steal_time(struct kvm_vcpu *vcpu) { struct kvm_host_map map; @@ -2719,7 +2725,7 @@ static void record_steal_time(struct kvm_vcpu *vcpu) trace_kvm_pv_tlb_flush(vcpu->vcpu_id, st->preempted & KVM_VCPU_FLUSH_TLB); if (xchg(&st->preempted, 0) & KVM_VCPU_FLUSH_TLB) - kvm_x86_ops.tlb_flush_guest(vcpu); + kvm_vcpu_flush_tlb_guest(vcpu); vcpu->arch.st.preempted = 0; @@ -8218,6 +8224,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_mmu_load_pgd(vcpu); if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) kvm_vcpu_flush_tlb(vcpu, true); + if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu)) + kvm_vcpu_flush_tlb_guest(vcpu); if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) { vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS; r = 0; From ad104b5e433a6851a3a9822663c1ca756832a5f0 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:11 -0700 Subject: [PATCH 0068/1043] KVM: VMX: Clean up vmx_flush_tlb_gva() Refactor vmx_flush_tlb_gva() to remove a superfluous local variable and clean up its comment, which is oddly located below the code it is commenting. No functional change intended. Reviewed-by: Vitaly Kuznetsov Reviewed-by: Paolo Bonzini Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-16-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 094acb53829b..8b55f43433a4 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2840,15 +2840,11 @@ static void exit_lmode(struct kvm_vcpu *vcpu) static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) { - int vpid = to_vmx(vcpu)->vpid; - - vpid_sync_vcpu_addr(vpid, addr); - /* - * If VPIDs are not supported or enabled, then the above is a no-op. - * But we don't really need a TLB flush in that case anyway, because - * each VM entry/exit includes an implicit flush when VPID is 0. + * vpid_sync_vcpu_addr() is a nop if vmx->vpid==0, see the comment in + * vmx_flush_tlb_guest() for an explanation of why this is ok. */ + vpid_sync_vcpu_addr(to_vmx(vcpu)->vpid, addr); } static void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu) From f55ac304ca47039368a5971fa61ebc8160c90659 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:12 -0700 Subject: [PATCH 0069/1043] KVM: x86: Drop @invalidate_gpa param from kvm_x86_ops' tlb_flush() Drop @invalidate_gpa from ->tlb_flush() and kvm_vcpu_flush_tlb() now that all callers pass %true for said param, or ignore the param (SVM has an internal call to svm_flush_tlb() in svm_flush_tlb_guest that somewhat arbitrarily passes %false). Remove __vmx_flush_tlb() as it is no longer used. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-17-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/svm/nested.c | 2 +- arch/x86/kvm/svm/svm.c | 6 ++--- arch/x86/kvm/svm/svm.h | 2 +- arch/x86/kvm/vmx/vmx.c | 4 ++-- arch/x86/kvm/vmx/vmx.h | 42 ++++++++++----------------------- arch/x86/kvm/x86.c | 6 ++--- 8 files changed, 24 insertions(+), 42 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 9951b01df57c..7156c749677a 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1103,7 +1103,7 @@ struct kvm_x86_ops { unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); - void (*tlb_flush)(struct kvm_vcpu *vcpu, bool invalidate_gpa); + void (*tlb_flush)(struct kvm_vcpu *vcpu); int (*tlb_remote_flush)(struct kvm *kvm); int (*tlb_remote_flush_with_range)(struct kvm *kvm, struct kvm_tlb_range *range); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 32c9f4b2a281..081f6b220b6e 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5179,7 +5179,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu) if (r) goto out; kvm_mmu_load_pgd(vcpu); - kvm_x86_ops.tlb_flush(vcpu, true); + kvm_x86_ops.tlb_flush(vcpu); out: return r; } diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 90a1ca939627..c62893102777 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -279,7 +279,7 @@ void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions; svm->nested.intercept = nested_vmcb->control.intercept; - svm_flush_tlb(&svm->vcpu, true); + svm_flush_tlb(&svm->vcpu); svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK; if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK) svm->vcpu.arch.hflags |= HF_VINTR_MASK; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 2c0fcb87a054..aafd18589fda 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1603,7 +1603,7 @@ int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) return 1; if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE)) - svm_flush_tlb(vcpu, true); + svm_flush_tlb(vcpu); vcpu->arch.cr4 = cr4; if (!npt_enabled) @@ -3153,7 +3153,7 @@ static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr) return 0; } -void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa) +void svm_flush_tlb(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); @@ -3172,7 +3172,7 @@ static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu) { - svm_flush_tlb(vcpu, false); + svm_flush_tlb(vcpu); } static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index df3474f4fb02..ca95204f9dde 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -360,7 +360,7 @@ u32 svm_msrpm_offset(u32 msr); void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer); void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); -void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa); +void svm_flush_tlb(struct kvm_vcpu *vcpu); void disable_nmi_singlestep(struct vcpu_svm *svm); /* nested.c */ diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 8b55f43433a4..d843ef34aafd 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6083,7 +6083,7 @@ void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) if (flexpriority_enabled) { sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; - vmx_flush_tlb(vcpu, true); + vmx_flush_tlb(vcpu); } break; case LAPIC_MODE_X2APIC: @@ -6101,7 +6101,7 @@ static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa) { if (!is_guest_mode(vcpu)) { vmcs_write64(APIC_ACCESS_ADDR, hpa); - vmx_flush_tlb(vcpu, true); + vmx_flush_tlb(vcpu); } } diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 1560296dde25..45978552524e 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -500,46 +500,28 @@ static inline struct vmcs *alloc_vmcs(bool shadow) u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa); -static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid, - bool invalidate_gpa) -{ - if (enable_ept && (invalidate_gpa || !enable_vpid)) { - if (!VALID_PAGE(vcpu->arch.mmu->root_hpa)) - return; - ept_sync_context(construct_eptp(vcpu, - vcpu->arch.mmu->root_hpa)); - } else { - vpid_sync_context(vpid); - } -} - -static inline void vmx_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa) +static inline void vmx_flush_tlb(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); /* - * Flush all EPTP/VPID contexts if the TLB flush _may_ have been - * invoked via kvm_flush_remote_tlbs(), which always passes %true for - * @invalidate_gpa. Flushing remote TLBs requires all contexts to be - * flushed, not just the active context. + * Flush all EPTP/VPID contexts, as the TLB flush _may_ have been + * invoked via kvm_flush_remote_tlbs(). Flushing remote TLBs requires + * all contexts to be flushed, not just the active context. * * Note, this also ensures a deferred TLB flush with VPID enabled and * EPT disabled invalidates the "correct" VPID, by nuking both L1 and * L2's VPIDs. */ - if (invalidate_gpa) { - if (enable_ept) { - ept_sync_global(); - } else if (enable_vpid) { - if (cpu_has_vmx_invvpid_global()) { - vpid_sync_vcpu_global(); - } else { - vpid_sync_vcpu_single(vmx->vpid); - vpid_sync_vcpu_single(vmx->nested.vpid02); - } + if (enable_ept) { + ept_sync_global(); + } else if (enable_vpid) { + if (cpu_has_vmx_invvpid_global()) { + vpid_sync_vcpu_global(); + } else { + vpid_sync_vcpu_single(vmx->vpid); + vpid_sync_vcpu_single(vmx->nested.vpid02); } - } else { - __vmx_flush_tlb(vcpu, vmx->vpid, false); } } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index a7df68af65e5..daf16247870c 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2690,10 +2690,10 @@ static void kvmclock_reset(struct kvm_vcpu *vcpu) vcpu->arch.time = 0; } -static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa) +static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu) { ++vcpu->stat.tlb_flush; - kvm_x86_ops.tlb_flush(vcpu, invalidate_gpa); + kvm_x86_ops.tlb_flush(vcpu); } static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu) @@ -8223,7 +8223,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu)) kvm_mmu_load_pgd(vcpu); if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) - kvm_vcpu_flush_tlb(vcpu, true); + kvm_vcpu_flush_tlb(vcpu); if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu)) kvm_vcpu_flush_tlb_guest(vcpu); if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) { From 72b38320872666f7fb6ff9564bf91139c685c234 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:13 -0700 Subject: [PATCH 0070/1043] KVM: SVM: Wire up ->tlb_flush_guest() directly to svm_flush_tlb() Use svm_flush_tlb() directly for kvm_x86_ops->tlb_flush_guest() now that the @invalidate_gpa param to ->tlb_flush() is gone, i.e. the wrapper for ->tlb_flush_guest() is no longer necessary. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-18-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index aafd18589fda..2ad6a8d1a77f 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3170,11 +3170,6 @@ static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva) invlpga(gva, svm->vmcb->control.asid); } -static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu) -{ - svm_flush_tlb(vcpu); -} - static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu) { } @@ -3944,7 +3939,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .tlb_flush = svm_flush_tlb, .tlb_flush_gva = svm_flush_tlb_gva, - .tlb_flush_guest = svm_flush_tlb_guest, + .tlb_flush_guest = svm_flush_tlb, .run = svm_vcpu_run, .handle_exit = handle_exit, From 5058b692c69997f9736b94df786509366c32f34d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:14 -0700 Subject: [PATCH 0071/1043] KVM: VMX: Move vmx_flush_tlb() to vmx.c Move vmx_flush_tlb() to vmx.c and make it non-inline static now that all its callers live in vmx.c. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-19-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 25 +++++++++++++++++++++++++ arch/x86/kvm/vmx/vmx.h | 25 ------------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d843ef34aafd..3cdca4d03c07 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2838,6 +2838,31 @@ static void exit_lmode(struct kvm_vcpu *vcpu) #endif +static void vmx_flush_tlb(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + /* + * Flush all EPTP/VPID contexts, as the TLB flush _may_ have been + * invoked via kvm_flush_remote_tlbs(). Flushing remote TLBs requires + * all contexts to be flushed, not just the active context. + * + * Note, this also ensures a deferred TLB flush with VPID enabled and + * EPT disabled invalidates the "correct" VPID, by nuking both L1 and + * L2's VPIDs. + */ + if (enable_ept) { + ept_sync_global(); + } else if (enable_vpid) { + if (cpu_has_vmx_invvpid_global()) { + vpid_sync_vcpu_global(); + } else { + vpid_sync_vcpu_single(vmx->vpid); + vpid_sync_vcpu_single(vmx->nested.vpid02); + } + } +} + static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) { /* diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 45978552524e..4c7b0713b438 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -500,31 +500,6 @@ static inline struct vmcs *alloc_vmcs(bool shadow) u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa); -static inline void vmx_flush_tlb(struct kvm_vcpu *vcpu) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - - /* - * Flush all EPTP/VPID contexts, as the TLB flush _may_ have been - * invoked via kvm_flush_remote_tlbs(). Flushing remote TLBs requires - * all contexts to be flushed, not just the active context. - * - * Note, this also ensures a deferred TLB flush with VPID enabled and - * EPT disabled invalidates the "correct" VPID, by nuking both L1 and - * L2's VPIDs. - */ - if (enable_ept) { - ept_sync_global(); - } else if (enable_vpid) { - if (cpu_has_vmx_invvpid_global()) { - vpid_sync_vcpu_global(); - } else { - vpid_sync_vcpu_single(vmx->vpid); - vpid_sync_vcpu_single(vmx->nested.vpid02); - } - } -} - static inline void decache_tsc_multiplier(struct vcpu_vmx *vmx) { vmx->current_tsc_ratio = vmx->vcpu.arch.tsc_scaling_ratio; From 25d8b84376e7c12ce4e4ec88c0f5bf0bfc9ff8f4 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:15 -0700 Subject: [PATCH 0072/1043] KVM: nVMX: Move nested_get_vpid02() to vmx/nested.h Move nested_get_vpid02() to vmx/nested.h so that a future patch can reference it from vmx.c to implement context-specific TLB flushing. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-20-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 7 ------- arch/x86/kvm/vmx/nested.h | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index cbc687e9209d..b18dc7bc3be7 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1132,13 +1132,6 @@ static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu) (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); } -static u16 nested_get_vpid02(struct kvm_vcpu *vcpu) -{ - struct vcpu_vmx *vmx = to_vmx(vcpu); - - return vmx->nested.vpid02 ? vmx->nested.vpid02 : vmx->vpid; -} - static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) { superset &= mask; diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index ac56aefa49e3..e8f4a74f7251 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -62,6 +62,13 @@ static inline int vmx_has_valid_vmcs12(struct kvm_vcpu *vcpu) vmx->nested.hv_evmcs; } +static inline u16 nested_get_vpid02(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + return vmx->nested.vpid02 ? vmx->nested.vpid02 : vmx->vpid; +} + static inline unsigned long nested_ept_get_eptp(struct kvm_vcpu *vcpu) { /* return the page table to be shadowed - in our case, EPT12 */ From 33d19ec9b14c0e304f5d44a0dd1403539360968b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:16 -0700 Subject: [PATCH 0073/1043] KVM: VMX: Introduce vmx_flush_tlb_current() Add a helper to flush TLB entries only for the current EPTP/VPID context and use it for the existing direct invocations of vmx_flush_tlb(). TLB flushes that are specific to the current vCPU state do not need to flush other contexts. Note, both converted call sites happen to be related to the APIC access page, this is purely coincidental. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-21-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 3cdca4d03c07..0f1dfbae649f 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2863,6 +2863,22 @@ static void vmx_flush_tlb(struct kvm_vcpu *vcpu) } } +static void vmx_flush_tlb_current(struct kvm_vcpu *vcpu) +{ + u64 root_hpa = vcpu->arch.mmu->root_hpa; + + /* No flush required if the current context is invalid. */ + if (!VALID_PAGE(root_hpa)) + return; + + if (enable_ept) + ept_sync_context(construct_eptp(vcpu, root_hpa)); + else if (!is_guest_mode(vcpu)) + vpid_sync_context(to_vmx(vcpu)->vpid); + else + vpid_sync_context(nested_get_vpid02(vcpu)); +} + static void vmx_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t addr) { /* @@ -6108,7 +6124,7 @@ void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) if (flexpriority_enabled) { sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; - vmx_flush_tlb(vcpu); + vmx_flush_tlb_current(vcpu); } break; case LAPIC_MODE_X2APIC: @@ -6126,7 +6142,7 @@ static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa) { if (!is_guest_mode(vcpu)) { vmcs_write64(APIC_ACCESS_ADDR, hpa); - vmx_flush_tlb(vcpu); + vmx_flush_tlb_current(vcpu); } } From 4a41e43cbe2c0f47d7ec80d156ab786d03cdaacd Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:17 -0700 Subject: [PATCH 0074/1043] KVM: SVM: Document the ASID logic in svm_flush_tlb() Add a comment in svm_flush_tlb() to document why it flushes only the current ASID, even when it is invoked when flushing remote TLBs. Cc: Tom Lendacky Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-22-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 2ad6a8d1a77f..fa5bb1b62059 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3157,6 +3157,13 @@ void svm_flush_tlb(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); + /* + * Flush only the current ASID even if the TLB flush was invoked via + * kvm_flush_remote_tlbs(). Although flushing remote TLBs requires all + * ASIDs to be flushed, KVM uses a single ASID for L1 and L2, and + * unconditionally does a TLB flush on both nested VM-Enter and nested + * VM-Exit (via kvm_mmu_reset_context()). + */ if (static_cpu_has(X86_FEATURE_FLUSHBYASID)) svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; else From 7780938cc70b848650722762fa4c7496fa68f9ec Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:18 -0700 Subject: [PATCH 0075/1043] KVM: x86: Rename ->tlb_flush() to ->tlb_flush_all() Rename ->tlb_flush() to ->tlb_flush_all() in preparation for adding a new hook to flush only the current ASID/context. Opportunstically replace the comment in vmx_flush_tlb() that explains why it flushes all EPTP/VPID contexts with a comment explaining why it unconditionally uses INVEPT when EPT is enabled. I.e. rely on the "all" part of the name to clarify why it does global INVEPT/INVVPID. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-23-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/vmx/vmx.c | 16 +++++++--------- arch/x86/kvm/x86.c | 6 +++--- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 7156c749677a..e6305d5e28fa 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1103,7 +1103,7 @@ struct kvm_x86_ops { unsigned long (*get_rflags)(struct kvm_vcpu *vcpu); void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); - void (*tlb_flush)(struct kvm_vcpu *vcpu); + void (*tlb_flush_all)(struct kvm_vcpu *vcpu); int (*tlb_remote_flush)(struct kvm *kvm); int (*tlb_remote_flush_with_range)(struct kvm *kvm, struct kvm_tlb_range *range); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 081f6b220b6e..85e17a057094 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5179,7 +5179,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu) if (r) goto out; kvm_mmu_load_pgd(vcpu); - kvm_x86_ops.tlb_flush(vcpu); + kvm_x86_ops.tlb_flush_all(vcpu); out: return r; } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index fa5bb1b62059..72b976e64df9 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3944,7 +3944,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .get_rflags = svm_get_rflags, .set_rflags = svm_set_rflags, - .tlb_flush = svm_flush_tlb, + .tlb_flush_all = svm_flush_tlb, .tlb_flush_gva = svm_flush_tlb_gva, .tlb_flush_guest = svm_flush_tlb, diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 0f1dfbae649f..da868846d96d 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2838,18 +2838,16 @@ static void exit_lmode(struct kvm_vcpu *vcpu) #endif -static void vmx_flush_tlb(struct kvm_vcpu *vcpu) +static void vmx_flush_tlb_all(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); /* - * Flush all EPTP/VPID contexts, as the TLB flush _may_ have been - * invoked via kvm_flush_remote_tlbs(). Flushing remote TLBs requires - * all contexts to be flushed, not just the active context. - * - * Note, this also ensures a deferred TLB flush with VPID enabled and - * EPT disabled invalidates the "correct" VPID, by nuking both L1 and - * L2's VPIDs. + * INVEPT must be issued when EPT is enabled, irrespective of VPID, as + * the CPU is not required to invalidate guest-physical mappings on + * VM-Entry, even if VPID is disabled. Guest-physical mappings are + * associated with the root EPT structure and not any particular VPID + * (INVVPID also isn't required to invalidate guest-physical mappings). */ if (enable_ept) { ept_sync_global(); @@ -7765,7 +7763,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .get_rflags = vmx_get_rflags, .set_rflags = vmx_set_rflags, - .tlb_flush = vmx_flush_tlb, + .tlb_flush_all = vmx_flush_tlb_all, .tlb_flush_gva = vmx_flush_tlb_gva, .tlb_flush_guest = vmx_flush_tlb_guest, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index daf16247870c..cd2a3d01bffb 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2690,10 +2690,10 @@ static void kvmclock_reset(struct kvm_vcpu *vcpu) vcpu->arch.time = 0; } -static void kvm_vcpu_flush_tlb(struct kvm_vcpu *vcpu) +static void kvm_vcpu_flush_tlb_all(struct kvm_vcpu *vcpu) { ++vcpu->stat.tlb_flush; - kvm_x86_ops.tlb_flush(vcpu); + kvm_x86_ops.tlb_flush_all(vcpu); } static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu) @@ -8223,7 +8223,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu)) kvm_mmu_load_pgd(vcpu); if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) - kvm_vcpu_flush_tlb(vcpu); + kvm_vcpu_flush_tlb_all(vcpu); if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu)) kvm_vcpu_flush_tlb_guest(vcpu); if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) { From 50b265a4eee0d9fb9581f1a300742515b80dffda Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:19 -0700 Subject: [PATCH 0076/1043] KVM: nVMX: Add helper to handle TLB flushes on nested VM-Enter/VM-Exit Add a helper to determine whether or not a full TLB flush needs to be performed on nested VM-Enter/VM-Exit, as the logic is identical for both flows and needs a fairly beefy comment to boot. This also provides a common point to make future adjustments to the logic. Handle vpid12 changes the new helper as well even though it is specific to VM-Enter. The vpid12 logic is an extension of the flushing logic, and it's worth the extra bool parameter to provide a single location for the flushing logic. Cc: Liran Alon Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-24-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 88 +++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index b18dc7bc3be7..ba1aedb535ef 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1132,6 +1132,48 @@ static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu) (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02); } +static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, + struct vmcs12 *vmcs12, + bool is_vmenter) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + /* + * If VPID is disabled, linear and combined mappings are flushed on + * VM-Enter/VM-Exit, and guest-physical mappings are valid only for + * their associated EPTP. + */ + if (!enable_vpid) + return; + + /* + * If vmcs12 doesn't use VPID, L1 expects linear and combined mappings + * for *all* contexts to be flushed on VM-Enter/VM-Exit. + * + * If VPID is enabled and used by vmc12, but L2 does not have a unique + * TLB tag (ASID), i.e. EPT is disabled and KVM was unable to allocate + * a VPID for L2, flush the TLB as the effective ASID is common to both + * L1 and L2. + * + * Defer the flush so that it runs after vmcs02.EPTP has been set by + * KVM_REQ_LOAD_MMU_PGD (if nested EPT is enabled) and to avoid + * redundant flushes further down the nested pipeline. + * + * If a TLB flush isn't required due to any of the above, and vpid12 is + * changing then the new "virtual" VPID (vpid12) will reuse the same + * "real" VPID (vpid02), and so needs to be sync'd. There is no direct + * mapping between vpid02 and vpid12, vpid02 is per-vCPU and reused for + * all nested vCPUs. + */ + if (!nested_cpu_has_vpid(vmcs12) || !nested_has_guest_tlb_tag(vcpu)) { + kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + } else if (is_vmenter && + vmcs12->virtual_processor_id != vmx->nested.last_vpid) { + vmx->nested.last_vpid = vmcs12->virtual_processor_id; + vpid_sync_context(nested_get_vpid02(vcpu)); + } +} + static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask) { superset &= mask; @@ -2440,32 +2482,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, if (kvm_has_tsc_control) decache_tsc_multiplier(vmx); - if (enable_vpid) { - /* - * There is no direct mapping between vpid02 and vpid12, the - * vpid02 is per-vCPU for L0 and reused while the value of - * vpid12 is changed w/ one invvpid during nested vmentry. - * The vpid12 is allocated by L1 for L2, so it will not - * influence global bitmap(for vpid01 and vpid02 allocation) - * even if spawn a lot of nested vCPUs. - */ - if (nested_cpu_has_vpid(vmcs12) && nested_has_guest_tlb_tag(vcpu)) { - if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) { - vmx->nested.last_vpid = vmcs12->virtual_processor_id; - vpid_sync_context(nested_get_vpid02(vcpu)); - } - } else { - /* - * If L1 use EPT, then L0 needs to execute INVEPT on - * EPTP02 instead of EPTP01. Therefore, delay TLB - * flush until vmcs02->eptp is fully updated by - * KVM_REQ_LOAD_MMU_PGD. Note that this assumes - * KVM_REQ_TLB_FLUSH is evaluated after - * KVM_REQ_LOAD_MMU_PGD in vcpu_enter_guest(). - */ - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); - } - } + nested_vmx_transition_tlb_flush(vcpu, vmcs12, true); if (nested_cpu_has_ept(vmcs12)) nested_ept_init_mmu_context(vcpu); @@ -4033,24 +4050,7 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, if (!enable_ept) vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault; - /* - * If vmcs01 doesn't use VPID, CPU flushes TLB on every - * VMEntry/VMExit. Thus, no need to flush TLB. - * - * If vmcs12 doesn't use VPID, L1 expects TLB to be - * flushed on every VMEntry/VMExit. - * - * Otherwise, we can preserve TLB entries as long as we are - * able to tag L1 TLB entries differently than L2 TLB entries. - * - * If vmcs12 uses EPT, we need to execute this flush on EPTP01 - * and therefore we request the TLB flush to happen only after VMCS EPTP - * has been set by KVM_REQ_LOAD_MMU_PGD. - */ - if (enable_vpid && - (!nested_cpu_has_vpid(vmcs12) || !nested_has_guest_tlb_tag(vcpu))) { - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); - } + nested_vmx_transition_tlb_flush(vcpu, vmcs12, false); vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs); vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp); From eeeb4f67a6cd437da1f5d1a20596cdc2d7b50551 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:20 -0700 Subject: [PATCH 0077/1043] KVM: x86: Introduce KVM_REQ_TLB_FLUSH_CURRENT to flush current ASID Add KVM_REQ_TLB_FLUSH_CURRENT to allow optimized TLB flushing of VMX's EPTP/VPID contexts[*] from the KVM MMU and/or in a deferred manner, e.g. to flush L2's context during nested VM-Enter. Convert KVM_REQ_TLB_FLUSH to KVM_REQ_TLB_FLUSH_CURRENT in flows where the flush is directly associated with vCPU-scoped instruction emulation, i.e. MOV CR3 and INVPCID. Add a comment in vmx_vcpu_load_vmcs() above its KVM_REQ_TLB_FLUSH to make it clear that it deliberately requests a flush of all contexts. Service any pending flush request on nested VM-Exit as it's possible a nested VM-Exit could occur after requesting a flush for L2. Add the same logic for nested VM-Enter even though it's _extremely_ unlikely for flush to be pending on nested VM-Enter, but theoretically possible (in the future) due to RSM (SMM) emulation. [*] Intel also has an Address Space Identifier (ASID) concept, e.g. EPTP+VPID+PCID == ASID, it's just not documented in the SDM because the rules of invalidation are different based on which piece of the ASID is being changed, i.e. whether the EPTP, VPID, or PCID context must be invalidated. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-25-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 4 +++- arch/x86/kvm/svm/svm.c | 1 + arch/x86/kvm/vmx/nested.c | 7 +++++++ arch/x86/kvm/vmx/vmx.c | 7 ++++++- arch/x86/kvm/x86.c | 11 +++++++++-- arch/x86/kvm/x86.h | 6 ++++++ 6 files changed, 32 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index e6305d5e28fa..72e9c4492f47 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -83,8 +83,9 @@ #define KVM_REQ_GET_VMCS12_PAGES KVM_ARCH_REQ(24) #define KVM_REQ_APICV_UPDATE \ KVM_ARCH_REQ_FLAGS(25, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) +#define KVM_REQ_TLB_FLUSH_CURRENT KVM_ARCH_REQ(26) #define KVM_REQ_HV_TLB_FLUSH \ - KVM_ARCH_REQ_FLAGS(26, KVM_REQUEST_NO_WAKEUP) + KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_NO_WAKEUP) #define CR0_RESERVED_BITS \ (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ @@ -1104,6 +1105,7 @@ struct kvm_x86_ops { void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags); void (*tlb_flush_all)(struct kvm_vcpu *vcpu); + void (*tlb_flush_current)(struct kvm_vcpu *vcpu); int (*tlb_remote_flush)(struct kvm *kvm); int (*tlb_remote_flush_with_range)(struct kvm *kvm, struct kvm_tlb_range *range); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 72b976e64df9..66123848448d 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3945,6 +3945,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .set_rflags = svm_set_rflags, .tlb_flush_all = svm_flush_tlb, + .tlb_flush_current = svm_flush_tlb, .tlb_flush_gva = svm_flush_tlb_gva, .tlb_flush_guest = svm_flush_tlb, diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index ba1aedb535ef..567a7a0f30e0 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3208,6 +3208,9 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, u32 exit_reason = EXIT_REASON_INVALID_STATE; u32 exit_qual; + if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu)) + kvm_vcpu_flush_tlb_current(vcpu); + evaluate_pending_interrupts = exec_controls_get(vmx) & (CPU_BASED_INTR_WINDOW_EXITING | CPU_BASED_NMI_WINDOW_EXITING); if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu)) @@ -4274,6 +4277,10 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, /* trying to cancel vmlaunch/vmresume is a bug */ WARN_ON_ONCE(vmx->nested.nested_run_pending); + /* Service the TLB flush request for L2 before switching to L1. */ + if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu)) + kvm_vcpu_flush_tlb_current(vcpu); + leave_guest_mode(vcpu); if (nested_cpu_has_preemption_timer(vmcs12)) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index da868846d96d..d958cee52f41 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1338,6 +1338,10 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu) void *gdt = get_current_gdt_ro(); unsigned long sysenter_esp; + /* + * Flush all EPTP/VPID contexts, the new pCPU may have stale + * TLB entries from its previous association with the vCPU. + */ kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); /* @@ -5468,7 +5472,7 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) if (kvm_get_active_pcid(vcpu) == operand.pcid) { kvm_mmu_sync_roots(vcpu); - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); } for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) @@ -7764,6 +7768,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .set_rflags = vmx_set_rflags, .tlb_flush_all = vmx_flush_tlb_all, + .tlb_flush_current = vmx_flush_tlb_current, .tlb_flush_gva = vmx_flush_tlb_gva, .tlb_flush_guest = vmx_flush_tlb_guest, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index cd2a3d01bffb..ebbe34d89469 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1019,7 +1019,7 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) { if (!skip_tlb_flush) { kvm_mmu_sync_roots(vcpu); - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); } return 0; } @@ -8222,10 +8222,17 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_mmu_sync_roots(vcpu); if (kvm_check_request(KVM_REQ_LOAD_MMU_PGD, vcpu)) kvm_mmu_load_pgd(vcpu); - if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) + if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { kvm_vcpu_flush_tlb_all(vcpu); + + /* Flushing all ASIDs flushes the current ASID... */ + kvm_clear_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); + } + if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu)) + kvm_vcpu_flush_tlb_current(vcpu); if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu)) kvm_vcpu_flush_tlb_guest(vcpu); + if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) { vcpu->run->exit_reason = KVM_EXIT_TPR_ACCESS; r = 0; diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index b968acc0516f..7b5ed8ed628e 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -125,6 +125,12 @@ static inline bool mmu_is_nested(struct kvm_vcpu *vcpu) return vcpu->arch.walk_mmu == &vcpu->arch.nested_mmu; } +static inline void kvm_vcpu_flush_tlb_current(struct kvm_vcpu *vcpu) +{ + ++vcpu->stat.tlb_flush; + kvm_x86_ops.tlb_flush_current(vcpu); +} + static inline int is_pae(struct kvm_vcpu *vcpu) { return kvm_read_cr4_bits(vcpu, X86_CR4_PAE); From 8c8560b833909cbf2e3a73d4dc8e65a60ba66f07 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:21 -0700 Subject: [PATCH 0078/1043] KVM: x86/mmu: Use KVM_REQ_TLB_FLUSH_CURRENT for MMU specific flushes Flush only the current ASID/context when requesting a TLB flush due to a change in the current vCPU's MMU to avoid blasting away TLB entries associated with other ASIDs/contexts, e.g. entries cached for L1 when a change in L2's MMU requires a flush. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-26-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 85e17a057094..575bd5de2118 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -2309,7 +2309,7 @@ static void kvm_mmu_flush_or_zap(struct kvm_vcpu *vcpu, return; if (local_flush) - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); } #ifdef CONFIG_KVM_MMU_AUDIT @@ -2516,11 +2516,11 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, break; WARN_ON(!list_empty(&invalid_list)); - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); } if (sp->unsync_children) - kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); __clear_sp_write_flooding_count(sp); trace_kvm_mmu_get_page(sp, false); @@ -3121,7 +3121,7 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, if (set_spte_ret & SET_SPTE_WRITE_PROTECTED_PT) { if (write_fault) ret = RET_PF_EMULATE; - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); } if (set_spte_ret & SET_SPTE_NEED_REMOTE_TLB_FLUSH || flush) @@ -4310,7 +4310,7 @@ static bool fast_cr3_switch(struct kvm_vcpu *vcpu, gpa_t new_cr3, kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); if (!skip_tlb_flush) { kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); } /* @@ -5179,7 +5179,7 @@ int kvm_mmu_load(struct kvm_vcpu *vcpu) if (r) goto out; kvm_mmu_load_pgd(vcpu); - kvm_x86_ops.tlb_flush_all(vcpu); + kvm_x86_ops.tlb_flush_current(vcpu); out: return r; } From c51e1ffee50121a2b2e00208f517a695493ee85d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:22 -0700 Subject: [PATCH 0079/1043] KVM: nVMX: Selectively use TLB_FLUSH_CURRENT for nested VM-Enter/VM-Exit Flush only the current context, as opposed to all contexts, when requesting a TLB flush to handle the scenario where a L1 does not expect a TLB flush, but one is required because L1 and L2 shared an ASID. This occurs if EPT is disabled (no per-EPTP tag), VPID is enabled (hardware doesn't flush unconditionally) and vmcs02 does not have its own VPID due to exhaustion of available VPIDs. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-27-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 567a7a0f30e0..deddf0f0f6e7 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1152,8 +1152,8 @@ static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, * * If VPID is enabled and used by vmc12, but L2 does not have a unique * TLB tag (ASID), i.e. EPT is disabled and KVM was unable to allocate - * a VPID for L2, flush the TLB as the effective ASID is common to both - * L1 and L2. + * a VPID for L2, flush the current context as the effective ASID is + * common to both L1 and L2. * * Defer the flush so that it runs after vmcs02.EPTP has been set by * KVM_REQ_LOAD_MMU_PGD (if nested EPT is enabled) and to avoid @@ -1165,8 +1165,10 @@ static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, * mapping between vpid02 and vpid12, vpid02 is per-vCPU and reused for * all nested vCPUs. */ - if (!nested_cpu_has_vpid(vmcs12) || !nested_has_guest_tlb_tag(vcpu)) { + if (!nested_cpu_has_vpid(vmcs12)) { kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); + } else if (!nested_has_guest_tlb_tag(vcpu)) { + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); } else if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) { vmx->nested.last_vpid = vmcs12->virtual_processor_id; From 1196cb970b996be69a2fcd9756117b394f8e7526 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:23 -0700 Subject: [PATCH 0080/1043] KVM: nVMX: Reload APIC access page on nested VM-Exit only if necessary Defer reloading L1's APIC page by logging the need for a reload and processing it during nested VM-Exit instead of unconditionally reloading the APIC page on nested VM-Exit. This eliminates a TLB flush on the majority of VM-Exits as the APIC page rarely needs to be reloaded. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-28-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 9 ++++----- arch/x86/kvm/vmx/vmx.c | 10 +++++++--- arch/x86/kvm/vmx/vmx.h | 1 + 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index deddf0f0f6e7..a7a526a21292 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -4346,11 +4346,10 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true); vmx->nested.pi_desc = NULL; - /* - * We are now running in L2, mmu_notifier will force to reload the - * page's hpa for L2 vmcs. Need to reload it for L1 before entering L1. - */ - kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); + if (vmx->nested.reload_vmcs01_apic_access_page) { + vmx->nested.reload_vmcs01_apic_access_page = false; + kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); + } if ((exit_reason != -1) && (enable_shadow_vmcs || vmx->nested.hv_evmcs)) vmx->nested.need_vmcs12_to_shadow_sync = true; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d958cee52f41..4739b780c74e 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6142,10 +6142,14 @@ void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa) { - if (!is_guest_mode(vcpu)) { - vmcs_write64(APIC_ACCESS_ADDR, hpa); - vmx_flush_tlb_current(vcpu); + /* Defer reload until vmcs01 is the current VMCS. */ + if (is_guest_mode(vcpu)) { + to_vmx(vcpu)->nested.reload_vmcs01_apic_access_page = true; + return; } + + vmcs_write64(APIC_ACCESS_ADDR, hpa); + vmx_flush_tlb_current(vcpu); } static void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 4c7b0713b438..31d7252df163 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -136,6 +136,7 @@ struct nested_vmx { bool vmcs02_initialized; bool change_vmcs01_virtual_apic_mode; + bool reload_vmcs01_apic_access_page; /* * Enlightened VMCS has been enabled. It does not mean that L1 has to From a4148b7ca2a5afe1295a41b5e30048cabcb74f8d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:24 -0700 Subject: [PATCH 0081/1043] KVM: VMX: Retrieve APIC access page HPA only when necessary Move the retrieval of the HPA associated with L1's APIC access page into VMX code to avoid unnecessarily calling gfn_to_page(), e.g. when the vCPU is in guest mode (L2). Alternatively, the optimization logic in VMX could be mirrored into the common x86 code, but that will get ugly fast when further optimizations are introduced. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-29-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/vmx/vmx.c | 16 ++++++++++++++-- arch/x86/kvm/x86.c | 13 +------------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 72e9c4492f47..541e2df8fc6e 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1152,7 +1152,7 @@ struct kvm_x86_ops { bool (*guest_apic_has_interrupt)(struct kvm_vcpu *vcpu); void (*load_eoi_exitmap)(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap); void (*set_virtual_apic_mode)(struct kvm_vcpu *vcpu); - void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu, hpa_t hpa); + void (*set_apic_access_page_addr)(struct kvm_vcpu *vcpu); int (*deliver_posted_interrupt)(struct kvm_vcpu *vcpu, int vector); int (*sync_pir_to_irr)(struct kvm_vcpu *vcpu); int (*set_tss_addr)(struct kvm *kvm, unsigned int addr); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 4739b780c74e..8550da629a61 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6140,16 +6140,28 @@ void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) vmx_update_msr_bitmap(vcpu); } -static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu, hpa_t hpa) +static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu) { + struct page *page; + /* Defer reload until vmcs01 is the current VMCS. */ if (is_guest_mode(vcpu)) { to_vmx(vcpu)->nested.reload_vmcs01_apic_access_page = true; return; } - vmcs_write64(APIC_ACCESS_ADDR, hpa); + page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); + if (is_error_page(page)) + return; + + vmcs_write64(APIC_ACCESS_ADDR, page_to_phys(page)); vmx_flush_tlb_current(vcpu); + + /* + * Do not pin apic access page in memory, the MMU notifier + * will call us again if it is migrated or swapped out. + */ + put_page(page); } static void vmx_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index ebbe34d89469..90aa4abbc0a6 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8157,24 +8157,13 @@ int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) { - struct page *page = NULL; - if (!lapic_in_kernel(vcpu)) return; if (!kvm_x86_ops.set_apic_access_page_addr) return; - page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); - if (is_error_page(page)) - return; - kvm_x86_ops.set_apic_access_page_addr(vcpu, page_to_phys(page)); - - /* - * Do not pin apic access page in memory, the MMU notifier - * will call us again if it is migrated or swapped out. - */ - put_page(page); + kvm_x86_ops.set_apic_access_page_addr(vcpu); } void __kvm_request_immediate_exit(struct kvm_vcpu *vcpu) From 4de1f9d469f4515464c2c4958a59479f641970ef Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:25 -0700 Subject: [PATCH 0082/1043] KVM: VMX: Don't reload APIC access page if its control is disabled Don't reload the APIC access page if its control is disabled, e.g. if the guest is running with x2APIC (likely) or with the local APIC disabled (unlikely), to avoid unnecessary TLB flushes and VMWRITEs. Unconditionally reload the APIC access page and flush the TLB when the guest's virtual APIC transitions to "xAPIC enabled", as any changes to the APIC access page's mapping will not be recorded while the guest's virtual APIC is disabled. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-30-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 8550da629a61..a87b8c1e8de6 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6126,7 +6126,15 @@ void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu) if (flexpriority_enabled) { sec_exec_control |= SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES; - vmx_flush_tlb_current(vcpu); + kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); + + /* + * Flush the TLB, reloading the APIC access page will + * only do so if its physical address has changed, but + * the guest may have inserted a non-APIC mapping into + * the TLB while the APIC access page was disabled. + */ + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); } break; case LAPIC_MODE_X2APIC: @@ -6150,6 +6158,10 @@ static void vmx_set_apic_access_page_addr(struct kvm_vcpu *vcpu) return; } + if (!(secondary_exec_controls_get(to_vmx(vcpu)) & + SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) + return; + page = gfn_to_page(vcpu->kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); if (is_error_page(page)) return; From b869855badd1387bd12415e4d5571e931825b546 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:26 -0700 Subject: [PATCH 0083/1043] KVM: x86/mmu: Move fast_cr3_switch() side effects to __kvm_mmu_new_cr3() Handle the side effects of a fast CR3 (PGD) switch up a level in __kvm_mmu_new_cr3(), which is the only caller of fast_cr3_switch(). This consolidates handling all side effects in __kvm_mmu_new_cr3() (where freeing the current root when KVM can't do a fast switch is already handled), and ameliorates the pain of adding a second boolean in a future patch to provide a separate "skip" override for the MMU sync. Cc: Vitaly Kuznetsov Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-31-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 69 +++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 38 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 575bd5de2118..6a939204d467 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4284,8 +4284,7 @@ static bool cached_root_available(struct kvm_vcpu *vcpu, gpa_t new_cr3, } static bool fast_cr3_switch(struct kvm_vcpu *vcpu, gpa_t new_cr3, - union kvm_mmu_page_role new_role, - bool skip_tlb_flush) + union kvm_mmu_page_role new_role) { struct kvm_mmu *mmu = vcpu->arch.mmu; @@ -4295,39 +4294,9 @@ static bool fast_cr3_switch(struct kvm_vcpu *vcpu, gpa_t new_cr3, * later if necessary. */ if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL && - mmu->root_level >= PT64_ROOT_4LEVEL) { - if (mmu_check_root(vcpu, new_cr3 >> PAGE_SHIFT)) - return false; - - if (cached_root_available(vcpu, new_cr3, new_role)) { - /* - * It is possible that the cached previous root page is - * obsolete because of a change in the MMU generation - * number. However, changing the generation number is - * accompanied by KVM_REQ_MMU_RELOAD, which will free - * the root set here and allocate a new one. - */ - kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); - if (!skip_tlb_flush) { - kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); - kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); - } - - /* - * The last MMIO access's GVA and GPA are cached in the - * VCPU. When switching to a new CR3, that GVA->GPA - * mapping may no longer be valid. So clear any cached - * MMIO info even when we don't need to sync the shadow - * page tables. - */ - vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY); - - __clear_sp_write_flooding_count( - page_header(mmu->root_hpa)); - - return true; - } - } + mmu->root_level >= PT64_ROOT_4LEVEL) + return !mmu_check_root(vcpu, new_cr3 >> PAGE_SHIFT) && + cached_root_available(vcpu, new_cr3, new_role); return false; } @@ -4336,9 +4305,33 @@ static void __kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, union kvm_mmu_page_role new_role, bool skip_tlb_flush) { - if (!fast_cr3_switch(vcpu, new_cr3, new_role, skip_tlb_flush)) - kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, - KVM_MMU_ROOT_CURRENT); + if (!fast_cr3_switch(vcpu, new_cr3, new_role)) { + kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, KVM_MMU_ROOT_CURRENT); + return; + } + + /* + * It's possible that the cached previous root page is obsolete because + * of a change in the MMU generation number. However, changing the + * generation number is accompanied by KVM_REQ_MMU_RELOAD, which will + * free the root set here and allocate a new one. + */ + kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); + + if (!skip_tlb_flush) { + kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); + } + + /* + * The last MMIO access's GVA and GPA are cached in the VCPU. When + * switching to a new CR3, that GVA->GPA mapping may no longer be + * valid. So clear any cached MMIO info even when we don't need to sync + * the shadow page tables. + */ + vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY); + + __clear_sp_write_flooding_count(page_header(vcpu->arch.mmu->root_hpa)); } void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush) From 4a632ac6ca66fb29b94a16495624c58f4d313f2f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:27 -0700 Subject: [PATCH 0084/1043] KVM: x86/mmu: Add separate override for MMU sync during fast CR3 switch Add a separate "skip" override for MMU sync, a future change to avoid TLB flushes on nested VMX transitions may need to sync the MMU even if the TLB flush is unnecessary. Suggested-by: Paolo Bonzini Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-32-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 3 ++- arch/x86/kvm/mmu/mmu.c | 13 +++++++------ arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/x86.c | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 541e2df8fc6e..ee5886a5006a 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1524,7 +1524,8 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva); void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, gva_t gva, hpa_t root_hpa); void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid); -void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush); +void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush, + bool skip_mmu_sync); void kvm_configure_mmu(bool enable_tdp, int tdp_page_level); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 6a939204d467..97d1e9b80b8e 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4303,7 +4303,7 @@ static bool fast_cr3_switch(struct kvm_vcpu *vcpu, gpa_t new_cr3, static void __kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, union kvm_mmu_page_role new_role, - bool skip_tlb_flush) + bool skip_tlb_flush, bool skip_mmu_sync) { if (!fast_cr3_switch(vcpu, new_cr3, new_role)) { kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, KVM_MMU_ROOT_CURRENT); @@ -4318,10 +4318,10 @@ static void __kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, */ kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); - if (!skip_tlb_flush) { + if (!skip_mmu_sync) kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); + if (!skip_tlb_flush) kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); - } /* * The last MMIO access's GVA and GPA are cached in the VCPU. When @@ -4334,10 +4334,11 @@ static void __kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, __clear_sp_write_flooding_count(page_header(vcpu->arch.mmu->root_hpa)); } -void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush) +void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush, + bool skip_mmu_sync) { __kvm_mmu_new_cr3(vcpu, new_cr3, kvm_mmu_calc_root_page_role(vcpu), - skip_tlb_flush); + skip_tlb_flush, skip_mmu_sync); } EXPORT_SYMBOL_GPL(kvm_mmu_new_cr3); @@ -5030,7 +5031,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty, execonly, level); - __kvm_mmu_new_cr3(vcpu, new_eptp, new_role.base, false); + __kvm_mmu_new_cr3(vcpu, new_eptp, new_role.base, false, false); if (new_role.as_u64 == context->mmu_role.as_u64) return; diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index a7a526a21292..bf5efb30cc21 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1101,7 +1101,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool ne } if (!nested_ept) - kvm_mmu_new_cr3(vcpu, cr3, false); + kvm_mmu_new_cr3(vcpu, cr3, false, false); vcpu->arch.cr3 = cr3; kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 90aa4abbc0a6..7bf5193ed9b4 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1031,7 +1031,7 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) return 1; - kvm_mmu_new_cr3(vcpu, cr3, skip_tlb_flush); + kvm_mmu_new_cr3(vcpu, cr3, skip_tlb_flush, skip_tlb_flush); vcpu->arch.cr3 = cr3; kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); From 71fe70130d88729abc0e658d3202618c340d2e71 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:28 -0700 Subject: [PATCH 0085/1043] KVM: x86/mmu: Add module param to force TLB flush on root reuse Add a module param, flush_on_reuse, to override skip_tlb_flush and skip_mmu_sync when performing a so called "fast cr3 switch", i.e. when reusing a cached root. The primary motiviation for the control is to provide a fallback mechanism in the event that TLB flushing and/or MMU sync bugs are exposed/introduced by upcoming changes to stop unconditionally flushing on nested VMX transitions. Suggested-by: Jim Mattson Suggested-by: Junaid Shahid Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-33-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 97d1e9b80b8e..c98948459414 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -78,6 +78,9 @@ module_param_cb(nx_huge_pages_recovery_ratio, &nx_huge_pages_recovery_ratio_ops, &nx_huge_pages_recovery_ratio, 0644); __MODULE_PARM_TYPE(nx_huge_pages_recovery_ratio, "uint"); +static bool __read_mostly force_flush_and_sync_on_reuse; +module_param_named(flush_on_reuse, force_flush_and_sync_on_reuse, bool, 0644); + /* * When setting this variable to true it enables Two-Dimensional-Paging * where the hardware walks 2 page tables: @@ -4318,9 +4321,9 @@ static void __kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, */ kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); - if (!skip_mmu_sync) + if (!skip_mmu_sync || force_flush_and_sync_on_reuse) kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); - if (!skip_tlb_flush) + if (!skip_tlb_flush || force_flush_and_sync_on_reuse) kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); /* From 41fab65e7c44ab1c3aa4b962e6f95649354db419 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:29 -0700 Subject: [PATCH 0086/1043] KVM: nVMX: Skip MMU sync on nested VMX transition when possible Skip the MMU sync when reusing a cached root if EPT is enabled or L1 enabled VPID for L2. If EPT is enabled, guest-physical mappings aren't flushed even if VPID is disabled, i.e. L1 can't expect stale TLB entries to be flushed if it has enabled EPT and L0 isn't shadowing PTEs (for L1 or L2) if L1 has EPT disabled. If VPID is enabled (and EPT is disabled), then L1 can't expect stale TLB entries to be flushed (for itself or L2). Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-34-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/vmx/nested.c | 48 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index c98948459414..34c015461c35 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5034,7 +5034,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty, execonly, level); - __kvm_mmu_new_cr3(vcpu, new_eptp, new_role.base, false, false); + __kvm_mmu_new_cr3(vcpu, new_eptp, new_role.base, false, true); if (new_role.as_u64 == context->mmu_role.as_u64) return; diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index bf5efb30cc21..a8290954caa7 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1073,6 +1073,48 @@ static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val) return (val & invalid_mask) == 0; } +/* + * Returns true if the MMU needs to be sync'd on nested VM-Enter/VM-Exit. + * tl;dr: the MMU needs a sync if L0 is using shadow paging and L1 didn't + * enable VPID for L2 (implying it expects a TLB flush on VMX transitions). + * Here's why. + * + * If EPT is enabled by L0 a sync is never needed: + * - if it is disabled by L1, then L0 is not shadowing L1 or L2 PTEs, there + * cannot be unsync'd SPTEs for either L1 or L2. + * + * - if it is also enabled by L1, then L0 doesn't need to sync on VM-Enter + * VM-Enter as VM-Enter isn't required to invalidate guest-physical mappings + * (irrespective of VPID), i.e. L1 can't rely on the (virtual) CPU to flush + * stale guest-physical mappings for L2 from the TLB. And as above, L0 isn't + * shadowing L1 PTEs so there are no unsync'd SPTEs to sync on VM-Exit. + * + * If EPT is disabled by L0: + * - if VPID is enabled by L1 (for L2), the situation is similar to when L1 + * enables EPT: L0 doesn't need to sync as VM-Enter and VM-Exit aren't + * required to invalidate linear mappings (EPT is disabled so there are + * no combined or guest-physical mappings), i.e. L1 can't rely on the + * (virtual) CPU to flush stale linear mappings for either L2 or itself (L1). + * + * - however if VPID is disabled by L1, then a sync is needed as L1 expects all + * linear mappings (EPT is disabled so there are no combined or guest-physical + * mappings) to be invalidated on both VM-Enter and VM-Exit. + * + * Note, this logic is subtly different than nested_has_guest_tlb_tag(), which + * additionally checks that L2 has been assigned a VPID (when EPT is disabled). + * Whether or not L2 has been assigned a VPID by L0 is irrelevant with respect + * to L1's expectations, e.g. L0 needs to invalidate hardware TLB entries if L2 + * doesn't have a unique VPID to prevent reusing L1's entries (assuming L1 has + * been assigned a VPID), but L0 doesn't need to do a MMU sync because L1 + * doesn't expect stale (virtual) TLB entries to be flushed, i.e. L1 doesn't + * know that L0 will flush the TLB and so L1 will do INVVPID as needed to flush + * stale TLB entries, at which point L0 will sync L2's MMU. + */ +static bool nested_vmx_transition_mmu_sync(struct kvm_vcpu *vcpu) +{ + return !enable_ept && !nested_cpu_has_vpid(get_vmcs12(vcpu)); +} + /* * Load guest's/host's cr3 at nested entry/exit. @nested_ept is true if we are * emulating VM-Entry into a guest with EPT enabled. On failure, the expected @@ -1100,8 +1142,12 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool ne } } + /* + * See nested_vmx_transition_mmu_sync for details on skipping the MMU sync. + */ if (!nested_ept) - kvm_mmu_new_cr3(vcpu, cr3, false, false); + kvm_mmu_new_cr3(vcpu, cr3, false, + !nested_vmx_transition_mmu_sync(vcpu)); vcpu->arch.cr3 = cr3; kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); From 9805c5f74b8a65da397ec6b70734e44f1f191c0b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:30 -0700 Subject: [PATCH 0087/1043] KVM: nVMX: Don't flush TLB on nested VMX transition Unconditionally skip the TLB flush triggered when reusing a root for a nested transition as nested_vmx_transition_tlb_flush() ensures the TLB is flushed when needed, regardless of whether the MMU can reuse a cached root (or the last root). Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-35-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 2 +- arch/x86/kvm/vmx/nested.c | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 34c015461c35..10fb9e7f1eca 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -5034,7 +5034,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty, execonly, level); - __kvm_mmu_new_cr3(vcpu, new_eptp, new_role.base, false, true); + __kvm_mmu_new_cr3(vcpu, new_eptp, new_role.base, true, true); if (new_role.as_u64 == context->mmu_role.as_u64) return; diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index a8290954caa7..237e2e32a659 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1143,10 +1143,12 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool ne } /* - * See nested_vmx_transition_mmu_sync for details on skipping the MMU sync. + * Unconditionally skip the TLB flush on fast CR3 switch, all TLB + * flushes are handled by nested_vmx_transition_tlb_flush(). See + * nested_vmx_transition_mmu_sync for details on skipping the MMU sync. */ if (!nested_ept) - kvm_mmu_new_cr3(vcpu, cr3, false, + kvm_mmu_new_cr3(vcpu, cr3, true, !nested_vmx_transition_mmu_sync(vcpu)); vcpu->arch.cr3 = cr3; From ce8fe7b77bd8ee405295e349c82d0ef8c9788200 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:31 -0700 Subject: [PATCH 0088/1043] KVM: nVMX: Free only the affected contexts when emulating INVEPT Add logic to handle_invept() to free only those roots that match the target EPT context when emulating a single-context INVEPT. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-36-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 42 ++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 237e2e32a659..7606d544ce9c 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5163,17 +5163,27 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu) return nested_vmx_succeed(vcpu); } +#define EPTP_PA_MASK GENMASK_ULL(51, 12) + +static bool nested_ept_root_matches(hpa_t root_hpa, u64 root_eptp, u64 eptp) +{ + return VALID_PAGE(root_hpa) && + ((root_eptp & EPTP_PA_MASK) == (eptp & EPTP_PA_MASK)); +} + /* Emulate the INVEPT instruction */ static int handle_invept(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); u32 vmx_instruction_info, types; - unsigned long type; + unsigned long type, roots_to_free; + struct kvm_mmu *mmu; gva_t gva; struct x86_exception e; struct { u64 eptp, gpa; } operand; + int i; if (!(vmx->nested.msrs.secondary_ctls_high & SECONDARY_EXEC_ENABLE_EPT) || @@ -5205,27 +5215,41 @@ static int handle_invept(struct kvm_vcpu *vcpu) return 1; } + /* + * Nested EPT roots are always held through guest_mmu, + * not root_mmu. + */ + mmu = &vcpu->arch.guest_mmu; + switch (type) { case VMX_EPT_EXTENT_CONTEXT: if (!nested_vmx_check_eptp(vcpu, operand.eptp)) return nested_vmx_failValid(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); - /* TODO: sync only the target EPTP context. */ - fallthrough; + roots_to_free = 0; + if (nested_ept_root_matches(mmu->root_hpa, mmu->root_cr3, + operand.eptp)) + roots_to_free |= KVM_MMU_ROOT_CURRENT; + + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { + if (nested_ept_root_matches(mmu->prev_roots[i].hpa, + mmu->prev_roots[i].cr3, + operand.eptp)) + roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); + } + break; case VMX_EPT_EXTENT_GLOBAL: - /* - * Nested EPT roots are always held through guest_mmu, - * not root_mmu. - */ - kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, - KVM_MMU_ROOTS_ALL); + roots_to_free = KVM_MMU_ROOTS_ALL; break; default: BUG_ON(1); break; } + if (roots_to_free) + kvm_mmu_free_roots(vcpu, mmu, roots_to_free); + return nested_vmx_succeed(vcpu); } From be01e8e2c632c41c69bb30e7196661ec6e8fdc10 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:32 -0700 Subject: [PATCH 0089/1043] KVM: x86: Replace "cr3" with "pgd" in "new cr3/pgd" related code Rename functions and variables in kvm_mmu_new_cr3() and related code to replace "cr3" with "pgd", i.e. continue the work started by commit 727a7e27cf88a ("KVM: x86: rename set_cr3 callback and related flags to load_mmu_pgd"). kvm_mmu_new_cr3() and company are not always loading a new CR3, e.g. when nested EPT is enabled "cr3" is actually an EPTP. Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-37-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 8 ++--- arch/x86/kvm/mmu/mmu.c | 58 ++++++++++++++++----------------- arch/x86/kvm/vmx/nested.c | 6 ++-- arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/kvm/x86.c | 2 +- 5 files changed, 38 insertions(+), 38 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index ee5886a5006a..d099749168f0 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -375,12 +375,12 @@ struct rsvd_bits_validate { }; struct kvm_mmu_root_info { - gpa_t cr3; + gpa_t pgd; hpa_t hpa; }; #define KVM_MMU_ROOT_INFO_INVALID \ - ((struct kvm_mmu_root_info) { .cr3 = INVALID_PAGE, .hpa = INVALID_PAGE }) + ((struct kvm_mmu_root_info) { .pgd = INVALID_PAGE, .hpa = INVALID_PAGE }) #define KVM_MMU_NUM_PREV_ROOTS 3 @@ -406,7 +406,7 @@ struct kvm_mmu { void (*update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, u64 *spte, const void *pte); hpa_t root_hpa; - gpa_t root_cr3; + gpa_t root_pgd; union kvm_mmu_role mmu_role; u8 root_level; u8 shadow_root_level; @@ -1524,7 +1524,7 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva); void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, gva_t gva, hpa_t root_hpa); void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid); -void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush, +void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, bool skip_tlb_flush, bool skip_mmu_sync); void kvm_configure_mmu(bool enable_tdp, int tdp_page_level); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 10fb9e7f1eca..3385d6dde541 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3665,7 +3665,7 @@ void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, &invalid_list); mmu->root_hpa = INVALID_PAGE; } - mmu->root_cr3 = 0; + mmu->root_pgd = 0; } kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list); @@ -3722,8 +3722,8 @@ static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu) } else BUG(); - /* root_cr3 is ignored for direct MMUs. */ - vcpu->arch.mmu->root_cr3 = 0; + /* root_pgd is ignored for direct MMUs. */ + vcpu->arch.mmu->root_pgd = 0; return 0; } @@ -3732,11 +3732,11 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) { struct kvm_mmu_page *sp; u64 pdptr, pm_mask; - gfn_t root_gfn, root_cr3; + gfn_t root_gfn, root_pgd; int i; - root_cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu); - root_gfn = root_cr3 >> PAGE_SHIFT; + root_pgd = vcpu->arch.mmu->get_guest_pgd(vcpu); + root_gfn = root_pgd >> PAGE_SHIFT; if (mmu_check_root(vcpu, root_gfn)) return 1; @@ -3761,7 +3761,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) ++sp->root_count; spin_unlock(&vcpu->kvm->mmu_lock); vcpu->arch.mmu->root_hpa = root; - goto set_root_cr3; + goto set_root_pgd; } /* @@ -3827,8 +3827,8 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->lm_root); } -set_root_cr3: - vcpu->arch.mmu->root_cr3 = root_cr3; +set_root_pgd: + vcpu->arch.mmu->root_pgd = root_pgd; return 0; } @@ -4244,49 +4244,49 @@ static void nonpaging_init_context(struct kvm_vcpu *vcpu, context->nx = false; } -static inline bool is_root_usable(struct kvm_mmu_root_info *root, gpa_t cr3, +static inline bool is_root_usable(struct kvm_mmu_root_info *root, gpa_t pgd, union kvm_mmu_page_role role) { - return (role.direct || cr3 == root->cr3) && + return (role.direct || pgd == root->pgd) && VALID_PAGE(root->hpa) && page_header(root->hpa) && role.word == page_header(root->hpa)->role.word; } /* - * Find out if a previously cached root matching the new CR3/role is available. + * Find out if a previously cached root matching the new pgd/role is available. * The current root is also inserted into the cache. * If a matching root was found, it is assigned to kvm_mmu->root_hpa and true is * returned. * Otherwise, the LRU root from the cache is assigned to kvm_mmu->root_hpa and * false is returned. This root should now be freed by the caller. */ -static bool cached_root_available(struct kvm_vcpu *vcpu, gpa_t new_cr3, +static bool cached_root_available(struct kvm_vcpu *vcpu, gpa_t new_pgd, union kvm_mmu_page_role new_role) { uint i; struct kvm_mmu_root_info root; struct kvm_mmu *mmu = vcpu->arch.mmu; - root.cr3 = mmu->root_cr3; + root.pgd = mmu->root_pgd; root.hpa = mmu->root_hpa; - if (is_root_usable(&root, new_cr3, new_role)) + if (is_root_usable(&root, new_pgd, new_role)) return true; for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { swap(root, mmu->prev_roots[i]); - if (is_root_usable(&root, new_cr3, new_role)) + if (is_root_usable(&root, new_pgd, new_role)) break; } mmu->root_hpa = root.hpa; - mmu->root_cr3 = root.cr3; + mmu->root_pgd = root.pgd; return i < KVM_MMU_NUM_PREV_ROOTS; } -static bool fast_cr3_switch(struct kvm_vcpu *vcpu, gpa_t new_cr3, +static bool fast_pgd_switch(struct kvm_vcpu *vcpu, gpa_t new_pgd, union kvm_mmu_page_role new_role) { struct kvm_mmu *mmu = vcpu->arch.mmu; @@ -4298,17 +4298,17 @@ static bool fast_cr3_switch(struct kvm_vcpu *vcpu, gpa_t new_cr3, */ if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL && mmu->root_level >= PT64_ROOT_4LEVEL) - return !mmu_check_root(vcpu, new_cr3 >> PAGE_SHIFT) && - cached_root_available(vcpu, new_cr3, new_role); + return !mmu_check_root(vcpu, new_pgd >> PAGE_SHIFT) && + cached_root_available(vcpu, new_pgd, new_role); return false; } -static void __kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, +static void __kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, union kvm_mmu_page_role new_role, bool skip_tlb_flush, bool skip_mmu_sync) { - if (!fast_cr3_switch(vcpu, new_cr3, new_role)) { + if (!fast_pgd_switch(vcpu, new_pgd, new_role)) { kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, KVM_MMU_ROOT_CURRENT); return; } @@ -4337,13 +4337,13 @@ static void __kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, __clear_sp_write_flooding_count(page_header(vcpu->arch.mmu->root_hpa)); } -void kvm_mmu_new_cr3(struct kvm_vcpu *vcpu, gpa_t new_cr3, bool skip_tlb_flush, +void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, bool skip_tlb_flush, bool skip_mmu_sync) { - __kvm_mmu_new_cr3(vcpu, new_cr3, kvm_mmu_calc_root_page_role(vcpu), + __kvm_mmu_new_pgd(vcpu, new_pgd, kvm_mmu_calc_root_page_role(vcpu), skip_tlb_flush, skip_mmu_sync); } -EXPORT_SYMBOL_GPL(kvm_mmu_new_cr3); +EXPORT_SYMBOL_GPL(kvm_mmu_new_pgd); static unsigned long get_cr3(struct kvm_vcpu *vcpu) { @@ -5034,7 +5034,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty, execonly, level); - __kvm_mmu_new_cr3(vcpu, new_eptp, new_role.base, true, true); + __kvm_mmu_new_pgd(vcpu, new_eptp, new_role.base, true, true); if (new_role.as_u64 == context->mmu_role.as_u64) return; @@ -5551,7 +5551,7 @@ void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid) for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { if (VALID_PAGE(mmu->prev_roots[i].hpa) && - pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].cr3)) { + pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd)) { mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa); tlb_flush = true; } @@ -5705,13 +5705,13 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu) vcpu->arch.walk_mmu = &vcpu->arch.root_mmu; vcpu->arch.root_mmu.root_hpa = INVALID_PAGE; - vcpu->arch.root_mmu.root_cr3 = 0; + vcpu->arch.root_mmu.root_pgd = 0; vcpu->arch.root_mmu.translate_gpa = translate_gpa; for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) vcpu->arch.root_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID; vcpu->arch.guest_mmu.root_hpa = INVALID_PAGE; - vcpu->arch.guest_mmu.root_cr3 = 0; + vcpu->arch.guest_mmu.root_pgd = 0; vcpu->arch.guest_mmu.translate_gpa = translate_gpa; for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) vcpu->arch.guest_mmu.prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID; diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 7606d544ce9c..aca57d8da400 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1148,7 +1148,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool ne * nested_vmx_transition_mmu_sync for details on skipping the MMU sync. */ if (!nested_ept) - kvm_mmu_new_cr3(vcpu, cr3, true, + kvm_mmu_new_pgd(vcpu, cr3, true, !nested_vmx_transition_mmu_sync(vcpu)); vcpu->arch.cr3 = cr3; @@ -5228,13 +5228,13 @@ static int handle_invept(struct kvm_vcpu *vcpu) VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); roots_to_free = 0; - if (nested_ept_root_matches(mmu->root_hpa, mmu->root_cr3, + if (nested_ept_root_matches(mmu->root_hpa, mmu->root_pgd, operand.eptp)) roots_to_free |= KVM_MMU_ROOT_CURRENT; for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { if (nested_ept_root_matches(mmu->prev_roots[i].hpa, - mmu->prev_roots[i].cr3, + mmu->prev_roots[i].pgd, operand.eptp)) roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index a87b8c1e8de6..c9c959fb8a66 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5476,7 +5476,7 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) } for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) - if (kvm_get_pcid(vcpu, vcpu->arch.mmu->prev_roots[i].cr3) + if (kvm_get_pcid(vcpu, vcpu->arch.mmu->prev_roots[i].pgd) == operand.pcid) roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 7bf5193ed9b4..de77bc9bd0d7 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1031,7 +1031,7 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) return 1; - kvm_mmu_new_cr3(vcpu, cr3, skip_tlb_flush, skip_tlb_flush); + kvm_mmu_new_pgd(vcpu, cr3, skip_tlb_flush, skip_tlb_flush); vcpu->arch.cr3 = cr3; kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); From be100ef136254ab4a3ff223e2288aae6c5809ac6 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 20 Mar 2020 14:28:33 -0700 Subject: [PATCH 0090/1043] KVM: VMX: Clean cr3/pgd handling in vmx_load_mmu_pgd() Rename @cr3 to @pgd in vmx_load_mmu_pgd() to reflect that it will be loaded into vmcs.EPT_POINTER and not vmcs.GUEST_CR3 when EPT is enabled. Similarly, load guest_cr3 with @pgd if and only if EPT is disabled. This fixes one of the last, if not _the_ last, cases in KVM where a variable that is not strictly a cr3 value uses "cr3" instead of "pgd". Signed-off-by: Sean Christopherson Message-Id: <20200320212833.3507-38-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index c9c959fb8a66..aa1b8cf7c915 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -3034,16 +3034,15 @@ u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa) return eptp; } -void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long cr3) +void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long pgd) { struct kvm *kvm = vcpu->kvm; bool update_guest_cr3 = true; unsigned long guest_cr3; u64 eptp; - guest_cr3 = cr3; if (enable_ept) { - eptp = construct_eptp(vcpu, cr3); + eptp = construct_eptp(vcpu, pgd); vmcs_write64(EPT_POINTER, eptp); if (kvm_x86_ops.tlb_remote_flush) { @@ -3064,6 +3063,8 @@ void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long cr3) else /* vmcs01.GUEST_CR3 is already up-to-date. */ update_guest_cr3 = false; ept_load_pdptrs(vcpu); + } else { + guest_cr3 = pgd; } if (update_guest_cr3) From 1c164cb3ffd084e5359a56abde715acf121e69a4 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Sat, 11 Apr 2020 17:36:27 +0200 Subject: [PATCH 0091/1043] KVM: SVM: Use do_machine_check to pass MCE to the host Use do_machine_check instead of INT $12 to pass MCE to the host, the same approach VMX uses. On a related note, there is no reason to limit the use of do_machine_check to 64 bit targets, as is currently done for VMX. MCE handling works for both target families. The patch is only compile tested, for both, 64 and 32 bit targets, someone should test the passing of the exception by injecting some MCEs into the guest. For future non-RFC patch, kvm_machine_check should be moved to some appropriate header file. Cc: Paolo Bonzini Cc: Joerg Roedel Cc: Sean Christopherson Signed-off-by: Uros Bizjak Message-Id: <20200411153627.3474710-1-ubizjak@gmail.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 66123848448d..747e60c65c98 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -1839,6 +1840,25 @@ static bool is_erratum_383(void) return true; } +/* + * Trigger machine check on the host. We assume all the MSRs are already set up + * by the CPU and that we still run on the same CPU as the MCE occurred on. + * We pass a fake environment to the machine check handler because we want + * the guest to be always treated like user space, no matter what context + * it used internally. + */ +static void kvm_machine_check(void) +{ +#if defined(CONFIG_X86_MCE) + struct pt_regs regs = { + .cs = 3, /* Fake ring 3 no matter what the guest ran on */ + .flags = X86_EFLAGS_IF, + }; + + do_machine_check(®s, 0); +#endif +} + static void svm_handle_mce(struct vcpu_svm *svm) { if (is_erratum_383()) { @@ -1857,11 +1877,7 @@ static void svm_handle_mce(struct vcpu_svm *svm) * On an #MC intercept the MCE handler is not called automatically in * the host. So do it by hand here. */ - asm volatile ( - "int $0x12\n"); - /* not sure if we ever come back to this point */ - - return; + kvm_machine_check(); } static int mc_interception(struct vcpu_svm *svm) From 63d04348371b7ea4a134bcf47c79763d969e9168 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 1 Apr 2020 00:42:22 +0200 Subject: [PATCH 0092/1043] KVM: x86: move kvm_create_vcpu_debugfs after last failure point The placement of kvm_create_vcpu_debugfs is more or less irrelevant, since it cannot fail and userspace should not care about the debugfs entries until it knows the vcpu has been created. Moving it after the last failure point removes the need to remove the directory when unwinding the creation. Reviewed-by: Emanuele Giuseppe Esposito Message-Id: <20200331224222.393439-1-pbonzini@redhat.com> Signed-off-by: Paolo Bonzini --- virt/kvm/kvm_main.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 03571f6acaa8..da8fd45e0e3e 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -3031,8 +3031,6 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) if (r) goto vcpu_free_run_page; - kvm_create_vcpu_debugfs(vcpu); - mutex_lock(&kvm->lock); if (kvm_get_vcpu_by_id(kvm, id)) { r = -EEXIST; @@ -3061,11 +3059,11 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) mutex_unlock(&kvm->lock); kvm_arch_vcpu_postcreate(vcpu); + kvm_create_vcpu_debugfs(vcpu); return r; unlock_vcpu_destroy: mutex_unlock(&kvm->lock); - debugfs_remove_recursive(vcpu->debugfs_dentry); kvm_arch_vcpu_destroy(vcpu); vcpu_free_run_page: free_page((unsigned long)vcpu->run); From 812756a82ea51e3c7ff7ba5e6fa3f34345234bc7 Mon Sep 17 00:00:00 2001 From: Emanuele Giuseppe Esposito Date: Tue, 14 Apr 2020 17:56:25 +0200 Subject: [PATCH 0093/1043] kvm_host: unify VM_STAT and VCPU_STAT definitions in a single place The macros VM_STAT and VCPU_STAT are redundantly implemented in multiple files, each used by a different architecure to initialize the debugfs entries for statistics. Since they all have the same purpose, they can be unified in a single common definition in include/linux/kvm_host.h Signed-off-by: Emanuele Giuseppe Esposito Message-Id: <20200414155625.20559-1-eesposit@redhat.com> Acked-by: Cornelia Huck Signed-off-by: Paolo Bonzini --- arch/arm64/kvm/guest.c | 23 ++--- arch/mips/kvm/mips.c | 61 ++++++------ arch/powerpc/kvm/book3s.c | 61 ++++++------ arch/powerpc/kvm/booke.c | 41 ++++---- arch/s390/kvm/kvm-s390.c | 203 +++++++++++++++++++------------------- arch/x86/kvm/x86.c | 80 +++++++-------- include/linux/kvm_host.h | 5 + 7 files changed, 231 insertions(+), 243 deletions(-) diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 23ebe51410f0..8417b200bec9 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -29,20 +29,17 @@ #include "trace.h" -#define VM_STAT(x) { #x, offsetof(struct kvm, stat.x), KVM_STAT_VM } -#define VCPU_STAT(x) { #x, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU } - struct kvm_stats_debugfs_item debugfs_entries[] = { - VCPU_STAT(halt_successful_poll), - VCPU_STAT(halt_attempted_poll), - VCPU_STAT(halt_poll_invalid), - VCPU_STAT(halt_wakeup), - VCPU_STAT(hvc_exit_stat), - VCPU_STAT(wfe_exit_stat), - VCPU_STAT(wfi_exit_stat), - VCPU_STAT(mmio_exit_user), - VCPU_STAT(mmio_exit_kernel), - VCPU_STAT(exits), + VCPU_STAT("halt_successful_poll", halt_successful_poll), + VCPU_STAT("halt_attempted_poll", halt_attempted_poll), + VCPU_STAT("halt_poll_invalid", halt_poll_invalid), + VCPU_STAT("halt_wakeup", halt_wakeup), + VCPU_STAT("hvc_exit_stat", hvc_exit_stat), + VCPU_STAT("wfe_exit_stat", wfe_exit_stat), + VCPU_STAT("wfi_exit_stat", wfi_exit_stat), + VCPU_STAT("mmio_exit_user", mmio_exit_user), + VCPU_STAT("mmio_exit_kernel", mmio_exit_kernel), + VCPU_STAT("exits", exits), { NULL } }; diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index 8f05dd0a0f4e..fdf1c14d9205 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -39,40 +39,39 @@ #define VECTORSPACING 0x100 /* for EI/VI mode */ #endif -#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x) struct kvm_stats_debugfs_item debugfs_entries[] = { - { "wait", VCPU_STAT(wait_exits), KVM_STAT_VCPU }, - { "cache", VCPU_STAT(cache_exits), KVM_STAT_VCPU }, - { "signal", VCPU_STAT(signal_exits), KVM_STAT_VCPU }, - { "interrupt", VCPU_STAT(int_exits), KVM_STAT_VCPU }, - { "cop_unusable", VCPU_STAT(cop_unusable_exits), KVM_STAT_VCPU }, - { "tlbmod", VCPU_STAT(tlbmod_exits), KVM_STAT_VCPU }, - { "tlbmiss_ld", VCPU_STAT(tlbmiss_ld_exits), KVM_STAT_VCPU }, - { "tlbmiss_st", VCPU_STAT(tlbmiss_st_exits), KVM_STAT_VCPU }, - { "addrerr_st", VCPU_STAT(addrerr_st_exits), KVM_STAT_VCPU }, - { "addrerr_ld", VCPU_STAT(addrerr_ld_exits), KVM_STAT_VCPU }, - { "syscall", VCPU_STAT(syscall_exits), KVM_STAT_VCPU }, - { "resvd_inst", VCPU_STAT(resvd_inst_exits), KVM_STAT_VCPU }, - { "break_inst", VCPU_STAT(break_inst_exits), KVM_STAT_VCPU }, - { "trap_inst", VCPU_STAT(trap_inst_exits), KVM_STAT_VCPU }, - { "msa_fpe", VCPU_STAT(msa_fpe_exits), KVM_STAT_VCPU }, - { "fpe", VCPU_STAT(fpe_exits), KVM_STAT_VCPU }, - { "msa_disabled", VCPU_STAT(msa_disabled_exits), KVM_STAT_VCPU }, - { "flush_dcache", VCPU_STAT(flush_dcache_exits), KVM_STAT_VCPU }, + VCPU_STAT("wait", wait_exits), + VCPU_STAT("cache", cache_exits), + VCPU_STAT("signal", signal_exits), + VCPU_STAT("interrupt", int_exits), + VCPU_STAT("cop_unusable", cop_unusable_exits), + VCPU_STAT("tlbmod", tlbmod_exits), + VCPU_STAT("tlbmiss_ld", tlbmiss_ld_exits), + VCPU_STAT("tlbmiss_st", tlbmiss_st_exits), + VCPU_STAT("addrerr_st", addrerr_st_exits), + VCPU_STAT("addrerr_ld", addrerr_ld_exits), + VCPU_STAT("syscall", syscall_exits), + VCPU_STAT("resvd_inst", resvd_inst_exits), + VCPU_STAT("break_inst", break_inst_exits), + VCPU_STAT("trap_inst", trap_inst_exits), + VCPU_STAT("msa_fpe", msa_fpe_exits), + VCPU_STAT("fpe", fpe_exits), + VCPU_STAT("msa_disabled", msa_disabled_exits), + VCPU_STAT("flush_dcache", flush_dcache_exits), #ifdef CONFIG_KVM_MIPS_VZ - { "vz_gpsi", VCPU_STAT(vz_gpsi_exits), KVM_STAT_VCPU }, - { "vz_gsfc", VCPU_STAT(vz_gsfc_exits), KVM_STAT_VCPU }, - { "vz_hc", VCPU_STAT(vz_hc_exits), KVM_STAT_VCPU }, - { "vz_grr", VCPU_STAT(vz_grr_exits), KVM_STAT_VCPU }, - { "vz_gva", VCPU_STAT(vz_gva_exits), KVM_STAT_VCPU }, - { "vz_ghfc", VCPU_STAT(vz_ghfc_exits), KVM_STAT_VCPU }, - { "vz_gpa", VCPU_STAT(vz_gpa_exits), KVM_STAT_VCPU }, - { "vz_resvd", VCPU_STAT(vz_resvd_exits), KVM_STAT_VCPU }, + VCPU_STAT("vz_gpsi", vz_gpsi_exits), + VCPU_STAT("vz_gsfc", vz_gsfc_exits), + VCPU_STAT("vz_hc", vz_hc_exits), + VCPU_STAT("vz_grr", vz_grr_exits), + VCPU_STAT("vz_gva", vz_gva_exits), + VCPU_STAT("vz_ghfc", vz_ghfc_exits), + VCPU_STAT("vz_gpa", vz_gpa_exits), + VCPU_STAT("vz_resvd", vz_resvd_exits), #endif - { "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU }, - { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll), KVM_STAT_VCPU }, - { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid), KVM_STAT_VCPU }, - { "halt_wakeup", VCPU_STAT(halt_wakeup), KVM_STAT_VCPU }, + VCPU_STAT("halt_successful_poll", halt_successful_poll), + VCPU_STAT("halt_attempted_poll", halt_attempted_poll), + VCPU_STAT("halt_poll_invalid", halt_poll_invalid), + VCPU_STAT("halt_wakeup", halt_wakeup), {NULL} }; diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c index 5690a1f9b976..37508a356f28 100644 --- a/arch/powerpc/kvm/book3s.c +++ b/arch/powerpc/kvm/book3s.c @@ -36,41 +36,38 @@ #include "book3s.h" #include "trace.h" -#define VM_STAT(x, ...) offsetof(struct kvm, stat.x), KVM_STAT_VM, ## __VA_ARGS__ -#define VCPU_STAT(x, ...) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__ - /* #define EXIT_DEBUG */ struct kvm_stats_debugfs_item debugfs_entries[] = { - { "exits", VCPU_STAT(sum_exits) }, - { "mmio", VCPU_STAT(mmio_exits) }, - { "sig", VCPU_STAT(signal_exits) }, - { "sysc", VCPU_STAT(syscall_exits) }, - { "inst_emu", VCPU_STAT(emulated_inst_exits) }, - { "dec", VCPU_STAT(dec_exits) }, - { "ext_intr", VCPU_STAT(ext_intr_exits) }, - { "queue_intr", VCPU_STAT(queue_intr) }, - { "halt_poll_success_ns", VCPU_STAT(halt_poll_success_ns) }, - { "halt_poll_fail_ns", VCPU_STAT(halt_poll_fail_ns) }, - { "halt_wait_ns", VCPU_STAT(halt_wait_ns) }, - { "halt_successful_poll", VCPU_STAT(halt_successful_poll), }, - { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll), }, - { "halt_successful_wait", VCPU_STAT(halt_successful_wait) }, - { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) }, - { "halt_wakeup", VCPU_STAT(halt_wakeup) }, - { "pf_storage", VCPU_STAT(pf_storage) }, - { "sp_storage", VCPU_STAT(sp_storage) }, - { "pf_instruc", VCPU_STAT(pf_instruc) }, - { "sp_instruc", VCPU_STAT(sp_instruc) }, - { "ld", VCPU_STAT(ld) }, - { "ld_slow", VCPU_STAT(ld_slow) }, - { "st", VCPU_STAT(st) }, - { "st_slow", VCPU_STAT(st_slow) }, - { "pthru_all", VCPU_STAT(pthru_all) }, - { "pthru_host", VCPU_STAT(pthru_host) }, - { "pthru_bad_aff", VCPU_STAT(pthru_bad_aff) }, - { "largepages_2M", VM_STAT(num_2M_pages, .mode = 0444) }, - { "largepages_1G", VM_STAT(num_1G_pages, .mode = 0444) }, + VCPU_STAT("exits", sum_exits), + VCPU_STAT("mmio", mmio_exits), + VCPU_STAT("sig", signal_exits), + VCPU_STAT("sysc", syscall_exits), + VCPU_STAT("inst_emu", emulated_inst_exits), + VCPU_STAT("dec", dec_exits), + VCPU_STAT("ext_intr", ext_intr_exits), + VCPU_STAT("queue_intr", queue_intr), + VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), + VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), + VCPU_STAT("halt_wait_ns", halt_wait_ns), + VCPU_STAT("halt_successful_poll", halt_successful_poll), + VCPU_STAT("halt_attempted_poll", halt_attempted_poll), + VCPU_STAT("halt_successful_wait", halt_successful_wait), + VCPU_STAT("halt_poll_invalid", halt_poll_invalid), + VCPU_STAT("halt_wakeup", halt_wakeup), + VCPU_STAT("pf_storage", pf_storage), + VCPU_STAT("sp_storage", sp_storage), + VCPU_STAT("pf_instruc", pf_instruc), + VCPU_STAT("sp_instruc", sp_instruc), + VCPU_STAT("ld", ld), + VCPU_STAT("ld_slow", ld_slow), + VCPU_STAT("st", st), + VCPU_STAT("st_slow", st_slow), + VCPU_STAT("pthru_all", pthru_all), + VCPU_STAT("pthru_host", pthru_host), + VCPU_STAT("pthru_bad_aff", pthru_bad_aff), + VM_STAT("largepages_2M", num_2M_pages, .mode = 0444), + VM_STAT("largepages_1G", num_1G_pages, .mode = 0444), { NULL } }; diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c index 6c18ea88fd25..c2984cb6dfa7 100644 --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -35,29 +35,26 @@ unsigned long kvmppc_booke_handlers; -#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM -#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU - struct kvm_stats_debugfs_item debugfs_entries[] = { - { "mmio", VCPU_STAT(mmio_exits) }, - { "sig", VCPU_STAT(signal_exits) }, - { "itlb_r", VCPU_STAT(itlb_real_miss_exits) }, - { "itlb_v", VCPU_STAT(itlb_virt_miss_exits) }, - { "dtlb_r", VCPU_STAT(dtlb_real_miss_exits) }, - { "dtlb_v", VCPU_STAT(dtlb_virt_miss_exits) }, - { "sysc", VCPU_STAT(syscall_exits) }, - { "isi", VCPU_STAT(isi_exits) }, - { "dsi", VCPU_STAT(dsi_exits) }, - { "inst_emu", VCPU_STAT(emulated_inst_exits) }, - { "dec", VCPU_STAT(dec_exits) }, - { "ext_intr", VCPU_STAT(ext_intr_exits) }, - { "halt_successful_poll", VCPU_STAT(halt_successful_poll) }, - { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) }, - { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) }, - { "halt_wakeup", VCPU_STAT(halt_wakeup) }, - { "doorbell", VCPU_STAT(dbell_exits) }, - { "guest doorbell", VCPU_STAT(gdbell_exits) }, - { "remote_tlb_flush", VM_STAT(remote_tlb_flush) }, + VCPU_STAT("mmio", mmio_exits), + VCPU_STAT("sig", signal_exits), + VCPU_STAT("itlb_r", itlb_real_miss_exits), + VCPU_STAT("itlb_v", itlb_virt_miss_exits), + VCPU_STAT("dtlb_r", dtlb_real_miss_exits), + VCPU_STAT("dtlb_v", dtlb_virt_miss_exits), + VCPU_STAT("sysc", syscall_exits), + VCPU_STAT("isi", isi_exits), + VCPU_STAT("dsi", dsi_exits), + VCPU_STAT("inst_emu", emulated_inst_exits), + VCPU_STAT("dec", dec_exits), + VCPU_STAT("ext_intr", ext_intr_exits), + VCPU_STAT("halt_successful_poll", halt_successful_poll), + VCPU_STAT("halt_attempted_poll", halt_attempted_poll), + VCPU_STAT("halt_poll_invalid", halt_poll_invalid), + VCPU_STAT("halt_wakeup", halt_wakeup), + VCPU_STAT("doorbell", dbell_exits), + VCPU_STAT("guest doorbell", gdbell_exits), + VM_STAT("remote_tlb_flush", remote_tlb_flush), { NULL } }; diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 5dcf9ff12828..5307929a6c89 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -57,110 +57,107 @@ #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \ (KVM_MAX_VCPUS + LOCAL_IRQS)) -#define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU -#define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM - struct kvm_stats_debugfs_item debugfs_entries[] = { - { "userspace_handled", VCPU_STAT(exit_userspace) }, - { "exit_null", VCPU_STAT(exit_null) }, - { "exit_validity", VCPU_STAT(exit_validity) }, - { "exit_stop_request", VCPU_STAT(exit_stop_request) }, - { "exit_external_request", VCPU_STAT(exit_external_request) }, - { "exit_io_request", VCPU_STAT(exit_io_request) }, - { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) }, - { "exit_instruction", VCPU_STAT(exit_instruction) }, - { "exit_pei", VCPU_STAT(exit_pei) }, - { "exit_program_interruption", VCPU_STAT(exit_program_interruption) }, - { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) }, - { "exit_operation_exception", VCPU_STAT(exit_operation_exception) }, - { "halt_successful_poll", VCPU_STAT(halt_successful_poll) }, - { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) }, - { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) }, - { "halt_no_poll_steal", VCPU_STAT(halt_no_poll_steal) }, - { "halt_wakeup", VCPU_STAT(halt_wakeup) }, - { "instruction_lctlg", VCPU_STAT(instruction_lctlg) }, - { "instruction_lctl", VCPU_STAT(instruction_lctl) }, - { "instruction_stctl", VCPU_STAT(instruction_stctl) }, - { "instruction_stctg", VCPU_STAT(instruction_stctg) }, - { "deliver_ckc", VCPU_STAT(deliver_ckc) }, - { "deliver_cputm", VCPU_STAT(deliver_cputm) }, - { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) }, - { "deliver_external_call", VCPU_STAT(deliver_external_call) }, - { "deliver_service_signal", VCPU_STAT(deliver_service_signal) }, - { "deliver_virtio", VCPU_STAT(deliver_virtio) }, - { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) }, - { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) }, - { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) }, - { "deliver_program", VCPU_STAT(deliver_program) }, - { "deliver_io", VCPU_STAT(deliver_io) }, - { "deliver_machine_check", VCPU_STAT(deliver_machine_check) }, - { "exit_wait_state", VCPU_STAT(exit_wait_state) }, - { "inject_ckc", VCPU_STAT(inject_ckc) }, - { "inject_cputm", VCPU_STAT(inject_cputm) }, - { "inject_external_call", VCPU_STAT(inject_external_call) }, - { "inject_float_mchk", VM_STAT(inject_float_mchk) }, - { "inject_emergency_signal", VCPU_STAT(inject_emergency_signal) }, - { "inject_io", VM_STAT(inject_io) }, - { "inject_mchk", VCPU_STAT(inject_mchk) }, - { "inject_pfault_done", VM_STAT(inject_pfault_done) }, - { "inject_program", VCPU_STAT(inject_program) }, - { "inject_restart", VCPU_STAT(inject_restart) }, - { "inject_service_signal", VM_STAT(inject_service_signal) }, - { "inject_set_prefix", VCPU_STAT(inject_set_prefix) }, - { "inject_stop_signal", VCPU_STAT(inject_stop_signal) }, - { "inject_pfault_init", VCPU_STAT(inject_pfault_init) }, - { "inject_virtio", VM_STAT(inject_virtio) }, - { "instruction_epsw", VCPU_STAT(instruction_epsw) }, - { "instruction_gs", VCPU_STAT(instruction_gs) }, - { "instruction_io_other", VCPU_STAT(instruction_io_other) }, - { "instruction_lpsw", VCPU_STAT(instruction_lpsw) }, - { "instruction_lpswe", VCPU_STAT(instruction_lpswe) }, - { "instruction_pfmf", VCPU_STAT(instruction_pfmf) }, - { "instruction_ptff", VCPU_STAT(instruction_ptff) }, - { "instruction_stidp", VCPU_STAT(instruction_stidp) }, - { "instruction_sck", VCPU_STAT(instruction_sck) }, - { "instruction_sckpf", VCPU_STAT(instruction_sckpf) }, - { "instruction_spx", VCPU_STAT(instruction_spx) }, - { "instruction_stpx", VCPU_STAT(instruction_stpx) }, - { "instruction_stap", VCPU_STAT(instruction_stap) }, - { "instruction_iske", VCPU_STAT(instruction_iske) }, - { "instruction_ri", VCPU_STAT(instruction_ri) }, - { "instruction_rrbe", VCPU_STAT(instruction_rrbe) }, - { "instruction_sske", VCPU_STAT(instruction_sske) }, - { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) }, - { "instruction_essa", VCPU_STAT(instruction_essa) }, - { "instruction_stsi", VCPU_STAT(instruction_stsi) }, - { "instruction_stfl", VCPU_STAT(instruction_stfl) }, - { "instruction_tb", VCPU_STAT(instruction_tb) }, - { "instruction_tpi", VCPU_STAT(instruction_tpi) }, - { "instruction_tprot", VCPU_STAT(instruction_tprot) }, - { "instruction_tsch", VCPU_STAT(instruction_tsch) }, - { "instruction_sthyi", VCPU_STAT(instruction_sthyi) }, - { "instruction_sie", VCPU_STAT(instruction_sie) }, - { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) }, - { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) }, - { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) }, - { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) }, - { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) }, - { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) }, - { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) }, - { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) }, - { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) }, - { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) }, - { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) }, - { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) }, - { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) }, - { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) }, - { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) }, - { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) }, - { "instruction_diag_10", VCPU_STAT(diagnose_10) }, - { "instruction_diag_44", VCPU_STAT(diagnose_44) }, - { "instruction_diag_9c", VCPU_STAT(diagnose_9c) }, - { "diag_9c_ignored", VCPU_STAT(diagnose_9c_ignored) }, - { "instruction_diag_258", VCPU_STAT(diagnose_258) }, - { "instruction_diag_308", VCPU_STAT(diagnose_308) }, - { "instruction_diag_500", VCPU_STAT(diagnose_500) }, - { "instruction_diag_other", VCPU_STAT(diagnose_other) }, + VCPU_STAT("userspace_handled", exit_userspace), + VCPU_STAT("exit_null", exit_null), + VCPU_STAT("exit_validity", exit_validity), + VCPU_STAT("exit_stop_request", exit_stop_request), + VCPU_STAT("exit_external_request", exit_external_request), + VCPU_STAT("exit_io_request", exit_io_request), + VCPU_STAT("exit_external_interrupt", exit_external_interrupt), + VCPU_STAT("exit_instruction", exit_instruction), + VCPU_STAT("exit_pei", exit_pei), + VCPU_STAT("exit_program_interruption", exit_program_interruption), + VCPU_STAT("exit_instr_and_program_int", exit_instr_and_program), + VCPU_STAT("exit_operation_exception", exit_operation_exception), + VCPU_STAT("halt_successful_poll", halt_successful_poll), + VCPU_STAT("halt_attempted_poll", halt_attempted_poll), + VCPU_STAT("halt_poll_invalid", halt_poll_invalid), + VCPU_STAT("halt_no_poll_steal", halt_no_poll_steal), + VCPU_STAT("halt_wakeup", halt_wakeup), + VCPU_STAT("instruction_lctlg", instruction_lctlg), + VCPU_STAT("instruction_lctl", instruction_lctl), + VCPU_STAT("instruction_stctl", instruction_stctl), + VCPU_STAT("instruction_stctg", instruction_stctg), + VCPU_STAT("deliver_ckc", deliver_ckc), + VCPU_STAT("deliver_cputm", deliver_cputm), + VCPU_STAT("deliver_emergency_signal", deliver_emergency_signal), + VCPU_STAT("deliver_external_call", deliver_external_call), + VCPU_STAT("deliver_service_signal", deliver_service_signal), + VCPU_STAT("deliver_virtio", deliver_virtio), + VCPU_STAT("deliver_stop_signal", deliver_stop_signal), + VCPU_STAT("deliver_prefix_signal", deliver_prefix_signal), + VCPU_STAT("deliver_restart_signal", deliver_restart_signal), + VCPU_STAT("deliver_program", deliver_program), + VCPU_STAT("deliver_io", deliver_io), + VCPU_STAT("deliver_machine_check", deliver_machine_check), + VCPU_STAT("exit_wait_state", exit_wait_state), + VCPU_STAT("inject_ckc", inject_ckc), + VCPU_STAT("inject_cputm", inject_cputm), + VCPU_STAT("inject_external_call", inject_external_call), + VM_STAT("inject_float_mchk", inject_float_mchk), + VCPU_STAT("inject_emergency_signal", inject_emergency_signal), + VM_STAT("inject_io", inject_io), + VCPU_STAT("inject_mchk", inject_mchk), + VM_STAT("inject_pfault_done", inject_pfault_done), + VCPU_STAT("inject_program", inject_program), + VCPU_STAT("inject_restart", inject_restart), + VM_STAT("inject_service_signal", inject_service_signal), + VCPU_STAT("inject_set_prefix", inject_set_prefix), + VCPU_STAT("inject_stop_signal", inject_stop_signal), + VCPU_STAT("inject_pfault_init", inject_pfault_init), + VM_STAT("inject_virtio", inject_virtio), + VCPU_STAT("instruction_epsw", instruction_epsw), + VCPU_STAT("instruction_gs", instruction_gs), + VCPU_STAT("instruction_io_other", instruction_io_other), + VCPU_STAT("instruction_lpsw", instruction_lpsw), + VCPU_STAT("instruction_lpswe", instruction_lpswe), + VCPU_STAT("instruction_pfmf", instruction_pfmf), + VCPU_STAT("instruction_ptff", instruction_ptff), + VCPU_STAT("instruction_stidp", instruction_stidp), + VCPU_STAT("instruction_sck", instruction_sck), + VCPU_STAT("instruction_sckpf", instruction_sckpf), + VCPU_STAT("instruction_spx", instruction_spx), + VCPU_STAT("instruction_stpx", instruction_stpx), + VCPU_STAT("instruction_stap", instruction_stap), + VCPU_STAT("instruction_iske", instruction_iske), + VCPU_STAT("instruction_ri", instruction_ri), + VCPU_STAT("instruction_rrbe", instruction_rrbe), + VCPU_STAT("instruction_sske", instruction_sske), + VCPU_STAT("instruction_ipte_interlock", instruction_ipte_interlock), + VCPU_STAT("instruction_essa", instruction_essa), + VCPU_STAT("instruction_stsi", instruction_stsi), + VCPU_STAT("instruction_stfl", instruction_stfl), + VCPU_STAT("instruction_tb", instruction_tb), + VCPU_STAT("instruction_tpi", instruction_tpi), + VCPU_STAT("instruction_tprot", instruction_tprot), + VCPU_STAT("instruction_tsch", instruction_tsch), + VCPU_STAT("instruction_sthyi", instruction_sthyi), + VCPU_STAT("instruction_sie", instruction_sie), + VCPU_STAT("instruction_sigp_sense", instruction_sigp_sense), + VCPU_STAT("instruction_sigp_sense_running", instruction_sigp_sense_running), + VCPU_STAT("instruction_sigp_external_call", instruction_sigp_external_call), + VCPU_STAT("instruction_sigp_emergency", instruction_sigp_emergency), + VCPU_STAT("instruction_sigp_cond_emergency", instruction_sigp_cond_emergency), + VCPU_STAT("instruction_sigp_start", instruction_sigp_start), + VCPU_STAT("instruction_sigp_stop", instruction_sigp_stop), + VCPU_STAT("instruction_sigp_stop_store_status", instruction_sigp_stop_store_status), + VCPU_STAT("instruction_sigp_store_status", instruction_sigp_store_status), + VCPU_STAT("instruction_sigp_store_adtl_status", instruction_sigp_store_adtl_status), + VCPU_STAT("instruction_sigp_set_arch", instruction_sigp_arch), + VCPU_STAT("instruction_sigp_set_prefix", instruction_sigp_prefix), + VCPU_STAT("instruction_sigp_restart", instruction_sigp_restart), + VCPU_STAT("instruction_sigp_cpu_reset", instruction_sigp_cpu_reset), + VCPU_STAT("instruction_sigp_init_cpu_reset", instruction_sigp_init_cpu_reset), + VCPU_STAT("instruction_sigp_unknown", instruction_sigp_unknown), + VCPU_STAT("instruction_diag_10", diagnose_10), + VCPU_STAT("instruction_diag_44", diagnose_44), + VCPU_STAT("instruction_diag_9c", diagnose_9c), + VCPU_STAT("diag_9c_ignored", diagnose_9c_ignored), + VCPU_STAT("instruction_diag_258", diagnose_258), + VCPU_STAT("instruction_diag_308", diagnose_308), + VCPU_STAT("instruction_diag_500", diagnose_500), + VCPU_STAT("instruction_diag_other", diagnose_other), { NULL } }; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index de77bc9bd0d7..8104dc9ff5b2 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -97,9 +97,6 @@ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS; -#define VM_STAT(x, ...) offsetof(struct kvm, stat.x), KVM_STAT_VM, ## __VA_ARGS__ -#define VCPU_STAT(x, ...) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__ - #define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK) @@ -194,45 +191,44 @@ u64 __read_mostly supported_xss; EXPORT_SYMBOL_GPL(supported_xss); struct kvm_stats_debugfs_item debugfs_entries[] = { - { "pf_fixed", VCPU_STAT(pf_fixed) }, - { "pf_guest", VCPU_STAT(pf_guest) }, - { "tlb_flush", VCPU_STAT(tlb_flush) }, - { "invlpg", VCPU_STAT(invlpg) }, - { "exits", VCPU_STAT(exits) }, - { "io_exits", VCPU_STAT(io_exits) }, - { "mmio_exits", VCPU_STAT(mmio_exits) }, - { "signal_exits", VCPU_STAT(signal_exits) }, - { "irq_window", VCPU_STAT(irq_window_exits) }, - { "nmi_window", VCPU_STAT(nmi_window_exits) }, - { "halt_exits", VCPU_STAT(halt_exits) }, - { "halt_successful_poll", VCPU_STAT(halt_successful_poll) }, - { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) }, - { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) }, - { "halt_wakeup", VCPU_STAT(halt_wakeup) }, - { "hypercalls", VCPU_STAT(hypercalls) }, - { "request_irq", VCPU_STAT(request_irq_exits) }, - { "irq_exits", VCPU_STAT(irq_exits) }, - { "host_state_reload", VCPU_STAT(host_state_reload) }, - { "fpu_reload", VCPU_STAT(fpu_reload) }, - { "insn_emulation", VCPU_STAT(insn_emulation) }, - { "insn_emulation_fail", VCPU_STAT(insn_emulation_fail) }, - { "irq_injections", VCPU_STAT(irq_injections) }, - { "nmi_injections", VCPU_STAT(nmi_injections) }, - { "req_event", VCPU_STAT(req_event) }, - { "l1d_flush", VCPU_STAT(l1d_flush) }, - { "mmu_shadow_zapped", VM_STAT(mmu_shadow_zapped) }, - { "mmu_pte_write", VM_STAT(mmu_pte_write) }, - { "mmu_pte_updated", VM_STAT(mmu_pte_updated) }, - { "mmu_pde_zapped", VM_STAT(mmu_pde_zapped) }, - { "mmu_flooded", VM_STAT(mmu_flooded) }, - { "mmu_recycled", VM_STAT(mmu_recycled) }, - { "mmu_cache_miss", VM_STAT(mmu_cache_miss) }, - { "mmu_unsync", VM_STAT(mmu_unsync) }, - { "remote_tlb_flush", VM_STAT(remote_tlb_flush) }, - { "largepages", VM_STAT(lpages, .mode = 0444) }, - { "nx_largepages_splitted", VM_STAT(nx_lpage_splits, .mode = 0444) }, - { "max_mmu_page_hash_collisions", - VM_STAT(max_mmu_page_hash_collisions) }, + VCPU_STAT("pf_fixed", pf_fixed), + VCPU_STAT("pf_guest", pf_guest), + VCPU_STAT("tlb_flush", tlb_flush), + VCPU_STAT("invlpg", invlpg), + VCPU_STAT("exits", exits), + VCPU_STAT("io_exits", io_exits), + VCPU_STAT("mmio_exits", mmio_exits), + VCPU_STAT("signal_exits", signal_exits), + VCPU_STAT("irq_window", irq_window_exits), + VCPU_STAT("nmi_window", nmi_window_exits), + VCPU_STAT("halt_exits", halt_exits), + VCPU_STAT("halt_successful_poll", halt_successful_poll), + VCPU_STAT("halt_attempted_poll", halt_attempted_poll), + VCPU_STAT("halt_poll_invalid", halt_poll_invalid), + VCPU_STAT("halt_wakeup", halt_wakeup), + VCPU_STAT("hypercalls", hypercalls), + VCPU_STAT("request_irq", request_irq_exits), + VCPU_STAT("irq_exits", irq_exits), + VCPU_STAT("host_state_reload", host_state_reload), + VCPU_STAT("fpu_reload", fpu_reload), + VCPU_STAT("insn_emulation", insn_emulation), + VCPU_STAT("insn_emulation_fail", insn_emulation_fail), + VCPU_STAT("irq_injections", irq_injections), + VCPU_STAT("nmi_injections", nmi_injections), + VCPU_STAT("req_event", req_event), + VCPU_STAT("l1d_flush", l1d_flush), + VM_STAT("mmu_shadow_zapped", mmu_shadow_zapped), + VM_STAT("mmu_pte_write", mmu_pte_write), + VM_STAT("mmu_pte_updated", mmu_pte_updated), + VM_STAT("mmu_pde_zapped", mmu_pde_zapped), + VM_STAT("mmu_flooded", mmu_flooded), + VM_STAT("mmu_recycled", mmu_recycled), + VM_STAT("mmu_cache_miss", mmu_cache_miss), + VM_STAT("mmu_unsync", mmu_unsync), + VM_STAT("remote_tlb_flush", remote_tlb_flush), + VM_STAT("largepages", lpages, .mode = 0444), + VM_STAT("nx_largepages_splitted", nx_lpage_splits, .mode = 0444), + VM_STAT("max_mmu_page_hash_collisions", max_mmu_page_hash_collisions), { NULL } }; diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 01276e3d01b9..658215f6102c 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1130,6 +1130,11 @@ struct kvm_stats_debugfs_item { #define KVM_DBGFS_GET_MODE(dbgfs_item) \ ((dbgfs_item)->mode ? (dbgfs_item)->mode : 0644) +#define VM_STAT(n, x, ...) \ + { n, offsetof(struct kvm, stat.x), KVM_STAT_VM, ## __VA_ARGS__ } +#define VCPU_STAT(n, x, ...) \ + { n, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__ } + extern struct kvm_stats_debugfs_item debugfs_entries[]; extern struct dentry *kvm_debugfs_dir; From 789afc5ccd4ef1374669f788c90a870d4e41074c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 10:55:10 -0700 Subject: [PATCH 0094/1043] KVM: nVMX: Move reflection check into nested_vmx_reflect_vmexit() Move the call to nested_vmx_exit_reflected() from vmx_handle_exit() into nested_vmx_reflect_vmexit() and change the semantics of the return value for nested_vmx_reflect_vmexit() to indicate whether or not the exit was reflected into L1. nested_vmx_exit_reflected() and nested_vmx_reflect_vmexit() are intrinsically tied together, calling one without simultaneously calling the other makes little sense. No functional change intended. Reviewed-by: Vitaly Kuznetsov Signed-off-by: Sean Christopherson Message-Id: <20200415175519.14230-2-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.h | 16 +++++++++++----- arch/x86/kvm/vmx/vmx.c | 4 ++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index e8f4a74f7251..d6112b3e9ecf 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -81,12 +81,16 @@ static inline bool nested_ept_ad_enabled(struct kvm_vcpu *vcpu) } /* - * Reflect a VM Exit into L1. + * Conditionally reflect a VM-Exit into L1. Returns %true if the VM-Exit was + * reflected into L1. */ -static inline int nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, - u32 exit_reason) +static inline bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, + u32 exit_reason) { - u32 exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + u32 exit_intr_info; + + if (!nested_vmx_exit_reflected(vcpu, exit_reason)) + return false; /* * At this point, the exit interruption info in exit_intr_info @@ -94,6 +98,8 @@ static inline int nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, * we need to query the in-kernel LAPIC. */ WARN_ON(exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT); + + exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); if ((exit_intr_info & (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) == (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) { @@ -105,7 +111,7 @@ static inline int nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, nested_vmx_vmexit(vcpu, exit_reason, exit_intr_info, vmcs_readl(EXIT_QUALIFICATION)); - return 1; + return true; } /* diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index aa1b8cf7c915..d92ed73b22da 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5905,8 +5905,8 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu, */ nested_mark_vmcs12_pages_dirty(vcpu); - if (nested_vmx_exit_reflected(vcpu, exit_reason)) - return nested_vmx_reflect_vmexit(vcpu, exit_reason); + if (nested_vmx_reflect_vmexit(vcpu, exit_reason)) + return 1; } if (exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) { From 7b7bd87dbd6aa8c09d5e8a8028bda69c3ab13969 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 10:55:11 -0700 Subject: [PATCH 0095/1043] KVM: nVMX: Uninline nested_vmx_reflect_vmexit(), i.e. move it to nested.c Uninline nested_vmx_reflect_vmexit() in preparation of refactoring nested_vmx_exit_reflected() to split up the reflection logic into more consumable chunks, e.g. VM-Fail vs. L1 wants the exit vs. L0 always handles the exit. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20200415175519.14230-3-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 34 +++++++++++++++++++++++++++++++++- arch/x86/kvm/vmx/nested.h | 36 +----------------------------------- 2 files changed, 34 insertions(+), 36 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index aca57d8da400..afc8c3538ab3 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5646,7 +5646,7 @@ static bool nested_vmx_exit_handled_mtf(struct vmcs12 *vmcs12) * should handle it ourselves in L0 (and then continue L2). Only call this * when in is_guest_mode (L2). */ -bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) +static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) { u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO); struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -5813,6 +5813,38 @@ bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) } } +/* + * Conditionally reflect a VM-Exit into L1. Returns %true if the VM-Exit was + * reflected into L1. + */ +bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason) +{ + u32 exit_intr_info; + + if (!nested_vmx_exit_reflected(vcpu, exit_reason)) + return false; + + /* + * At this point, the exit interruption info in exit_intr_info + * is only valid for EXCEPTION_NMI exits. For EXTERNAL_INTERRUPT + * we need to query the in-kernel LAPIC. + */ + WARN_ON(exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT); + + exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + if ((exit_intr_info & + (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) == + (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) { + struct vmcs12 *vmcs12 = get_vmcs12(vcpu); + + vmcs12->vm_exit_intr_error_code = + vmcs_read32(VM_EXIT_INTR_ERROR_CODE); + } + + nested_vmx_vmexit(vcpu, exit_reason, exit_intr_info, + vmcs_readl(EXIT_QUALIFICATION)); + return true; +} static int vmx_get_nested_state(struct kvm_vcpu *vcpu, struct kvm_nested_state __user *user_kvm_nested_state, diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index d6112b3e9ecf..bd959bd2eb58 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -25,7 +25,7 @@ void nested_vmx_set_vmcs_shadowing_bitmap(void); void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu); enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry); -bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason); +bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason); void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, u32 exit_intr_info, unsigned long exit_qualification); void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu); @@ -80,40 +80,6 @@ static inline bool nested_ept_ad_enabled(struct kvm_vcpu *vcpu) return nested_ept_get_eptp(vcpu) & VMX_EPTP_AD_ENABLE_BIT; } -/* - * Conditionally reflect a VM-Exit into L1. Returns %true if the VM-Exit was - * reflected into L1. - */ -static inline bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, - u32 exit_reason) -{ - u32 exit_intr_info; - - if (!nested_vmx_exit_reflected(vcpu, exit_reason)) - return false; - - /* - * At this point, the exit interruption info in exit_intr_info - * is only valid for EXCEPTION_NMI exits. For EXTERNAL_INTERRUPT - * we need to query the in-kernel LAPIC. - */ - WARN_ON(exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT); - - exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); - if ((exit_intr_info & - (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) == - (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) { - struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - - vmcs12->vm_exit_intr_error_code = - vmcs_read32(VM_EXIT_INTR_ERROR_CODE); - } - - nested_vmx_vmexit(vcpu, exit_reason, exit_intr_info, - vmcs_readl(EXIT_QUALIFICATION)); - return true; -} - /* * Return the cr0 value that a nested guest would read. This is a combination * of the real cr0 used to run the guest (guest_cr0), and the bits shadowed by From fbdd50250396d55bf9986b2b9f271a7e91c111ea Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 10:55:12 -0700 Subject: [PATCH 0096/1043] KVM: nVMX: Move VM-Fail check out of nested_vmx_exit_reflected() Check for VM-Fail on nested VM-Enter in nested_vmx_reflect_vmexit() in preparation for separating nested_vmx_exit_reflected() into separate "L0 wants exit exit" and "L1 wants the exit" helpers. Explicitly set exit_intr_info and exit_qual to zero instead of reading them from vmcs02, as they are invalid on VM-Fail (and thankfully ignored by nested_vmx_vmexit() for nested VM-Fail). Signed-off-by: Sean Christopherson Message-Id: <20200415175519.14230-4-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index afc8c3538ab3..cb7fd78ad9da 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5652,15 +5652,6 @@ static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) struct vcpu_vmx *vmx = to_vmx(vcpu); struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - WARN_ON_ONCE(vmx->nested.nested_run_pending); - - if (unlikely(vmx->fail)) { - trace_kvm_nested_vmenter_failed( - "hardware VM-instruction error: ", - vmcs_read32(VM_INSTRUCTION_ERROR)); - return true; - } - trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason, vmcs_readl(EXIT_QUALIFICATION), vmx->idt_vectoring_info, @@ -5819,7 +5810,23 @@ static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) */ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason) { - u32 exit_intr_info; + struct vcpu_vmx *vmx = to_vmx(vcpu); + u32 exit_intr_info, exit_qual; + + WARN_ON_ONCE(vmx->nested.nested_run_pending); + + /* + * Late nested VM-Fail shares the same flow as nested VM-Exit since KVM + * has already loaded L2's state. + */ + if (unlikely(vmx->fail)) { + trace_kvm_nested_vmenter_failed( + "hardware VM-instruction error: ", + vmcs_read32(VM_INSTRUCTION_ERROR)); + exit_intr_info = 0; + exit_qual = 0; + goto reflect_vmexit; + } if (!nested_vmx_exit_reflected(vcpu, exit_reason)) return false; @@ -5840,9 +5847,10 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason) vmcs12->vm_exit_intr_error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); } + exit_qual = vmcs_readl(EXIT_QUALIFICATION); - nested_vmx_vmexit(vcpu, exit_reason, exit_intr_info, - vmcs_readl(EXIT_QUALIFICATION)); +reflect_vmexit: + nested_vmx_vmexit(vcpu, exit_reason, exit_intr_info, exit_qual); return true; } From 236871b6744a6b3c8f9653a429cc041bba8908d5 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 10:55:13 -0700 Subject: [PATCH 0097/1043] KVM: nVMX: Move nested VM-Exit tracepoint into nested_vmx_reflect_vmexit() Move the tracepoint for nested VM-Exits in preparation of splitting the reflection logic into L1 wants the exit vs. L0 always handles the exit. Signed-off-by: Sean Christopherson Message-Id: <20200415175519.14230-5-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index cb7fd78ad9da..de122ef3eeed 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5648,19 +5648,13 @@ static bool nested_vmx_exit_handled_mtf(struct vmcs12 *vmcs12) */ static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) { - u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO); struct vcpu_vmx *vmx = to_vmx(vcpu); struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - - trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason, - vmcs_readl(EXIT_QUALIFICATION), - vmx->idt_vectoring_info, - intr_info, - vmcs_read32(VM_EXIT_INTR_ERROR_CODE), - KVM_ISA_VMX); + u32 intr_info; switch (exit_reason) { case EXIT_REASON_EXCEPTION_NMI: + intr_info = vmcs_read32(VM_EXIT_INTR_INFO); if (is_nmi(intr_info)) return false; else if (is_page_fault(intr_info)) @@ -5828,6 +5822,14 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason) goto reflect_vmexit; } + exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + exit_qual = vmcs_readl(EXIT_QUALIFICATION); + + trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason, exit_qual, + vmx->idt_vectoring_info, exit_intr_info, + vmcs_read32(VM_EXIT_INTR_ERROR_CODE), + KVM_ISA_VMX); + if (!nested_vmx_exit_reflected(vcpu, exit_reason)) return false; @@ -5838,7 +5840,6 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason) */ WARN_ON(exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT); - exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); if ((exit_intr_info & (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) == (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) { @@ -5847,7 +5848,6 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason) vmcs12->vm_exit_intr_error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE); } - exit_qual = vmcs_readl(EXIT_QUALIFICATION); reflect_vmexit: nested_vmx_vmexit(vcpu, exit_reason, exit_intr_info, exit_qual); From 2c1f3323802e4c4495bc8cc55d4867ded6014274 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 10:55:14 -0700 Subject: [PATCH 0098/1043] KVM: nVMX: Split VM-Exit reflection logic into L0 vs. L1 wants Split the logic that determines whether a nested VM-Exit is reflected into L1 into "L0 wants" and "L1 wants" to document the core control flow at a high level. If L0 wants the VM-Exit, e.g. because the exit is due to a hardware event that isn't passed through to L1, then KVM should handle the exit in L0 without considering L1's configuration. Then, if L0 doesn't want the exit, KVM needs to query L1's wants to determine whether or not L1 "caused" the exit, e.g. by setting an exiting control, versus the exit occurring due to an L0 setting, e.g. when L0 intercepts an action that L1 chose to pass-through. Note, this adds an extra read on vmcs.VM_EXIT_INTR_INFO for exception. This will be addressed in a future patch via a VMX-wide enhancement, rather than pile on another case where vmx->exit_intr_info is conditionally available. Suggested-by: Vitaly Kuznetsov Signed-off-by: Sean Christopherson Message-Id: <20200415175519.14230-6-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 119 ++++++++++++++++++++++++-------------- 1 file changed, 74 insertions(+), 45 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index de122ef3eeed..c2745cd9e022 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5642,13 +5642,71 @@ static bool nested_vmx_exit_handled_mtf(struct vmcs12 *vmcs12) } /* - * Return true if we should exit from L2 to L1 to handle an exit, or false if we - * should handle it ourselves in L0 (and then continue L2). Only call this - * when in is_guest_mode (L2). + * Return true if L0 wants to handle an exit from L2 regardless of whether or not + * L1 wants the exit. Only call this when in is_guest_mode (L2). */ -static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) +static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) +{ + u32 intr_info; + + switch (exit_reason) { + case EXIT_REASON_EXCEPTION_NMI: + intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + if (is_nmi(intr_info)) + return true; + else if (is_page_fault(intr_info)) + return vcpu->arch.apf.host_apf_reason || !enable_ept; + else if (is_debug(intr_info) && + vcpu->guest_debug & + (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) + return true; + else if (is_breakpoint(intr_info) && + vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) + return true; + return false; + case EXIT_REASON_EXTERNAL_INTERRUPT: + return true; + case EXIT_REASON_MCE_DURING_VMENTRY: + return true; + case EXIT_REASON_EPT_VIOLATION: + /* + * L0 always deals with the EPT violation. If nested EPT is + * used, and the nested mmu code discovers that the address is + * missing in the guest EPT table (EPT12), the EPT violation + * will be injected with nested_ept_inject_page_fault() + */ + return true; + case EXIT_REASON_EPT_MISCONFIG: + /* + * L2 never uses directly L1's EPT, but rather L0's own EPT + * table (shadow on EPT) or a merged EPT table that L0 built + * (EPT on EPT). So any problems with the structure of the + * table is L0's fault. + */ + return true; + case EXIT_REASON_PREEMPTION_TIMER: + return true; + case EXIT_REASON_PML_FULL: + /* We emulate PML support to L1. */ + return true; + case EXIT_REASON_VMFUNC: + /* VM functions are emulated through L2->L0 vmexits. */ + return true; + case EXIT_REASON_ENCLS: + /* SGX is never exposed to L1 */ + return true; + default: + break; + } + return false; +} + +/* + * Return 1 if L1 wants to intercept an exit from L2. Only call this when in + * is_guest_mode (L2). + */ +static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) { - struct vcpu_vmx *vmx = to_vmx(vcpu); struct vmcs12 *vmcs12 = get_vmcs12(vcpu); u32 intr_info; @@ -5656,20 +5714,13 @@ static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) case EXIT_REASON_EXCEPTION_NMI: intr_info = vmcs_read32(VM_EXIT_INTR_INFO); if (is_nmi(intr_info)) - return false; + return true; else if (is_page_fault(intr_info)) - return !vmx->vcpu.arch.apf.host_apf_reason && enable_ept; - else if (is_debug(intr_info) && - vcpu->guest_debug & - (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) - return false; - else if (is_breakpoint(intr_info) && - vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) - return false; + return true; return vmcs12->exception_bitmap & (1u << (intr_info & INTR_INFO_VECTOR_MASK)); case EXIT_REASON_EXTERNAL_INTERRUPT: - return false; + return nested_exit_on_intr(vcpu); case EXIT_REASON_TRIPLE_FAULT: return true; case EXIT_REASON_INTERRUPT_WINDOW: @@ -5734,7 +5785,7 @@ static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) nested_cpu_has2(vmcs12, SECONDARY_EXEC_PAUSE_LOOP_EXITING); case EXIT_REASON_MCE_DURING_VMENTRY: - return false; + return true; case EXIT_REASON_TPR_BELOW_THRESHOLD: return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW); case EXIT_REASON_APIC_ACCESS: @@ -5746,22 +5797,6 @@ static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) * delivery" only come from vmcs12. */ return true; - case EXIT_REASON_EPT_VIOLATION: - /* - * L0 always deals with the EPT violation. If nested EPT is - * used, and the nested mmu code discovers that the address is - * missing in the guest EPT table (EPT12), the EPT violation - * will be injected with nested_ept_inject_page_fault() - */ - return false; - case EXIT_REASON_EPT_MISCONFIG: - /* - * L2 never uses directly L1's EPT, but rather L0's own EPT - * table (shadow on EPT) or a merged EPT table that L0 built - * (EPT on EPT). So any problems with the structure of the - * table is L0's fault. - */ - return false; case EXIT_REASON_INVPCID: return nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) && @@ -5778,17 +5813,6 @@ static bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason) * the XSS exit bitmap in vmcs12. */ return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES); - case EXIT_REASON_PREEMPTION_TIMER: - return false; - case EXIT_REASON_PML_FULL: - /* We emulate PML support to L1. */ - return false; - case EXIT_REASON_VMFUNC: - /* VM functions are emulated through L2->L0 vmexits. */ - return false; - case EXIT_REASON_ENCLS: - /* SGX is never exposed to L1 */ - return false; case EXIT_REASON_UMWAIT: case EXIT_REASON_TPAUSE: return nested_cpu_has2(vmcs12, @@ -5830,7 +5854,12 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason) vmcs_read32(VM_EXIT_INTR_ERROR_CODE), KVM_ISA_VMX); - if (!nested_vmx_exit_reflected(vcpu, exit_reason)) + /* If L0 (KVM) wants the exit, it trumps L1's desires. */ + if (nested_vmx_l0_wants_exit(vcpu, exit_reason)) + return false; + + /* If L1 doesn't want the exit, handle it in L0. */ + if (!nested_vmx_l1_wants_exit(vcpu, exit_reason)) return false; /* @@ -6162,7 +6191,7 @@ void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps) * reason is that if one of these bits is necessary, it will appear * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control * fields of vmcs01 and vmcs02, will turn these bits off - and - * nested_vmx_exit_reflected() will not pass related exits to L1. + * nested_vmx_l1_wants_exit() will not pass related exits to L1. * These rules have exceptions below. */ From 1d283062c949f16fcc01d8a508b680e50c7d5ea0 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 10:55:15 -0700 Subject: [PATCH 0099/1043] KVM: nVMX: Drop a superfluous WARN on reflecting EXTERNAL_INTERRUPT Drop the WARN in nested_vmx_reflect_vmexit() that fires if KVM attempts to reflect an external interrupt. The WARN is blatantly impossible to hit now that nested_vmx_l0_wants_exit() is called from nested_vmx_reflect_vmexit() unconditionally returns true for EXTERNAL_INTERRUPT. Reviewed-by: Vitaly Kuznetsov Signed-off-by: Sean Christopherson Message-Id: <20200415175519.14230-7-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index c2745cd9e022..e8db560ee3eb 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5863,12 +5863,11 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason) return false; /* - * At this point, the exit interruption info in exit_intr_info - * is only valid for EXCEPTION_NMI exits. For EXTERNAL_INTERRUPT - * we need to query the in-kernel LAPIC. + * vmcs.VM_EXIT_INTR_INFO is only valid for EXCEPTION_NMI exits. For + * EXTERNAL_INTERRUPT, the value for vmcs12->vm_exit_intr_info would + * need to be synthesized by querying the in-kernel LAPIC, but external + * interrupts are never reflected to L1 so it's a non-issue. */ - WARN_ON(exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT); - if ((exit_intr_info & (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) == (INTR_INFO_VALID_MASK | INTR_INFO_DELIVER_CODE_MASK)) { From f47baaed4fef3e31baa750cf337fe62bfcaca31f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 10:55:16 -0700 Subject: [PATCH 0100/1043] KVM: nVMX: Pull exit_reason from vcpu_vmx in nested_vmx_reflect_vmexit() Grab the exit reason from the vcpu struct in nested_vmx_reflect_vmexit() instead of having the exit reason explicitly passed from the caller. This fixes a discrepancy between VM-Fail and VM-Exit handling, as the VM-Fail case is already handled by checking vcpu_vmx, e.g. the exit reason previously passed on the stack is bogus if vmx->fail is set. Not taking the exit reason on the stack also avoids having to document that nested_vmx_reflect_vmexit() requires the full exit reason, as opposed to just the basic exit reason, which is not at all obvious since the only usages of the full exit reason are for tracing and way down in prepare_vmcs12() where it's propagated to vmcs12. No functional change intended. Reviewed-by: Vitaly Kuznetsov Signed-off-by: Sean Christopherson Message-Id: <20200415175519.14230-8-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 3 ++- arch/x86/kvm/vmx/nested.h | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index e8db560ee3eb..67d27bc40143 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5826,9 +5826,10 @@ static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) * Conditionally reflect a VM-Exit into L1. Returns %true if the VM-Exit was * reflected into L1. */ -bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason) +bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); + u32 exit_reason = vmx->exit_reason; u32 exit_intr_info, exit_qual; WARN_ON_ONCE(vmx->nested.nested_run_pending); diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index bd959bd2eb58..61cafee13ece 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -25,7 +25,7 @@ void nested_vmx_set_vmcs_shadowing_bitmap(void); void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu); enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry); -bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason); +bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu); void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, u32 exit_intr_info, unsigned long exit_qualification); void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d92ed73b22da..19a82f846e8e 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5905,7 +5905,7 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu, */ nested_mark_vmcs12_pages_dirty(vcpu); - if (nested_vmx_reflect_vmexit(vcpu, exit_reason)) + if (nested_vmx_reflect_vmexit(vcpu)) return 1; } From 2a7833899f6ab08d654a51f763a85ccf6c744090 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 10:55:17 -0700 Subject: [PATCH 0101/1043] KVM: nVMX: Cast exit_reason to u16 to check for nested EXTERNAL_INTERRUPT Explicitly check only the basic exit reason when emulating an external interrupt VM-Exit in nested_vmx_vmexit(). Checking the full exit reason doesn't currently cause problems, but only because the only exit reason modifier support by KVM is FAILED_VMENTRY, which is mutually exclusive with EXTERNAL_INTERRUPT. Future modifiers, e.g. ENCLAVE_MODE, will coexist with EXTERNAL_INTERRUPT. Reviewed-by: Vitaly Kuznetsov Signed-off-by: Sean Christopherson Message-Id: <20200415175519.14230-9-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 67d27bc40143..23d84d063197 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -4406,7 +4406,7 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; if (likely(!vmx->fail)) { - if (exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && + if ((u16)exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && nested_exit_intr_ack_set(vcpu)) { int irq = kvm_cpu_get_interrupt(vcpu); WARN_ON(irq < 0); From 4dcefa312a6c1cc2aa2dd8a9ba76d97d63b3bd17 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 10:55:18 -0700 Subject: [PATCH 0102/1043] KVM: nVMX: Rename exit_reason to vm_exit_reason for nested VM-Exit Use "vm_exit_reason" for code related to injecting a nested VM-Exit to VM-Exits to make it clear that nested_vmx_vmexit() expects the full exit eason, not just the basic exit reason. The basic exit reason (bits 15:0 of vmcs.VM_EXIT_REASON) is colloquially referred to as simply "exit reason". Note, other flows, e.g. vmx_handle_exit(), are intentionally left as is. A future patch will convert vmx->exit_reason to a union + bit-field, and the exempted flows will interact with the unionized of "exit_reason". Reviewed-by: Vitaly Kuznetsov Signed-off-by: Sean Christopherson Message-Id: <20200415175519.14230-10-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 29 +++++++++++++++-------------- arch/x86/kvm/vmx/nested.h | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 23d84d063197..6f303ccd478d 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -328,19 +328,19 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, { struct vmcs12 *vmcs12 = get_vmcs12(vcpu); struct vcpu_vmx *vmx = to_vmx(vcpu); - u32 exit_reason; + u32 vm_exit_reason; unsigned long exit_qualification = vcpu->arch.exit_qualification; if (vmx->nested.pml_full) { - exit_reason = EXIT_REASON_PML_FULL; + vm_exit_reason = EXIT_REASON_PML_FULL; vmx->nested.pml_full = false; exit_qualification &= INTR_INFO_UNBLOCK_NMI; } else if (fault->error_code & PFERR_RSVD_MASK) - exit_reason = EXIT_REASON_EPT_MISCONFIG; + vm_exit_reason = EXIT_REASON_EPT_MISCONFIG; else - exit_reason = EXIT_REASON_EPT_VIOLATION; + vm_exit_reason = EXIT_REASON_EPT_VIOLATION; - nested_vmx_vmexit(vcpu, exit_reason, 0, exit_qualification); + nested_vmx_vmexit(vcpu, vm_exit_reason, 0, exit_qualification); vmcs12->guest_physical_address = fault->address; } @@ -4002,11 +4002,11 @@ static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) * which already writes to vmcs12 directly. */ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, - u32 exit_reason, u32 exit_intr_info, + u32 vm_exit_reason, u32 exit_intr_info, unsigned long exit_qualification) { /* update exit information fields: */ - vmcs12->vm_exit_reason = exit_reason; + vmcs12->vm_exit_reason = vm_exit_reason; vmcs12->exit_qualification = exit_qualification; vmcs12->vm_exit_intr_info = exit_intr_info; @@ -4318,7 +4318,7 @@ vmabort: * and modify vmcs12 to make it see what it would expect to see there if * L2 was its real guest. Must only be called when in L2 (is_guest_mode()) */ -void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, +void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, u32 exit_intr_info, unsigned long exit_qualification) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -4342,9 +4342,9 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, if (likely(!vmx->fail)) { sync_vmcs02_to_vmcs12(vcpu, vmcs12); - if (exit_reason != -1) - prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info, - exit_qualification); + if (vm_exit_reason != -1) + prepare_vmcs12(vcpu, vmcs12, vm_exit_reason, + exit_intr_info, exit_qualification); /* * Must happen outside of sync_vmcs02_to_vmcs12() as it will @@ -4399,14 +4399,15 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu); } - if ((exit_reason != -1) && (enable_shadow_vmcs || vmx->nested.hv_evmcs)) + if ((vm_exit_reason != -1) && + (enable_shadow_vmcs || vmx->nested.hv_evmcs)) vmx->nested.need_vmcs12_to_shadow_sync = true; /* in case we halted in L2 */ vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; if (likely(!vmx->fail)) { - if ((u16)exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && + if ((u16)vm_exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT && nested_exit_intr_ack_set(vcpu)) { int irq = kvm_cpu_get_interrupt(vcpu); WARN_ON(irq < 0); @@ -4414,7 +4415,7 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR; } - if (exit_reason != -1) + if (vm_exit_reason != -1) trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason, vmcs12->exit_qualification, vmcs12->idt_vectoring_info_field, diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index 61cafee13ece..1514ff4db77f 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -26,7 +26,7 @@ void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu); enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, bool from_vmentry); bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu); -void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason, +void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, u32 exit_intr_info, unsigned long exit_qualification); void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu); int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); From 9932b49e5abef0218254d15b8278e3dbee5ceea3 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 13:34:50 -0700 Subject: [PATCH 0103/1043] KVM: nVMX: Invoke ept_save_pdptrs() if and only if PAE paging is enabled Invoke ept_save_pdptrs() when restoring L1's host state on a "late" VM-Fail if and only if PAE paging is enabled. This saves a CALL in the common case where L1 is a 64-bit host, and avoids incorrectly marking the PDPTRs as dirty. WARN if ept_save_pdptrs() is called with PAE disabled now that the nested usage pre-checks is_pae_paging(). Barring a bug in KVM's MMU, attempting to read the PDPTRs with PAE disabled is now impossible. Signed-off-by: Sean Christopherson Message-Id: <20200415203454.8296-2-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/vmx/vmx.c | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 6f303ccd478d..5a214cbf243c 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -4250,7 +4250,7 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu) * VMFail, like everything else we just need to ensure our * software model is up-to-date. */ - if (enable_ept) + if (enable_ept && is_pae_paging(vcpu)) ept_save_pdptrs(vcpu); kvm_mmu_reset_context(vcpu); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 19a82f846e8e..f9bee922f1e6 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2937,12 +2937,13 @@ void ept_save_pdptrs(struct kvm_vcpu *vcpu) { struct kvm_mmu *mmu = vcpu->arch.walk_mmu; - if (is_pae_paging(vcpu)) { - mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0); - mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1); - mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2); - mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3); - } + if (WARN_ON_ONCE(!is_pae_paging(vcpu))) + return; + + mmu->pdptrs[0] = vmcs_read64(GUEST_PDPTR0); + mmu->pdptrs[1] = vmcs_read64(GUEST_PDPTR1); + mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2); + mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3); kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); } From e5d03de5937e915c8e41b6357c337529dfae6797 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 13:34:51 -0700 Subject: [PATCH 0104/1043] KVM: nVMX: Reset register cache (available and dirty masks) on VMCS switch Reset the per-vCPU available and dirty register masks when switching between vmcs01 and vmcs02, as the masks track state relative to the current VMCS. The stale masks don't cause problems in the current code base because the registers are either unconditionally written on nested transitions or, in the case of segment registers, have an additional tracker that is manually reset. Note, by dropping (previously implicitly, now explicitly) the dirty mask when switching the active VMCS, KVM is technically losing writes to the associated fields. But, the only regs that can be dirtied (RIP, RSP and PDPTRs) are unconditionally written on nested transitions, e.g. explicit writeback is a waste of cycles, and a WARN_ON would be rather pointless. Signed-off-by: Sean Christopherson Message-Id: <20200415203454.8296-3-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 1 + arch/x86/kvm/vmx/vmx.c | 7 +------ arch/x86/kvm/vmx/vmx.h | 11 +++++++++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 5a214cbf243c..c01005ec445f 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -307,6 +307,7 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) vmx_sync_vmcs_host_state(vmx, prev); put_cpu(); + vmx_register_cache_reset(vcpu); vmx_segment_cache_clear(vmx); } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index f9bee922f1e6..629c89314d44 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6711,12 +6711,7 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) loadsegment(es, __USER_DS); #endif - vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP) - | (1 << VCPU_EXREG_RFLAGS) - | (1 << VCPU_EXREG_PDPTR) - | (1 << VCPU_EXREG_SEGMENTS) - | (1 << VCPU_EXREG_CR3)); - vcpu->arch.regs_dirty = 0; + vmx_register_cache_reset(vcpu); pt_guest_exit(vmx); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 31d7252df163..6b668b604898 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -8,6 +8,7 @@ #include #include "capabilities.h" +#include "kvm_cache_regs.h" #include "ops.h" #include "vmcs.h" @@ -447,6 +448,16 @@ static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx) vmx->segment_cache.bitmask = 0; } +static inline void vmx_register_cache_reset(struct kvm_vcpu *vcpu) +{ + vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP) + | (1 << VCPU_EXREG_RFLAGS) + | (1 << VCPU_EXREG_PDPTR) + | (1 << VCPU_EXREG_SEGMENTS) + | (1 << VCPU_EXREG_CR3)); + vcpu->arch.regs_dirty = 0; +} + static inline u32 vmx_vmentry_ctrl(void) { u32 vmentry_ctrl = vmcs_config.vmentry_ctrl; From ec0241f3bbe155a58455ce4a6057be5db6529b0f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 13:34:52 -0700 Subject: [PATCH 0105/1043] KVM: nVMX: Drop manual clearing of segment cache on nested VMCS switch Drop the call to vmx_segment_cache_clear() in vmx_switch_vmcs() now that the entire register cache is reset when switching the active VMCS, e.g. vmx_segment_cache_test_set() will reset the segment cache due to VCPU_EXREG_SEGMENTS being unavailable. Move vmx_segment_cache_clear() to vmx.c now that it's no longer invoked by the nested code. Signed-off-by: Sean Christopherson Message-Id: <20200415203454.8296-4-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 1 - arch/x86/kvm/vmx/vmx.c | 5 +++++ arch/x86/kvm/vmx/vmx.h | 5 ----- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index c01005ec445f..dad6b05e35ae 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -308,7 +308,6 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) put_cpu(); vmx_register_cache_reset(vcpu); - vmx_segment_cache_clear(vmx); } /* diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 629c89314d44..e0587f532b64 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -437,6 +437,11 @@ static const struct kvm_vmx_segment_field { VMX_SEGMENT_FIELD(LDTR), }; +static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx) +{ + vmx->segment_cache.bitmask = 0; +} + static unsigned long host_idt_base; /* diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 6b668b604898..78c99a60fa49 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -443,11 +443,6 @@ BUILD_CONTROLS_SHADOW(pin, PIN_BASED_VM_EXEC_CONTROL) BUILD_CONTROLS_SHADOW(exec, CPU_BASED_VM_EXEC_CONTROL) BUILD_CONTROLS_SHADOW(secondary_exec, SECONDARY_VM_EXEC_CONTROL) -static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx) -{ - vmx->segment_cache.bitmask = 0; -} - static inline void vmx_register_cache_reset(struct kvm_vcpu *vcpu) { vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP) From 5addc235199f15ae964e7ac6b20cf43f4a661573 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 13:34:53 -0700 Subject: [PATCH 0106/1043] KVM: VMX: Cache vmcs.EXIT_QUALIFICATION using arch avail_reg flags Introduce a new "extended register" type, EXIT_INFO_1 (to pair with the nomenclature in .get_exit_info()), and use it to cache VMX's vmcs.EXIT_QUALIFICATION. Signed-off-by: Sean Christopherson Message-Id: <20200415203454.8296-5-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/vmx/nested.c | 20 ++++++++++---------- arch/x86/kvm/vmx/vmx.c | 28 ++++++++++++++-------------- arch/x86/kvm/vmx/vmx.h | 15 ++++++++++++++- 4 files changed, 39 insertions(+), 25 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index d099749168f0..9673ec700cd0 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -170,6 +170,7 @@ enum kvm_reg { VCPU_EXREG_CR3, VCPU_EXREG_RFLAGS, VCPU_EXREG_SEGMENTS, + VCPU_EXREG_EXIT_INFO_1, }; enum { diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index dad6b05e35ae..dc16b2f8ce97 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -4604,7 +4604,7 @@ static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer) gva_t gva; struct x86_exception e; - if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), + if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), vmcs_read32(VMX_INSTRUCTION_INFO), false, sizeof(*vmpointer), &gva)) return 1; @@ -4869,7 +4869,7 @@ static int handle_vmread(struct kvm_vcpu *vcpu) { struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) : get_vmcs12(vcpu); - unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + unsigned long exit_qualification = vmx_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); struct vcpu_vmx *vmx = to_vmx(vcpu); struct x86_exception e; @@ -4955,7 +4955,7 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu) { struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu) : get_vmcs12(vcpu); - unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + unsigned long exit_qualification = vmx_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); struct vcpu_vmx *vmx = to_vmx(vcpu); struct x86_exception e; @@ -5140,7 +5140,7 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) /* Emulate the VMPTRST instruction */ static int handle_vmptrst(struct kvm_vcpu *vcpu) { - unsigned long exit_qual = vmcs_readl(EXIT_QUALIFICATION); + unsigned long exit_qual = vmx_get_exit_qual(vcpu); u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO); gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; struct x86_exception e; @@ -5208,7 +5208,7 @@ static int handle_invept(struct kvm_vcpu *vcpu) /* According to the Intel VMX instruction reference, the memory * operand is read even if it isn't needed (e.g., for type==global) */ - if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), + if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { @@ -5290,7 +5290,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) /* according to the intel vmx instruction reference, the memory * operand is read even if it isn't needed (e.g., for type==global) */ - if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), + if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { @@ -5420,7 +5420,7 @@ static int handle_vmfunc(struct kvm_vcpu *vcpu) fail: nested_vmx_vmexit(vcpu, vmx->exit_reason, vmcs_read32(VM_EXIT_INTR_INFO), - vmcs_readl(EXIT_QUALIFICATION)); + vmx_get_exit_qual(vcpu)); return 1; } @@ -5471,7 +5471,7 @@ static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu, if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS)) return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING); - exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + exit_qualification = vmx_get_exit_qual(vcpu); port = exit_qualification >> 16; size = (exit_qualification & 7) + 1; @@ -5525,7 +5525,7 @@ static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu, static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { - unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + unsigned long exit_qualification = vmx_get_exit_qual(vcpu); int cr = exit_qualification & 15; int reg; unsigned long val; @@ -5849,7 +5849,7 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) } exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); - exit_qual = vmcs_readl(EXIT_QUALIFICATION); + exit_qual = vmx_get_exit_qual(vcpu); trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason, exit_qual, vmx->idt_vectoring_info, exit_intr_info, diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index e0587f532b64..b7aab2284a9c 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4698,7 +4698,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) } if (is_page_fault(intr_info)) { - cr2 = vmcs_readl(EXIT_QUALIFICATION); + cr2 = vmx_get_exit_qual(vcpu); /* EPT won't cause page fault directly */ WARN_ON_ONCE(!vcpu->arch.apf.host_apf_reason && enable_ept); return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0); @@ -4714,7 +4714,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) kvm_queue_exception_e(vcpu, AC_VECTOR, error_code); return 1; case DB_VECTOR: - dr6 = vmcs_readl(EXIT_QUALIFICATION); + dr6 = vmx_get_exit_qual(vcpu); if (!(vcpu->guest_debug & (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { vcpu->arch.dr6 &= ~DR_TRAP_BITS; @@ -4769,7 +4769,7 @@ static int handle_io(struct kvm_vcpu *vcpu) int size, in, string; unsigned port; - exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + exit_qualification = vmx_get_exit_qual(vcpu); string = (exit_qualification & 16) != 0; ++vcpu->stat.io_exits; @@ -4860,7 +4860,7 @@ static int handle_cr(struct kvm_vcpu *vcpu) int err; int ret; - exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + exit_qualification = vmx_get_exit_qual(vcpu); cr = exit_qualification & 15; reg = (exit_qualification >> 8) & 15; switch ((exit_qualification >> 4) & 3) { @@ -4937,7 +4937,7 @@ static int handle_dr(struct kvm_vcpu *vcpu) unsigned long exit_qualification; int dr, dr7, reg; - exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + exit_qualification = vmx_get_exit_qual(vcpu); dr = exit_qualification & DEBUG_REG_ACCESS_NUM; /* First, if DR does not exist, trigger UD */ @@ -5050,7 +5050,7 @@ static int handle_invd(struct kvm_vcpu *vcpu) static int handle_invlpg(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + unsigned long exit_qualification = vmx_get_exit_qual(vcpu); kvm_mmu_invlpg(vcpu, exit_qualification); return kvm_skip_emulated_instruction(vcpu); @@ -5082,7 +5082,7 @@ static int handle_xsetbv(struct kvm_vcpu *vcpu) static int handle_apic_access(struct kvm_vcpu *vcpu) { if (likely(fasteoi)) { - unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + unsigned long exit_qualification = vmx_get_exit_qual(vcpu); int access_type, offset; access_type = exit_qualification & APIC_ACCESS_TYPE; @@ -5103,7 +5103,7 @@ static int handle_apic_access(struct kvm_vcpu *vcpu) static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + unsigned long exit_qualification = vmx_get_exit_qual(vcpu); int vector = exit_qualification & 0xff; /* EOI-induced VM exit is trap-like and thus no need to adjust IP */ @@ -5113,7 +5113,7 @@ static int handle_apic_eoi_induced(struct kvm_vcpu *vcpu) static int handle_apic_write(struct kvm_vcpu *vcpu) { - unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + unsigned long exit_qualification = vmx_get_exit_qual(vcpu); u32 offset = exit_qualification & 0xfff; /* APIC-write VM exit is trap-like and thus no need to adjust IP */ @@ -5134,7 +5134,7 @@ static int handle_task_switch(struct kvm_vcpu *vcpu) idt_index = (vmx->idt_vectoring_info & VECTORING_INFO_VECTOR_MASK); type = (vmx->idt_vectoring_info & VECTORING_INFO_TYPE_MASK); - exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + exit_qualification = vmx_get_exit_qual(vcpu); reason = (u32)exit_qualification >> 30; if (reason == TASK_SWITCH_GATE && idt_v) { @@ -5184,7 +5184,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu) gpa_t gpa; u64 error_code; - exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + exit_qualification = vmx_get_exit_qual(vcpu); /* * EPT violation happened while executing iret from NMI, @@ -5444,7 +5444,7 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) /* According to the Intel instruction reference, the memory operand * is read even if it isn't needed (e.g., for type==all) */ - if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION), + if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; @@ -5520,7 +5520,7 @@ static int handle_pml_full(struct kvm_vcpu *vcpu) trace_kvm_pml_full(vcpu->vcpu_id); - exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + exit_qualification = vmx_get_exit_qual(vcpu); /* * PML buffer FULL happened while executing iret from NMI, @@ -5634,7 +5634,7 @@ static const int kvm_vmx_max_exit_handlers = static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2) { - *info1 = vmcs_readl(EXIT_QUALIFICATION); + *info1 = vmx_get_exit_qual(vcpu); *info2 = vmcs_read32(VM_EXIT_INTR_INFO); } diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 78c99a60fa49..a13eafec67fc 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -210,6 +210,7 @@ struct vcpu_vmx { */ bool guest_state_loaded; + unsigned long exit_qualification; u32 exit_intr_info; u32 idt_vectoring_info; ulong rflags; @@ -449,7 +450,8 @@ static inline void vmx_register_cache_reset(struct kvm_vcpu *vcpu) | (1 << VCPU_EXREG_RFLAGS) | (1 << VCPU_EXREG_PDPTR) | (1 << VCPU_EXREG_SEGMENTS) - | (1 << VCPU_EXREG_CR3)); + | (1 << VCPU_EXREG_CR3) + | (1 << VCPU_EXREG_EXIT_INFO_1)); vcpu->arch.regs_dirty = 0; } @@ -493,6 +495,17 @@ static inline struct pi_desc *vcpu_to_pi_desc(struct kvm_vcpu *vcpu) return &(to_vmx(vcpu)->pi_desc); } +static inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + if (!kvm_register_is_available(vcpu, VCPU_EXREG_EXIT_INFO_1)) { + kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_1); + vmx->exit_qualification = vmcs_readl(EXIT_QUALIFICATION); + } + return vmx->exit_qualification; +} + struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags); void free_vmcs(struct vmcs *vmcs); int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs); From 8791585837f659943936b8e1cce9d039436ad1ca Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 13:34:54 -0700 Subject: [PATCH 0107/1043] KVM: VMX: Cache vmcs.EXIT_INTR_INFO using arch avail_reg flags Introduce a new "extended register" type, EXIT_INFO_2 (to pair with the nomenclature in .get_exit_info()), and use it to cache VMX's vmcs.EXIT_INTR_INFO. Drop a comment in vmx_recover_nmi_blocking() that is obsoleted by the generic caching mechanism. Signed-off-by: Sean Christopherson Message-Id: <20200415203454.8296-6-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/vmx/nested.c | 10 +++++----- arch/x86/kvm/vmx/vmx.c | 20 ++++++++------------ arch/x86/kvm/vmx/vmx.h | 14 +++++++++++++- 4 files changed, 27 insertions(+), 18 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 9673ec700cd0..ceba205d32a2 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -171,6 +171,7 @@ enum kvm_reg { VCPU_EXREG_RFLAGS, VCPU_EXREG_SEGMENTS, VCPU_EXREG_EXIT_INFO_1, + VCPU_EXREG_EXIT_INFO_2, }; enum { diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index dc16b2f8ce97..78744c6e4a04 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5419,7 +5419,7 @@ static int handle_vmfunc(struct kvm_vcpu *vcpu) fail: nested_vmx_vmexit(vcpu, vmx->exit_reason, - vmcs_read32(VM_EXIT_INTR_INFO), + vmx_get_intr_info(vcpu), vmx_get_exit_qual(vcpu)); return 1; } @@ -5652,7 +5652,7 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) switch (exit_reason) { case EXIT_REASON_EXCEPTION_NMI: - intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + intr_info = vmx_get_intr_info(vcpu); if (is_nmi(intr_info)) return true; else if (is_page_fault(intr_info)) @@ -5708,12 +5708,12 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) */ static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) { + u32 intr_info = vmx_get_intr_info(vcpu); struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - u32 intr_info; switch (exit_reason) { case EXIT_REASON_EXCEPTION_NMI: - intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + intr_info = vmx_get_intr_info(vcpu); if (is_nmi(intr_info)) return true; else if (is_page_fault(intr_info)) @@ -5848,7 +5848,7 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) goto reflect_vmexit; } - exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + exit_intr_info = vmx_get_intr_info(vcpu); exit_qual = vmx_get_exit_qual(vcpu); trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason, exit_qual, diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index b7aab2284a9c..50951cdbe76f 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5635,7 +5635,7 @@ static const int kvm_vmx_max_exit_handlers = static void vmx_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2) { *info1 = vmx_get_exit_qual(vcpu); - *info2 = vmcs_read32(VM_EXIT_INTR_INFO); + *info2 = vmx_get_intr_info(vcpu); } static void vmx_destroy_pml_buffer(struct vcpu_vmx *vmx) @@ -6298,16 +6298,16 @@ static void vmx_apicv_post_state_restore(struct kvm_vcpu *vcpu) static void handle_exception_nmi_irqoff(struct vcpu_vmx *vmx) { - vmx->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + u32 intr_info = vmx_get_intr_info(&vmx->vcpu); /* if exit due to PF check for async PF */ - if (is_page_fault(vmx->exit_intr_info)) { + if (is_page_fault(intr_info)) { vmx->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason(); /* Handle machine checks before interrupts are enabled */ - } else if (is_machine_check(vmx->exit_intr_info)) { + } else if (is_machine_check(intr_info)) { kvm_machine_check(); /* We need to handle NMIs before interrupts are enabled */ - } else if (is_nmi(vmx->exit_intr_info)) { + } else if (is_nmi(intr_info)) { kvm_before_interrupt(&vmx->vcpu); asm("int $2"); kvm_after_interrupt(&vmx->vcpu); @@ -6322,9 +6322,8 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu) unsigned long tmp; #endif gate_desc *desc; - u32 intr_info; + u32 intr_info = vmx_get_intr_info(vcpu); - intr_info = vmcs_read32(VM_EXIT_INTR_INFO); if (WARN_ONCE(!is_external_intr(intr_info), "KVM: unexpected VM-Exit interrupt info: 0x%x", intr_info)) return; @@ -6405,11 +6404,8 @@ static void vmx_recover_nmi_blocking(struct vcpu_vmx *vmx) if (enable_vnmi) { if (vmx->loaded_vmcs->nmi_known_unmasked) return; - /* - * Can't use vmx->exit_intr_info since we're not sure what - * the exit reason is. - */ - exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + + exit_intr_info = vmx_get_intr_info(&vmx->vcpu); unblock_nmi = (exit_intr_info & INTR_INFO_UNBLOCK_NMI) != 0; vector = exit_intr_info & INTR_INFO_VECTOR_MASK; /* diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index a13eafec67fc..edfb739e5907 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -451,7 +451,8 @@ static inline void vmx_register_cache_reset(struct kvm_vcpu *vcpu) | (1 << VCPU_EXREG_PDPTR) | (1 << VCPU_EXREG_SEGMENTS) | (1 << VCPU_EXREG_CR3) - | (1 << VCPU_EXREG_EXIT_INFO_1)); + | (1 << VCPU_EXREG_EXIT_INFO_1) + | (1 << VCPU_EXREG_EXIT_INFO_2)); vcpu->arch.regs_dirty = 0; } @@ -506,6 +507,17 @@ static inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu) return vmx->exit_qualification; } +static inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + + if (!kvm_register_is_available(vcpu, VCPU_EXREG_EXIT_INFO_2)) { + kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_2); + vmx->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO); + } + return vmx->exit_intr_info; +} + struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags); void free_vmcs(struct vmcs *vmcs); int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs); From c583eed6d72bfea853ca7aa645d9295dd2720c6a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 14:44:13 -0700 Subject: [PATCH 0108/1043] KVM: x86/mmu: Set @writable to false for non-visible accesses by L2 Explicitly set @writable to false in try_async_pf() if the GFN->PFN translation is short-circuited due to the requested GFN not being visible to L2. Leaving @writable ('map_writable' in the callers) uninitialized is ok in that it's never actually consumed, but one has to track it all the way through set_spte() being short-circuited by set_mmio_spte() to understand that the uninitialized variable is benign, and relying on @writable being ignored is an unnecessary risk. Explicitly setting @writable also aligns try_async_pf() with __gfn_to_pfn_memslot(). Jim Mattson Signed-off-by: Sean Christopherson Message-Id: <20200415214414.10194-2-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 3385d6dde541..b67046fa0ac4 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4090,6 +4090,7 @@ static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, */ if (is_guest_mode(vcpu) && !kvm_is_visible_gfn(vcpu->kvm, gfn)) { *pfn = KVM_PFN_NOSLOT; + *writable = false; return false; } From c36b71503a2268206ebeda6697094ffb4e7e94c2 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 16 Apr 2020 09:48:07 -0400 Subject: [PATCH 0109/1043] KVM: x86/mmu: Avoid an extra memslot lookup in try_async_pf() for L2 Create a new function kvm_is_visible_memslot() and use it from kvm_is_visible_gfn(); use the new function in try_async_pf() too, to avoid an extra memslot lookup. Opportunistically squish a multi-line comment into a single-line comment. Note, the end result, KVM_PFN_NOSLOT, is unchanged. Cc: Jim Mattson Cc: Rick Edgecombe Suggested-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 9 +++------ include/linux/kvm_host.h | 6 ++++++ virt/kvm/kvm_main.c | 6 +----- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index b67046fa0ac4..e618472c572b 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4082,19 +4082,16 @@ static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, gpa_t cr2_or_gpa, kvm_pfn_t *pfn, bool write, bool *writable) { - struct kvm_memory_slot *slot; + struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); bool async; - /* - * Don't expose private memslots to L2. - */ - if (is_guest_mode(vcpu) && !kvm_is_visible_gfn(vcpu->kvm, gfn)) { + /* Don't expose private memslots to L2. */ + if (is_guest_mode(vcpu) && !kvm_is_visible_memslot(slot)) { *pfn = KVM_PFN_NOSLOT; *writable = false; return false; } - slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); async = false; *pfn = __gfn_to_pfn_memslot(slot, gfn, false, &async, write, writable); if (!async) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 658215f6102c..7d4f1eb70274 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1357,6 +1357,12 @@ static inline void kvm_vcpu_set_dy_eligible(struct kvm_vcpu *vcpu, bool val) } #endif /* CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT */ +static inline bool kvm_is_visible_memslot(struct kvm_memory_slot *memslot) +{ + return (memslot && memslot->id < KVM_USER_MEM_SLOTS && + !(memslot->flags & KVM_MEMSLOT_INVALID)); +} + struct kvm_vcpu *kvm_get_running_vcpu(void); struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index da8fd45e0e3e..8aa577db131e 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1607,11 +1607,7 @@ bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) { struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn); - if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS || - memslot->flags & KVM_MEMSLOT_INVALID) - return false; - - return true; + return kvm_is_visible_memslot(memslot); } EXPORT_SYMBOL_GPL(kvm_is_visible_gfn); From b8d295f96b6c9745d7eebee622690e0e835c63af Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 15 Apr 2020 17:07:39 -0700 Subject: [PATCH 0110/1043] KVM: nVMX: Remove non-functional "support" for CR3 target values Remove all references to cr3_target_value[0-3] and replace the fields in vmcs12 with "dead_space" to preserve the vmcs12 layout. KVM doesn't support emulating CR3-target values, despite a variety of code that implies otherwise, as KVM unconditionally reports '0' for the number of supported CR3-target values. This technically fixes a bug where KVM would incorrectly allow VMREAD and VMWRITE to nonexistent fields, i.e. cr3_target_value[0-3]. Per Intel's SDM, the number of supported CR3-target values reported in VMX_MISC also enumerates the existence of the associated VMCS fields: If a future implementation supports more than 4 CR3-target values, they will be encoded consecutively following the 4 encodings given here. Alternatively, the "bug" could be fixed by actually advertisting support for 4 CR3-target values, but that'd likely just enable kvm-unit-tests given that no one has complained about lack of support for going on ten years, e.g. KVM, Xen and HyperV don't use CR3-target values. Signed-off-by: Sean Christopherson Message-Id: <20200416000739.9012-1-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/nested-vmx.rst | 5 +---- arch/x86/kvm/vmx/evmcs.c | 8 -------- arch/x86/kvm/vmx/nested.c | 17 ----------------- arch/x86/kvm/vmx/vmcs12.c | 4 ---- arch/x86/kvm/vmx/vmcs12.h | 10 ++-------- arch/x86/kvm/vmx/vmx.c | 9 --------- 6 files changed, 3 insertions(+), 50 deletions(-) diff --git a/Documentation/virt/kvm/nested-vmx.rst b/Documentation/virt/kvm/nested-vmx.rst index 592b0ab6970b..89851cbb7df9 100644 --- a/Documentation/virt/kvm/nested-vmx.rst +++ b/Documentation/virt/kvm/nested-vmx.rst @@ -116,10 +116,7 @@ struct shadow_vmcs is ever changed. natural_width cr4_guest_host_mask; natural_width cr0_read_shadow; natural_width cr4_read_shadow; - natural_width cr3_target_value0; - natural_width cr3_target_value1; - natural_width cr3_target_value2; - natural_width cr3_target_value3; + natural_width dead_space[4]; /* Last remnants of cr3_target_value[0-3]. */ natural_width exit_qualification; natural_width guest_linear_address; natural_width guest_cr0; diff --git a/arch/x86/kvm/vmx/evmcs.c b/arch/x86/kvm/vmx/evmcs.c index 303813423c3e..73f3e07c1852 100644 --- a/arch/x86/kvm/vmx/evmcs.c +++ b/arch/x86/kvm/vmx/evmcs.c @@ -160,14 +160,6 @@ const struct evmcs_field vmcs_field_to_evmcs_1[] = { HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), EVMCS1_FIELD(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr, HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), - EVMCS1_FIELD(CR3_TARGET_VALUE0, cr3_target_value0, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), - EVMCS1_FIELD(CR3_TARGET_VALUE1, cr3_target_value1, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), - EVMCS1_FIELD(CR3_TARGET_VALUE2, cr3_target_value2, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), - EVMCS1_FIELD(CR3_TARGET_VALUE3, cr3_target_value3, - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL), /* 32 bit rw */ EVMCS1_FIELD(TPR_THRESHOLD, tpr_threshold, diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 78744c6e4a04..f228339cd0a0 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1785,10 +1785,6 @@ static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr; * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr; * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr; - * vmcs12->cr3_target_value0 = evmcs->cr3_target_value0; - * vmcs12->cr3_target_value1 = evmcs->cr3_target_value1; - * vmcs12->cr3_target_value2 = evmcs->cr3_target_value2; - * vmcs12->cr3_target_value3 = evmcs->cr3_target_value3; * vmcs12->page_fault_error_code_mask = * evmcs->page_fault_error_code_mask; * vmcs12->page_fault_error_code_match = @@ -1862,10 +1858,6 @@ static int copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr; * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr; * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr; - * evmcs->cr3_target_value0 = vmcs12->cr3_target_value0; - * evmcs->cr3_target_value1 = vmcs12->cr3_target_value1; - * evmcs->cr3_target_value2 = vmcs12->cr3_target_value2; - * evmcs->cr3_target_value3 = vmcs12->cr3_target_value3; * evmcs->tpr_threshold = vmcs12->tpr_threshold; * evmcs->virtual_processor_id = vmcs12->virtual_processor_id; * evmcs->exception_bitmap = vmcs12->exception_bitmap; @@ -5541,15 +5533,6 @@ static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu, return true; break; case 3: - if ((vmcs12->cr3_target_count >= 1 && - vmcs12->cr3_target_value0 == val) || - (vmcs12->cr3_target_count >= 2 && - vmcs12->cr3_target_value1 == val) || - (vmcs12->cr3_target_count >= 3 && - vmcs12->cr3_target_value2 == val) || - (vmcs12->cr3_target_count >= 4 && - vmcs12->cr3_target_value3 == val)) - return false; if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING)) return true; break; diff --git a/arch/x86/kvm/vmx/vmcs12.c b/arch/x86/kvm/vmx/vmcs12.c index 53dfb401316d..c8e51c004f78 100644 --- a/arch/x86/kvm/vmx/vmcs12.c +++ b/arch/x86/kvm/vmx/vmcs12.c @@ -115,10 +115,6 @@ const unsigned short vmcs_field_to_offset_table[] = { FIELD(CR4_GUEST_HOST_MASK, cr4_guest_host_mask), FIELD(CR0_READ_SHADOW, cr0_read_shadow), FIELD(CR4_READ_SHADOW, cr4_read_shadow), - FIELD(CR3_TARGET_VALUE0, cr3_target_value0), - FIELD(CR3_TARGET_VALUE1, cr3_target_value1), - FIELD(CR3_TARGET_VALUE2, cr3_target_value2), - FIELD(CR3_TARGET_VALUE3, cr3_target_value3), FIELD(EXIT_QUALIFICATION, exit_qualification), FIELD(GUEST_LINEAR_ADDRESS, guest_linear_address), FIELD(GUEST_CR0, guest_cr0), diff --git a/arch/x86/kvm/vmx/vmcs12.h b/arch/x86/kvm/vmx/vmcs12.h index d0c6df373f67..80232daf00ff 100644 --- a/arch/x86/kvm/vmx/vmcs12.h +++ b/arch/x86/kvm/vmx/vmcs12.h @@ -80,10 +80,7 @@ struct __packed vmcs12 { natural_width cr4_guest_host_mask; natural_width cr0_read_shadow; natural_width cr4_read_shadow; - natural_width cr3_target_value0; - natural_width cr3_target_value1; - natural_width cr3_target_value2; - natural_width cr3_target_value3; + natural_width dead_space[4]; /* Last remnants of cr3_target_value[0-3]. */ natural_width exit_qualification; natural_width guest_linear_address; natural_width guest_cr0; @@ -263,10 +260,7 @@ static inline void vmx_check_vmcs12_offsets(void) CHECK_OFFSET(cr4_guest_host_mask, 352); CHECK_OFFSET(cr0_read_shadow, 360); CHECK_OFFSET(cr4_read_shadow, 368); - CHECK_OFFSET(cr3_target_value0, 376); - CHECK_OFFSET(cr3_target_value1, 384); - CHECK_OFFSET(cr3_target_value2, 392); - CHECK_OFFSET(cr3_target_value3, 400); + CHECK_OFFSET(dead_space, 376); CHECK_OFFSET(exit_qualification, 408); CHECK_OFFSET(guest_linear_address, 416); CHECK_OFFSET(guest_cr0, 424); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 50951cdbe76f..281436b08e13 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5717,7 +5717,6 @@ void dump_vmcs(void) u32 cpu_based_exec_ctrl, pin_based_exec_ctrl, secondary_exec_control; unsigned long cr4; u64 efer; - int i, n; if (!dump_invalid_vmcs) { pr_warn_ratelimited("set kvm_intel.dump_invalid_vmcs=1 to dump internal KVM state.\n"); @@ -5854,14 +5853,6 @@ void dump_vmcs(void) pr_err("PostedIntrVec = 0x%02x\n", vmcs_read16(POSTED_INTR_NV)); if ((secondary_exec_control & SECONDARY_EXEC_ENABLE_EPT)) pr_err("EPT pointer = 0x%016llx\n", vmcs_read64(EPT_POINTER)); - n = vmcs_read32(CR3_TARGET_COUNT); - for (i = 0; i + 1 < n; i += 4) - pr_err("CR3 target%u=%016lx target%u=%016lx\n", - i, vmcs_readl(CR3_TARGET_VALUE0 + i * 2), - i + 1, vmcs_readl(CR3_TARGET_VALUE0 + i * 2 + 2)); - if (i < n) - pr_err("CR3 target%u=%016lx\n", - i, vmcs_readl(CR3_TARGET_VALUE0 + i * 2)); if (secondary_exec_control & SECONDARY_EXEC_PAUSE_LOOP_EXITING) pr_err("PLE Gap=%08x Window=%08x\n", vmcs_read32(PLE_GAP), vmcs_read32(PLE_WINDOW)); From 873e1da16918a83cc7440e299dda0dc7ab6db34c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 10 Apr 2020 10:47:02 -0700 Subject: [PATCH 0111/1043] KVM: VMX: Optimize handling of VM-Entry failures in vmx_vcpu_run() Mark the VM-Fail, VM-Exit on VM-Enter, and #MC on VM-Enter paths as 'unlikely' so as to improve code generation so that it favors successful VM-Enter. The performance of successful VM-Enter is for more important, irrespective of whether or not success is actually likely. Signed-off-by: Sean Christopherson Message-Id: <20200410174703.1138-2-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 281436b08e13..1fc10b821b78 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6724,11 +6724,16 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) vmx->nested.nested_run_pending = 0; vmx->idt_vectoring_info = 0; - vmx->exit_reason = vmx->fail ? 0xdead : vmcs_read32(VM_EXIT_REASON); - if ((u16)vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY) + if (unlikely(vmx->fail)) { + vmx->exit_reason = 0xdead; + return; + } + + vmx->exit_reason = vmcs_read32(VM_EXIT_REASON); + if (unlikely((u16)vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY)) kvm_machine_check(); - if (vmx->fail || (vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) + if (unlikely(vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) return; vmx->loaded_vmcs->launched = 1; From a9ab13ff6e844ad5b3ed39339e6db9a76bb539ad Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Fri, 10 Apr 2020 10:47:03 -0700 Subject: [PATCH 0112/1043] KVM: X86: Improve latency for single target IPI fastpath IPI and Timer cause the main MSRs write vmexits in cloud environment observation, let's optimize virtual IPI latency more aggressively to inject target IPI as soon as possible. Running kvm-unit-tests/vmexit.flat IPI testing on SKX server, disable adaptive advance lapic timer and adaptive halt-polling to avoid the interference, this patch can give another 7% improvement. w/o fastpath -> x86.c fastpath 4238 -> 3543 16.4% x86.c fastpath -> vmx.c fastpath 3543 -> 3293 7% w/o fastpath -> vmx.c fastpath 4238 -> 3293 22.3% Cc: Haiwei Li Signed-off-by: Wanpeng Li Signed-off-by: Sean Christopherson Message-Id: <20200410174703.1138-3-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 5 ++--- arch/x86/kvm/svm/svm.c | 24 ++++++++++++++++-------- arch/x86/kvm/vmx/vmx.c | 22 +++++++++++++--------- arch/x86/kvm/x86.c | 6 +++--- 4 files changed, 34 insertions(+), 23 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index ceba205d32a2..f26df2cb0591 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1126,7 +1126,7 @@ struct kvm_x86_ops { */ void (*tlb_flush_guest)(struct kvm_vcpu *vcpu); - void (*run)(struct kvm_vcpu *vcpu); + enum exit_fastpath_completion (*run)(struct kvm_vcpu *vcpu); int (*handle_exit)(struct kvm_vcpu *vcpu, enum exit_fastpath_completion exit_fastpath); int (*skip_emulated_instruction)(struct kvm_vcpu *vcpu); @@ -1176,8 +1176,7 @@ struct kvm_x86_ops { struct x86_instruction_info *info, enum x86_intercept_stage stage, struct x86_exception *exception); - void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu, - enum exit_fastpath_completion *exit_fastpath); + void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu); int (*check_nested_events)(struct kvm_vcpu *vcpu); void (*request_immediate_exit)(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 747e60c65c98..a6f4e1bdb045 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3299,10 +3299,21 @@ static void svm_cancel_injection(struct kvm_vcpu *vcpu) svm_complete_interrupts(svm); } +static enum exit_fastpath_completion svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu) +{ + if (!is_guest_mode(vcpu) && + to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR && + to_svm(vcpu)->vmcb->control.exit_info_1) + return handle_fastpath_set_msr_irqoff(vcpu); + + return EXIT_FASTPATH_NONE; +} + void __svm_vcpu_run(unsigned long vmcb_pa, unsigned long *regs); -static void svm_vcpu_run(struct kvm_vcpu *vcpu) +static enum exit_fastpath_completion svm_vcpu_run(struct kvm_vcpu *vcpu) { + enum exit_fastpath_completion exit_fastpath; struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; @@ -3314,7 +3325,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) * again. */ if (unlikely(svm->nested.exit_required)) - return; + return EXIT_FASTPATH_NONE; /* * Disable singlestep if we're injecting an interrupt/exception. @@ -3398,6 +3409,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) stgi(); /* Any pending NMI will happen here */ + exit_fastpath = svm_exit_handlers_fastpath(vcpu); if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) kvm_after_interrupt(&svm->vcpu); @@ -3426,6 +3438,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu) svm_handle_mce(svm); mark_all_clean(svm->vmcb); + return exit_fastpath; } static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long root) @@ -3727,13 +3740,8 @@ out: return ret; } -static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu, - enum exit_fastpath_completion *exit_fastpath) +static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu) { - if (!is_guest_mode(vcpu) && - to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR && - to_svm(vcpu)->vmcb->control.exit_info_1) - *exit_fastpath = handle_fastpath_set_msr_irqoff(vcpu); } static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 1fc10b821b78..766303b31949 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6350,8 +6350,7 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu) } STACK_FRAME_NON_STANDARD(handle_external_interrupt_irqoff); -static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu, - enum exit_fastpath_completion *exit_fastpath) +static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -6359,9 +6358,6 @@ static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu, handle_external_interrupt_irqoff(vcpu); else if (vmx->exit_reason == EXIT_REASON_EXCEPTION_NMI) handle_exception_nmi_irqoff(vmx); - else if (!is_guest_mode(vcpu) && - vmx->exit_reason == EXIT_REASON_MSR_WRITE) - *exit_fastpath = handle_fastpath_set_msr_irqoff(vcpu); } static bool vmx_has_emulated_msr(int index) @@ -6565,8 +6561,9 @@ void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) bool __vmx_vcpu_run(struct vcpu_vmx *vmx, unsigned long *regs, bool launched); -static void vmx_vcpu_run(struct kvm_vcpu *vcpu) +static enum exit_fastpath_completion vmx_vcpu_run(struct kvm_vcpu *vcpu) { + enum exit_fastpath_completion exit_fastpath; struct vcpu_vmx *vmx = to_vmx(vcpu); unsigned long cr3, cr4; @@ -6578,7 +6575,7 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) /* Don't enter VMX if guest state is invalid, let the exit handler start emulation until we arrive back to a valid state */ if (vmx->emulation_required) - return; + return EXIT_FASTPATH_NONE; if (vmx->ple_window_dirty) { vmx->ple_window_dirty = false; @@ -6726,7 +6723,7 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) if (unlikely(vmx->fail)) { vmx->exit_reason = 0xdead; - return; + return EXIT_FASTPATH_NONE; } vmx->exit_reason = vmcs_read32(VM_EXIT_REASON); @@ -6734,13 +6731,20 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu) kvm_machine_check(); if (unlikely(vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) - return; + return EXIT_FASTPATH_NONE; + + if (!is_guest_mode(vcpu) && vmx->exit_reason == EXIT_REASON_MSR_WRITE) + exit_fastpath = handle_fastpath_set_msr_irqoff(vcpu); + else + exit_fastpath = EXIT_FASTPATH_NONE; vmx->loaded_vmcs->launched = 1; vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); vmx_recover_nmi_blocking(vmx); vmx_complete_interrupts(vmx); + + return exit_fastpath; } static void vmx_free_vcpu(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 8104dc9ff5b2..8d9bcd5faac5 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8179,7 +8179,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) bool req_int_win = dm_request_for_irq_injection(vcpu) && kvm_cpu_accept_dm_intr(vcpu); - enum exit_fastpath_completion exit_fastpath = EXIT_FASTPATH_NONE; + enum exit_fastpath_completion exit_fastpath; bool req_immediate_exit = false; @@ -8406,7 +8406,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_RELOAD; } - kvm_x86_ops.run(vcpu); + exit_fastpath = kvm_x86_ops.run(vcpu); /* * Do this here before restoring debug registers on the host. And @@ -8438,7 +8438,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) vcpu->mode = OUTSIDE_GUEST_MODE; smp_wmb(); - kvm_x86_ops.handle_exit_irqoff(vcpu, &exit_fastpath); + kvm_x86_ops.handle_exit_irqoff(vcpu); /* * Consume any pending interrupts, including the possible source of From 4f233371f6bb1578340b6c7034ddf7b76606d4ae Mon Sep 17 00:00:00 2001 From: Krish Sadhukhan Date: Thu, 9 Apr 2020 16:50:33 -0400 Subject: [PATCH 0113/1043] KVM: nSVM: Check for CR0.CD and CR0.NW on VMRUN of nested guests According to section "Canonicalization and Consistency Checks" in APM vol. 2, the following guest state combination is illegal: "CR0.CD is zero and CR0.NW is set" Signed-off-by: Krish Sadhukhan Message-Id: <20200409205035.16830-2-krish.sadhukhan@oracle.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index c62893102777..3e5bd739a6f6 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -207,6 +207,10 @@ static bool nested_vmcb_checks(struct vmcb *vmcb) if ((vmcb->save.efer & EFER_SVME) == 0) return false; + if (((vmcb->save.cr0 & X86_CR0_CD) == 0) && + (vmcb->save.cr0 & X86_CR0_NW)) + return false; + if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0) return false; From 1b94f6f81007b4afaea3480ec018bc9236148961 Mon Sep 17 00:00:00 2001 From: Tianjia Zhang Date: Thu, 16 Apr 2020 13:10:57 +0800 Subject: [PATCH 0114/1043] KVM: Remove redundant argument to kvm_arch_vcpu_ioctl_run In earlier versions of kvm, 'kvm_run' was an independent structure and was not included in the vcpu structure. At present, 'kvm_run' is already included in the vcpu structure, so the parameter 'kvm_run' is redundant. This patch simplifies the function definition, removes the extra 'kvm_run' parameter, and extracts it from the 'kvm_vcpu' structure if necessary. Signed-off-by: Tianjia Zhang Message-Id: <20200416051057.26526-1-tianjia.zhang@linux.alibaba.com> Signed-off-by: Paolo Bonzini --- arch/mips/kvm/mips.c | 3 ++- arch/powerpc/kvm/powerpc.c | 3 ++- arch/s390/kvm/kvm-s390.c | 3 ++- arch/x86/kvm/x86.c | 11 ++++++----- include/linux/kvm_host.h | 2 +- virt/kvm/arm/arm.c | 6 +++--- virt/kvm/kvm_main.c | 2 +- 7 files changed, 17 insertions(+), 13 deletions(-) diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index fdf1c14d9205..9f50ceef9978 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -438,8 +438,9 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, return -ENOIOCTLCMD; } -int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { + struct kvm_run *run = vcpu->run; int r = -EINTR; vcpu_load(vcpu); diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index e15166b0a16d..7e24691e138a 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -1764,8 +1764,9 @@ int kvm_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg) return r; } -int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { + struct kvm_run *run = vcpu->run; int r; vcpu_load(vcpu); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 5307929a6c89..75471b646fd7 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -4333,8 +4333,9 @@ static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) store_regs_fmt2(vcpu, kvm_run); } -int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { + struct kvm_run *kvm_run = vcpu->run; int rc; if (kvm_run->immediate_exit) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 8d9bcd5faac5..59958ce2b681 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8710,8 +8710,9 @@ static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu) trace_kvm_fpu(0); } -int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { + struct kvm_run *kvm_run = vcpu->run; int r; vcpu_load(vcpu); @@ -8729,18 +8730,18 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) r = -EAGAIN; if (signal_pending(current)) { r = -EINTR; - vcpu->run->exit_reason = KVM_EXIT_INTR; + kvm_run->exit_reason = KVM_EXIT_INTR; ++vcpu->stat.signal_exits; } goto out; } - if (vcpu->run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) { + if (kvm_run->kvm_valid_regs & ~KVM_SYNC_X86_VALID_FIELDS) { r = -EINVAL; goto out; } - if (vcpu->run->kvm_dirty_regs) { + if (kvm_run->kvm_dirty_regs) { r = sync_regs(vcpu); if (r != 0) goto out; @@ -8770,7 +8771,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) out: kvm_put_guest_fpu(vcpu); - if (vcpu->run->kvm_valid_regs) + if (kvm_run->kvm_valid_regs) store_regs(vcpu); post_kvm_run_save(vcpu); kvm_sigset_deactivate(vcpu); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 7d4f1eb70274..5285a5568208 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -866,7 +866,7 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, struct kvm_mp_state *mp_state); int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg); -int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run); +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu); int kvm_arch_init(void *opaque); void kvm_arch_exit(void); diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c index 48d0ec44ad77..f5390ac2165b 100644 --- a/virt/kvm/arm/arm.c +++ b/virt/kvm/arm/arm.c @@ -639,7 +639,6 @@ static void check_vcpu_requests(struct kvm_vcpu *vcpu) /** * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code * @vcpu: The VCPU pointer - * @run: The kvm_run structure pointer used for userspace state exchange * * This function is called through the VCPU_RUN ioctl called from user space. It * will execute VM code in a loop until the time slice for the process is used @@ -647,8 +646,9 @@ static void check_vcpu_requests(struct kvm_vcpu *vcpu) * return with return value 0 and with the kvm_run structure filled in with the * required data for the requested emulation. */ -int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) +int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) { + struct kvm_run *run = vcpu->run; int ret; if (unlikely(!kvm_vcpu_initialized(vcpu))) @@ -659,7 +659,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) return ret; if (run->exit_reason == KVM_EXIT_MMIO) { - ret = kvm_handle_mmio_return(vcpu, vcpu->run); + ret = kvm_handle_mmio_return(vcpu, run); if (ret) return ret; } diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 8aa577db131e..e2f60e313c87 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -3129,7 +3129,7 @@ static long kvm_vcpu_ioctl(struct file *filp, synchronize_rcu(); put_pid(oldpid); } - r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run); + r = kvm_arch_vcpu_ioctl_run(vcpu); trace_kvm_userspace_exit(vcpu->run->exit_reason, r); break; } From da1fda288943c37de8e1513b98f6dda40c8cd421 Mon Sep 17 00:00:00 2001 From: Stefan Raspl Date: Thu, 2 Apr 2020 10:57:03 +0200 Subject: [PATCH 0115/1043] tools/kvm_stat: add command line switch '-z' to skip zero records When running in logging mode, skip records with all zeros (=empty records) to preserve space when logging to files. Signed-off-by: Stefan Raspl Message-Id: <20200402085705.61155-2-raspl@linux.ibm.com> Signed-off-by: Paolo Bonzini --- tools/kvm/kvm_stat/kvm_stat | 28 ++++++++++++++++++++-------- tools/kvm/kvm_stat/kvm_stat.txt | 4 ++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/tools/kvm/kvm_stat/kvm_stat b/tools/kvm/kvm_stat/kvm_stat index e83fc8e868f4..d6cced4e1ef4 100755 --- a/tools/kvm/kvm_stat/kvm_stat +++ b/tools/kvm/kvm_stat/kvm_stat @@ -1500,8 +1500,7 @@ class StdFormat(object): def get_banner(self): return self._banner - @staticmethod - def get_statline(keys, s): + def get_statline(self, keys, s): res = '' for key in keys: res += ' %9d' % s[key].delta @@ -1517,8 +1516,7 @@ class CSVFormat(object): def get_banner(self): return self._banner - @staticmethod - def get_statline(keys, s): + def get_statline(self, keys, s): return reduce(lambda res, key: "{},{!s}".format(res, s[key].delta), keys, '') @@ -1527,14 +1525,21 @@ def log(stats, opts, frmt, keys): """Prints statistics as reiterating key block, multiple value blocks.""" line = 0 banner_repeat = 20 + banner_printed = False + while True: try: time.sleep(opts.set_delay) - if line % banner_repeat == 0: + if line % banner_repeat == 0 and not banner_printed: print(frmt.get_banner()) - print(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + - frmt.get_statline(keys, stats.get())) - line += 1 + banner_printed = True + values = stats.get() + if (not opts.skip_zero_records or + any(values[k].delta != 0 for k in keys)): + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + frmt.get_statline(keys, values)) + line += 1 + banner_printed = False except KeyboardInterrupt: break @@ -1655,9 +1660,16 @@ Press any other key to refresh statistics immediately. default=False, help='retrieve statistics from tracepoints', ) + argparser.add_argument('-z', '--skip-zero-records', + action='store_true', + default=False, + help='omit records with all zeros in logging mode', + ) options = argparser.parse_args() if options.csv and not options.log: sys.exit('Error: Option -c/--csv requires -l/--log') + if options.skip_zero_records and not options.log: + sys.exit('Error: Option -z/--skip-zero-records requires -l/--log') try: # verify that we were passed a valid regex up front re.compile(options.fields) diff --git a/tools/kvm/kvm_stat/kvm_stat.txt b/tools/kvm/kvm_stat/kvm_stat.txt index a97ded2aedad..24296dccc00a 100644 --- a/tools/kvm/kvm_stat/kvm_stat.txt +++ b/tools/kvm/kvm_stat/kvm_stat.txt @@ -104,6 +104,10 @@ OPTIONS --tracepoints:: retrieve statistics from tracepoints +*z*:: +--skip-zero-records:: + omit records with all zeros in logging mode + SEE ALSO -------- 'perf'(1), 'trace-cmd'(1) From 3754afe7cf7cc3693a9c9ff795e9bd97175ca639 Mon Sep 17 00:00:00 2001 From: Stefan Raspl Date: Thu, 2 Apr 2020 10:57:04 +0200 Subject: [PATCH 0116/1043] tools/kvm_stat: Add command line switch '-L' to log to file To integrate with logrotate, we have a signal handler that will re-open the logfile. Assuming we have a systemd unit file with ExecStart=kvm_stat -dtc -s 10 -L /var/log/kvm_stat.csv ExecReload=/bin/kill -HUP $MAINPID and a logrotate config featuring postrotate /bin/systemctl reload kvm_stat.service endscript Then the overall flow will look like this: (1) systemd starts kvm_stat, logging to A. (2) At some point, logrotate runs, moving A to B. kvm_stat continues to write to B at this point. (3) After rotating, logrotate restarts the kvm_stat unit via systemctl. (4) The kvm_stat unit sends a SIGHUP to kvm_stat, finally making it switch over to writing to A again. Note that in order to keep the structure of the cvs output in tact, we make sure to, in contrast to the standard log format, only write the header once at the beginning of a file. This implies that the header is suppressed when appending to an existing file. Unlike with the standard format, where we append to an existing file by starting out with a header. Signed-off-by: Stefan Raspl Message-Id: <20200402085705.61155-3-raspl@linux.ibm.com> Signed-off-by: Paolo Bonzini --- tools/kvm/kvm_stat/kvm_stat | 68 ++++++++++++++++++++++++++++----- tools/kvm/kvm_stat/kvm_stat.txt | 11 +++++- 2 files changed, 67 insertions(+), 12 deletions(-) diff --git a/tools/kvm/kvm_stat/kvm_stat b/tools/kvm/kvm_stat/kvm_stat index d6cced4e1ef4..d199a3694be8 100755 --- a/tools/kvm/kvm_stat/kvm_stat +++ b/tools/kvm/kvm_stat/kvm_stat @@ -32,6 +32,7 @@ import resource import struct import re import subprocess +import signal from collections import defaultdict, namedtuple from functools import reduce from datetime import datetime @@ -228,6 +229,8 @@ IOCTL_NUMBERS = { 'RESET': 0x00002403, } +signal_received = False + ENCODING = locale.getpreferredencoding(False) TRACE_FILTER = re.compile(r'^[^\(]*$') @@ -1523,26 +1526,64 @@ class CSVFormat(object): def log(stats, opts, frmt, keys): """Prints statistics as reiterating key block, multiple value blocks.""" + global signal_received line = 0 banner_repeat = 20 - banner_printed = False + f = None + def do_banner(opts): + nonlocal f + if opts.log_to_file: + if not f: + try: + f = open(opts.log_to_file, 'a') + except (IOError, OSError): + sys.exit("Error: Could not open file: %s" % + opts.log_to_file) + if isinstance(frmt, CSVFormat) and f.tell() != 0: + return + print(frmt.get_banner(), file=f or sys.stdout) + + def do_statline(opts, values): + statline = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + \ + frmt.get_statline(keys, values) + print(statline, file=f or sys.stdout) + + do_banner(opts) + banner_printed = True while True: try: time.sleep(opts.set_delay) - if line % banner_repeat == 0 and not banner_printed: - print(frmt.get_banner()) + if signal_received: + banner_printed = True + line = 0 + f.close() + do_banner(opts) + signal_received = False + if (line % banner_repeat == 0 and not banner_printed and + not (opts.log_to_file and isinstance(frmt, CSVFormat))): + do_banner(opts) banner_printed = True values = stats.get() if (not opts.skip_zero_records or any(values[k].delta != 0 for k in keys)): - print(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + - frmt.get_statline(keys, values)) + do_statline(opts, values) line += 1 banner_printed = False except KeyboardInterrupt: break + if opts.log_to_file: + f.close() + + +def handle_signal(sig, frame): + global signal_received + + signal_received = True + + return + def is_delay_valid(delay): """Verify delay is in valid value range.""" @@ -1615,7 +1656,7 @@ Press any other key to refresh statistics immediately. argparser.add_argument('-c', '--csv', action='store_true', default=False, - help='log in csv format - requires option -l/--log', + help='log in csv format - requires option -l/-L', ) argparser.add_argument('-d', '--debugfs', action='store_true', @@ -1643,6 +1684,11 @@ Press any other key to refresh statistics immediately. default=False, help='run in logging mode (like vmstat)', ) + argparser.add_argument('-L', '--log-to-file', + type=str, + metavar='FILE', + help="like '--log', but logging to a file" + ) argparser.add_argument('-p', '--pid', type=int, default=0, @@ -1666,10 +1712,10 @@ Press any other key to refresh statistics immediately. help='omit records with all zeros in logging mode', ) options = argparser.parse_args() - if options.csv and not options.log: + if options.csv and not (options.log or options.log_to_file): sys.exit('Error: Option -c/--csv requires -l/--log') - if options.skip_zero_records and not options.log: - sys.exit('Error: Option -z/--skip-zero-records requires -l/--log') + if options.skip_zero_records and not (options.log or options.log_to_file): + sys.exit('Error: Option -z/--skip-zero-records requires -l/-L') try: # verify that we were passed a valid regex up front re.compile(options.fields) @@ -1749,7 +1795,9 @@ def main(): sys.stdout.write(' ' + '\n '.join(sorted(set(event_list))) + '\n') sys.exit(0) - if options.log: + if options.log or options.log_to_file: + if options.log_to_file: + signal.signal(signal.SIGHUP, handle_signal) keys = sorted(stats.get().keys()) if options.csv: frmt = CSVFormat(keys) diff --git a/tools/kvm/kvm_stat/kvm_stat.txt b/tools/kvm/kvm_stat/kvm_stat.txt index 24296dccc00a..feaf46451e83 100644 --- a/tools/kvm/kvm_stat/kvm_stat.txt +++ b/tools/kvm/kvm_stat/kvm_stat.txt @@ -65,8 +65,10 @@ OPTIONS run in batch mode for one second -c:: ---csv=:: - log in csv format - requires option -l/--log +--csv:: + log in csv format. Requires option -l/--log or -L/--log-to-file. + When used with option -L/--log-to-file, the header is only ever + written to start of file to preserve the format. -d:: --debugfs:: @@ -92,6 +94,11 @@ OPTIONS --log:: run in logging mode (like vmstat) + +-L:: +--log-to-file=:: + like -l/--log, but logging to a file. Appends to existing files. + -p:: --pid=:: limit statistics to one virtual machine (pid) From 997b7e98990cd44243651827e4efa366d9885907 Mon Sep 17 00:00:00 2001 From: Stefan Raspl Date: Thu, 2 Apr 2020 10:57:05 +0200 Subject: [PATCH 0117/1043] tools/kvm_stat: add sample systemd unit file Add a sample unit file as a basis for systemd integration of kvm_stat logs. Signed-off-by: Stefan Raspl Message-Id: <20200402085705.61155-4-raspl@linux.ibm.com> Signed-off-by: Paolo Bonzini --- tools/kvm/kvm_stat/kvm_stat.service | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tools/kvm/kvm_stat/kvm_stat.service diff --git a/tools/kvm/kvm_stat/kvm_stat.service b/tools/kvm/kvm_stat/kvm_stat.service new file mode 100644 index 000000000000..71aabaffe779 --- /dev/null +++ b/tools/kvm/kvm_stat/kvm_stat.service @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only + +[Unit] +Description=Service that logs KVM kernel module trace events +Before=qemu-kvm.service + +[Service] +Type=simple +ExecStart=/usr/bin/kvm_stat -dtcz -s 10 -L /var/log/kvm_stat.csv +ExecReload=/bin/kill -HUP $MAINPID +Restart=always +SyslogIdentifier=kvm_stat +SyslogLevel=debug + +[Install] +WantedBy=multi-user.target From e72436bc3a5206f95bb384e741154166ddb3202e Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 17 Apr 2020 12:21:06 -0400 Subject: [PATCH 0118/1043] KVM: SVM: avoid infinite loop on NPF from bad address When a nested page fault is taken from an address that does not have a memslot associated to it, kvm_mmu_do_page_fault returns RET_PF_EMULATE (via mmu_set_spte) and kvm_mmu_page_fault then invokes svm_need_emulation_on_page_fault. The default answer there is to return false, but in this case this just causes the page fault to be retried ad libitum. Since this is not a fast path, and the only other case where it is taken is an erratum, just stick a kvm_vcpu_gfn_to_memslot check in there to detect the common case where the erratum is not happening. This fixes an infinite loop in the new set_memory_region_test. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 7 +++++++ virt/kvm/kvm_main.c | 1 + 2 files changed, 8 insertions(+) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index a6f4e1bdb045..eb95283ab68d 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3837,6 +3837,13 @@ static bool svm_need_emulation_on_page_fault(struct kvm_vcpu *vcpu) bool smap = cr4 & X86_CR4_SMAP; bool is_user = svm_get_cpl(vcpu) == 3; + /* + * If RIP is invalid, go ahead with emulation which will cause an + * internal error exit. + */ + if (!kvm_vcpu_gfn_to_memslot(vcpu, kvm_rip_read(vcpu) >> PAGE_SHIFT)) + return true; + /* * Detect and workaround Errata 1096 Fam_17h_00_0Fh. * diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index e2f60e313c87..e7436d054305 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1602,6 +1602,7 @@ struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn { return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn); } +EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_memslot); bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn) { From 1d2c6c9bd4b768bb665eeeb793dd50c2cebcbf0c Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 17 Apr 2020 12:35:38 -0400 Subject: [PATCH 0119/1043] selftests: kvm/set_memory_region_test: do not check RIP if the guest shuts down On AMD, the state of the VMCB is undefined after a shutdown VMEXIT. KVM takes a very conservative approach to that and resets the guest altogether when that happens. This causes the set_memory_region_test to fail because the RIP is 0xfff0 (the reset vector). Restrict the RIP test to KVM_EXIT_INTERNAL_ERROR in order to fix this. Signed-off-by: Paolo Bonzini --- .../testing/selftests/kvm/set_memory_region_test.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index 260e638826dc..b3ece55a2da6 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -287,10 +287,15 @@ static void test_delete_memory_region(void) vcpu_regs_get(vm, VCPU_ID, ®s); - TEST_ASSERT(regs.rip >= final_rip_start && - regs.rip < final_rip_end, - "Bad rip, expected 0x%lx - 0x%lx, got 0x%llx\n", - final_rip_start, final_rip_end, regs.rip); + /* + * On AMD, after KVM_EXIT_SHUTDOWN the VMCB has been reinitialized already, + * so the instruction pointer would point to the reset vector. + */ + if (run->exit_reason == KVM_EXIT_INTERNAL_ERROR) + TEST_ASSERT(regs.rip >= final_rip_start && + regs.rip < final_rip_end, + "Bad rip, expected 0x%lx - 0x%lx, got 0x%llx\n", + final_rip_start, final_rip_end, regs.rip); kvm_vm_free(vm); } From 2c66e281c0330289aad93e6dd722b25fee29306c Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 21 Apr 2020 19:59:44 +0800 Subject: [PATCH 0120/1043] MIPS: Do not initialise globals to 0 Fix the following checkpatch error: ERROR: do not initialise globals to 0 #834: FILE: arch/mips/kernel/setup.c:834: +int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */ Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/setup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index 10bef8f78e7c..b1e2d4360383 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -831,7 +831,7 @@ arch_initcall(debugfs_mips); /* User defined DMA coherency from command line. */ enum coherent_io_user_state coherentio = IO_COHERENCE_DEFAULT; EXPORT_SYMBOL_GPL(coherentio); -int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */ +int hw_coherentio; /* Actual hardware supported DMA coherency setting. */ static int __init setcoherentio(char *str) { From 2a3d47c94ebdfb97bd8aa95343c58b679cb11724 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 21 Apr 2020 19:59:45 +0800 Subject: [PATCH 0121/1043] MIPS: Cleanup code about plat_mem_setup() In the current code, plat_mem_setup() is called by arch_mem_init() instead of setup_arch() and has been declared in asm/bootinfo.h, so modify the code comment to reflect the reality and remove the useless duplicate declartion in arch/mips/kernel/setup.c. Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/bootinfo.h | 2 +- arch/mips/kernel/setup.c | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/mips/include/asm/bootinfo.h b/arch/mips/include/asm/bootinfo.h index 61727785a247..dcd6a75277d1 100644 --- a/arch/mips/include/asm/bootinfo.h +++ b/arch/mips/include/asm/bootinfo.h @@ -121,7 +121,7 @@ extern unsigned long fw_passed_dtb; #endif /* - * Platform memory detection hook called by setup_arch + * Platform memory detection hook called by arch_mem_init() */ extern void plat_mem_setup(void); diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index b1e2d4360383..5481a0c68521 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -654,8 +654,6 @@ static void __init bootcmdline_init(char **cmdline_p) */ static void __init arch_mem_init(char **cmdline_p) { - extern void plat_mem_setup(void); - /* call board setup routine */ plat_mem_setup(); memblock_set_bottom_up(true); From 269b3a9ac538c4ae87f84be640b9fa89914a2489 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 21 Apr 2020 19:59:46 +0800 Subject: [PATCH 0122/1043] MIPS: Make sparse_init() using top-down allocation In the current code, if CONFIG_SWIOTLB is set, when failed to get IO TLB memory from the low pages by plat_swiotlb_setup(), it may lead to the boot process failed with kernel panic. (1) On the Loongson and SiByte platform arch/mips/loongson64/dma.c arch/mips/sibyte/common/dma.c void __init plat_swiotlb_setup(void) { swiotlb_init(1); } kernel/dma/swiotlb.c void __init swiotlb_init(int verbose) { ... vstart = memblock_alloc_low(PAGE_ALIGN(bytes), PAGE_SIZE); if (vstart && !swiotlb_init_with_tbl(vstart, io_tlb_nslabs, verbose)) return; ... pr_warn("Cannot allocate buffer"); no_iotlb_memory = true; } phys_addr_t swiotlb_tbl_map_single() { ... if (no_iotlb_memory) panic("Can not allocate SWIOTLB buffer earlier ..."); ... } (2) On the Cavium OCTEON platform arch/mips/cavium-octeon/dma-octeon.c void __init plat_swiotlb_setup(void) { ... octeon_swiotlb = memblock_alloc_low(swiotlbsize, PAGE_SIZE); if (!octeon_swiotlb) panic("%s: Failed to allocate %zu bytes align=%lx\n", __func__, swiotlbsize, PAGE_SIZE); ... } Because IO_TLB_DEFAULT_SIZE is 64M, if the rest size of low memory is less than 64M when call plat_swiotlb_setup(), we can easily reproduce the panic case. In order to reduce the possibility of kernel panic when failed to get IO TLB memory under CONFIG_SWIOTLB, it is better to allocate low memory as small as possible before plat_swiotlb_setup(), so make sparse_init() using top-down allocation. Reported-by: Juxin Gao Co-developed-by: Juxin Gao Signed-off-by: Juxin Gao Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/setup.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index 5481a0c68521..8db533cd816c 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -700,7 +700,17 @@ static void __init arch_mem_init(char **cmdline_p) memblock_reserve(crashk_res.start, resource_size(&crashk_res)); #endif device_tree_init(); + + /* + * In order to reduce the possibility of kernel panic when failed to + * get IO TLB memory under CONFIG_SWIOTLB, it is better to allocate + * low memory as small as possible before plat_swiotlb_setup(), so + * make sparse_init() using top-down allocation. + */ + memblock_set_bottom_up(false); sparse_init(); + memblock_set_bottom_up(true); + plat_swiotlb_setup(); dma_contiguous_reserve(PFN_PHYS(max_low_pfn)); From 6abf4a2f80eab562a41dff1ade14e2d85743bc9f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 21 Apr 2020 19:11:36 +0200 Subject: [PATCH 0123/1043] ASoC: txx9: add back the hack for a too small resource_size_t Looks like I misread the Kconfig magic and this driver can be compiled into 32-bit kernels. Add back the hack to extent the range of the resource_size_t, and include the header with the txx9-specific ioremap magic for that. Fixes: acfaaf52ebfd ("ASoC: txx9: don't work around too small resource_size_t") Signed-off-by: Christoph Hellwig Acked-by: Mark Brown Signed-off-by: Thomas Bogendoerfer --- sound/soc/txx9/txx9aclc-ac97.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sound/soc/txx9/txx9aclc-ac97.c b/sound/soc/txx9/txx9aclc-ac97.c index 7402448bdb09..d9e348444bd0 100644 --- a/sound/soc/txx9/txx9aclc-ac97.c +++ b/sound/soc/txx9/txx9aclc-ac97.c @@ -14,6 +14,7 @@ #include #include #include +#include /* for TXX9_DIRECTMAP_BASE */ #include #include #include @@ -175,8 +176,6 @@ static int txx9aclc_ac97_dev_probe(struct platform_device *pdev) int err; int irq; - BUILD_BUG_ON(sizeof(drvdata->physbase) > sizeof(r->start)); - irq = platform_get_irq(pdev, 0); if (irq < 0) return irq; @@ -192,6 +191,10 @@ static int txx9aclc_ac97_dev_probe(struct platform_device *pdev) platform_set_drvdata(pdev, drvdata); drvdata->physbase = r->start; + if (sizeof(drvdata->physbase) > sizeof(r->start) && + r->start >= TXX9_DIRECTMAP_BASE && + r->start < TXX9_DIRECTMAP_BASE + 0x400000) + drvdata->physbase |= 0xf00000000ull; err = devm_request_irq(&pdev->dev, irq, txx9aclc_ac97_irq, 0, dev_name(&pdev->dev), drvdata); if (err < 0) From 3054d06719079388a543de6adb812638675ad8f5 Mon Sep 17 00:00:00 2001 From: Paul Moore Date: Tue, 21 Apr 2020 09:10:56 -0400 Subject: [PATCH 0124/1043] audit: fix a net reference leak in audit_list_rules_send() If audit_list_rules_send() fails when trying to create a new thread to send the rules it also fails to cleanup properly, leaking a reference to a net structure. This patch fixes the error patch and renames audit_send_list() to audit_send_list_thread() to better match its cousin, audit_send_reply_thread(). Reported-by: teroincn@gmail.com Reviewed-by: Richard Guy Briggs Signed-off-by: Paul Moore --- kernel/audit.c | 2 +- kernel/audit.h | 2 +- kernel/auditfilter.c | 16 +++++++--------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/kernel/audit.c b/kernel/audit.c index 66b81358b64f..622c30246d19 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -880,7 +880,7 @@ main_queue: return 0; } -int audit_send_list(void *_dest) +int audit_send_list_thread(void *_dest) { struct audit_netlink_list *dest = _dest; struct sk_buff *skb; diff --git a/kernel/audit.h b/kernel/audit.h index 2eed4d231624..f0233dc40b17 100644 --- a/kernel/audit.h +++ b/kernel/audit.h @@ -229,7 +229,7 @@ struct audit_netlink_list { struct sk_buff_head q; }; -int audit_send_list(void *_dest); +int audit_send_list_thread(void *_dest); extern int selinux_audit_rule_update(void); diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c index 026e34da4ace..a10e2997aa6c 100644 --- a/kernel/auditfilter.c +++ b/kernel/auditfilter.c @@ -1161,11 +1161,8 @@ int audit_rule_change(int type, int seq, void *data, size_t datasz) */ int audit_list_rules_send(struct sk_buff *request_skb, int seq) { - u32 portid = NETLINK_CB(request_skb).portid; - struct net *net = sock_net(NETLINK_CB(request_skb).sk); struct task_struct *tsk; struct audit_netlink_list *dest; - int err = 0; /* We can't just spew out the rules here because we might fill * the available socket buffer space and deadlock waiting for @@ -1173,25 +1170,26 @@ int audit_list_rules_send(struct sk_buff *request_skb, int seq) * happen if we're actually running in the context of auditctl * trying to _send_ the stuff */ - dest = kmalloc(sizeof(struct audit_netlink_list), GFP_KERNEL); + dest = kmalloc(sizeof(*dest), GFP_KERNEL); if (!dest) return -ENOMEM; - dest->net = get_net(net); - dest->portid = portid; + dest->net = get_net(sock_net(NETLINK_CB(request_skb).sk)); + dest->portid = NETLINK_CB(request_skb).portid; skb_queue_head_init(&dest->q); mutex_lock(&audit_filter_mutex); audit_list_rules(seq, &dest->q); mutex_unlock(&audit_filter_mutex); - tsk = kthread_run(audit_send_list, dest, "audit_send_list"); + tsk = kthread_run(audit_send_list_thread, dest, "audit_send_list"); if (IS_ERR(tsk)) { skb_queue_purge(&dest->q); + put_net(dest->net); kfree(dest); - err = PTR_ERR(tsk); + return PTR_ERR(tsk); } - return err; + return 0; } int audit_comparator(u32 left, u32 op, u32 right) From 9521eb3ea19a828d8fd59a2785338fd742dbcf31 Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Mon, 20 Apr 2020 15:27:31 +0200 Subject: [PATCH 0125/1043] selinux: don't produce incorrect filename_trans_count I thought I fixed the counting in filename_trans_read_helper() to count the compat rule count correctly in the final version, but it's still wrong. To really count the same thing as in the compat path, we'd need to add up the cardinalities of stype bitmaps of all datums. Since the kernel currently doesn't implement an ebitmap_cardinality() function (and computing the proper count would just waste CPU cycles anyway), just document that we use the field only in case of the old format and stop updating it in filename_trans_read_helper(). Fixes: 430059024389 ("selinux: implement new format of filename transitions") Signed-off-by: Ondrej Mosnacek Signed-off-by: Paul Moore --- security/selinux/ss/policydb.c | 11 +++-------- security/selinux/ss/policydb.h | 3 ++- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index ef8d5b46b05b..1c0041576643 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -2016,12 +2016,7 @@ static int filename_trans_read_helper(struct policydb *p, void *fp) if (rc) goto out; - rc = ebitmap_set_bit(&p->filename_trans_ttypes, ttype, 1); - if (rc) - return rc; - - p->filename_trans_count += ndatum; - return 0; + return ebitmap_set_bit(&p->filename_trans_ttypes, ttype, 1); out: kfree(ft); @@ -2051,7 +2046,7 @@ static int filename_trans_read(struct policydb *p, void *fp) nel = le32_to_cpu(buf[0]); if (p->policyvers < POLICYDB_VERSION_COMP_FTRANS) { - p->filename_trans_count = nel; + p->compat_filename_trans_count = nel; p->filename_trans = hashtab_create(filenametr_hash, filenametr_cmp, (1 << 11)); if (!p->filename_trans) @@ -3568,7 +3563,7 @@ static int filename_trans_write(struct policydb *p, void *fp) return 0; if (p->policyvers < POLICYDB_VERSION_COMP_FTRANS) { - buf[0] = cpu_to_le32(p->filename_trans_count); + buf[0] = cpu_to_le32(p->compat_filename_trans_count); rc = put_entry(buf, sizeof(u32), 1, fp); if (rc) return rc; diff --git a/security/selinux/ss/policydb.h b/security/selinux/ss/policydb.h index d3adb522d3f3..35dc6aa7904d 100644 --- a/security/selinux/ss/policydb.h +++ b/security/selinux/ss/policydb.h @@ -270,7 +270,8 @@ struct policydb { struct ebitmap filename_trans_ttypes; /* actual set of filename_trans rules */ struct hashtab *filename_trans; - u32 filename_trans_count; + /* only used if policyvers < POLICYDB_VERSION_COMP_FTRANS */ + u32 compat_filename_trans_count; /* bools indexed by (value - 1) */ struct cond_bool_datum **bool_val_to_struct; From 56083bdf67c78030f11cdaed5b2b54959a329b02 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 17 Apr 2020 10:32:53 -0400 Subject: [PATCH 0126/1043] KVM: x86: check_nested_events is never NULL Both Intel and AMD now implement it, so there is no need to check if the callback is implemented. Reviewed-by: Vitaly Kuznetsov Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 59958ce2b681..0492baeb78ab 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7699,7 +7699,7 @@ static int inject_pending_event(struct kvm_vcpu *vcpu) * from L2 to L1 due to pending L1 events which require exit * from L2 to L1. */ - if (is_guest_mode(vcpu) && kvm_x86_ops.check_nested_events) { + if (is_guest_mode(vcpu)) { r = kvm_x86_ops.check_nested_events(vcpu); if (r != 0) return r; @@ -7761,7 +7761,7 @@ static int inject_pending_event(struct kvm_vcpu *vcpu) * proposal and current concerns. Perhaps we should be setting * KVM_REQ_EVENT only on certain events and not unconditionally? */ - if (is_guest_mode(vcpu) && kvm_x86_ops.check_nested_events) { + if (is_guest_mode(vcpu)) { r = kvm_x86_ops.check_nested_events(vcpu); if (r != 0) return r; @@ -8527,7 +8527,7 @@ static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu) static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu) { - if (is_guest_mode(vcpu) && kvm_x86_ops.check_nested_events) + if (is_guest_mode(vcpu)) kvm_x86_ops.check_nested_events(vcpu); return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && From 25091990ef289a5962b5bddd8c68de3f691c9e3c Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 17 Apr 2020 10:34:29 -0400 Subject: [PATCH 0127/1043] KVM: eVMCS: check if nesting is enabled In the next patch nested_get_evmcs_version will be always set in kvm_x86_ops for VMX, even if nesting is disabled. Therefore, check whether VMX (aka nesting) is available in the function, the caller will not do the check anymore. Reviewed-by: Vitaly Kuznetsov Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/evmcs.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/arch/x86/kvm/vmx/evmcs.c b/arch/x86/kvm/vmx/evmcs.c index 73f3e07c1852..e5325bd0f304 100644 --- a/arch/x86/kvm/vmx/evmcs.c +++ b/arch/x86/kvm/vmx/evmcs.c @@ -4,6 +4,7 @@ #include #include "../hyperv.h" +#include "../cpuid.h" #include "evmcs.h" #include "vmcs.h" #include "vmx.h" @@ -326,17 +327,18 @@ bool nested_enlightened_vmentry(struct kvm_vcpu *vcpu, u64 *evmcs_gpa) uint16_t nested_get_evmcs_version(struct kvm_vcpu *vcpu) { - struct vcpu_vmx *vmx = to_vmx(vcpu); - /* - * vmcs_version represents the range of supported Enlightened VMCS - * versions: lower 8 bits is the minimal version, higher 8 bits is the - * maximum supported version. KVM supports versions from 1 to - * KVM_EVMCS_VERSION. - */ - if (vmx->nested.enlightened_vmcs_enabled) - return (KVM_EVMCS_VERSION << 8) | 1; + struct vcpu_vmx *vmx = to_vmx(vcpu); + /* + * vmcs_version represents the range of supported Enlightened VMCS + * versions: lower 8 bits is the minimal version, higher 8 bits is the + * maximum supported version. KVM supports versions from 1 to + * KVM_EVMCS_VERSION. + */ + if (kvm_cpu_cap_get(X86_FEATURE_VMX) && + vmx->nested.enlightened_vmcs_enabled) + return (KVM_EVMCS_VERSION << 8) | 1; - return 0; + return 0; } void nested_evmcs_filter_control_msr(u32 msr_index, u64 *pdata) From 33b22172452f05c351fd2fa24c28d2e76c7b0692 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 17 Apr 2020 10:24:18 -0400 Subject: [PATCH 0128/1043] KVM: x86: move nested-related kvm_x86_ops to a separate struct Clean up some of the patching of kvm_x86_ops, by moving kvm_x86_ops related to nested virtualization into a separate struct. As a result, these ops will always be non-NULL on VMX. This is not a problem: * check_nested_events is only called if is_guest_mode(vcpu) returns true * get_nested_state treats VMXOFF state the same as nested being disabled * set_nested_state fails if you attempt to set nested state while nesting is disabled * nested_enable_evmcs could already be called on a CPU without VMX enabled in CPUID. * nested_get_evmcs_version was fixed in the previous patch Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 29 ++++++++++++++++------------- arch/x86/kvm/hyperv.c | 4 ++-- arch/x86/kvm/svm/nested.c | 6 +++++- arch/x86/kvm/svm/svm.c | 13 +++++-------- arch/x86/kvm/svm/svm.h | 3 ++- arch/x86/kvm/vmx/nested.c | 16 +++++++++------- arch/x86/kvm/vmx/nested.h | 2 ++ arch/x86/kvm/vmx/vmx.c | 7 +------ arch/x86/kvm/x86.c | 28 ++++++++++++++-------------- 9 files changed, 56 insertions(+), 52 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index f26df2cb0591..a239a297be33 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1178,7 +1178,6 @@ struct kvm_x86_ops { struct x86_exception *exception); void (*handle_exit_irqoff)(struct kvm_vcpu *vcpu); - int (*check_nested_events)(struct kvm_vcpu *vcpu); void (*request_immediate_exit)(struct kvm_vcpu *vcpu); void (*sched_in)(struct kvm_vcpu *kvm, int cpu); @@ -1211,6 +1210,7 @@ struct kvm_x86_ops { /* pmu operations of sub-arch */ const struct kvm_pmu_ops *pmu_ops; + const struct kvm_x86_nested_ops *nested_ops; /* * Architecture specific hooks for vCPU blocking due to @@ -1238,14 +1238,6 @@ struct kvm_x86_ops { void (*setup_mce)(struct kvm_vcpu *vcpu); - int (*get_nested_state)(struct kvm_vcpu *vcpu, - struct kvm_nested_state __user *user_kvm_nested_state, - unsigned user_data_size); - int (*set_nested_state)(struct kvm_vcpu *vcpu, - struct kvm_nested_state __user *user_kvm_nested_state, - struct kvm_nested_state *kvm_state); - bool (*get_vmcs12_pages)(struct kvm_vcpu *vcpu); - int (*smi_allowed)(struct kvm_vcpu *vcpu); int (*pre_enter_smm)(struct kvm_vcpu *vcpu, char *smstate); int (*pre_leave_smm)(struct kvm_vcpu *vcpu, const char *smstate); @@ -1257,16 +1249,27 @@ struct kvm_x86_ops { int (*get_msr_feature)(struct kvm_msr_entry *entry); - int (*nested_enable_evmcs)(struct kvm_vcpu *vcpu, - uint16_t *vmcs_version); - uint16_t (*nested_get_evmcs_version)(struct kvm_vcpu *vcpu); - bool (*need_emulation_on_page_fault)(struct kvm_vcpu *vcpu); bool (*apic_init_signal_blocked)(struct kvm_vcpu *vcpu); int (*enable_direct_tlbflush)(struct kvm_vcpu *vcpu); }; +struct kvm_x86_nested_ops { + int (*check_events)(struct kvm_vcpu *vcpu); + int (*get_state)(struct kvm_vcpu *vcpu, + struct kvm_nested_state __user *user_kvm_nested_state, + unsigned user_data_size); + int (*set_state)(struct kvm_vcpu *vcpu, + struct kvm_nested_state __user *user_kvm_nested_state, + struct kvm_nested_state *kvm_state); + bool (*get_vmcs12_pages)(struct kvm_vcpu *vcpu); + + int (*enable_evmcs)(struct kvm_vcpu *vcpu, + uint16_t *vmcs_version); + uint16_t (*get_evmcs_version)(struct kvm_vcpu *vcpu); +}; + struct kvm_x86_init_ops { int (*cpu_has_kvm_support)(void); int (*disabled_by_bios)(void); diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index b850f676abe4..2f96ff9e60ee 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1799,8 +1799,8 @@ int kvm_vcpu_ioctl_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, }; int i, nent = ARRAY_SIZE(cpuid_entries); - if (kvm_x86_ops.nested_get_evmcs_version) - evmcs_ver = kvm_x86_ops.nested_get_evmcs_version(vcpu); + if (kvm_x86_ops.nested_ops->get_evmcs_version) + evmcs_ver = kvm_x86_ops.nested_ops->get_evmcs_version(vcpu); /* Skip NESTED_FEATURES if eVMCS is not supported */ if (!evmcs_ver) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 3e5bd739a6f6..6ea047e6882e 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -784,7 +784,7 @@ static bool nested_exit_on_intr(struct vcpu_svm *svm) return (svm->nested.intercept & 1ULL); } -int svm_check_nested_events(struct kvm_vcpu *vcpu) +static int svm_check_nested_events(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); bool block_nested_events = @@ -825,3 +825,7 @@ int nested_svm_exit_special(struct vcpu_svm *svm) return NESTED_EXIT_CONTINUE; } + +struct kvm_x86_nested_ops svm_nested_ops = { + .check_events = svm_check_nested_events, +}; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index eb95283ab68d..c86f7278509b 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3902,9 +3902,9 @@ static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu) /* * TODO: Last condition latch INIT signals on vCPU when * vCPU is in guest-mode and vmcb12 defines intercept on INIT. - * To properly emulate the INIT intercept, SVM should implement - * kvm_x86_ops.check_nested_events() and call nested_svm_vmexit() - * there if an INIT signal is pending. + * To properly emulate the INIT intercept, + * svm_check_nested_events() should call nested_svm_vmexit() + * if an INIT signal is pending. */ return !gif_set(svm) || (svm->vmcb->control.intercept & (1ULL << INTERCEPT_INIT)); @@ -4032,6 +4032,8 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .sched_in = svm_sched_in, .pmu_ops = &amd_pmu_ops, + .nested_ops = &svm_nested_ops, + .deliver_posted_interrupt = svm_deliver_avic_intr, .dy_apicv_has_pending_interrupt = svm_dy_apicv_has_pending_interrupt, .update_pi_irte = svm_update_pi_irte, @@ -4046,14 +4048,9 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .mem_enc_reg_region = svm_register_enc_region, .mem_enc_unreg_region = svm_unregister_enc_region, - .nested_enable_evmcs = NULL, - .nested_get_evmcs_version = NULL, - .need_emulation_on_page_fault = svm_need_emulation_on_page_fault, .apic_init_signal_blocked = svm_apic_init_signal_blocked, - - .check_nested_events = svm_check_nested_events, }; static struct kvm_x86_init_ops svm_init_ops __initdata = { diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index ca95204f9dde..98c2890d561d 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -398,9 +398,10 @@ int nested_svm_exit_handled(struct vcpu_svm *svm); int nested_svm_check_permissions(struct vcpu_svm *svm); int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, bool has_error_code, u32 error_code); -int svm_check_nested_events(struct kvm_vcpu *vcpu); int nested_svm_exit_special(struct vcpu_svm *svm); +extern struct kvm_x86_nested_ops svm_nested_ops; + /* avic.c */ #define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK (0xFF) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index f228339cd0a0..56074d3443e0 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -6440,12 +6440,14 @@ __init int nested_vmx_hardware_setup(struct kvm_x86_ops *ops, exit_handlers[EXIT_REASON_INVVPID] = handle_invvpid; exit_handlers[EXIT_REASON_VMFUNC] = handle_vmfunc; - ops->check_nested_events = vmx_check_nested_events; - ops->get_nested_state = vmx_get_nested_state; - ops->set_nested_state = vmx_set_nested_state; - ops->get_vmcs12_pages = nested_get_vmcs12_pages; - ops->nested_enable_evmcs = nested_enable_evmcs; - ops->nested_get_evmcs_version = nested_get_evmcs_version; - return 0; } + +struct kvm_x86_nested_ops vmx_nested_ops = { + .check_events = vmx_check_nested_events, + .get_state = vmx_get_nested_state, + .set_state = vmx_set_nested_state, + .get_vmcs12_pages = nested_get_vmcs12_pages, + .enable_evmcs = nested_enable_evmcs, + .get_evmcs_version = nested_get_evmcs_version, +}; diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index 1514ff4db77f..7ce9572c3d3a 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -278,4 +278,6 @@ static inline bool nested_cr4_valid(struct kvm_vcpu *vcpu, unsigned long val) #define nested_guest_cr4_valid nested_cr4_valid #define nested_host_cr4_valid nested_cr4_valid +extern struct kvm_x86_nested_ops vmx_nested_ops; + #endif /* __KVM_X86_VMX_NESTED_H */ diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 766303b31949..455cd2c8dbce 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7862,6 +7862,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .post_block = vmx_post_block, .pmu_ops = &intel_pmu_ops, + .nested_ops = &vmx_nested_ops, .update_pi_irte = vmx_update_pi_irte, @@ -7877,12 +7878,6 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .pre_leave_smm = vmx_pre_leave_smm, .enable_smi_window = enable_smi_window, - .check_nested_events = NULL, - .get_nested_state = NULL, - .set_nested_state = NULL, - .get_vmcs12_pages = NULL, - .nested_enable_evmcs = NULL, - .nested_get_evmcs_version = NULL, .need_emulation_on_page_fault = vmx_need_emulation_on_page_fault, .apic_init_signal_blocked = vmx_apic_init_signal_blocked, }; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 0492baeb78ab..8c0b77ac8dc6 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3442,14 +3442,14 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = KVM_X2APIC_API_VALID_FLAGS; break; case KVM_CAP_NESTED_STATE: - r = kvm_x86_ops.get_nested_state ? - kvm_x86_ops.get_nested_state(NULL, NULL, 0) : 0; + r = kvm_x86_ops.nested_ops->get_state ? + kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0; break; case KVM_CAP_HYPERV_DIRECT_TLBFLUSH: r = kvm_x86_ops.enable_direct_tlbflush != NULL; break; case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: - r = kvm_x86_ops.nested_enable_evmcs != NULL; + r = kvm_x86_ops.nested_ops->enable_evmcs != NULL; break; default: break; @@ -4235,9 +4235,9 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, return kvm_hv_activate_synic(vcpu, cap->cap == KVM_CAP_HYPERV_SYNIC2); case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: - if (!kvm_x86_ops.nested_enable_evmcs) + if (!kvm_x86_ops.nested_ops->enable_evmcs) return -ENOTTY; - r = kvm_x86_ops.nested_enable_evmcs(vcpu, &vmcs_version); + r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version); if (!r) { user_ptr = (void __user *)(uintptr_t)cap->args[0]; if (copy_to_user(user_ptr, &vmcs_version, @@ -4552,7 +4552,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, u32 user_data_size; r = -EINVAL; - if (!kvm_x86_ops.get_nested_state) + if (!kvm_x86_ops.nested_ops->get_state) break; BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size)); @@ -4560,8 +4560,8 @@ long kvm_arch_vcpu_ioctl(struct file *filp, if (get_user(user_data_size, &user_kvm_nested_state->size)) break; - r = kvm_x86_ops.get_nested_state(vcpu, user_kvm_nested_state, - user_data_size); + r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state, + user_data_size); if (r < 0) break; @@ -4582,7 +4582,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, int idx; r = -EINVAL; - if (!kvm_x86_ops.set_nested_state) + if (!kvm_x86_ops.nested_ops->set_state) break; r = -EFAULT; @@ -4604,7 +4604,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, break; idx = srcu_read_lock(&vcpu->kvm->srcu); - r = kvm_x86_ops.set_nested_state(vcpu, user_kvm_nested_state, &kvm_state); + r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state); srcu_read_unlock(&vcpu->kvm->srcu, idx); break; } @@ -7700,7 +7700,7 @@ static int inject_pending_event(struct kvm_vcpu *vcpu) * from L2 to L1. */ if (is_guest_mode(vcpu)) { - r = kvm_x86_ops.check_nested_events(vcpu); + r = kvm_x86_ops.nested_ops->check_events(vcpu); if (r != 0) return r; } @@ -7762,7 +7762,7 @@ static int inject_pending_event(struct kvm_vcpu *vcpu) * KVM_REQ_EVENT only on certain events and not unconditionally? */ if (is_guest_mode(vcpu)) { - r = kvm_x86_ops.check_nested_events(vcpu); + r = kvm_x86_ops.nested_ops->check_events(vcpu); if (r != 0) return r; } @@ -8185,7 +8185,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_request_pending(vcpu)) { if (kvm_check_request(KVM_REQ_GET_VMCS12_PAGES, vcpu)) { - if (unlikely(!kvm_x86_ops.get_vmcs12_pages(vcpu))) { + if (unlikely(!kvm_x86_ops.nested_ops->get_vmcs12_pages(vcpu))) { r = 0; goto out; } @@ -8528,7 +8528,7 @@ static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu) static inline bool kvm_vcpu_running(struct kvm_vcpu *vcpu) { if (is_guest_mode(vcpu)) - kvm_x86_ops.check_nested_events(vcpu); + kvm_x86_ops.nested_ops->check_events(vcpu); return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && !vcpu->arch.apf.halted); From ac0f7d42584125dab8039e60ab4ade48cc2db61c Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Mon, 6 Apr 2020 12:41:50 +0200 Subject: [PATCH 0129/1043] Drivers: hv: copy from message page only what's needed Hyper-V Interrupt Message Page (SIMP) has 16 256-byte slots for messages. Each message comes with a header (16 bytes) which specifies the payload length (up to 240 bytes). vmbus_on_msg_dpc(), however, doesn't look at the real message length and copies the whole slot to a temporary buffer before passing it to message handlers. This is potentially dangerous as hypervisor doesn't have to clean the whole slot when putting a new message there and a message handler can get access to some data which belongs to a previous message. Note, this is not currently a problem because all message handlers are in-kernel but eventually we may e.g. get this exported to userspace. Note also, that this is not a performance critical path: messages (unlike events) represent rare events so it doesn't really matter (from performance point of view) if we copy too much. Fix the issue by taking into account the real message length. The temporary buffer allocated by vmbus_on_msg_dpc() remains fixed size for now. Also, check that the supplied payload length is valid (<= 240 bytes). Signed-off-by: Vitaly Kuznetsov Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200406104154.45010-2-vkuznets@redhat.com Signed-off-by: Wei Liu --- drivers/hv/vmbus_drv.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index a68bce4d0ddb..183a5b07c3ad 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1060,6 +1060,12 @@ void vmbus_on_msg_dpc(unsigned long data) goto msg_handled; } + if (msg->header.payload_size > HV_MESSAGE_PAYLOAD_BYTE_COUNT) { + WARN_ONCE(1, "payload size is too large (%d)\n", + msg->header.payload_size); + goto msg_handled; + } + entry = &channel_message_table[hdr->msgtype]; if (!entry->message_handler) @@ -1071,7 +1077,8 @@ void vmbus_on_msg_dpc(unsigned long data) return; INIT_WORK(&ctx->work, vmbus_onmessage_work); - memcpy(&ctx->msg, msg, sizeof(*msg)); + memcpy(&ctx->msg, msg, sizeof(msg->header) + + msg->header.payload_size); /* * The host can generate a rescind message while we From a276463b7aeb6186e7e4315cccb032773fb31b5d Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Mon, 6 Apr 2020 12:41:51 +0200 Subject: [PATCH 0130/1043] Drivers: hv: allocate the exact needed memory for messages When we need to pass a buffer with Hyper-V message we don't need to always allocate 256 bytes for the message: the real message length is known from the header. Change 'struct onmessage_work_context' to make it possible to not over-allocate. Signed-off-by: Vitaly Kuznetsov Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200406104154.45010-3-vkuznets@redhat.com Signed-off-by: Wei Liu --- drivers/hv/vmbus_drv.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 183a5b07c3ad..e356ea4752c0 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1019,7 +1019,10 @@ static struct bus_type hv_bus = { struct onmessage_work_context { struct work_struct work; - struct hv_message msg; + struct { + struct hv_message_header header; + u8 payload[]; + } msg; }; static void vmbus_onmessage_work(struct work_struct *work) @@ -1072,7 +1075,8 @@ void vmbus_on_msg_dpc(unsigned long data) goto msg_handled; if (entry->handler_type == VMHT_BLOCKING) { - ctx = kmalloc(sizeof(*ctx), GFP_ATOMIC); + ctx = kmalloc(sizeof(*ctx) + msg->header.payload_size, + GFP_ATOMIC); if (ctx == NULL) return; @@ -1126,10 +1130,11 @@ static void vmbus_force_channel_rescinded(struct vmbus_channel *channel) WARN_ON(!is_hvsock_channel(channel)); /* - * sizeof(*ctx) is small and the allocation should really not fail, + * Allocation size is small and the allocation should really not fail, * otherwise the state of the hv_sock connections ends up in limbo. */ - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL | __GFP_NOFAIL); + ctx = kzalloc(sizeof(*ctx) + sizeof(*rescind), + GFP_KERNEL | __GFP_NOFAIL); /* * So far, these are not really used by Linux. Just set them to the @@ -1139,7 +1144,7 @@ static void vmbus_force_channel_rescinded(struct vmbus_channel *channel) ctx->msg.header.payload_size = sizeof(*rescind); /* These values are actually used by Linux. */ - rescind = (struct vmbus_channel_rescind_offer *)ctx->msg.u.payload; + rescind = (struct vmbus_channel_rescind_offer *)ctx->msg.payload; rescind->header.msgtype = CHANNELMSG_RESCIND_CHANNELOFFER; rescind->child_relid = channel->offermsg.child_relid; From 5cc415001bca8fe0e3f0ee6d58a953a314dd9751 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Mon, 6 Apr 2020 12:41:52 +0200 Subject: [PATCH 0131/1043] Drivers: hv: avoid passing opaque pointer to vmbus_onmessage() vmbus_onmessage() doesn't need the header of the message, it only uses it to get to the payload, we can pass the pointer to the payload directly. Signed-off-by: Vitaly Kuznetsov Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200406104154.45010-4-vkuznets@redhat.com Signed-off-by: Wei Liu --- drivers/hv/channel_mgmt.c | 7 +------ drivers/hv/vmbus_drv.c | 3 ++- include/linux/hyperv.h | 2 +- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index 501c43c5851d..da93b5f1b143 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -1363,13 +1363,8 @@ channel_message_table[CHANNELMSG_COUNT] = { * * This is invoked in the vmbus worker thread context. */ -void vmbus_onmessage(void *context) +void vmbus_onmessage(struct vmbus_channel_message_header *hdr) { - struct hv_message *msg = context; - struct vmbus_channel_message_header *hdr; - - hdr = (struct vmbus_channel_message_header *)msg->u.payload; - trace_vmbus_on_message(hdr); /* diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index e356ea4752c0..8f08e8273dd8 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1035,7 +1035,8 @@ static void vmbus_onmessage_work(struct work_struct *work) ctx = container_of(work, struct onmessage_work_context, work); - vmbus_onmessage(&ctx->msg); + vmbus_onmessage((struct vmbus_channel_message_header *) + &ctx->msg.payload); kfree(ctx); } diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 692c89ccf5df..cbd24f4e68d1 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -1017,7 +1017,7 @@ static inline void clear_low_latency_mode(struct vmbus_channel *c) c->low_latency = false; } -void vmbus_onmessage(void *context); +void vmbus_onmessage(struct vmbus_channel_message_header *hdr); int vmbus_request_offers(void); From b0a284dc65b401a508dc2c5ed7d465884220f607 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Mon, 6 Apr 2020 12:43:15 +0200 Subject: [PATCH 0132/1043] Drivers: hv: make sure that 'struct vmbus_channel_message_header' compiles correctly Strictly speaking, compiler is free to use something different from 'u32' for 'enum vmbus_channel_message_type' (e.g. char) but it doesn't happen in real life, just add a BUILD_BUG_ON() guardian. Signed-off-by: Vitaly Kuznetsov Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200406104316.45303-1-vkuznets@redhat.com Signed-off-by: Wei Liu --- drivers/hv/vmbus_drv.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 8f08e8273dd8..6e54adb1dd33 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1051,6 +1051,13 @@ void vmbus_on_msg_dpc(unsigned long data) struct onmessage_work_context *ctx; u32 message_type = msg->header.message_type; + /* + * 'enum vmbus_channel_message_type' is supposed to always be 'u32' as + * it is being used in 'struct vmbus_channel_message_header' definition + * which is supposed to match hypervisor ABI. + */ + BUILD_BUG_ON(sizeof(enum vmbus_channel_message_type) != sizeof(u32)); + if (message_type == HVMSG_NONE) /* no msg */ return; From 52c7803f9bd4b1f0ac6e2e3e6051415198cc06bd Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Mon, 6 Apr 2020 12:43:26 +0200 Subject: [PATCH 0133/1043] Drivers: hv: check VMBus messages lengths VMBus message handlers (channel_message_table) receive a pointer to 'struct vmbus_channel_message_header' and cast it to a structure of their choice, which is sometimes longer than the header. We, however, don't check that the message is long enough so in case hypervisor screws up we'll be accessing memory beyond what was allocated for temporary buffer. Previously, we used to always allocate and copy 256 bytes from message page to temporary buffer but this is hardly better: in case the message is shorter than we expect we'll be trying to consume garbage as some real data and no memory guarding technique will be able to identify an issue. Introduce 'min_payload_len' to 'struct vmbus_channel_message_table_entry' and check against it in vmbus_on_msg_dpc(). Note, we can't require the exact length as new hypervisor versions may add extra fields to messages, we only check that the message is not shorter than we expect. Signed-off-by: Vitaly Kuznetsov Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200406104326.45361-1-vkuznets@redhat.com Signed-off-by: Wei Liu --- drivers/hv/channel_mgmt.c | 54 ++++++++++++++++++++++----------------- drivers/hv/hyperv_vmbus.h | 1 + drivers/hv/vmbus_drv.c | 6 +++++ 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index da93b5f1b143..3a5700c8486a 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -1332,30 +1332,36 @@ static void vmbus_onversion_response( /* Channel message dispatch table */ const struct vmbus_channel_message_table_entry channel_message_table[CHANNELMSG_COUNT] = { - { CHANNELMSG_INVALID, 0, NULL }, - { CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer }, - { CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind }, - { CHANNELMSG_REQUESTOFFERS, 0, NULL }, - { CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered }, - { CHANNELMSG_OPENCHANNEL, 0, NULL }, - { CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result }, - { CHANNELMSG_CLOSECHANNEL, 0, NULL }, - { CHANNELMSG_GPADL_HEADER, 0, NULL }, - { CHANNELMSG_GPADL_BODY, 0, NULL }, - { CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created }, - { CHANNELMSG_GPADL_TEARDOWN, 0, NULL }, - { CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown }, - { CHANNELMSG_RELID_RELEASED, 0, NULL }, - { CHANNELMSG_INITIATE_CONTACT, 0, NULL }, - { CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response }, - { CHANNELMSG_UNLOAD, 0, NULL }, - { CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response }, - { CHANNELMSG_18, 0, NULL }, - { CHANNELMSG_19, 0, NULL }, - { CHANNELMSG_20, 0, NULL }, - { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL }, - { CHANNELMSG_22, 0, NULL }, - { CHANNELMSG_TL_CONNECT_RESULT, 0, NULL }, + { CHANNELMSG_INVALID, 0, NULL, 0}, + { CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer, + sizeof(struct vmbus_channel_offer_channel)}, + { CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind, + sizeof(struct vmbus_channel_rescind_offer) }, + { CHANNELMSG_REQUESTOFFERS, 0, NULL, 0}, + { CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered, 0}, + { CHANNELMSG_OPENCHANNEL, 0, NULL, 0}, + { CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result, + sizeof(struct vmbus_channel_open_result)}, + { CHANNELMSG_CLOSECHANNEL, 0, NULL, 0}, + { CHANNELMSG_GPADL_HEADER, 0, NULL, 0}, + { CHANNELMSG_GPADL_BODY, 0, NULL, 0}, + { CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created, + sizeof(struct vmbus_channel_gpadl_created)}, + { CHANNELMSG_GPADL_TEARDOWN, 0, NULL, 0}, + { CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown, + sizeof(struct vmbus_channel_gpadl_torndown) }, + { CHANNELMSG_RELID_RELEASED, 0, NULL, 0}, + { CHANNELMSG_INITIATE_CONTACT, 0, NULL, 0}, + { CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response, + sizeof(struct vmbus_channel_version_response)}, + { CHANNELMSG_UNLOAD, 0, NULL, 0}, + { CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response, 0}, + { CHANNELMSG_18, 0, NULL, 0}, + { CHANNELMSG_19, 0, NULL, 0}, + { CHANNELMSG_20, 0, NULL, 0}, + { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL, 0}, + { CHANNELMSG_22, 0, NULL, 0}, + { CHANNELMSG_TL_CONNECT_RESULT, 0, NULL, 0}, }; /* diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h index 70b30e223a57..ab560ac9c040 100644 --- a/drivers/hv/hyperv_vmbus.h +++ b/drivers/hv/hyperv_vmbus.h @@ -317,6 +317,7 @@ struct vmbus_channel_message_table_entry { enum vmbus_channel_message_type message_type; enum vmbus_message_handler_type handler_type; void (*message_handler)(struct vmbus_channel_message_header *msg); + u32 min_payload_len; }; extern const struct vmbus_channel_message_table_entry diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 6e54adb1dd33..3a05e1bc359c 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1082,6 +1082,12 @@ void vmbus_on_msg_dpc(unsigned long data) if (!entry->message_handler) goto msg_handled; + if (msg->header.payload_size < entry->min_payload_len) { + WARN_ONCE(1, "message too short: msgtype=%d len=%d\n", + hdr->msgtype, msg->header.payload_size); + goto msg_handled; + } + if (entry->handler_type == VMHT_BLOCKING) { ctx = kmalloc(sizeof(*ctx) + msg->header.payload_size, GFP_ATOMIC); From 8a857c55420f29da4fc131adc22b12d474c48f4c Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Mon, 6 Apr 2020 02:15:04 +0200 Subject: [PATCH 0134/1043] Drivers: hv: vmbus: Always handle the VMBus messages on CPU0 A Linux guest have to pick a "connect CPU" to communicate with the Hyper-V host. This CPU can not be taken offline because Hyper-V does not provide a way to change that CPU assignment. Current code sets the connect CPU to whatever CPU ends up running the function vmbus_negotiate_version(), and this will generate problems if that CPU is taken offine. Establish CPU0 as the connect CPU, and add logics to prevents the connect CPU from being taken offline. We could pick some other CPU, and we could pick that "other CPU" dynamically if there was a reason to do so at some point in the future. But for now, #defining the connect CPU to 0 is the most straightforward and least complex solution. While on this, add inline comments explaining "why" offer and rescind messages should not be handled by a same serialized work queue. Suggested-by: Dexuan Cui Signed-off-by: Andrea Parri (Microsoft) Reviewed-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20200406001514.19876-2-parri.andrea@gmail.com Reviewed-by: Michael Kelley Signed-off-by: Wei Liu --- drivers/hv/connection.c | 20 +------------------- drivers/hv/hv.c | 7 +++++++ drivers/hv/hyperv_vmbus.h | 11 ++++++----- drivers/hv/vmbus_drv.c | 20 +++++++++++++++++--- 4 files changed, 31 insertions(+), 27 deletions(-) diff --git a/drivers/hv/connection.c b/drivers/hv/connection.c index 74e77de89b4f..f4bd306d2cef 100644 --- a/drivers/hv/connection.c +++ b/drivers/hv/connection.c @@ -69,7 +69,6 @@ MODULE_PARM_DESC(max_version, int vmbus_negotiate_version(struct vmbus_channel_msginfo *msginfo, u32 version) { int ret = 0; - unsigned int cur_cpu; struct vmbus_channel_initiate_contact *msg; unsigned long flags; @@ -102,24 +101,7 @@ int vmbus_negotiate_version(struct vmbus_channel_msginfo *msginfo, u32 version) msg->monitor_page1 = virt_to_phys(vmbus_connection.monitor_pages[0]); msg->monitor_page2 = virt_to_phys(vmbus_connection.monitor_pages[1]); - /* - * We want all channel messages to be delivered on CPU 0. - * This has been the behavior pre-win8. This is not - * perf issue and having all channel messages delivered on CPU 0 - * would be ok. - * For post win8 hosts, we support receiving channel messagges on - * all the CPUs. This is needed for kexec to work correctly where - * the CPU attempting to connect may not be CPU 0. - */ - if (version >= VERSION_WIN8_1) { - cur_cpu = get_cpu(); - msg->target_vcpu = hv_cpu_number_to_vp_number(cur_cpu); - vmbus_connection.connect_cpu = cur_cpu; - put_cpu(); - } else { - msg->target_vcpu = 0; - vmbus_connection.connect_cpu = 0; - } + msg->target_vcpu = hv_cpu_number_to_vp_number(VMBUS_CONNECT_CPU); /* * Add to list before we send the request since we may diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c index 6098e0cbdb4b..e2b331045464 100644 --- a/drivers/hv/hv.c +++ b/drivers/hv/hv.c @@ -249,6 +249,13 @@ int hv_synic_cleanup(unsigned int cpu) bool channel_found = false; unsigned long flags; + /* + * Hyper-V does not provide a way to change the connect CPU once + * it is set; we must prevent the connect CPU from going offline. + */ + if (cpu == VMBUS_CONNECT_CPU) + return -EBUSY; + /* * Search for channels which are bound to the CPU we're about to * cleanup. In case we find one and vmbus is still connected we need to diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h index ab560ac9c040..000740d053a2 100644 --- a/drivers/hv/hyperv_vmbus.h +++ b/drivers/hv/hyperv_vmbus.h @@ -212,12 +212,13 @@ enum vmbus_connect_state { #define MAX_SIZE_CHANNEL_MESSAGE HV_MESSAGE_PAYLOAD_BYTE_COUNT -struct vmbus_connection { - /* - * CPU on which the initial host contact was made. - */ - int connect_cpu; +/* + * The CPU that Hyper-V will interrupt for VMBUS messages, such as + * CHANNELMSG_OFFERCHANNEL and CHANNELMSG_RESCIND_CHANNELOFFER. + */ +#define VMBUS_CONNECT_CPU 0 +struct vmbus_connection { u32 msg_conn_id; atomic_t offer_in_progress; diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 3a05e1bc359c..495337ec68f1 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1109,14 +1109,28 @@ void vmbus_on_msg_dpc(unsigned long data) /* * If we are handling the rescind message; * schedule the work on the global work queue. + * + * The OFFER message and the RESCIND message should + * not be handled by the same serialized work queue, + * because the OFFER handler may call vmbus_open(), + * which tries to open the channel by sending an + * OPEN_CHANNEL message to the host and waits for + * the host's response; however, if the host has + * rescinded the channel before it receives the + * OPEN_CHANNEL message, the host just silently + * ignores the OPEN_CHANNEL message; as a result, + * the guest's OFFER handler hangs for ever, if we + * handle the RESCIND message in the same serialized + * work queue: the RESCIND handler can not start to + * run before the OFFER handler finishes. */ - schedule_work_on(vmbus_connection.connect_cpu, + schedule_work_on(VMBUS_CONNECT_CPU, &ctx->work); break; case CHANNELMSG_OFFERCHANNEL: atomic_inc(&vmbus_connection.offer_in_progress); - queue_work_on(vmbus_connection.connect_cpu, + queue_work_on(VMBUS_CONNECT_CPU, vmbus_connection.work_queue, &ctx->work); break; @@ -1164,7 +1178,7 @@ static void vmbus_force_channel_rescinded(struct vmbus_channel *channel) INIT_WORK(&ctx->work, vmbus_onmessage_work); - queue_work_on(vmbus_connection.connect_cpu, + queue_work_on(VMBUS_CONNECT_CPU, vmbus_connection.work_queue, &ctx->work); } From b9fa1b8797dcb579a9642a502769e1a5c3adc0d2 Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Mon, 6 Apr 2020 02:15:05 +0200 Subject: [PATCH 0135/1043] Drivers: hv: vmbus: Don't bind the offer&rescind works to a specific CPU The offer and rescind works are currently scheduled on the so called "connect CPU". However, this is not really needed: we can synchronize the works by relying on the usage of the offer_in_progress counter and of the channel_mutex mutex. This synchronization is already in place. So, remove this unnecessary "bind to the connect CPU" constraint and update the inline comments accordingly. Suggested-by: Dexuan Cui Signed-off-by: Andrea Parri (Microsoft) Link: https://lore.kernel.org/r/20200406001514.19876-3-parri.andrea@gmail.com Reviewed-by: Michael Kelley Signed-off-by: Wei Liu --- drivers/hv/channel_mgmt.c | 21 ++++++++++++++++----- drivers/hv/vmbus_drv.c | 39 ++++++++++++++++++++++++++++----------- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index 3a5700c8486a..0ced32ef2715 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -1028,11 +1028,22 @@ static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr) * offer comes in first and then the rescind. * Since we process these events in work elements, * and with preemption, we may end up processing - * the events out of order. Given that we handle these - * work elements on the same CPU, this is possible only - * in the case of preemption. In any case wait here - * until the offer processing has moved beyond the - * point where the channel is discoverable. + * the events out of order. We rely on the synchronization + * provided by offer_in_progress and by channel_mutex for + * ordering these events: + * + * { Initially: offer_in_progress = 1 } + * + * CPU1 CPU2 + * + * [vmbus_process_offer()] [vmbus_onoffer_rescind()] + * + * LOCK channel_mutex WAIT_ON offer_in_progress == 0 + * DECREMENT offer_in_progress LOCK channel_mutex + * INSERT chn_list SEARCH chn_list + * UNLOCK channel_mutex UNLOCK channel_mutex + * + * Forbids: CPU2's SEARCH from *not* seeing CPU1's INSERT */ while (atomic_read(&vmbus_connection.offer_in_progress) != 0) { diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 495337ec68f1..1f3e69d18d35 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1101,8 +1101,9 @@ void vmbus_on_msg_dpc(unsigned long data) /* * The host can generate a rescind message while we * may still be handling the original offer. We deal with - * this condition by ensuring the processing is done on the - * same CPU. + * this condition by relying on the synchronization provided + * by offer_in_progress and by channel_mutex. See also the + * inline comments in vmbus_onoffer_rescind(). */ switch (hdr->msgtype) { case CHANNELMSG_RESCIND_CHANNELOFFER: @@ -1124,16 +1125,34 @@ void vmbus_on_msg_dpc(unsigned long data) * work queue: the RESCIND handler can not start to * run before the OFFER handler finishes. */ - schedule_work_on(VMBUS_CONNECT_CPU, - &ctx->work); + schedule_work(&ctx->work); break; case CHANNELMSG_OFFERCHANNEL: + /* + * The host sends the offer message of a given channel + * before sending the rescind message of the same + * channel. These messages are sent to the guest's + * connect CPU; the guest then starts processing them + * in the tasklet handler on this CPU: + * + * VMBUS_CONNECT_CPU + * + * [vmbus_on_msg_dpc()] + * atomic_inc() // CHANNELMSG_OFFERCHANNEL + * queue_work() + * ... + * [vmbus_on_msg_dpc()] + * schedule_work() // CHANNELMSG_RESCIND_CHANNELOFFER + * + * We rely on the memory-ordering properties of the + * queue_work() and schedule_work() primitives, which + * guarantee that the atomic increment will be visible + * to the CPUs which will execute the offer & rescind + * works by the time these works will start execution. + */ atomic_inc(&vmbus_connection.offer_in_progress); - queue_work_on(VMBUS_CONNECT_CPU, - vmbus_connection.work_queue, - &ctx->work); - break; + fallthrough; default: queue_work(vmbus_connection.work_queue, &ctx->work); @@ -1178,9 +1197,7 @@ static void vmbus_force_channel_rescinded(struct vmbus_channel *channel) INIT_WORK(&ctx->work, vmbus_onmessage_work); - queue_work_on(VMBUS_CONNECT_CPU, - vmbus_connection.work_queue, - &ctx->work); + queue_work(vmbus_connection.work_queue, &ctx->work); } #endif /* CONFIG_PM_SLEEP */ From 8b6a877c060ed6b86878fe66c7c6493a6054cf23 Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Mon, 6 Apr 2020 02:15:06 +0200 Subject: [PATCH 0136/1043] Drivers: hv: vmbus: Replace the per-CPU channel lists with a global array of channels When Hyper-V sends an interrupt to the guest, the guest has to figure out which channel the interrupt is associated with. Hyper-V sets a bit in a memory page that is shared with the guest, indicating a particular "relid" that the interrupt is associated with. The current Linux code then uses a set of per-CPU linked lists to map a given "relid" to a pointer to a channel structure. This design introduces a synchronization problem if the CPU that Hyper-V will interrupt for a certain channel is changed. If the interrupt comes on the "old CPU" and the channel was already moved to the per-CPU list of the "new CPU", then the relid -> channel mapping will fail and the interrupt is dropped. Similarly, if the interrupt comes on the new CPU but the channel was not moved to the per-CPU list of the new CPU, then the mapping will fail and the interrupt is dropped. Relids are integers ranging from 0 to 2047. The mapping from relids to channel structures can be done by setting up an array with 2048 entries, each entry being a pointer to a channel structure (hence total size ~16K bytes, which is not a problem). The array is global, so there are no per-CPU linked lists to update. The array can be searched and updated by loading from/storing to the array at the specified index. With no per-CPU data structures, the above mentioned synchronization problem is avoided and the relid2channel() function gets simpler. Suggested-by: Michael Kelley Signed-off-by: Andrea Parri (Microsoft) Link: https://lore.kernel.org/r/20200406001514.19876-4-parri.andrea@gmail.com Reviewed-by: Michael Kelley Signed-off-by: Wei Liu --- drivers/hv/channel_mgmt.c | 179 ++++++++++++++++++++++++-------------- drivers/hv/connection.c | 38 +++----- drivers/hv/hv.c | 2 - drivers/hv/hyperv_vmbus.h | 14 +-- drivers/hv/vmbus_drv.c | 48 ++++++---- include/linux/hyperv.h | 5 -- 6 files changed, 160 insertions(+), 126 deletions(-) diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index 0ced32ef2715..bbd0eee4362f 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -319,7 +319,6 @@ static struct vmbus_channel *alloc_channel(void) init_completion(&channel->rescind_event); INIT_LIST_HEAD(&channel->sc_list); - INIT_LIST_HEAD(&channel->percpu_list); tasklet_init(&channel->callback_event, vmbus_on_event, (unsigned long)channel); @@ -340,23 +339,49 @@ static void free_channel(struct vmbus_channel *channel) kobject_put(&channel->kobj); } -static void percpu_channel_enq(void *arg) +void vmbus_channel_map_relid(struct vmbus_channel *channel) { - struct vmbus_channel *channel = arg; - struct hv_per_cpu_context *hv_cpu - = this_cpu_ptr(hv_context.cpu_context); - - list_add_tail_rcu(&channel->percpu_list, &hv_cpu->chan_list); + if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS)) + return; + /* + * The mapping of the channel's relid is visible from the CPUs that + * execute vmbus_chan_sched() by the time that vmbus_chan_sched() will + * execute: + * + * (a) In the "normal (i.e., not resuming from hibernation)" path, + * the full barrier in smp_store_mb() guarantees that the store + * is propagated to all CPUs before the add_channel_work work + * is queued. In turn, add_channel_work is queued before the + * channel's ring buffer is allocated/initialized and the + * OPENCHANNEL message for the channel is sent in vmbus_open(). + * Hyper-V won't start sending the interrupts for the channel + * before the OPENCHANNEL message is acked. The memory barrier + * in vmbus_chan_sched() -> sync_test_and_clear_bit() ensures + * that vmbus_chan_sched() must find the channel's relid in + * recv_int_page before retrieving the channel pointer from the + * array of channels. + * + * (b) In the "resuming from hibernation" path, the smp_store_mb() + * guarantees that the store is propagated to all CPUs before + * the VMBus connection is marked as ready for the resume event + * (cf. check_ready_for_resume_event()). The interrupt handler + * of the VMBus driver and vmbus_chan_sched() can not run before + * vmbus_bus_resume() has completed execution (cf. resume_noirq). + */ + smp_store_mb( + vmbus_connection.channels[channel->offermsg.child_relid], + channel); } -static void percpu_channel_deq(void *arg) +void vmbus_channel_unmap_relid(struct vmbus_channel *channel) { - struct vmbus_channel *channel = arg; - - list_del_rcu(&channel->percpu_list); + if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS)) + return; + WRITE_ONCE( + vmbus_connection.channels[channel->offermsg.child_relid], + NULL); } - static void vmbus_release_relid(u32 relid) { struct vmbus_channel_relid_released msg; @@ -376,17 +401,25 @@ void hv_process_channel_removal(struct vmbus_channel *channel) struct vmbus_channel *primary_channel; unsigned long flags; - BUG_ON(!mutex_is_locked(&vmbus_connection.channel_mutex)); + lockdep_assert_held(&vmbus_connection.channel_mutex); BUG_ON(!channel->rescind); - if (channel->target_cpu != get_cpu()) { - put_cpu(); - smp_call_function_single(channel->target_cpu, - percpu_channel_deq, channel, true); - } else { - percpu_channel_deq(channel); - put_cpu(); - } + /* + * hv_process_channel_removal() could find INVALID_RELID only for + * hv_sock channels. See the inline comments in vmbus_onoffer(). + */ + WARN_ON(channel->offermsg.child_relid == INVALID_RELID && + !is_hvsock_channel(channel)); + + /* + * Upon suspend, an in-use hv_sock channel is removed from the array of + * channels and the relid is invalidated. After hibernation, when the + * user-space appplication destroys the channel, it's unnecessary and + * unsafe to remove the channel from the array of channels. See also + * the inline comments before the call of vmbus_release_relid() below. + */ + if (channel->offermsg.child_relid != INVALID_RELID) + vmbus_channel_unmap_relid(channel); if (channel->primary_channel == NULL) { list_del(&channel->listentry); @@ -447,16 +480,6 @@ static void vmbus_add_channel_work(struct work_struct *work) init_vp_index(newchannel, dev_type); - if (newchannel->target_cpu != get_cpu()) { - put_cpu(); - smp_call_function_single(newchannel->target_cpu, - percpu_channel_enq, - newchannel, true); - } else { - percpu_channel_enq(newchannel); - put_cpu(); - } - /* * This state is used to indicate a successful open * so that when we do close the channel normally, we @@ -523,17 +546,10 @@ err_deq_chan: spin_unlock_irqrestore(&primary_channel->lock, flags); } - mutex_unlock(&vmbus_connection.channel_mutex); + /* vmbus_process_offer() has mapped the channel. */ + vmbus_channel_unmap_relid(newchannel); - if (newchannel->target_cpu != get_cpu()) { - put_cpu(); - smp_call_function_single(newchannel->target_cpu, - percpu_channel_deq, - newchannel, true); - } else { - percpu_channel_deq(newchannel); - put_cpu(); - } + mutex_unlock(&vmbus_connection.channel_mutex); vmbus_release_relid(newchannel->offermsg.child_relid); @@ -599,6 +615,8 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel) spin_unlock_irqrestore(&channel->lock, flags); } + vmbus_channel_map_relid(newchannel); + mutex_unlock(&vmbus_connection.channel_mutex); /* @@ -940,8 +958,6 @@ static void vmbus_onoffer(struct vmbus_channel_message_header *hdr) oldchannel = find_primary_channel_by_offer(offer); if (oldchannel != NULL) { - atomic_dec(&vmbus_connection.offer_in_progress); - /* * We're resuming from hibernation: all the sub-channel and * hv_sock channels we had before the hibernation should have @@ -949,36 +965,65 @@ static void vmbus_onoffer(struct vmbus_channel_message_header *hdr) * primary channel that we had before the hibernation. */ + /* + * { Initially: channel relid = INVALID_RELID, + * channels[valid_relid] = NULL } + * + * CPU1 CPU2 + * + * [vmbus_onoffer()] [vmbus_device_release()] + * + * LOCK channel_mutex LOCK channel_mutex + * STORE channel relid = valid_relid LOAD r1 = channel relid + * MAP_RELID channel if (r1 != INVALID_RELID) + * UNLOCK channel_mutex UNMAP_RELID channel + * UNLOCK channel_mutex + * + * Forbids: r1 == valid_relid && + * channels[valid_relid] == channel + * + * Note. r1 can be INVALID_RELID only for an hv_sock channel. + * None of the hv_sock channels which were present before the + * suspend are re-offered upon the resume. See the WARN_ON() + * in hv_process_channel_removal(). + */ + mutex_lock(&vmbus_connection.channel_mutex); + + atomic_dec(&vmbus_connection.offer_in_progress); + WARN_ON(oldchannel->offermsg.child_relid != INVALID_RELID); /* Fix up the relid. */ oldchannel->offermsg.child_relid = offer->child_relid; offer_sz = sizeof(*offer); - if (memcmp(offer, &oldchannel->offermsg, offer_sz) == 0) { - check_ready_for_resume_event(); - return; + if (memcmp(offer, &oldchannel->offermsg, offer_sz) != 0) { + /* + * This is not an error, since the host can also change + * the other field(s) of the offer, e.g. on WS RS5 + * (Build 17763), the offer->connection_id of the + * Mellanox VF vmbus device can change when the host + * reoffers the device upon resume. + */ + pr_debug("vmbus offer changed: relid=%d\n", + offer->child_relid); + + print_hex_dump_debug("Old vmbus offer: ", + DUMP_PREFIX_OFFSET, 16, 4, + &oldchannel->offermsg, offer_sz, + false); + print_hex_dump_debug("New vmbus offer: ", + DUMP_PREFIX_OFFSET, 16, 4, + offer, offer_sz, false); + + /* Fix up the old channel. */ + vmbus_setup_channel_state(oldchannel, offer); } - /* - * This is not an error, since the host can also change the - * other field(s) of the offer, e.g. on WS RS5 (Build 17763), - * the offer->connection_id of the Mellanox VF vmbus device - * can change when the host reoffers the device upon resume. - */ - pr_debug("vmbus offer changed: relid=%d\n", - offer->child_relid); - - print_hex_dump_debug("Old vmbus offer: ", DUMP_PREFIX_OFFSET, - 16, 4, &oldchannel->offermsg, offer_sz, - false); - print_hex_dump_debug("New vmbus offer: ", DUMP_PREFIX_OFFSET, - 16, 4, offer, offer_sz, false); - - /* Fix up the old channel. */ - vmbus_setup_channel_state(oldchannel, offer); - + /* Add the channel back to the array of channels. */ + vmbus_channel_map_relid(oldchannel); check_ready_for_resume_event(); + mutex_unlock(&vmbus_connection.channel_mutex); return; } @@ -1036,14 +1081,14 @@ static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr) * * CPU1 CPU2 * - * [vmbus_process_offer()] [vmbus_onoffer_rescind()] + * [vmbus_onoffer()] [vmbus_onoffer_rescind()] * * LOCK channel_mutex WAIT_ON offer_in_progress == 0 * DECREMENT offer_in_progress LOCK channel_mutex - * INSERT chn_list SEARCH chn_list + * STORE channels[] LOAD channels[] * UNLOCK channel_mutex UNLOCK channel_mutex * - * Forbids: CPU2's SEARCH from *not* seeing CPU1's INSERT + * Forbids: CPU2's LOAD from *not* seeing CPU1's STORE */ while (atomic_read(&vmbus_connection.offer_in_progress) != 0) { diff --git a/drivers/hv/connection.c b/drivers/hv/connection.c index f4bd306d2cef..11170d9a2e1a 100644 --- a/drivers/hv/connection.c +++ b/drivers/hv/connection.c @@ -248,6 +248,14 @@ int vmbus_connect(void) pr_info("Vmbus version:%d.%d\n", version >> 16, version & 0xFFFF); + vmbus_connection.channels = kcalloc(MAX_CHANNEL_RELIDS, + sizeof(struct vmbus_channel *), + GFP_KERNEL); + if (vmbus_connection.channels == NULL) { + ret = -ENOMEM; + goto cleanup; + } + kfree(msginfo); return 0; @@ -295,33 +303,9 @@ void vmbus_disconnect(void) */ struct vmbus_channel *relid2channel(u32 relid) { - struct vmbus_channel *channel; - struct vmbus_channel *found_channel = NULL; - struct list_head *cur, *tmp; - struct vmbus_channel *cur_sc; - - BUG_ON(!mutex_is_locked(&vmbus_connection.channel_mutex)); - - list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { - if (channel->offermsg.child_relid == relid) { - found_channel = channel; - break; - } else if (!list_empty(&channel->sc_list)) { - /* - * Deal with sub-channels. - */ - list_for_each_safe(cur, tmp, &channel->sc_list) { - cur_sc = list_entry(cur, struct vmbus_channel, - sc_list); - if (cur_sc->offermsg.child_relid == relid) { - found_channel = cur_sc; - break; - } - } - } - } - - return found_channel; + if (WARN_ON(relid >= MAX_CHANNEL_RELIDS)) + return NULL; + return READ_ONCE(vmbus_connection.channels[relid]); } /* diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c index e2b331045464..17bf1f229152 100644 --- a/drivers/hv/hv.c +++ b/drivers/hv/hv.c @@ -117,8 +117,6 @@ int hv_synic_alloc(void) pr_err("Unable to allocate post msg page\n"); goto err; } - - INIT_LIST_HEAD(&hv_cpu->chan_list); } return 0; diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h index 000740d053a2..3edf993b0fd9 100644 --- a/drivers/hv/hyperv_vmbus.h +++ b/drivers/hv/hyperv_vmbus.h @@ -132,12 +132,6 @@ struct hv_per_cpu_context { * basis. */ struct tasklet_struct msg_dpc; - - /* - * To optimize the mapping of relid to channel, maintain - * per-cpu list of the channels based on their CPU affinity. - */ - struct list_head chan_list; }; struct hv_context { @@ -202,6 +196,8 @@ int hv_ringbuffer_read(struct vmbus_channel *channel, /* TODO: Need to make this configurable */ #define MAX_NUM_CHANNELS_SUPPORTED 256 +#define MAX_CHANNEL_RELIDS \ + max(MAX_NUM_CHANNELS_SUPPORTED, HV_EVENT_FLAGS_COUNT) enum vmbus_connect_state { DISCONNECTED, @@ -251,6 +247,9 @@ struct vmbus_connection { struct list_head chn_list; struct mutex channel_mutex; + /* Array of channels */ + struct vmbus_channel **channels; + /* * An offer message is handled first on the work_queue, and then * is further handled on handle_primary_chan_wq or @@ -338,6 +337,9 @@ int vmbus_add_channel_kobj(struct hv_device *device_obj, void vmbus_remove_channel_attr_group(struct vmbus_channel *channel); +void vmbus_channel_map_relid(struct vmbus_channel *channel); +void vmbus_channel_unmap_relid(struct vmbus_channel *channel); + struct vmbus_channel *relid2channel(u32 relid); void vmbus_free_channels(void); diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 1f3e69d18d35..21a01bca7fcd 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1252,33 +1252,39 @@ static void vmbus_chan_sched(struct hv_per_cpu_context *hv_cpu) if (relid == 0) continue; + /* + * Pairs with the kfree_rcu() in vmbus_chan_release(). + * Guarantees that the channel data structure doesn't + * get freed while the channel pointer below is being + * dereferenced. + */ rcu_read_lock(); /* Find channel based on relid */ - list_for_each_entry_rcu(channel, &hv_cpu->chan_list, percpu_list) { - if (channel->offermsg.child_relid != relid) - continue; + channel = relid2channel(relid); + if (channel == NULL) + goto sched_unlock_rcu; - if (channel->rescind) - continue; + if (channel->rescind) + goto sched_unlock_rcu; - trace_vmbus_chan_sched(channel); + trace_vmbus_chan_sched(channel); - ++channel->interrupts; + ++channel->interrupts; - switch (channel->callback_mode) { - case HV_CALL_ISR: - vmbus_channel_isr(channel); - break; + switch (channel->callback_mode) { + case HV_CALL_ISR: + vmbus_channel_isr(channel); + break; - case HV_CALL_BATCHED: - hv_begin_read(&channel->inbound); - /* fallthrough */ - case HV_CALL_DIRECT: - tasklet_schedule(&channel->callback_event); - } + case HV_CALL_BATCHED: + hv_begin_read(&channel->inbound); + fallthrough; + case HV_CALL_DIRECT: + tasklet_schedule(&channel->callback_event); } +sched_unlock_rcu: rcu_read_unlock(); } } @@ -2264,9 +2270,12 @@ static int vmbus_bus_suspend(struct device *dev) list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { /* - * Invalidate the field. Upon resume, vmbus_onoffer() will fix - * up the field, and the other fields (if necessary). + * Remove the channel from the array of channels and invalidate + * the channel's relid. Upon resume, vmbus_onoffer() will fix + * up the relid (and other fields, if necessary) and add the + * channel back to the array. */ + vmbus_channel_unmap_relid(channel); channel->offermsg.child_relid = INVALID_RELID; if (is_hvsock_channel(channel)) { @@ -2502,6 +2511,7 @@ static void __exit vmbus_exit(void) hv_debug_rm_all_dir(); vmbus_free_channels(); + kfree(vmbus_connection.channels); if (ms_hyperv.misc_features & HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE) { kmsg_dump_unregister(&hv_kmsg_dumper); diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index cbd24f4e68d1..f5b3f008c55a 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -854,11 +854,6 @@ struct vmbus_channel { * Support per-channel state for use by vmbus drivers. */ void *per_channel_state; - /* - * To support per-cpu lookup mapping of relid to channel, - * link up channels based on their CPU affinity. - */ - struct list_head percpu_list; /* * Defer freeing channel until after all cpu's have From ac5047671758ad4be9f93898247b3a8b6dfde4c7 Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Mon, 6 Apr 2020 02:15:07 +0200 Subject: [PATCH 0137/1043] hv_netvsc: Disable NAPI before closing the VMBus channel vmbus_chan_sched() might call the netvsc driver callback function that ends up scheduling NAPI work. This "work" can access the channel ring buffer, so we must ensure that any such work is completed and that the ring buffer is no longer being accessed before freeing the ring buffer data structure in the channel closure path. To this end, disable NAPI before calling vmbus_close() in netvsc_device_remove(). Suggested-by: Michael Kelley Signed-off-by: Andrea Parri (Microsoft) Acked-by: Stephen Hemminger Cc: "David S. Miller" Cc: Link: https://lore.kernel.org/r/20200406001514.19876-5-parri.andrea@gmail.com Reviewed-by: Michael Kelley Signed-off-by: Wei Liu --- drivers/hv/channel.c | 6 ++++++ drivers/net/hyperv/netvsc.c | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c index 23f358cb7f49..256ee90c7446 100644 --- a/drivers/hv/channel.c +++ b/drivers/hv/channel.c @@ -609,6 +609,12 @@ void vmbus_reset_channel_cb(struct vmbus_channel *channel) * the former is accessing channel->inbound.ring_buffer, the latter * could be freeing the ring_buffer pages, so here we must stop it * first. + * + * vmbus_chan_sched() might call the netvsc driver callback function + * that ends up scheduling NAPI work that accesses the ring buffer. + * At this point, we have to ensure that any such work is completed + * and that the channel ring buffer is no longer being accessed, cf. + * the calls to napi_disable() in netvsc_device_remove(). */ tasklet_disable(&channel->callback_event); diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c index ca68aa1df801..41f5cf0bb997 100644 --- a/drivers/net/hyperv/netvsc.c +++ b/drivers/net/hyperv/netvsc.c @@ -636,9 +636,12 @@ void netvsc_device_remove(struct hv_device *device) RCU_INIT_POINTER(net_device_ctx->nvdev, NULL); - /* And disassociate NAPI context from device */ - for (i = 0; i < net_device->num_chn; i++) + /* Disable NAPI and disassociate its context from the device. */ + for (i = 0; i < net_device->num_chn; i++) { + /* See also vmbus_reset_channel_cb(). */ + napi_disable(&net_device->chan_table[i].napi); netif_napi_del(&net_device->chan_table[i].napi); + } /* * At this point, no one should be accessing net_device From 238d2ed8f7d1b1ca0c13334bb8197a42654af946 Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Mon, 6 Apr 2020 02:15:08 +0200 Subject: [PATCH 0138/1043] hv_utils: Always execute the fcopy and vss callbacks in a tasklet The fcopy and vss callback functions could be running in a tasklet at the same time they are called in hv_poll_channel(). Current code serializes the invocations of these functions, and their accesses to the channel ring buffer, by sending an IPI to the CPU that is allowed to access the ring buffer, cf. hv_poll_channel(). This IPI mechanism becomes infeasible if we allow changing the CPU that a channel will interrupt. Instead modify the callback wrappers to always execute the fcopy and vss callbacks in a tasklet, thus mirroring the solution for the kvp callback functions adopted since commit a3ade8cc474d8 ("HV: properly delay KVP packets when negotiation is in progress"). This will ensure that the callback function can't run on two CPUs at the same time. Suggested-by: Michael Kelley Signed-off-by: Andrea Parri (Microsoft) Link: https://lore.kernel.org/r/20200406001514.19876-6-parri.andrea@gmail.com Reviewed-by: Michael Kelley Signed-off-by: Wei Liu --- drivers/hv/hv_fcopy.c | 2 +- drivers/hv/hv_snapshot.c | 2 +- drivers/hv/hyperv_vmbus.h | 7 +------ 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/drivers/hv/hv_fcopy.c b/drivers/hv/hv_fcopy.c index bb9ba3f7c794..5040d7e0cd9e 100644 --- a/drivers/hv/hv_fcopy.c +++ b/drivers/hv/hv_fcopy.c @@ -71,7 +71,7 @@ static void fcopy_poll_wrapper(void *channel) { /* Transaction is finished, reset the state here to avoid races. */ fcopy_transaction.state = HVUTIL_READY; - hv_fcopy_onchannelcallback(channel); + tasklet_schedule(&((struct vmbus_channel *)channel)->callback_event); } static void fcopy_timeout_func(struct work_struct *dummy) diff --git a/drivers/hv/hv_snapshot.c b/drivers/hv/hv_snapshot.c index 1c75b38f0d6d..783779e4cc1a 100644 --- a/drivers/hv/hv_snapshot.c +++ b/drivers/hv/hv_snapshot.c @@ -80,7 +80,7 @@ static void vss_poll_wrapper(void *channel) { /* Transaction is finished, reset the state here to avoid races. */ vss_transaction.state = HVUTIL_READY; - hv_vss_onchannelcallback(channel); + tasklet_schedule(&((struct vmbus_channel *)channel)->callback_event); } /* diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h index 3edf993b0fd9..5e5cebe5d048 100644 --- a/drivers/hv/hyperv_vmbus.h +++ b/drivers/hv/hyperv_vmbus.h @@ -378,12 +378,7 @@ static inline void hv_poll_channel(struct vmbus_channel *channel, { if (!channel) return; - - if (in_interrupt() && (channel->target_cpu == smp_processor_id())) { - cb(channel); - return; - } - smp_call_function_single(channel->target_cpu, cb, channel, true); + cb(channel); } enum hvutil_device_state { From 9403b66e6161130ebae7e55a97491c84c1ad6f9f Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Mon, 6 Apr 2020 02:15:09 +0200 Subject: [PATCH 0139/1043] Drivers: hv: vmbus: Use a spin lock for synchronizing channel scheduling vs. channel removal Since vmbus_chan_sched() dereferences the ring buffer pointer, we have to make sure that the ring buffer data structures don't get freed while such dereferencing is happening. Current code does this by sending an IPI to the CPU that is allowed to access that ring buffer from interrupt level, cf., vmbus_reset_channel_cb(). But with the new functionality to allow changing the CPU that a channel will interrupt, we can't be sure what CPU will be running the vmbus_chan_sched() function for a particular channel, so the current IPI mechanism is infeasible. Instead synchronize vmbus_chan_sched() and vmbus_reset_channel_cb() by using the (newly introduced) per-channel spin lock "sched_lock". Move the test for onchannel_callback being NULL before the "switch" control statement in vmbus_chan_sched(), in order to not access the ring buffer if the vmbus_reset_channel_cb() has been completed on the channel. Suggested-by: Michael Kelley Signed-off-by: Andrea Parri (Microsoft) Link: https://lore.kernel.org/r/20200406001514.19876-7-parri.andrea@gmail.com Reviewed-by: Michael Kelley Signed-off-by: Wei Liu --- drivers/hv/channel.c | 24 +++++++----------------- drivers/hv/channel_mgmt.c | 1 + drivers/hv/vmbus_drv.c | 30 +++++++++++++++++------------- include/linux/hyperv.h | 6 ++++++ 4 files changed, 31 insertions(+), 30 deletions(-) diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c index 256ee90c7446..132e476f87b2 100644 --- a/drivers/hv/channel.c +++ b/drivers/hv/channel.c @@ -594,15 +594,10 @@ post_msg_err: } EXPORT_SYMBOL_GPL(vmbus_teardown_gpadl); -static void reset_channel_cb(void *arg) -{ - struct vmbus_channel *channel = arg; - - channel->onchannel_callback = NULL; -} - void vmbus_reset_channel_cb(struct vmbus_channel *channel) { + unsigned long flags; + /* * vmbus_on_event(), running in the per-channel tasklet, can race * with vmbus_close_internal() in the case of SMP guest, e.g., when @@ -618,17 +613,12 @@ void vmbus_reset_channel_cb(struct vmbus_channel *channel) */ tasklet_disable(&channel->callback_event); - channel->sc_creation_callback = NULL; + /* See the inline comments in vmbus_chan_sched(). */ + spin_lock_irqsave(&channel->sched_lock, flags); + channel->onchannel_callback = NULL; + spin_unlock_irqrestore(&channel->sched_lock, flags); - /* Stop the callback asap */ - if (channel->target_cpu != get_cpu()) { - put_cpu(); - smp_call_function_single(channel->target_cpu, reset_channel_cb, - channel, true); - } else { - reset_channel_cb(channel); - put_cpu(); - } + channel->sc_creation_callback = NULL; /* Re-enable tasklet for use on re-open */ tasklet_enable(&channel->callback_event); diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index bbd0eee4362f..2e730f84654b 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -315,6 +315,7 @@ static struct vmbus_channel *alloc_channel(void) if (!channel) return NULL; + spin_lock_init(&channel->sched_lock); spin_lock_init(&channel->lock); init_completion(&channel->rescind_event); diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 21a01bca7fcd..0f7dfa507a40 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1201,18 +1201,6 @@ static void vmbus_force_channel_rescinded(struct vmbus_channel *channel) } #endif /* CONFIG_PM_SLEEP */ -/* - * Direct callback for channels using other deferred processing - */ -static void vmbus_channel_isr(struct vmbus_channel *channel) -{ - void (*callback_fn)(void *); - - callback_fn = READ_ONCE(channel->onchannel_callback); - if (likely(callback_fn != NULL)) - (*callback_fn)(channel->channel_callback_context); -} - /* * Schedule all channels with events pending */ @@ -1243,6 +1231,7 @@ static void vmbus_chan_sched(struct hv_per_cpu_context *hv_cpu) return; for_each_set_bit(relid, recv_int_page, maxbits) { + void (*callback_fn)(void *context); struct vmbus_channel *channel; if (!sync_test_and_clear_bit(relid, recv_int_page)) @@ -1268,13 +1257,26 @@ static void vmbus_chan_sched(struct hv_per_cpu_context *hv_cpu) if (channel->rescind) goto sched_unlock_rcu; + /* + * Make sure that the ring buffer data structure doesn't get + * freed while we dereference the ring buffer pointer. Test + * for the channel's onchannel_callback being NULL within a + * sched_lock critical section. See also the inline comments + * in vmbus_reset_channel_cb(). + */ + spin_lock(&channel->sched_lock); + + callback_fn = channel->onchannel_callback; + if (unlikely(callback_fn == NULL)) + goto sched_unlock; + trace_vmbus_chan_sched(channel); ++channel->interrupts; switch (channel->callback_mode) { case HV_CALL_ISR: - vmbus_channel_isr(channel); + (*callback_fn)(channel->channel_callback_context); break; case HV_CALL_BATCHED: @@ -1284,6 +1286,8 @@ static void vmbus_chan_sched(struct hv_per_cpu_context *hv_cpu) tasklet_schedule(&channel->callback_event); } +sched_unlock: + spin_unlock(&channel->sched_lock); sched_unlock_rcu: rcu_read_unlock(); } diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index f5b3f008c55a..62b2c11d0875 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -771,6 +771,12 @@ struct vmbus_channel { void (*onchannel_callback)(void *context); void *channel_callback_context; + /* + * Synchronize channel scheduling and channel removal; see the inline + * comments in vmbus_chan_sched() and vmbus_reset_channel_cb(). + */ + spinlock_t sched_lock; + /* * A channel can be marked for one of three modes of reading: * BATCHED - callback called from taslket and should read From 240ad77cb50d9f0a961fcb0f21e67939cf7a9c04 Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Mon, 6 Apr 2020 02:15:10 +0200 Subject: [PATCH 0140/1043] PCI: hv: Prepare hv_compose_msi_msg() for the VMBus-channel-interrupt-to-vCPU reassignment functionality The current implementation of hv_compose_msi_msg() is incompatible with the new functionality that allows changing the vCPU a VMBus channel will interrupt: if this function always calls hv_pci_onchannelcallback() in the polling loop, the interrupt going to a different CPU could cause hv_pci_onchannelcallback() to be running simultaneously in a tasklet, which will break. The current code also has a problem in that it is not synchronized with vmbus_reset_channel_cb(): hv_compose_msi_msg() could be accessing the ring buffer via the call of hv_pci_onchannelcallback() well after the time that vmbus_reset_channel_cb() has finished. Fix these issues as follows. Disable the channel tasklet before entering the polling loop in hv_compose_msi_msg() and re-enable it when done. This will prevent hv_pci_onchannelcallback() from running in a tasklet on a different CPU. Moreover, poll by always calling hv_pci_onchannelcallback(), but check the channel callback function for NULL and invoke the callback within a sched_lock critical section. This will prevent hv_compose_msi_msg() from accessing the ring buffer after vmbus_reset_channel_cb() has acquired the sched_lock spinlock. Suggested-by: Michael Kelley Signed-off-by: Andrea Parri (Microsoft) Cc: Lorenzo Pieralisi Cc: Andrew Murray Cc: Bjorn Helgaas Cc: Link: https://lore.kernel.org/r/20200406001514.19876-8-parri.andrea@gmail.com Reviewed-by: Michael Kelley Signed-off-by: Wei Liu --- drivers/pci/controller/pci-hyperv.c | 44 ++++++++++++++++++----------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index e15022ff63e3..222ff5639ebe 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -1356,11 +1356,11 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) { struct irq_cfg *cfg = irqd_cfg(data); struct hv_pcibus_device *hbus; + struct vmbus_channel *channel; struct hv_pci_dev *hpdev; struct pci_bus *pbus; struct pci_dev *pdev; struct cpumask *dest; - unsigned long flags; struct compose_comp_ctxt comp; struct tran_int_desc *int_desc; struct { @@ -1378,6 +1378,7 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) dest = irq_data_get_effective_affinity_mask(data); pbus = pdev->bus; hbus = container_of(pbus->sysdata, struct hv_pcibus_device, sysdata); + channel = hbus->hdev->channel; hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn)); if (!hpdev) goto return_null_message; @@ -1435,43 +1436,52 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) goto free_int_desc; } + /* + * Prevents hv_pci_onchannelcallback() from running concurrently + * in the tasklet. + */ + tasklet_disable(&channel->callback_event); + /* * Since this function is called with IRQ locks held, can't * do normal wait for completion; instead poll. */ while (!try_wait_for_completion(&comp.comp_pkt.host_event)) { + unsigned long flags; + /* 0xFFFF means an invalid PCI VENDOR ID. */ if (hv_pcifront_get_vendor_id(hpdev) == 0xFFFF) { dev_err_once(&hbus->hdev->device, "the device has gone\n"); - goto free_int_desc; + goto enable_tasklet; } /* - * When the higher level interrupt code calls us with - * interrupt disabled, we must poll the channel by calling - * the channel callback directly when channel->target_cpu is - * the current CPU. When the higher level interrupt code - * calls us with interrupt enabled, let's add the - * local_irq_save()/restore() to avoid race: - * hv_pci_onchannelcallback() can also run in tasklet. + * Make sure that the ring buffer data structure doesn't get + * freed while we dereference the ring buffer pointer. Test + * for the channel's onchannel_callback being NULL within a + * sched_lock critical section. See also the inline comments + * in vmbus_reset_channel_cb(). */ - local_irq_save(flags); - - if (hbus->hdev->channel->target_cpu == smp_processor_id()) - hv_pci_onchannelcallback(hbus); - - local_irq_restore(flags); + spin_lock_irqsave(&channel->sched_lock, flags); + if (unlikely(channel->onchannel_callback == NULL)) { + spin_unlock_irqrestore(&channel->sched_lock, flags); + goto enable_tasklet; + } + hv_pci_onchannelcallback(hbus); + spin_unlock_irqrestore(&channel->sched_lock, flags); if (hpdev->state == hv_pcichild_ejecting) { dev_err_once(&hbus->hdev->device, "the device is being ejected\n"); - goto free_int_desc; + goto enable_tasklet; } udelay(100); } + tasklet_enable(&channel->callback_event); + if (comp.comp_pkt.completion_status < 0) { dev_err(&hbus->hdev->device, "Request for interrupt failed: 0x%x", @@ -1495,6 +1505,8 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) put_pcichild(hpdev); return; +enable_tasklet: + tasklet_enable(&channel->callback_event); free_int_desc: kfree(int_desc); drop_reference: From 8ef4c4abbbcdcd9d4bc0fd9454df03e6dac24b73 Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Mon, 6 Apr 2020 02:15:11 +0200 Subject: [PATCH 0141/1043] Drivers: hv: vmbus: Remove the unused HV_LOCALIZED channel affinity logic The logic is unused since commit 509879bdb30b8 ("Drivers: hv: Introduce a policy for controlling channel affinity"). This logic assumes that a channel target_cpu doesn't change during the lifetime of a channel, but this assumption is incompatible with the new functionality that allows changing the vCPU a channel will interrupt. Signed-off-by: Andrea Parri (Microsoft) Link: https://lore.kernel.org/r/20200406001514.19876-9-parri.andrea@gmail.com Reviewed-by: Michael Kelley Signed-off-by: Wei Liu --- drivers/hv/channel_mgmt.c | 107 +++++++++----------------------------- include/linux/hyperv.h | 27 ---------- 2 files changed, 26 insertions(+), 108 deletions(-) diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index 2e730f84654b..a19dc28fdf77 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -433,14 +433,6 @@ void hv_process_channel_removal(struct vmbus_channel *channel) spin_unlock_irqrestore(&primary_channel->lock, flags); } - /* - * We need to free the bit for init_vp_index() to work in the case - * of sub-channel, when we reload drivers like hv_netvsc. - */ - if (channel->affinity_policy == HV_LOCALIZED) - cpumask_clear_cpu(channel->target_cpu, - &primary_channel->alloced_cpus_in_node); - /* * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and * the relid is invalidated; after hibernation, when the user-space app @@ -662,20 +654,21 @@ static DEFINE_SPINLOCK(bind_channel_to_cpu_lock); /* * Starting with Win8, we can statically distribute the incoming * channel interrupt load by binding a channel to VCPU. - * We distribute the interrupt loads to one or more NUMA nodes based on - * the channel's affinity_policy. * * For pre-win8 hosts or non-performance critical channels we assign the * first CPU in the first NUMA node. + * + * Starting with win8, performance critical channels will be distributed + * evenly among all the available NUMA nodes. Once the node is assigned, + * we will assign the CPU based on a simple round robin scheme. */ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type) { - u32 cur_cpu; bool perf_chn = vmbus_devs[dev_type].perf_device; - struct vmbus_channel *primary = channel->primary_channel; - int next_node; cpumask_var_t available_mask; struct cpumask *alloced_mask; + u32 target_cpu; + int numa_node; if ((vmbus_proto_version == VERSION_WS2008) || (vmbus_proto_version == VERSION_WIN7) || (!perf_chn) || @@ -693,31 +686,27 @@ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type) return; } + /* + * Serializes the accesses to the global variable next_numa_node_id. + * See also the header comment of the spin lock declaration. + */ spin_lock(&bind_channel_to_cpu_lock); - /* - * Based on the channel affinity policy, we will assign the NUMA - * nodes. - */ - - if ((channel->affinity_policy == HV_BALANCED) || (!primary)) { - while (true) { - next_node = next_numa_node_id++; - if (next_node == nr_node_ids) { - next_node = next_numa_node_id = 0; - continue; - } - if (cpumask_empty(cpumask_of_node(next_node))) - continue; - break; + while (true) { + numa_node = next_numa_node_id++; + if (numa_node == nr_node_ids) { + next_numa_node_id = 0; + continue; } - channel->numa_node = next_node; - primary = channel; + if (cpumask_empty(cpumask_of_node(numa_node))) + continue; + break; } - alloced_mask = &hv_context.hv_numa_map[primary->numa_node]; + channel->numa_node = numa_node; + alloced_mask = &hv_context.hv_numa_map[numa_node]; if (cpumask_weight(alloced_mask) == - cpumask_weight(cpumask_of_node(primary->numa_node))) { + cpumask_weight(cpumask_of_node(numa_node))) { /* * We have cycled through all the CPUs in the node; * reset the alloced map. @@ -725,57 +714,13 @@ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type) cpumask_clear(alloced_mask); } - cpumask_xor(available_mask, alloced_mask, - cpumask_of_node(primary->numa_node)); + cpumask_xor(available_mask, alloced_mask, cpumask_of_node(numa_node)); - cur_cpu = -1; + target_cpu = cpumask_first(available_mask); + cpumask_set_cpu(target_cpu, alloced_mask); - if (primary->affinity_policy == HV_LOCALIZED) { - /* - * Normally Hyper-V host doesn't create more subchannels - * than there are VCPUs on the node but it is possible when not - * all present VCPUs on the node are initialized by guest. - * Clear the alloced_cpus_in_node to start over. - */ - if (cpumask_equal(&primary->alloced_cpus_in_node, - cpumask_of_node(primary->numa_node))) - cpumask_clear(&primary->alloced_cpus_in_node); - } - - while (true) { - cur_cpu = cpumask_next(cur_cpu, available_mask); - if (cur_cpu >= nr_cpu_ids) { - cur_cpu = -1; - cpumask_copy(available_mask, - cpumask_of_node(primary->numa_node)); - continue; - } - - if (primary->affinity_policy == HV_LOCALIZED) { - /* - * NOTE: in the case of sub-channel, we clear the - * sub-channel related bit(s) in - * primary->alloced_cpus_in_node in - * hv_process_channel_removal(), so when we - * reload drivers like hv_netvsc in SMP guest, here - * we're able to re-allocate - * bit from primary->alloced_cpus_in_node. - */ - if (!cpumask_test_cpu(cur_cpu, - &primary->alloced_cpus_in_node)) { - cpumask_set_cpu(cur_cpu, - &primary->alloced_cpus_in_node); - cpumask_set_cpu(cur_cpu, alloced_mask); - break; - } - } else { - cpumask_set_cpu(cur_cpu, alloced_mask); - break; - } - } - - channel->target_cpu = cur_cpu; - channel->target_vp = hv_cpu_number_to_vp_number(cur_cpu); + channel->target_cpu = target_cpu; + channel->target_vp = hv_cpu_number_to_vp_number(target_cpu); spin_unlock(&bind_channel_to_cpu_lock); diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 62b2c11d0875..247356dbd742 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -689,11 +689,6 @@ union hv_connection_id { } u; }; -enum hv_numa_policy { - HV_BALANCED = 0, - HV_LOCALIZED, -}; - enum vmbus_device_type { HV_IDE = 0, HV_SCSI, @@ -808,10 +803,6 @@ struct vmbus_channel { u32 target_vp; /* The corresponding CPUID in the guest */ u32 target_cpu; - /* - * State to manage the CPU affiliation of channels. - */ - struct cpumask alloced_cpus_in_node; int numa_node; /* * Support for sub-channels. For high performance devices, @@ -898,18 +889,6 @@ struct vmbus_channel { */ bool low_latency; - /* - * NUMA distribution policy: - * We support two policies: - * 1) Balanced: Here all performance critical channels are - * distributed evenly amongst all the NUMA nodes. - * This policy will be the default policy. - * 2) Localized: All channels of a given instance of a - * performance critical service will be assigned CPUs - * within a selected NUMA node. - */ - enum hv_numa_policy affinity_policy; - bool probe_done; /* @@ -965,12 +944,6 @@ static inline bool is_sub_channel(const struct vmbus_channel *c) return c->offermsg.offer.sub_channel_index != 0; } -static inline void set_channel_affinity_state(struct vmbus_channel *c, - enum hv_numa_policy policy) -{ - c->affinity_policy = policy; -} - static inline void set_channel_read_mode(struct vmbus_channel *c, enum hv_callback_mode mode) { From d570aec0f2154e1bfba14ffd0df164a185e363b5 Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Mon, 6 Apr 2020 02:15:12 +0200 Subject: [PATCH 0142/1043] Drivers: hv: vmbus: Synchronize init_vp_index() vs. CPU hotplug init_vp_index() may access the cpu_online_mask mask via its calls of cpumask_of_node(). Make sure to protect these accesses with a cpus_read_lock() critical section. Also, remove some (hardcoded) instances of CPU(0) from init_vp_index() and replace them with VMBUS_CONNECT_CPU. The connect CPU can not go offline, since Hyper-V does not provide a way to change it. Finally, order the accesses of target_cpu from init_vp_index() and hv_synic_cleanup() by relying on the channel_mutex; this is achieved by moving the call of init_vp_index() into vmbus_process_offer(). Signed-off-by: Andrea Parri (Microsoft) Link: https://lore.kernel.org/r/20200406001514.19876-10-parri.andrea@gmail.com Reviewed-by: Michael Kelley Signed-off-by: Wei Liu --- drivers/hv/channel_mgmt.c | 47 ++++++++++++++++++++++++++++----------- drivers/hv/hv.c | 7 +++--- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index a19dc28fdf77..2db3823f0e59 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -466,13 +467,8 @@ static void vmbus_add_channel_work(struct work_struct *work) container_of(work, struct vmbus_channel, add_channel_work); struct vmbus_channel *primary_channel = newchannel->primary_channel; unsigned long flags; - u16 dev_type; int ret; - dev_type = hv_get_dev_type(newchannel); - - init_vp_index(newchannel, dev_type); - /* * This state is used to indicate a successful open * so that when we do close the channel normally, we @@ -504,7 +500,7 @@ static void vmbus_add_channel_work(struct work_struct *work) if (!newchannel->device_obj) goto err_deq_chan; - newchannel->device_obj->device_id = dev_type; + newchannel->device_obj->device_id = hv_get_dev_type(newchannel); /* * Add the new device to the bus. This will kick off device-driver * binding which eventually invokes the device driver's AddDevice() @@ -560,6 +556,25 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel) unsigned long flags; bool fnew = true; + /* + * Initialize the target_CPU before inserting the channel in + * the chn_list and sc_list lists, within the channel_mutex + * critical section: + * + * CPU1 CPU2 + * + * [vmbus_process_offer()] [hv_syninc_cleanup()] + * + * STORE target_cpu LOCK channel_mutex + * LOCK channel_mutex SEARCH chn_list + * INSERT chn_list LOAD target_cpu + * UNLOCK channel_mutex UNLOCK channel_mutex + * + * Forbids: CPU2's SEARCH from seeing CPU1's INSERT && + * CPU2's LOAD from *not* seing CPU1's STORE + */ + init_vp_index(newchannel, hv_get_dev_type(newchannel)); + mutex_lock(&vmbus_connection.channel_mutex); /* Remember the channels that should be cleaned up upon suspend. */ @@ -656,7 +671,7 @@ static DEFINE_SPINLOCK(bind_channel_to_cpu_lock); * channel interrupt load by binding a channel to VCPU. * * For pre-win8 hosts or non-performance critical channels we assign the - * first CPU in the first NUMA node. + * VMBUS_CONNECT_CPU. * * Starting with win8, performance critical channels will be distributed * evenly among all the available NUMA nodes. Once the node is assigned, @@ -675,17 +690,22 @@ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type) !alloc_cpumask_var(&available_mask, GFP_KERNEL)) { /* * Prior to win8, all channel interrupts are - * delivered on cpu 0. + * delivered on VMBUS_CONNECT_CPU. * Also if the channel is not a performance critical - * channel, bind it to cpu 0. - * In case alloc_cpumask_var() fails, bind it to cpu 0. + * channel, bind it to VMBUS_CONNECT_CPU. + * In case alloc_cpumask_var() fails, bind it to + * VMBUS_CONNECT_CPU. */ - channel->numa_node = 0; - channel->target_cpu = 0; - channel->target_vp = hv_cpu_number_to_vp_number(0); + channel->numa_node = cpu_to_node(VMBUS_CONNECT_CPU); + channel->target_cpu = VMBUS_CONNECT_CPU; + channel->target_vp = + hv_cpu_number_to_vp_number(VMBUS_CONNECT_CPU); return; } + /* No CPUs can come up or down during this. */ + cpus_read_lock(); + /* * Serializes the accesses to the global variable next_numa_node_id. * See also the header comment of the spin lock declaration. @@ -723,6 +743,7 @@ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type) channel->target_vp = hv_cpu_number_to_vp_number(target_cpu); spin_unlock(&bind_channel_to_cpu_lock); + cpus_read_unlock(); free_cpumask_var(available_mask); } diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c index 17bf1f229152..188b42b07f07 100644 --- a/drivers/hv/hv.c +++ b/drivers/hv/hv.c @@ -256,9 +256,10 @@ int hv_synic_cleanup(unsigned int cpu) /* * Search for channels which are bound to the CPU we're about to - * cleanup. In case we find one and vmbus is still connected we need to - * fail, this will effectively prevent CPU offlining. There is no way - * we can re-bind channels to different CPUs for now. + * cleanup. In case we find one and vmbus is still connected, we + * fail; this will effectively prevent CPU offlining. + * + * TODO: Re-bind the channels to different CPUs. */ mutex_lock(&vmbus_connection.channel_mutex); list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { From 7527810573436f00e582d3d5ef2eb3c027c98d7d Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Mon, 6 Apr 2020 02:15:13 +0200 Subject: [PATCH 0143/1043] Drivers: hv: vmbus: Introduce the CHANNELMSG_MODIFYCHANNEL message type VMBus version 4.1 and later support the CHANNELMSG_MODIFYCHANNEL(22) message type which can be used to request Hyper-V to change the vCPU that a channel will interrupt. Introduce the CHANNELMSG_MODIFYCHANNEL message type, and define the vmbus_send_modifychannel() function to send CHANNELMSG_MODIFYCHANNEL requests to the host via a hypercall. The function is then used to define a sysfs "store" operation, which allows to change the (v)CPU the channel will interrupt by using the sysfs interface. The feature can be used for load balancing or other purposes. One interesting catch here is that Hyper-V can *not* currently ACK CHANNELMSG_MODIFYCHANNEL messages with the promise that (after the ACK is sent) the channel won't send any more interrupts to the "old" CPU. The peculiarity of the CHANNELMSG_MODIFYCHANNEL messages is problematic if the user want to take a CPU offline, since we don't want to take a CPU offline (and, potentially, "lose" channel interrupts on such CPU) if the host is still processing a CHANNELMSG_MODIFYCHANNEL message associated to that CPU. It is worth mentioning, however, that we have been unable to observe the above mentioned "race": in all our tests, CHANNELMSG_MODIFYCHANNEL requests appeared *as if* they were processed synchronously by the host. Suggested-by: Michael Kelley Signed-off-by: Andrea Parri (Microsoft) Link: https://lore.kernel.org/r/20200406001514.19876-11-parri.andrea@gmail.com Reviewed-by: Michael Kelley [ wei: fix conflict in channel_mgmt.c ] Signed-off-by: Wei Liu --- drivers/hv/channel.c | 28 ++++++++++ drivers/hv/channel_mgmt.c | 2 +- drivers/hv/hv_trace.h | 19 +++++++ drivers/hv/vmbus_drv.c | 108 +++++++++++++++++++++++++++++++++++++- include/linux/hyperv.h | 10 +++- 5 files changed, 163 insertions(+), 4 deletions(-) diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c index 132e476f87b2..90070b337c10 100644 --- a/drivers/hv/channel.c +++ b/drivers/hv/channel.c @@ -289,6 +289,34 @@ int vmbus_send_tl_connect_request(const guid_t *shv_guest_servie_id, } EXPORT_SYMBOL_GPL(vmbus_send_tl_connect_request); +/* + * Set/change the vCPU (@target_vp) the channel (@child_relid) will interrupt. + * + * CHANNELMSG_MODIFYCHANNEL messages are aynchronous. Also, Hyper-V does not + * ACK such messages. IOW we can't know when the host will stop interrupting + * the "old" vCPU and start interrupting the "new" vCPU for the given channel. + * + * The CHANNELMSG_MODIFYCHANNEL message type is supported since VMBus version + * VERSION_WIN10_V4_1. + */ +int vmbus_send_modifychannel(u32 child_relid, u32 target_vp) +{ + struct vmbus_channel_modifychannel conn_msg; + int ret; + + memset(&conn_msg, 0, sizeof(conn_msg)); + conn_msg.header.msgtype = CHANNELMSG_MODIFYCHANNEL; + conn_msg.child_relid = child_relid; + conn_msg.target_vp = target_vp; + + ret = vmbus_post_msg(&conn_msg, sizeof(conn_msg), true); + + trace_vmbus_send_modifychannel(&conn_msg, ret); + + return ret; +} +EXPORT_SYMBOL_GPL(vmbus_send_modifychannel); + /* * create_gpadl_header - Creates a gpadl for the specified buffer */ diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index 2db3823f0e59..ffd7fffa5f83 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -1383,7 +1383,7 @@ channel_message_table[CHANNELMSG_COUNT] = { { CHANNELMSG_19, 0, NULL, 0}, { CHANNELMSG_20, 0, NULL, 0}, { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL, 0}, - { CHANNELMSG_22, 0, NULL, 0}, + { CHANNELMSG_MODIFYCHANNEL, 0, NULL, 0}, { CHANNELMSG_TL_CONNECT_RESULT, 0, NULL, 0}, }; diff --git a/drivers/hv/hv_trace.h b/drivers/hv/hv_trace.h index e70783e33680..a43bc76c2d5d 100644 --- a/drivers/hv/hv_trace.h +++ b/drivers/hv/hv_trace.h @@ -296,6 +296,25 @@ TRACE_EVENT(vmbus_send_tl_connect_request, ) ); +TRACE_EVENT(vmbus_send_modifychannel, + TP_PROTO(const struct vmbus_channel_modifychannel *msg, + int ret), + TP_ARGS(msg, ret), + TP_STRUCT__entry( + __field(u32, child_relid) + __field(u32, target_vp) + __field(int, ret) + ), + TP_fast_assign( + __entry->child_relid = msg->child_relid; + __entry->target_vp = msg->target_vp; + __entry->ret = ret; + ), + TP_printk("binding child_relid 0x%x to target_vp 0x%x, ret %d", + __entry->child_relid, __entry->target_vp, __entry->ret + ) + ); + DECLARE_EVENT_CLASS(vmbus_channel, TP_PROTO(const struct vmbus_channel *channel), TP_ARGS(channel), diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 0f7dfa507a40..5d24b25fb5aa 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1606,8 +1606,24 @@ static ssize_t vmbus_chan_attr_show(struct kobject *kobj, return attribute->show(chan, buf); } +static ssize_t vmbus_chan_attr_store(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + const struct vmbus_chan_attribute *attribute + = container_of(attr, struct vmbus_chan_attribute, attr); + struct vmbus_channel *chan + = container_of(kobj, struct vmbus_channel, kobj); + + if (!attribute->store) + return -EIO; + + return attribute->store(chan, buf, count); +} + static const struct sysfs_ops vmbus_chan_sysfs_ops = { .show = vmbus_chan_attr_show, + .store = vmbus_chan_attr_store, }; static ssize_t out_mask_show(struct vmbus_channel *channel, char *buf) @@ -1678,11 +1694,99 @@ static ssize_t write_avail_show(struct vmbus_channel *channel, char *buf) } static VMBUS_CHAN_ATTR_RO(write_avail); -static ssize_t show_target_cpu(struct vmbus_channel *channel, char *buf) +static ssize_t target_cpu_show(struct vmbus_channel *channel, char *buf) { return sprintf(buf, "%u\n", channel->target_cpu); } -static VMBUS_CHAN_ATTR(cpu, S_IRUGO, show_target_cpu, NULL); +static ssize_t target_cpu_store(struct vmbus_channel *channel, + const char *buf, size_t count) +{ + ssize_t ret = count; + u32 target_cpu; + + if (vmbus_proto_version < VERSION_WIN10_V4_1) + return -EIO; + + if (sscanf(buf, "%uu", &target_cpu) != 1) + return -EIO; + + /* Validate target_cpu for the cpumask_test_cpu() operation below. */ + if (target_cpu >= nr_cpumask_bits) + return -EINVAL; + + /* No CPUs should come up or down during this. */ + cpus_read_lock(); + + if (!cpumask_test_cpu(target_cpu, cpu_online_mask)) { + cpus_read_unlock(); + return -EINVAL; + } + + /* + * Synchronizes target_cpu_store() and channel closure: + * + * { Initially: state = CHANNEL_OPENED } + * + * CPU1 CPU2 + * + * [target_cpu_store()] [vmbus_disconnect_ring()] + * + * LOCK channel_mutex LOCK channel_mutex + * LOAD r1 = state LOAD r2 = state + * IF (r1 == CHANNEL_OPENED) IF (r2 == CHANNEL_OPENED) + * SEND MODIFYCHANNEL STORE state = CHANNEL_OPEN + * [...] SEND CLOSECHANNEL + * UNLOCK channel_mutex UNLOCK channel_mutex + * + * Forbids: r1 == r2 == CHANNEL_OPENED (i.e., CPU1's LOCK precedes + * CPU2's LOCK) && CPU2's SEND precedes CPU1's SEND + * + * Note. The host processes the channel messages "sequentially", in + * the order in which they are received on a per-partition basis. + */ + mutex_lock(&vmbus_connection.channel_mutex); + + /* + * Hyper-V will ignore MODIFYCHANNEL messages for "non-open" channels; + * avoid sending the message and fail here for such channels. + */ + if (channel->state != CHANNEL_OPENED_STATE) { + ret = -EIO; + goto cpu_store_unlock; + } + + if (channel->target_cpu == target_cpu) + goto cpu_store_unlock; + + if (vmbus_send_modifychannel(channel->offermsg.child_relid, + hv_cpu_number_to_vp_number(target_cpu))) { + ret = -EIO; + goto cpu_store_unlock; + } + + /* + * Warning. At this point, there is *no* guarantee that the host will + * have successfully processed the vmbus_send_modifychannel() request. + * See the header comment of vmbus_send_modifychannel() for more info. + * + * Lags in the processing of the above vmbus_send_modifychannel() can + * result in missed interrupts if the "old" target CPU is taken offline + * before Hyper-V starts sending interrupts to the "new" target CPU. + * But apart from this offlining scenario, the code tolerates such + * lags. It will function correctly even if a channel interrupt comes + * in on a CPU that is different from the channel target_cpu value. + */ + + channel->target_cpu = target_cpu; + channel->target_vp = hv_cpu_number_to_vp_number(target_cpu); + channel->numa_node = cpu_to_node(target_cpu); + +cpu_store_unlock: + mutex_unlock(&vmbus_connection.channel_mutex); + cpus_read_unlock(); + return ret; +} +static VMBUS_CHAN_ATTR(cpu, 0644, target_cpu_show, target_cpu_store); static ssize_t channel_pending_show(struct vmbus_channel *channel, char *buf) diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 247356dbd742..b85d7580f2c1 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -425,7 +425,7 @@ enum vmbus_channel_message_type { CHANNELMSG_19 = 19, CHANNELMSG_20 = 20, CHANNELMSG_TL_CONNECT_REQUEST = 21, - CHANNELMSG_22 = 22, + CHANNELMSG_MODIFYCHANNEL = 22, CHANNELMSG_TL_CONNECT_RESULT = 23, CHANNELMSG_COUNT }; @@ -620,6 +620,13 @@ struct vmbus_channel_tl_connect_request { guid_t host_service_id; } __packed; +/* Modify Channel parameters, cf. vmbus_send_modifychannel() */ +struct vmbus_channel_modifychannel { + struct vmbus_channel_message_header header; + u32 child_relid; + u32 target_vp; +} __packed; + struct vmbus_channel_version_response { struct vmbus_channel_message_header header; u8 version_supported; @@ -1505,6 +1512,7 @@ extern __u32 vmbus_proto_version; int vmbus_send_tl_connect_request(const guid_t *shv_guest_servie_id, const guid_t *shv_host_servie_id); +int vmbus_send_modifychannel(u32 child_relid, u32 target_vp); void vmbus_set_event(struct vmbus_channel *channel); /* Get the start of the ring buffer. */ From 9bd4af240f4db39e754081d135e6ef7a54bb6828 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 21 Apr 2020 00:53:27 -0700 Subject: [PATCH 0144/1043] KVM: nVMX: Drop a redundant call to vmx_get_intr_info() Drop nested_vmx_l1_wants_exit()'s initialization of intr_info from vmx_get_intr_info() that was inadvertantly introduced along with the caching mechanism. EXIT_REASON_EXCEPTION_NMI, the only consumer of intr_info, populates the variable before using it. Fixes: bb53120d67cd ("KVM: VMX: Cache vmcs.EXIT_INTR_INFO using arch avail_reg flags") Signed-off-by: Sean Christopherson Message-Id: <20200421075328.14458-2-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 56074d3443e0..12b78468317c 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5691,8 +5691,8 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) */ static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) { - u32 intr_info = vmx_get_intr_info(vcpu); struct vmcs12 *vmcs12 = get_vmcs12(vcpu); + u32 intr_info; switch (exit_reason) { case EXIT_REASON_EXCEPTION_NMI: From c05b5940d94019383dadb77ba614248d5a2323e1 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Fri, 24 Apr 2020 18:56:44 +0800 Subject: [PATCH 0145/1043] MIPS: Fix the declaration conflict of mm_isBranchInstr() mm_isBranchInstr() is declared both in branch.h and in fpu_emulator.h but the two declarations are conflict. If both of them are included by a same file, they will cause a build error: ./arch/mips/include/asm/branch.h:33:19: error: static declaration of 'mm_isBranchInstr' follows non-static declaration static inline int mm_isBranchInstr(struct pt_regs *regs, ^ ./arch/mips/include/asm/fpu_emulator.h:177:5: note: previous declaration of 'mm_isBranchInstr' was here int mm_isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, Fix this error by removing both isBranchInstr() and mm_isBranchInstr() in fpu_emulator.h, and declaring both of them in branch.h. Signed-off-by: Huacai Chen Reviewed-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/branch.h | 3 +++ arch/mips/include/asm/fpu_emulator.h | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/mips/include/asm/branch.h b/arch/mips/include/asm/branch.h index da80878f2c0d..fa3dcbf56fa9 100644 --- a/arch/mips/include/asm/branch.h +++ b/arch/mips/include/asm/branch.h @@ -27,6 +27,9 @@ extern int __MIPS16e_compute_return_epc(struct pt_regs *regs); #define MM_POOL32A_MINOR_SHIFT 0x6 #define MM_MIPS32_COND_FC 0x30 +int isBranchInstr(struct pt_regs *regs, + struct mm_decoded_insn dec_insn, unsigned long *contpc); + extern int __mm_isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, unsigned long *contpc); diff --git a/arch/mips/include/asm/fpu_emulator.h b/arch/mips/include/asm/fpu_emulator.h index bb7c71ffe5b7..f67759e81210 100644 --- a/arch/mips/include/asm/fpu_emulator.h +++ b/arch/mips/include/asm/fpu_emulator.h @@ -172,10 +172,6 @@ void force_fcr31_sig(unsigned long fcr31, void __user *fault_addr, struct task_struct *tsk); int process_fpemu_return(int sig, void __user *fault_addr, unsigned long fcr31); -int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, - unsigned long *contpc); -int mm_isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, - unsigned long *contpc); /* * Mask the FCSR Cause bits according to the Enable bits, observing From d339cd02b888eb8c4508fd772120782eac59a9fa Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Fri, 24 Apr 2020 18:56:45 +0800 Subject: [PATCH 0146/1043] MIPS: Move unaligned load/store helpers to inst.h Move unaligned load/store helpers from unaligned.c to inst.h, then other parts of the kernel can use these helpers. Use __ASSEMBLY__ to guard the definition of "LONG" in asm.h to avoid build error on IPxx platforms. Signed-off-by: Huacai Chen Signed-off-by: Pei Huang Reviewed-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/asm.h | 4 + arch/mips/include/asm/inst.h | 773 ++++++++++++++++++++++++++++++++++ arch/mips/kernel/unaligned.c | 775 ----------------------------------- 3 files changed, 777 insertions(+), 775 deletions(-) diff --git a/arch/mips/include/asm/asm.h b/arch/mips/include/asm/asm.h index c23527ba65d0..934465de470e 100644 --- a/arch/mips/include/asm/asm.h +++ b/arch/mips/include/asm/asm.h @@ -202,7 +202,9 @@ symbol = value #define LONG_SRA sra #define LONG_SRAV srav +#ifdef __ASSEMBLY__ #define LONG .word +#endif #define LONGSIZE 4 #define LONGMASK 3 #define LONGLOG 2 @@ -225,7 +227,9 @@ symbol = value #define LONG_SRA dsra #define LONG_SRAV dsrav +#ifdef __ASSEMBLY__ #define LONG .dword +#endif #define LONGSIZE 8 #define LONGMASK 7 #define LONGLOG 3 diff --git a/arch/mips/include/asm/inst.h b/arch/mips/include/asm/inst.h index 22912f78401c..82545454ad3e 100644 --- a/arch/mips/include/asm/inst.h +++ b/arch/mips/include/asm/inst.h @@ -11,6 +11,7 @@ #ifndef _ASM_INST_H #define _ASM_INST_H +#include #include /* HACHACHAHCAHC ... */ @@ -85,4 +86,776 @@ struct mm_decoded_insn { /* Recode table from 16-bit register notation to 32-bit GPR. Do NOT export!!! */ extern const int reg16to32[]; +#ifdef __BIG_ENDIAN +#define _LoadHW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ (".set\tnoat\n" \ + "1:\t"type##_lb("%0", "0(%2)")"\n" \ + "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + "3:\t.set\tat\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR +#define _LoadW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + "1:\t"type##_lwl("%0", "(%2)")"\n" \ + "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ +/* For CPUs without lwl instruction */ +#define _LoadW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n" \ + ".set\tnoat\n\t" \ + "1:"type##_lb("%0", "0(%2)")"\n\t" \ + "2:"type##_lbu("$1", "1(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:"type##_lbu("$1", "2(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:"type##_lbu("$1", "3(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ + +#define _LoadHWU(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tnoat\n" \ + "1:\t"type##_lbu("%0", "0(%2)")"\n" \ + "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".set\tat\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR +#define _LoadWU(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + "1:\t"type##_lwl("%0", "(%2)")"\n" \ + "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\ + "dsll\t%0, %0, 32\n\t" \ + "dsrl\t%0, %0, 32\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + "\t.section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#define _LoadDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + "1:\tldl\t%0, (%2)\n" \ + "2:\tldr\t%0, 7(%2)\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + "\t.section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ +/* For CPUs without lwl and ldl instructions */ +#define _LoadWU(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:"type##_lbu("%0", "0(%2)")"\n\t" \ + "2:"type##_lbu("$1", "1(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:"type##_lbu("$1", "2(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:"type##_lbu("$1", "3(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#define _LoadDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:lb\t%0, 0(%2)\n\t" \ + "2:lbu\t $1, 1(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:lbu\t$1, 2(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:lbu\t$1, 3(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "5:lbu\t$1, 4(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "6:lbu\t$1, 5(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "7:lbu\t$1, 6(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "8:lbu\t$1, 7(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n\t" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + STR(PTR)"\t5b, 11b\n\t" \ + STR(PTR)"\t6b, 11b\n\t" \ + STR(PTR)"\t7b, 11b\n\t" \ + STR(PTR)"\t8b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ + + +#define _StoreHW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tnoat\n" \ + "1:\t"type##_sb("%1", "1(%2)")"\n" \ + "srl\t$1, %1, 0x8\n" \ + "2:\t"type##_sb("$1", "0(%2)")"\n" \ + ".set\tat\n\t" \ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT));\ +} while (0) + +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR +#define _StoreW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + "1:\t"type##_swl("%1", "(%2)")"\n" \ + "2:\t"type##_swr("%1", "3(%2)")"\n\t"\ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#define _StoreDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + "1:\tsdl\t%1,(%2)\n" \ + "2:\tsdr\t%1, 7(%2)\n\t" \ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ +#define _StoreW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:"type##_sb("%1", "3(%2)")"\n\t" \ + "srl\t$1, %1, 0x8\n\t" \ + "2:"type##_sb("$1", "2(%2)")"\n\t" \ + "srl\t$1, $1, 0x8\n\t" \ + "3:"type##_sb("$1", "1(%2)")"\n\t" \ + "srl\t$1, $1, 0x8\n\t" \ + "4:"type##_sb("$1", "0(%2)")"\n\t" \ + ".set\tpop\n\t" \ + "li\t%0, 0\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%0, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT) \ + : "memory"); \ +} while (0) + +#define _StoreDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:sb\t%1, 7(%2)\n\t" \ + "dsrl\t$1, %1, 0x8\n\t" \ + "2:sb\t$1, 6(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "3:sb\t$1, 5(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "4:sb\t$1, 4(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "5:sb\t$1, 3(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "6:sb\t$1, 2(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "7:sb\t$1, 1(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "8:sb\t$1, 0(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + ".set\tpop\n\t" \ + "li\t%0, 0\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%0, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + STR(PTR)"\t5b, 11b\n\t" \ + STR(PTR)"\t6b, 11b\n\t" \ + STR(PTR)"\t7b, 11b\n\t" \ + STR(PTR)"\t8b, 11b\n\t" \ + ".previous" \ + : "=&r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT) \ + : "memory"); \ +} while (0) + +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ + +#else /* __BIG_ENDIAN */ + +#define _LoadHW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ (".set\tnoat\n" \ + "1:\t"type##_lb("%0", "1(%2)")"\n" \ + "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + "3:\t.set\tat\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR +#define _LoadW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + "1:\t"type##_lwl("%0", "3(%2)")"\n" \ + "2:\t"type##_lwr("%0", "(%2)")"\n\t"\ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ +/* For CPUs without lwl instruction */ +#define _LoadW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n" \ + ".set\tnoat\n\t" \ + "1:"type##_lb("%0", "3(%2)")"\n\t" \ + "2:"type##_lbu("$1", "2(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:"type##_lbu("$1", "1(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:"type##_lbu("$1", "0(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ + + +#define _LoadHWU(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tnoat\n" \ + "1:\t"type##_lbu("%0", "1(%2)")"\n" \ + "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".set\tat\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR +#define _LoadWU(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + "1:\t"type##_lwl("%0", "3(%2)")"\n" \ + "2:\t"type##_lwr("%0", "(%2)")"\n\t"\ + "dsll\t%0, %0, 32\n\t" \ + "dsrl\t%0, %0, 32\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + "\t.section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#define _LoadDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + "1:\tldl\t%0, 7(%2)\n" \ + "2:\tldr\t%0, (%2)\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + "\t.section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ +/* For CPUs without lwl and ldl instructions */ +#define _LoadWU(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:"type##_lbu("%0", "3(%2)")"\n\t" \ + "2:"type##_lbu("$1", "2(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:"type##_lbu("$1", "1(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:"type##_lbu("$1", "0(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#define _LoadDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:lb\t%0, 7(%2)\n\t" \ + "2:lbu\t$1, 6(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:lbu\t$1, 5(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:lbu\t$1, 4(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "5:lbu\t$1, 3(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "6:lbu\t$1, 2(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "7:lbu\t$1, 1(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "8:lbu\t$1, 0(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n\t" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + STR(PTR)"\t5b, 11b\n\t" \ + STR(PTR)"\t6b, 11b\n\t" \ + STR(PTR)"\t7b, 11b\n\t" \ + STR(PTR)"\t8b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ + +#define _StoreHW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tnoat\n" \ + "1:\t"type##_sb("%1", "0(%2)")"\n" \ + "srl\t$1,%1, 0x8\n" \ + "2:\t"type##_sb("$1", "1(%2)")"\n" \ + ".set\tat\n\t" \ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT));\ +} while (0) + +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR +#define _StoreW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + "1:\t"type##_swl("%1", "3(%2)")"\n" \ + "2:\t"type##_swr("%1", "(%2)")"\n\t"\ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#define _StoreDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + "1:\tsdl\t%1, 7(%2)\n" \ + "2:\tsdr\t%1, (%2)\n\t" \ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ +/* For CPUs without swl and sdl instructions */ +#define _StoreW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:"type##_sb("%1", "0(%2)")"\n\t" \ + "srl\t$1, %1, 0x8\n\t" \ + "2:"type##_sb("$1", "1(%2)")"\n\t" \ + "srl\t$1, $1, 0x8\n\t" \ + "3:"type##_sb("$1", "2(%2)")"\n\t" \ + "srl\t$1, $1, 0x8\n\t" \ + "4:"type##_sb("$1", "3(%2)")"\n\t" \ + ".set\tpop\n\t" \ + "li\t%0, 0\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%0, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT) \ + : "memory"); \ +} while (0) + +#define _StoreDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:sb\t%1, 0(%2)\n\t" \ + "dsrl\t$1, %1, 0x8\n\t" \ + "2:sb\t$1, 1(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "3:sb\t$1, 2(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "4:sb\t$1, 3(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "5:sb\t$1, 4(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "6:sb\t$1, 5(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "7:sb\t$1, 6(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "8:sb\t$1, 7(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + ".set\tpop\n\t" \ + "li\t%0, 0\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%0, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + STR(PTR)"\t5b, 11b\n\t" \ + STR(PTR)"\t6b, 11b\n\t" \ + STR(PTR)"\t7b, 11b\n\t" \ + STR(PTR)"\t8b, 11b\n\t" \ + ".previous" \ + : "=&r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT) \ + : "memory"); \ +} while (0) + +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ +#endif + +#define LoadHWU(addr, value, res) _LoadHWU(addr, value, res, kernel) +#define LoadHWUE(addr, value, res) _LoadHWU(addr, value, res, user) +#define LoadWU(addr, value, res) _LoadWU(addr, value, res, kernel) +#define LoadWUE(addr, value, res) _LoadWU(addr, value, res, user) +#define LoadHW(addr, value, res) _LoadHW(addr, value, res, kernel) +#define LoadHWE(addr, value, res) _LoadHW(addr, value, res, user) +#define LoadW(addr, value, res) _LoadW(addr, value, res, kernel) +#define LoadWE(addr, value, res) _LoadW(addr, value, res, user) +#define LoadDW(addr, value, res) _LoadDW(addr, value, res) + +#define StoreHW(addr, value, res) _StoreHW(addr, value, res, kernel) +#define StoreHWE(addr, value, res) _StoreHW(addr, value, res, user) +#define StoreW(addr, value, res) _StoreW(addr, value, res, kernel) +#define StoreWE(addr, value, res) _StoreW(addr, value, res, user) +#define StoreDW(addr, value, res) _StoreDW(addr, value, res) + #endif /* _ASM_INST_H */ diff --git a/arch/mips/kernel/unaligned.c b/arch/mips/kernel/unaligned.c index ca6fc4762d97..19b906a31c81 100644 --- a/arch/mips/kernel/unaligned.c +++ b/arch/mips/kernel/unaligned.c @@ -92,9 +92,6 @@ #include #include -#define STR(x) __STR(x) -#define __STR(x) #x - enum { UNALIGNED_ACTION_QUIET, UNALIGNED_ACTION_SIGNAL, @@ -108,778 +105,6 @@ static u32 unaligned_action; #endif extern void show_registers(struct pt_regs *regs); -#ifdef __BIG_ENDIAN -#define _LoadHW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ (".set\tnoat\n" \ - "1:\t"type##_lb("%0", "0(%2)")"\n" \ - "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - "3:\t.set\tat\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#ifndef CONFIG_CPU_NO_LOAD_STORE_LR -#define _LoadW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - "1:\t"type##_lwl("%0", "(%2)")"\n" \ - "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ -/* For CPUs without lwl instruction */ -#define _LoadW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n" \ - ".set\tnoat\n\t" \ - "1:"type##_lb("%0", "0(%2)")"\n\t" \ - "2:"type##_lbu("$1", "1(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "3:"type##_lbu("$1", "2(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "4:"type##_lbu("$1", "3(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - ".set\tpop\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%1, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ - -#define _LoadHWU(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tnoat\n" \ - "1:\t"type##_lbu("%0", "0(%2)")"\n" \ - "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".set\tat\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#ifndef CONFIG_CPU_NO_LOAD_STORE_LR -#define _LoadWU(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - "1:\t"type##_lwl("%0", "(%2)")"\n" \ - "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\ - "dsll\t%0, %0, 32\n\t" \ - "dsrl\t%0, %0, 32\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - "\t.section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#define _LoadDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - "1:\tldl\t%0, (%2)\n" \ - "2:\tldr\t%0, 7(%2)\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - "\t.section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ -/* For CPUs without lwl and ldl instructions */ -#define _LoadWU(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:"type##_lbu("%0", "0(%2)")"\n\t" \ - "2:"type##_lbu("$1", "1(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "3:"type##_lbu("$1", "2(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "4:"type##_lbu("$1", "3(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - ".set\tpop\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%1, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#define _LoadDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:lb\t%0, 0(%2)\n\t" \ - "2:lbu\t $1, 1(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "3:lbu\t$1, 2(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "4:lbu\t$1, 3(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "5:lbu\t$1, 4(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "6:lbu\t$1, 5(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "7:lbu\t$1, 6(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "8:lbu\t$1, 7(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - ".set\tpop\n\t" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%1, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ - STR(PTR)"\t5b, 11b\n\t" \ - STR(PTR)"\t6b, 11b\n\t" \ - STR(PTR)"\t7b, 11b\n\t" \ - STR(PTR)"\t8b, 11b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ - - -#define _StoreHW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tnoat\n" \ - "1:\t"type##_sb("%1", "1(%2)")"\n" \ - "srl\t$1, %1, 0x8\n" \ - "2:\t"type##_sb("$1", "0(%2)")"\n" \ - ".set\tat\n\t" \ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT));\ -} while(0) - -#ifndef CONFIG_CPU_NO_LOAD_STORE_LR -#define _StoreW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - "1:\t"type##_swl("%1", "(%2)")"\n" \ - "2:\t"type##_swr("%1", "3(%2)")"\n\t"\ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#define _StoreDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - "1:\tsdl\t%1,(%2)\n" \ - "2:\tsdr\t%1, 7(%2)\n\t" \ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ -#define _StoreW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:"type##_sb("%1", "3(%2)")"\n\t" \ - "srl\t$1, %1, 0x8\n\t" \ - "2:"type##_sb("$1", "2(%2)")"\n\t" \ - "srl\t$1, $1, 0x8\n\t" \ - "3:"type##_sb("$1", "1(%2)")"\n\t" \ - "srl\t$1, $1, 0x8\n\t" \ - "4:"type##_sb("$1", "0(%2)")"\n\t" \ - ".set\tpop\n\t" \ - "li\t%0, 0\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%0, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ - ".previous" \ - : "=&r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT) \ - : "memory"); \ -} while(0) - -#define _StoreDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:sb\t%1, 7(%2)\n\t" \ - "dsrl\t$1, %1, 0x8\n\t" \ - "2:sb\t$1, 6(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "3:sb\t$1, 5(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "4:sb\t$1, 4(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "5:sb\t$1, 3(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "6:sb\t$1, 2(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "7:sb\t$1, 1(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "8:sb\t$1, 0(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - ".set\tpop\n\t" \ - "li\t%0, 0\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%0, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ - STR(PTR)"\t5b, 11b\n\t" \ - STR(PTR)"\t6b, 11b\n\t" \ - STR(PTR)"\t7b, 11b\n\t" \ - STR(PTR)"\t8b, 11b\n\t" \ - ".previous" \ - : "=&r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT) \ - : "memory"); \ -} while(0) - -#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ - -#else /* __BIG_ENDIAN */ - -#define _LoadHW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ (".set\tnoat\n" \ - "1:\t"type##_lb("%0", "1(%2)")"\n" \ - "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - "3:\t.set\tat\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#ifndef CONFIG_CPU_NO_LOAD_STORE_LR -#define _LoadW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - "1:\t"type##_lwl("%0", "3(%2)")"\n" \ - "2:\t"type##_lwr("%0", "(%2)")"\n\t"\ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ -/* For CPUs without lwl instruction */ -#define _LoadW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n" \ - ".set\tnoat\n\t" \ - "1:"type##_lb("%0", "3(%2)")"\n\t" \ - "2:"type##_lbu("$1", "2(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "3:"type##_lbu("$1", "1(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "4:"type##_lbu("$1", "0(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - ".set\tpop\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%1, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ - - -#define _LoadHWU(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tnoat\n" \ - "1:\t"type##_lbu("%0", "1(%2)")"\n" \ - "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".set\tat\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#ifndef CONFIG_CPU_NO_LOAD_STORE_LR -#define _LoadWU(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - "1:\t"type##_lwl("%0", "3(%2)")"\n" \ - "2:\t"type##_lwr("%0", "(%2)")"\n\t"\ - "dsll\t%0, %0, 32\n\t" \ - "dsrl\t%0, %0, 32\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - "\t.section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#define _LoadDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - "1:\tldl\t%0, 7(%2)\n" \ - "2:\tldr\t%0, (%2)\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - "\t.section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ -/* For CPUs without lwl and ldl instructions */ -#define _LoadWU(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:"type##_lbu("%0", "3(%2)")"\n\t" \ - "2:"type##_lbu("$1", "2(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "3:"type##_lbu("$1", "1(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "4:"type##_lbu("$1", "0(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - ".set\tpop\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%1, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#define _LoadDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:lb\t%0, 7(%2)\n\t" \ - "2:lbu\t$1, 6(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "3:lbu\t$1, 5(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "4:lbu\t$1, 4(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "5:lbu\t$1, 3(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "6:lbu\t$1, 2(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "7:lbu\t$1, 1(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "8:lbu\t$1, 0(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - ".set\tpop\n\t" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%1, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ - STR(PTR)"\t5b, 11b\n\t" \ - STR(PTR)"\t6b, 11b\n\t" \ - STR(PTR)"\t7b, 11b\n\t" \ - STR(PTR)"\t8b, 11b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while(0) -#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ - -#define _StoreHW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tnoat\n" \ - "1:\t"type##_sb("%1", "0(%2)")"\n" \ - "srl\t$1,%1, 0x8\n" \ - "2:\t"type##_sb("$1", "1(%2)")"\n" \ - ".set\tat\n\t" \ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT));\ -} while(0) - -#ifndef CONFIG_CPU_NO_LOAD_STORE_LR -#define _StoreW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - "1:\t"type##_swl("%1", "3(%2)")"\n" \ - "2:\t"type##_swr("%1", "(%2)")"\n\t"\ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#define _StoreDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - "1:\tsdl\t%1, 7(%2)\n" \ - "2:\tsdr\t%1, (%2)\n\t" \ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); \ -} while(0) - -#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ -/* For CPUs without swl and sdl instructions */ -#define _StoreW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:"type##_sb("%1", "0(%2)")"\n\t" \ - "srl\t$1, %1, 0x8\n\t" \ - "2:"type##_sb("$1", "1(%2)")"\n\t" \ - "srl\t$1, $1, 0x8\n\t" \ - "3:"type##_sb("$1", "2(%2)")"\n\t" \ - "srl\t$1, $1, 0x8\n\t" \ - "4:"type##_sb("$1", "3(%2)")"\n\t" \ - ".set\tpop\n\t" \ - "li\t%0, 0\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%0, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ - ".previous" \ - : "=&r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT) \ - : "memory"); \ -} while(0) - -#define _StoreDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:sb\t%1, 0(%2)\n\t" \ - "dsrl\t$1, %1, 0x8\n\t" \ - "2:sb\t$1, 1(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "3:sb\t$1, 2(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "4:sb\t$1, 3(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "5:sb\t$1, 4(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "6:sb\t$1, 5(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "7:sb\t$1, 6(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "8:sb\t$1, 7(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - ".set\tpop\n\t" \ - "li\t%0, 0\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%0, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ - STR(PTR)"\t5b, 11b\n\t" \ - STR(PTR)"\t6b, 11b\n\t" \ - STR(PTR)"\t7b, 11b\n\t" \ - STR(PTR)"\t8b, 11b\n\t" \ - ".previous" \ - : "=&r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT) \ - : "memory"); \ -} while(0) - -#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ -#endif - -#define LoadHWU(addr, value, res) _LoadHWU(addr, value, res, kernel) -#define LoadHWUE(addr, value, res) _LoadHWU(addr, value, res, user) -#define LoadWU(addr, value, res) _LoadWU(addr, value, res, kernel) -#define LoadWUE(addr, value, res) _LoadWU(addr, value, res, user) -#define LoadHW(addr, value, res) _LoadHW(addr, value, res, kernel) -#define LoadHWE(addr, value, res) _LoadHW(addr, value, res, user) -#define LoadW(addr, value, res) _LoadW(addr, value, res, kernel) -#define LoadWE(addr, value, res) _LoadW(addr, value, res, user) -#define LoadDW(addr, value, res) _LoadDW(addr, value, res) - -#define StoreHW(addr, value, res) _StoreHW(addr, value, res, kernel) -#define StoreHWE(addr, value, res) _StoreHW(addr, value, res, user) -#define StoreW(addr, value, res) _StoreW(addr, value, res, kernel) -#define StoreWE(addr, value, res) _StoreW(addr, value, res, user) -#define StoreDW(addr, value, res) _StoreDW(addr, value, res) - static void emulate_load_store_insn(struct pt_regs *regs, void __user *addr, unsigned int __user *pc) { From f83e4f9896eff614d0f2547a561fa5f39f9cddde Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Fri, 24 Apr 2020 18:56:46 +0800 Subject: [PATCH 0147/1043] MIPS: Loongson-3: Add some unaligned instructions emulation 1, Add unaligned gslq, gssq, gslqc1, gssqc1 emulation; 2, Add unaligned gsl{h, w, d}x, gss{h, w, d}x emulation; 3, Add unaligned gslwxc1, gsswxc1, gsldxc1, gssdxc1 emulation. Signed-off-by: Huacai Chen Signed-off-by: Pei Huang Reviewed-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/uapi/asm/inst.h | 26 +++ arch/mips/loongson64/cop2-ex.c | 289 +++++++++++++++++++++++++++++- 2 files changed, 314 insertions(+), 1 deletion(-) diff --git a/arch/mips/include/uapi/asm/inst.h b/arch/mips/include/uapi/asm/inst.h index eaa3a80affdf..98f97c85e059 100644 --- a/arch/mips/include/uapi/asm/inst.h +++ b/arch/mips/include/uapi/asm/inst.h @@ -988,6 +988,30 @@ struct mm16_r5_format { /* Load/store from stack pointer format */ ;)))) }; +/* + * Loongson-3 overridden COP2 instruction formats (32-bit length) + */ +struct loongson3_lswc2_format { /* Loongson-3 overridden lwc2/swc2 Load/Store format */ + __BITFIELD_FIELD(unsigned int opcode : 6, + __BITFIELD_FIELD(unsigned int base : 5, + __BITFIELD_FIELD(unsigned int rt : 5, + __BITFIELD_FIELD(unsigned int fr : 1, + __BITFIELD_FIELD(unsigned int offset : 9, + __BITFIELD_FIELD(unsigned int ls : 1, + __BITFIELD_FIELD(unsigned int rq : 5, + ;))))))) +}; + +struct loongson3_lsdc2_format { /* Loongson-3 overridden ldc2/sdc2 Load/Store format */ + __BITFIELD_FIELD(unsigned int opcode : 6, + __BITFIELD_FIELD(unsigned int base : 5, + __BITFIELD_FIELD(unsigned int rt : 5, + __BITFIELD_FIELD(unsigned int index : 5, + __BITFIELD_FIELD(unsigned int offset : 8, + __BITFIELD_FIELD(unsigned int opcode1 : 3, + ;)))))) +}; + /* * MIPS16e instruction formats (16-bit length) */ @@ -1088,6 +1112,8 @@ union mips_instruction { struct mm16_rb_format mm16_rb_format; struct mm16_r3_format mm16_r3_format; struct mm16_r5_format mm16_r5_format; + struct loongson3_lswc2_format loongson3_lswc2_format; + struct loongson3_lsdc2_format loongson3_lsdc2_format; }; union mips16e_instruction { diff --git a/arch/mips/loongson64/cop2-ex.c b/arch/mips/loongson64/cop2-ex.c index 9efdfe430ff0..af0600dfe83c 100644 --- a/arch/mips/loongson64/cop2-ex.c +++ b/arch/mips/loongson64/cop2-ex.c @@ -14,17 +14,29 @@ #include #include #include +#include +#include #include #include +#include +#include #include #include static int loongson_cu2_call(struct notifier_block *nfb, unsigned long action, void *data) { - int fpu_owned; + unsigned int res, fpu_owned; + unsigned long ra, value, value_next; + union mips_instruction insn; int fr = !test_thread_flag(TIF_32BIT_FPREGS); + struct pt_regs *regs = (struct pt_regs *)data; + void __user *addr = (void __user *)regs->cp0_badvaddr; + unsigned int __user *pc = (unsigned int __user *)exception_epc(regs); + + ra = regs->regs[31]; + __get_user(insn.word, pc); switch (action) { case CU2_EXCEPTION: @@ -49,9 +61,284 @@ static int loongson_cu2_call(struct notifier_block *nfb, unsigned long action, preempt_enable(); return NOTIFY_STOP; /* Don't call default notifier */ + + case CU2_LWC2_OP: + if (insn.loongson3_lswc2_format.ls == 0) + goto sigbus; + + if (insn.loongson3_lswc2_format.fr == 0) { /* gslq */ + if (!access_ok(addr, 16)) + goto sigbus; + + LoadDW(addr, value, res); + if (res) + goto fault; + + LoadDW(addr + 8, value_next, res); + if (res) + goto fault; + + regs->regs[insn.loongson3_lswc2_format.rt] = value; + regs->regs[insn.loongson3_lswc2_format.rq] = value_next; + compute_return_epc(regs); + } else { /* gslqc1 */ + if (!access_ok(addr, 16)) + goto sigbus; + + lose_fpu(1); + LoadDW(addr, value, res); + if (res) + goto fault; + + LoadDW(addr + 8, value_next, res); + if (res) + goto fault; + + set_fpr64(current->thread.fpu.fpr, + insn.loongson3_lswc2_format.rt, value); + set_fpr64(current->thread.fpu.fpr, + insn.loongson3_lswc2_format.rq, value_next); + compute_return_epc(regs); + own_fpu(1); + } + return NOTIFY_STOP; /* Don't call default notifier */ + + case CU2_SWC2_OP: + if (insn.loongson3_lswc2_format.ls == 0) + goto sigbus; + + if (insn.loongson3_lswc2_format.fr == 0) { /* gssq */ + if (!access_ok(addr, 16)) + goto sigbus; + + /* write upper 8 bytes first */ + value_next = regs->regs[insn.loongson3_lswc2_format.rq]; + + StoreDW(addr + 8, value_next, res); + if (res) + goto fault; + value = regs->regs[insn.loongson3_lswc2_format.rt]; + + StoreDW(addr, value, res); + if (res) + goto fault; + + compute_return_epc(regs); + } else { /* gssqc1 */ + if (!access_ok(addr, 16)) + goto sigbus; + + lose_fpu(1); + value_next = get_fpr64(current->thread.fpu.fpr, + insn.loongson3_lswc2_format.rq); + + StoreDW(addr + 8, value_next, res); + if (res) + goto fault; + + value = get_fpr64(current->thread.fpu.fpr, + insn.loongson3_lswc2_format.rt); + + StoreDW(addr, value, res); + if (res) + goto fault; + + compute_return_epc(regs); + own_fpu(1); + } + return NOTIFY_STOP; /* Don't call default notifier */ + + case CU2_LDC2_OP: + switch (insn.loongson3_lsdc2_format.opcode1) { + /* + * Loongson-3 overridden ldc2 instructions. + * opcode1 instruction + * 0x1 gslhx: load 2 bytes to GPR + * 0x2 gslwx: load 4 bytes to GPR + * 0x3 gsldx: load 8 bytes to GPR + * 0x6 gslwxc1: load 4 bytes to FPR + * 0x7 gsldxc1: load 8 bytes to FPR + */ + case 0x1: + if (!access_ok(addr, 2)) + goto sigbus; + + LoadHW(addr, value, res); + if (res) + goto fault; + + compute_return_epc(regs); + regs->regs[insn.loongson3_lsdc2_format.rt] = value; + break; + case 0x2: + if (!access_ok(addr, 4)) + goto sigbus; + + LoadW(addr, value, res); + if (res) + goto fault; + + compute_return_epc(regs); + regs->regs[insn.loongson3_lsdc2_format.rt] = value; + break; + case 0x3: + if (!access_ok(addr, 8)) + goto sigbus; + + LoadDW(addr, value, res); + if (res) + goto fault; + + compute_return_epc(regs); + regs->regs[insn.loongson3_lsdc2_format.rt] = value; + break; + case 0x6: + die_if_kernel("Unaligned FP access in kernel code", regs); + BUG_ON(!used_math()); + if (!access_ok(addr, 4)) + goto sigbus; + + lose_fpu(1); + LoadW(addr, value, res); + if (res) + goto fault; + + set_fpr64(current->thread.fpu.fpr, + insn.loongson3_lsdc2_format.rt, value); + compute_return_epc(regs); + own_fpu(1); + + break; + case 0x7: + die_if_kernel("Unaligned FP access in kernel code", regs); + BUG_ON(!used_math()); + if (!access_ok(addr, 8)) + goto sigbus; + + lose_fpu(1); + LoadDW(addr, value, res); + if (res) + goto fault; + + set_fpr64(current->thread.fpu.fpr, + insn.loongson3_lsdc2_format.rt, value); + compute_return_epc(regs); + own_fpu(1); + break; + + } + return NOTIFY_STOP; /* Don't call default notifier */ + + case CU2_SDC2_OP: + switch (insn.loongson3_lsdc2_format.opcode1) { + /* + * Loongson-3 overridden sdc2 instructions. + * opcode1 instruction + * 0x1 gsshx: store 2 bytes from GPR + * 0x2 gsswx: store 4 bytes from GPR + * 0x3 gssdx: store 8 bytes from GPR + * 0x6 gsswxc1: store 4 bytes from FPR + * 0x7 gssdxc1: store 8 bytes from FPR + */ + case 0x1: + if (!access_ok(addr, 2)) + goto sigbus; + + compute_return_epc(regs); + value = regs->regs[insn.loongson3_lsdc2_format.rt]; + + StoreHW(addr, value, res); + if (res) + goto fault; + + break; + case 0x2: + if (!access_ok(addr, 4)) + goto sigbus; + + compute_return_epc(regs); + value = regs->regs[insn.loongson3_lsdc2_format.rt]; + + StoreW(addr, value, res); + if (res) + goto fault; + + break; + case 0x3: + if (!access_ok(addr, 8)) + goto sigbus; + + compute_return_epc(regs); + value = regs->regs[insn.loongson3_lsdc2_format.rt]; + + StoreDW(addr, value, res); + if (res) + goto fault; + + break; + + case 0x6: + die_if_kernel("Unaligned FP access in kernel code", regs); + BUG_ON(!used_math()); + + if (!access_ok(addr, 4)) + goto sigbus; + + lose_fpu(1); + value = get_fpr64(current->thread.fpu.fpr, + insn.loongson3_lsdc2_format.rt); + + StoreW(addr, value, res); + if (res) + goto fault; + + compute_return_epc(regs); + own_fpu(1); + + break; + case 0x7: + die_if_kernel("Unaligned FP access in kernel code", regs); + BUG_ON(!used_math()); + + if (!access_ok(addr, 8)) + goto sigbus; + + lose_fpu(1); + value = get_fpr64(current->thread.fpu.fpr, + insn.loongson3_lsdc2_format.rt); + + StoreDW(addr, value, res); + if (res) + goto fault; + + compute_return_epc(regs); + own_fpu(1); + + break; + } + return NOTIFY_STOP; /* Don't call default notifier */ } return NOTIFY_OK; /* Let default notifier send signals */ + +fault: + /* roll back jump/branch */ + regs->regs[31] = ra; + regs->cp0_epc = (unsigned long)pc; + /* Did we have an exception handler installed? */ + if (fixup_exception(regs)) + return NOTIFY_STOP; /* Don't call default notifier */ + + die_if_kernel("Unhandled kernel unaligned access", regs); + force_sig(SIGSEGV); + + return NOTIFY_STOP; /* Don't call default notifier */ + +sigbus: + die_if_kernel("Unhandled kernel unaligned access", regs); + force_sig(SIGBUS); + + return NOTIFY_STOP; /* Don't call default notifier */ } static int __init loongson_cu2_setup(void) From d82d500f5118f9d07477d597fe75c3bd73b44c8e Mon Sep 17 00:00:00 2001 From: Liangliang Huang Date: Thu, 23 Apr 2020 19:44:21 -0400 Subject: [PATCH 0148/1043] MIPS: arch_send_call_function_single_ipi() calling conventions change Use mp_ops->send_ipi_single() instead of mp_ops->send_ipi_mask() in arch_send_call_function_single_ipi(). send_ipi_single() can send IPI signal to a special cpu more efficiently. Signed-off-by: Liangliang Huang Reviewed-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/smp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/include/asm/smp.h b/arch/mips/include/asm/smp.h index 7990c1c70471..5d9ff61004ca 100644 --- a/arch/mips/include/asm/smp.h +++ b/arch/mips/include/asm/smp.h @@ -125,7 +125,7 @@ static inline void arch_send_call_function_single_ipi(int cpu) { extern const struct plat_smp_ops *mp_ops; /* private */ - mp_ops->send_ipi_mask(cpumask_of(cpu), SMP_CALL_FUNCTION); + mp_ops->send_ipi_single(cpu, SMP_CALL_FUNCTION); } static inline void arch_send_call_function_ipi_mask(const struct cpumask *mask) From c213db614685e8dc9f910bca5cf934816ba423fe Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Wed, 22 Apr 2020 22:45:34 +0800 Subject: [PATCH 0149/1043] MIPS: Clear XContext at boot time XContext might be dirty at boot time. We need to clear it to ensure early stackframe is safe. Signed-off-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/head.S | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/mips/kernel/head.S b/arch/mips/kernel/head.S index 351d40fe0859..3b02ffe46304 100644 --- a/arch/mips/kernel/head.S +++ b/arch/mips/kernel/head.S @@ -132,6 +132,9 @@ dtb_found: #endif MTC0 zero, CP0_CONTEXT # clear context register +#ifdef CONFIG_64BIT + MTC0 zero, CP0_XCONTEXT +#endif PTR_LA $28, init_thread_union /* Set the SP after an empty pt_regs. */ PTR_LI sp, _THREAD_SIZE - 32 - PT_SIZE From 87796555d48ca1cc681515e065b824ecab9c4282 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 22 Apr 2020 17:11:27 -0700 Subject: [PATCH 0150/1043] KVM: nVMX: Store vmcs.EXIT_QUALIFICATION as an unsigned long, not u32 Use an unsigned long for 'exit_qual' in nested_vmx_reflect_vmexit(), the EXIT_QUALIFICATION field is naturally sized, not a 32-bit field. The bug is most easily observed by doing VMXON (or any VMX instruction) in L2 with a negative displacement, in which case dropping the upper bits on nested VM-Exit results in L1 calculating the wrong virtual address for the memory operand, e.g. "vmxon -0x8(%rbp)" yields: Unhandled cpu exception 14 #PF at ip 0000000000400553 rbp=0000000000537000 cr2=0000000100536ff8 Fixes: fbdd50250396d ("KVM: nVMX: Move VM-Fail check out of nested_vmx_exit_reflected()") Signed-off-by: Sean Christopherson Message-Id: <20200423001127.13490-1-sean.j.christopherson@intel.com> Reviewed-by: Vitaly Kuznetsov Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 12b78468317c..b516c24494e3 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5814,7 +5814,8 @@ bool nested_vmx_reflect_vmexit(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); u32 exit_reason = vmx->exit_reason; - u32 exit_intr_info, exit_qual; + unsigned long exit_qual; + u32 exit_intr_info; WARN_ON_ONCE(vmx->nested.nested_run_pending); From acd05785e48c01edb2c4f4d014d28478b5f19fb5 Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 17 Apr 2020 15:14:46 -0700 Subject: [PATCH 0151/1043] kvm: add capability for halt polling KVM_CAP_HALT_POLL is a per-VM capability that lets userspace control the halt-polling time, allowing halt-polling to be tuned or disabled on particular VMs. With dynamic halt-polling, a VM's VCPUs can poll from anywhere from [0, halt_poll_ns] on each halt. KVM_CAP_HALT_POLL sets the upper limit on the poll time. Signed-off-by: David Matlack Signed-off-by: Jon Cargille Reviewed-by: Jim Mattson Message-Id: <20200417221446.108733-1-jcargill@google.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 17 +++++++++++++++++ include/linux/kvm_host.h | 1 + include/uapi/linux/kvm.h | 1 + virt/kvm/kvm_main.c | 19 +++++++++++++++---- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index efbbe570aa9b..d871dacb984e 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -5802,6 +5802,23 @@ If present, this capability can be enabled for a VM, meaning that KVM will allow the transition to secure guest mode. Otherwise KVM will veto the transition. +7.20 KVM_CAP_HALT_POLL +---------------------- + +:Architectures: all +:Target: VM +:Parameters: args[0] is the maximum poll time in nanoseconds +:Returns: 0 on success; -1 on error + +This capability overrides the kvm module parameter halt_poll_ns for the +target VM. + +VCPU polling allows a VCPU to poll for wakeup events instead of immediately +scheduling during guest halts. The maximum time a VCPU can spend polling is +controlled by the kvm module parameter halt_poll_ns. This capability allows +the maximum halt time to specified on a per-VM basis, effectively overriding +the module parameter for the target VM. + 8. Other capabilities. ====================== diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 5285a5568208..3cc6ccbb1183 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -503,6 +503,7 @@ struct kvm { struct srcu_struct srcu; struct srcu_struct irq_srcu; pid_t userspace_pid; + unsigned int max_halt_poll_ns; }; #define kvm_err(fmt, ...) \ diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 428c7dde6b4b..ac9eba0289d1 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1017,6 +1017,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_S390_VCPU_RESETS 179 #define KVM_CAP_S390_PROTECTED 180 #define KVM_CAP_PPC_SECURE_GUEST 181 +#define KVM_CAP_HALT_POLL 182 #ifdef KVM_CAP_IRQ_ROUTING diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index e7436d054305..33e1eee96f75 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -710,6 +710,8 @@ static struct kvm *kvm_create_vm(unsigned long type) goto out_err_no_arch_destroy_vm; } + kvm->max_halt_poll_ns = halt_poll_ns; + r = kvm_arch_init_vm(kvm, type); if (r) goto out_err_no_arch_destroy_vm; @@ -2713,15 +2715,16 @@ out: if (!kvm_arch_no_poll(vcpu)) { if (!vcpu_valid_wakeup(vcpu)) { shrink_halt_poll_ns(vcpu); - } else if (halt_poll_ns) { + } else if (vcpu->kvm->max_halt_poll_ns) { if (block_ns <= vcpu->halt_poll_ns) ; /* we had a long block, shrink polling */ - else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns) + else if (vcpu->halt_poll_ns && + block_ns > vcpu->kvm->max_halt_poll_ns) shrink_halt_poll_ns(vcpu); /* we had a short halt and our poll time is too small */ - else if (vcpu->halt_poll_ns < halt_poll_ns && - block_ns < halt_poll_ns) + else if (vcpu->halt_poll_ns < vcpu->kvm->max_halt_poll_ns && + block_ns < vcpu->kvm->max_halt_poll_ns) grow_halt_poll_ns(vcpu); } else { vcpu->halt_poll_ns = 0; @@ -3510,6 +3513,7 @@ static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) case KVM_CAP_IOEVENTFD_ANY_LENGTH: case KVM_CAP_CHECK_EXTENSION_VM: case KVM_CAP_ENABLE_CAP_VM: + case KVM_CAP_HALT_POLL: return 1; #ifdef CONFIG_KVM_MMIO case KVM_CAP_COALESCED_MMIO: @@ -3560,6 +3564,13 @@ static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm, return 0; } #endif + case KVM_CAP_HALT_POLL: { + if (cap->flags || cap->args[0] != (unsigned int)cap->args[0]) + return -EINVAL; + + kvm->max_halt_poll_ns = cap->args[0]; + return 0; + } default: return kvm_vm_ioctl_enable_cap(kvm, cap); } From 7c67f54661fcc8d141fb11abbab1739f32e13b03 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 23 Apr 2020 10:52:48 -0400 Subject: [PATCH 0152/1043] KVM: SVM: do not allow VMRUN inside SMM VMRUN is not supported inside the SMM handler and the behavior is undefined. Just raise a #UD. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 6ea047e6882e..a7c3b3030e59 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -345,8 +345,12 @@ int nested_svm_vmrun(struct vcpu_svm *svm) struct kvm_host_map map; u64 vmcb_gpa; - vmcb_gpa = svm->vmcb->save.rax; + if (is_smm(&svm->vcpu)) { + kvm_queue_exception(&svm->vcpu, UD_VECTOR); + return 1; + } + vmcb_gpa = svm->vmcb->save.rax; ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb_gpa), &map); if (ret == -EINVAL) { kvm_inject_gp(&svm->vcpu, 0); From a44f83092d716d8a5e5e1a2620b68cf720c2ed2a Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Sun, 26 Apr 2020 19:09:52 +0800 Subject: [PATCH 0153/1043] MIPS: Rename the "Fill" cache ops to avoid build failure MIPS define a "Fill" macro as a cache operation in cacheops.h, this will cause build failure under some special configurations because in seq_file.c there is a "Fill" label. To avoid this failure we rename the "Fill" macro to "Fill_I" which has the same coding style as other cache operations in cacheops.h (we think renaming the "Fill" macro is more reasonable than renaming the "Fill" label). Callers of "Fill" macro is also updated. Signed-off-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/cacheops.h | 2 +- arch/mips/mm/c-r4k.c | 2 +- arch/mips/pmcs-msp71xx/msp_setup.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/mips/include/asm/cacheops.h b/arch/mips/include/asm/cacheops.h index 8031fbc6b69a..50253efecb56 100644 --- a/arch/mips/include/asm/cacheops.h +++ b/arch/mips/include/asm/cacheops.h @@ -48,7 +48,7 @@ * R4000-specific cacheops */ #define Create_Dirty_Excl_D (Cache_D | 0x0c) -#define Fill (Cache_I | 0x14) +#define Fill_I (Cache_I | 0x14) #define Hit_Writeback_I (Cache_I | Hit_Writeback) #define Hit_Writeback_D (Cache_D | Hit_Writeback) diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index 36a311348739..89b6839140d7 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -1049,7 +1049,7 @@ static inline void rm7k_erratum31(void) "cache\t%1, 0x3000(%0)\n\t" ".set pop\n" : - : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill)); + : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill_I)); } } diff --git a/arch/mips/pmcs-msp71xx/msp_setup.c b/arch/mips/pmcs-msp71xx/msp_setup.c index d1e59cec116e..e0f20f487d96 100644 --- a/arch/mips/pmcs-msp71xx/msp_setup.c +++ b/arch/mips/pmcs-msp71xx/msp_setup.c @@ -55,7 +55,7 @@ void msp7120_reset(void) for (iptr = (void *)((unsigned int)start & ~(L1_CACHE_BYTES - 1)); iptr < end; iptr += L1_CACHE_BYTES) - cache_op(Fill, iptr); + cache_op(Fill_I, iptr); __asm__ __volatile__ ( "startpoint: \n" From e2e13925ff74afa011e8c667cd9ceda38f254fc5 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Sun, 26 Apr 2020 17:30:45 +0800 Subject: [PATCH 0154/1043] MIPS: Loongson: Add support for perf tool In order to use perf tool on the Loongson platform, we should enable kernel support for various performance events provided by software and hardware, so add CONFIG_PERF_EVENTS=y to loongson3_defconfig. E.g. without this patch: [loongson@localhost perf]$ ./perf list List of pre-defined events (to be used in -e): duration_time [Tool event] rNNN [Raw hardware event descriptor] cpu/t1=v1[,t2=v2,t3 ...]/modifier [Raw hardware event descriptor] (see 'man perf-list' on how to encode it) mem:[/len][:access] [Hardware breakpoint] With this patch: [loongson@localhost perf]$ ./perf list List of pre-defined events (to be used in -e): branch-instructions OR branches [Hardware event] branch-misses [Hardware event] cpu-cycles OR cycles [Hardware event] instructions [Hardware event] alignment-faults [Software event] bpf-output [Software event] context-switches OR cs [Software event] cpu-clock [Software event] cpu-migrations OR migrations [Software event] dummy [Software event] emulation-faults [Software event] major-faults [Software event] minor-faults [Software event] page-faults OR faults [Software event] task-clock [Software event] duration_time [Tool event] L1-dcache-load-misses [Hardware cache event] L1-dcache-store-misses [Hardware cache event] L1-icache-load-misses [Hardware cache event] branch-load-misses [Hardware cache event] branch-loads [Hardware cache event] dTLB-load-misses [Hardware cache event] dTLB-store-misses [Hardware cache event] iTLB-load-misses [Hardware cache event] rNNN [Raw hardware event descriptor] cpu/t1=v1[,t2=v2,t3 ...]/modifier [Raw hardware event descriptor] (see 'man perf-list' on how to encode it) mem:[/len][:access] [Hardware breakpoint] Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/configs/loongson3_defconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/mips/configs/loongson3_defconfig b/arch/mips/configs/loongson3_defconfig index 51675f5000d6..6768c1682671 100644 --- a/arch/mips/configs/loongson3_defconfig +++ b/arch/mips/configs/loongson3_defconfig @@ -21,6 +21,7 @@ CONFIG_SYSFS_DEPRECATED=y CONFIG_RELAY=y CONFIG_BLK_DEV_INITRD=y CONFIG_EMBEDDED=y +CONFIG_PERF_EVENTS=y CONFIG_MACH_LOONGSON64=y CONFIG_SMP=y CONFIG_HZ_256=y From 0cf2ea1121aa14f6873ed2907a3e27b62c87fcbe Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Wed, 22 Apr 2020 22:43:44 +0800 Subject: [PATCH 0155/1043] MIPS: Kernel: Identify Loongson-2K processors Loongson-2K (Loongson64 Reduced) is a family of SoC shipped with gs264e core. Signed-off-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/cpu.h | 4 ++++ arch/mips/kernel/cpu-probe.c | 19 ++++++++++++++++++- arch/mips/kernel/idle.c | 3 ++- arch/mips/mm/c-r4k.c | 12 +++++++++--- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/arch/mips/include/asm/cpu.h b/arch/mips/include/asm/cpu.h index 216a22916740..46c190e78acf 100644 --- a/arch/mips/include/asm/cpu.h +++ b/arch/mips/include/asm/cpu.h @@ -250,6 +250,10 @@ #define PRID_REV_LOONGSON1C 0x0020 /* Same as Loongson-1B */ #define PRID_REV_LOONGSON2E 0x0002 #define PRID_REV_LOONGSON2F 0x0003 +#define PRID_REV_LOONGSON2K_R1_0 0x0000 +#define PRID_REV_LOONGSON2K_R1_1 0x0001 +#define PRID_REV_LOONGSON2K_R1_2 0x0002 +#define PRID_REV_LOONGSON2K_R1_3 0x0003 #define PRID_REV_LOONGSON3A_R1 0x0005 #define PRID_REV_LOONGSON3B_R1 0x0006 #define PRID_REV_LOONGSON3B_R2 0x0007 diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index f21a2304401f..6de14b527c68 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -1935,7 +1935,24 @@ platform: static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) { switch (c->processor_id & PRID_IMP_MASK) { - case PRID_IMP_LOONGSON_64C: /* Loongson-2/3 */ + case PRID_IMP_LOONGSON_64R: /* Loongson-64 Reduced */ + switch (c->processor_id & PRID_REV_MASK) { + case PRID_REV_LOONGSON2K_R1_0: + case PRID_REV_LOONGSON2K_R1_1: + case PRID_REV_LOONGSON2K_R1_2: + case PRID_REV_LOONGSON2K_R1_3: + c->cputype = CPU_LOONGSON64; + __cpu_name[cpu] = "Loongson-2K"; + set_elf_platform(cpu, "gs264e"); + set_isa(c, MIPS_CPU_ISA_M64R2); + break; + } + decode_configs(c); + c->writecombine = _CACHE_UNCACHED_ACCELERATED; + c->ases |= (MIPS_ASE_LOONGSON_MMI | MIPS_ASE_LOONGSON_EXT | + MIPS_ASE_LOONGSON_EXT2); + break; + case PRID_IMP_LOONGSON_64C: /* Loongson-3 Classic */ switch (c->processor_id & PRID_REV_MASK) { case PRID_REV_LOONGSON3A_R2_0: case PRID_REV_LOONGSON3A_R2_1: diff --git a/arch/mips/kernel/idle.c b/arch/mips/kernel/idle.c index 37f8e78e2869..60d8c2a380fe 100644 --- a/arch/mips/kernel/idle.c +++ b/arch/mips/kernel/idle.c @@ -180,7 +180,8 @@ void __init check_wait(void) break; case CPU_LOONGSON64: if ((c->processor_id & (PRID_IMP_MASK | PRID_REV_MASK)) >= - (PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_0)) + (PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_0) || + (c->processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64R) cpu_wait = r4k_wait; break; diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index 89b6839140d7..85eb62e40e2b 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -1303,7 +1303,8 @@ static void probe_pcache(void) c->dcache.linesz; c->dcache.waybit = 0; if ((c->processor_id & (PRID_IMP_MASK | PRID_REV_MASK)) >= - (PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_0)) + (PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_0) || + (c->processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64R) c->options |= MIPS_CPU_PREFETCH; break; @@ -1629,8 +1630,13 @@ static void __init loongson3_sc_init(void) scache_size = c->scache.sets * c->scache.ways * c->scache.linesz; - /* Loongson-3 has 4 cores, 1MB scache for each. scaches are shared */ - scache_size *= 4; + + /* Loongson-3 has 4-Scache banks, while Loongson-2K have only 2 banks */ + if ((c->processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64R) + scache_size *= 2; + else + scache_size *= 4; + c->scache.waybit = 0; c->scache.waysize = scache_size / c->scache.ways; pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n", From 25259f7a5de2de9d67793dc584b15c83a3134c93 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Sun, 26 Apr 2020 22:43:56 +1000 Subject: [PATCH 0156/1043] clocksource/drivers/timer-microchip-pit64b: Select CONFIG_TIMER_OF This driver is an OF driver, it depends on OF, and uses TIMER_OF_DECLARE, so it should select CONFIG_TIMER_OF. Without CONFIG_TIMER_OF enabled this can lead to warnings such as: powerpc-linux-ld: warning: orphan section `__timer_of_table' from `drivers/clocksource/timer-microchip-pit64b.o' being placed in section `__timer_of_table'. Because TIMER_OF_TABLES in vmlinux.lds.h doesn't emit anything into the linker script when CONFIG_TIMER_OF is not enabled. Fixes: 625022a5f160 ("clocksource/drivers/timer-microchip-pit64b: Add Microchip PIT64B support") Cc: stable@vger.kernel.org # v5.6+ Reported-by: kbuild test robot Signed-off-by: Michael Ellerman Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200426124356.3929682-1-mpe@ellerman.id.au --- drivers/clocksource/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index f2142e6bbea3..f225c27b70f7 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -709,6 +709,7 @@ config MICROCHIP_PIT64B bool "Microchip PIT64B support" depends on OF || COMPILE_TEST select CLKSRC_MMIO + select TIMER_OF help This option enables Microchip PIT64B timer for Atmel based system. It supports the oneshot, the periodic From bfed0eded1ce00bda5cc2d2939b017f88e6b1fd0 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Fri, 17 Apr 2020 16:20:45 -0500 Subject: [PATCH 0157/1043] clocksource/drivers/versatile: Allow CONFIG_CLKSRC_VERSATILE to be disabled The timer-versatile driver provides a sched_clock for certain Arm Ltd. reference platforms. Specifically, it is used on Versatile and 32-bit VExpress. It is not needed for those platforms with an arch timer (all the 64-bit ones) yet CONFIG_MFD_VEXPRESS_SYSREG does still need to be enabled. In that case, the timer-versatile can only be disabled when COMPILE_TEST is enabled which is not desirable. Let's use the sub-arch kconfig symbols instead. Realview platforms don't have the sysregs that this driver uses so correct the help text. Cc: Daniel Lezcano Cc: Thomas Gleixner Cc: Linus Walleij Signed-off-by: Rob Herring Reviewed-by: Linus Walleij Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200417212045.16917-1-robh@kernel.org --- drivers/clocksource/Kconfig | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index f225c27b70f7..9c2d72b33e89 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -562,12 +562,11 @@ config CLKSRC_VERSATILE bool "ARM Versatile (Express) reference platforms clock source" if COMPILE_TEST depends on GENERIC_SCHED_CLOCK && !ARCH_USES_GETTIMEOFFSET select TIMER_OF - default y if MFD_VEXPRESS_SYSREG + default y if (ARCH_VEXPRESS || ARCH_VERSATILE) && ARM help This option enables clock source based on free running counter available in the "System Registers" block of - ARM Versatile, RealView and Versatile Express reference - platforms. + ARM Versatile and Versatile Express reference platforms. config CLKSRC_MIPS_GIC bool From 9d2161bed4e39ef7a5e5f18f69c4a57d001051b9 Mon Sep 17 00:00:00 2001 From: Richard Guy Briggs Date: Wed, 22 Apr 2020 17:37:04 -0400 Subject: [PATCH 0158/1043] audit: log audit netlink multicast bind and unbind Log information about programs connecting to and disconnecting from the audit netlink multicast socket. This is needed so that during investigations a security officer can tell who or what had access to the audit trail. This helps to meet the FAU_SAR.2 requirement for Common Criteria. Here is the systemd startup event: type=PROCTITLE msg=audit(2020-04-22 10:10:21.787:10) : proctitle=/init type=SYSCALL msg=audit(2020-04-22 10:10:21.787:10) : arch=x86_64 syscall=bind success=yes exit=0 a0=0x19 a1=0x555f4aac7e90 a2=0xc a3=0x7ffcb792ff44 items=0 ppid=0 pid=1 auid=unset uid=root gid=root euid=root suid=root fsuid=root egid=root sgid=root fsgid=root tty=(none) ses=unset comm=systemd exe=/usr/lib/systemd/systemd subj=kernel key=(null) type=UNKNOWN[1335] msg=audit(2020-04-22 10:10:21.787:10) : pid=1 uid=root auid=unset tty=(none) ses=unset subj=kernel comm=systemd exe=/usr/lib/systemd/systemd nl-mcgrp=1 op=connect res=yes And events from the test suite that just uses close(): type=PROCTITLE msg=audit(2020-04-22 11:47:08.501:442) : proctitle=/usr/bin/perl -w amcast_joinpart/test type=SYSCALL msg=audit(2020-04-22 11:47:08.501:442) : arch=x86_64 syscall=bind success=yes exit=0 a0=0x7 a1=0x563004378760 a2=0xc a3=0x0 items=0 ppid=815 pid=818 auid=root uid=root gid=root euid=root suid=root fsuid=root egid=root sgid=root fsgid=root tty=ttyS0 ses=1 comm=perl exe=/usr/bin/perl subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 key=(null) type=UNKNOWN[1335] msg=audit(2020-04-22 11:47:08.501:442) : pid=818 uid=root auid=root tty=ttyS0 ses=1 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 comm=perl exe=/usr/bin/perl nl-mcgrp=1 op=connect res=yes type=UNKNOWN[1335] msg=audit(2020-04-22 11:47:08.501:443) : pid=818 uid=root auid=root tty=ttyS0 ses=1 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 comm=perl exe=/usr/bin/perl nl-mcgrp=1 op=disconnect res=yes And the events from the test suite using setsockopt with NETLINK_DROP_MEMBERSHIP: type=PROCTITLE msg=audit(2020-04-22 11:39:53.291:439) : proctitle=/usr/bin/perl -w amcast_joinpart/test type=SYSCALL msg=audit(2020-04-22 11:39:53.291:439) : arch=x86_64 syscall=bind success=yes exit=0 a0=0x7 a1=0x5560877c2d20 a2=0xc a3=0x0 items=0 ppid=772 pid=775 auid=root uid=root gid=root euid=root suid=root fsuid=root egid=root sgid=root fsgid=root tty=ttyS0 ses=1 comm=perl exe=/usr/bin/perl subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 key=(null) type=UNKNOWN[1335] msg=audit(2020-04-22 11:39:53.291:439) : pid=775 uid=root auid=root tty=ttyS0 ses=1 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 comm=perl exe=/usr/bin/perl nl-mcgrp=1 op=connect res=yes type=PROCTITLE msg=audit(2020-04-22 11:39:53.292:440) : proctitle=/usr/bin/perl -w amcast_joinpart/test type=SYSCALL msg=audit(2020-04-22 11:39:53.292:440) : arch=x86_64 syscall=setsockopt success=yes exit=0 a0=0x7 a1=SOL_NETLINK a2=0x2 a3=0x7ffc8366f000 items=0 ppid=772 pid=775 auid=root uid=root gid=root euid=root suid=root fsuid=root egid=root sgid=root fsgid=root tty=ttyS0 ses=1 comm=perl exe=/usr/bin/perl subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 key=(null) type=UNKNOWN[1335] msg=audit(2020-04-22 11:39:53.292:440) : pid=775 uid=root auid=root tty=ttyS0 ses=1 subj=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 comm=perl exe=/usr/bin/perl nl-mcgrp=1 op=disconnect res=yes Please see the upstream issue tracker at https://github.com/linux-audit/audit-kernel/issues/28 With the feature description at https://github.com/linux-audit/audit-kernel/wiki/RFE-Audit-Multicast-Socket-Join-Part The testsuite support is at https://github.com/rgbriggs/audit-testsuite/compare/ghak28-mcast-part-join https://github.com/linux-audit/audit-testsuite/pull/93 And the userspace support patch is at https://github.com/linux-audit/audit-userspace/pull/114 Signed-off-by: Richard Guy Briggs Signed-off-by: Paul Moore --- include/uapi/linux/audit.h | 1 + kernel/audit.c | 52 +++++++++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/include/uapi/linux/audit.h b/include/uapi/linux/audit.h index a534d71e689a..9b6a973f4cc3 100644 --- a/include/uapi/linux/audit.h +++ b/include/uapi/linux/audit.h @@ -117,6 +117,7 @@ #define AUDIT_TIME_INJOFFSET 1332 /* Timekeeping offset injected */ #define AUDIT_TIME_ADJNTPVAL 1333 /* NTP value adjustment */ #define AUDIT_BPF 1334 /* BPF subsystem */ +#define AUDIT_EVENT_LISTENER 1335 /* Task joined multicast read socket */ #define AUDIT_AVC 1400 /* SE Linux avc denial or grant */ #define AUDIT_SELINUX_ERR 1401 /* Internal SE Linux Errors */ diff --git a/kernel/audit.c b/kernel/audit.c index 622c30246d19..e33460e01b3b 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -1530,20 +1530,60 @@ static void audit_receive(struct sk_buff *skb) audit_ctl_unlock(); } -/* Run custom bind function on netlink socket group connect or bind requests. */ -static int audit_bind(struct net *net, int group) +/* Log information about who is connecting to the audit multicast socket */ +static void audit_log_multicast(int group, const char *op, int err) { - if (!capable(CAP_AUDIT_READ)) - return -EPERM; + const struct cred *cred; + struct tty_struct *tty; + char comm[sizeof(current->comm)]; + struct audit_buffer *ab; - return 0; + if (!audit_enabled) + return; + + ab = audit_log_start(audit_context(), GFP_KERNEL, AUDIT_EVENT_LISTENER); + if (!ab) + return; + + cred = current_cred(); + tty = audit_get_tty(); + audit_log_format(ab, "pid=%u uid=%u auid=%u tty=%s ses=%u", + task_pid_nr(current), + from_kuid(&init_user_ns, cred->uid), + from_kuid(&init_user_ns, audit_get_loginuid(current)), + tty ? tty_name(tty) : "(none)", + audit_get_sessionid(current)); + audit_put_tty(tty); + audit_log_task_context(ab); /* subj= */ + audit_log_format(ab, " comm="); + audit_log_untrustedstring(ab, get_task_comm(comm, current)); + audit_log_d_path_exe(ab, current->mm); /* exe= */ + audit_log_format(ab, " nl-mcgrp=%d op=%s res=%d", group, op, !err); + audit_log_end(ab); +} + +/* Run custom bind function on netlink socket group connect or bind requests. */ +static int audit_multicast_bind(struct net *net, int group) +{ + int err = 0; + + if (!capable(CAP_AUDIT_READ)) + err = -EPERM; + audit_log_multicast(group, "connect", err); + return err; +} + +static void audit_multicast_unbind(struct net *net, int group) +{ + audit_log_multicast(group, "disconnect", 0); } static int __net_init audit_net_init(struct net *net) { struct netlink_kernel_cfg cfg = { .input = audit_receive, - .bind = audit_bind, + .bind = audit_multicast_bind, + .unbind = audit_multicast_unbind, .flags = NL_CFG_F_NONROOT_RECV, .groups = AUDIT_NLGRP_MAX, }; From 618c35556e5f2156c8bc5cd43664363838e38e45 Mon Sep 17 00:00:00 2001 From: Jason Yan Date: Tue, 28 Apr 2020 14:32:54 +0800 Subject: [PATCH 0159/1043] MIPS: oprofile: remove unneeded semicolon in common.c Fix the following coccicheck warning: arch/mips/oprofile/common.c:113:2-3: Unneeded semicolon Signed-off-by: Jason Yan Signed-off-by: Thomas Bogendoerfer --- arch/mips/oprofile/common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/oprofile/common.c b/arch/mips/oprofile/common.c index 03db268cba5c..d3996c4c6440 100644 --- a/arch/mips/oprofile/common.c +++ b/arch/mips/oprofile/common.c @@ -110,7 +110,7 @@ int __init oprofile_arch_init(struct oprofile_operations *ops) case CPU_LOONGSON64: lmodel = &op_model_loongson3_ops; break; - }; + } /* * Always set the backtrace. This allows unsupported CPU types to still From 8c88cc53ffa62b4bb05931c878bcf3dd10db66b5 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Fri, 3 Apr 2020 17:29:49 +0800 Subject: [PATCH 0160/1043] MIPS: Loongson: Get host bridge information Read the address of host bridge configuration space to get the vendor ID and device ID of host bridge, and then we can distinguish various types of host bridge such as LS7A or RS780E. Signed-off-by: Tiezhu Yang Reviewed-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- .../include/asm/mach-loongson64/boot_param.h | 6 ++++++ arch/mips/loongson64/env.c | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/arch/mips/include/asm/mach-loongson64/boot_param.h b/arch/mips/include/asm/mach-loongson64/boot_param.h index 2ed483e32d8c..fc9f14bc5777 100644 --- a/arch/mips/include/asm/mach-loongson64/boot_param.h +++ b/arch/mips/include/asm/mach-loongson64/boot_param.h @@ -192,6 +192,11 @@ struct boot_params { struct efi_reset_system_t reset_system; }; +enum loongson_bridge_type { + RS780E = 1, + LS7A = 2 +}; + struct loongson_system_configuration { u32 nr_cpus; u32 nr_nodes; @@ -200,6 +205,7 @@ struct loongson_system_configuration { u16 boot_cpu_id; u16 reserved_cpus_mask; enum loongson_cpu_type cputype; + enum loongson_bridge_type bridgetype; u64 ht_control_base; u64 pci_mem_start_addr; u64 pci_mem_end_addr; diff --git a/arch/mips/loongson64/env.c b/arch/mips/loongson64/env.c index 2554ef11170d..71f4aaf58791 100644 --- a/arch/mips/loongson64/env.c +++ b/arch/mips/loongson64/env.c @@ -14,12 +14,15 @@ * Author: Wu Zhangjin, wuzhangjin@gmail.com */ #include +#include #include #include #include #include #include +#define HOST_BRIDGE_CONFIG_ADDR ((void __iomem *)TO_UNCAC(0x1a000000)) + u32 cpu_clock_freq; EXPORT_SYMBOL(cpu_clock_freq); struct efi_memory_map_loongson *loongson_memmap; @@ -43,6 +46,8 @@ void __init prom_init_env(void) struct system_loongson *esys; struct efi_cpuinfo_loongson *ecpu; struct irq_source_routing_table *eirq_source; + u32 id; + u16 vendor, device; /* firmware arguments are initialized in head.S */ boot_p = (struct boot_params *)fw_arg2; @@ -178,4 +183,17 @@ void __init prom_init_env(void) memcpy(loongson_sysconf.sensors, esys->sensors, sizeof(struct sensor_device) * loongson_sysconf.nr_sensors); pr_info("CpuClock = %u\n", cpu_clock_freq); + + /* Read the ID of PCI host bridge to detect bridge type */ + id = readl(HOST_BRIDGE_CONFIG_ADDR); + vendor = id & 0xffff; + device = (id >> 16) & 0xffff; + + if (vendor == PCI_VENDOR_ID_LOONGSON && device == 0x7a00) { + pr_info("The bridge chip is LS7A\n"); + loongson_sysconf.bridgetype = LS7A; + } else { + pr_info("The bridge chip is RS780E or SR5690\n"); + loongson_sysconf.bridgetype = RS780E; + } } From c4dad0aab3fca0c1f0baa4cc84b6ec91b7ebf426 Mon Sep 17 00:00:00 2001 From: Richard Guy Briggs Date: Wed, 22 Apr 2020 17:39:28 -0400 Subject: [PATCH 0161/1043] audit: tidy and extend netfilter_cfg x_tables NETFILTER_CFG record generation was inconsistent for x_tables and ebtables configuration changes. The call was needlessly messy and there were supporting records missing at times while they were produced when not requested. Simplify the logging call into a new audit_log_nfcfg call. Honour the audit_enabled setting while more consistently recording information including supporting records by tidying up dummy checks. Add an op= field that indicates the operation being performed (register or replace). Here is the enhanced sample record: type=NETFILTER_CFG msg=audit(1580905834.919:82970): table=filter family=2 entries=83 op=replace Generate audit NETFILTER_CFG records on ebtables table registration. Previously this was being done for x_tables registration and replacement operations and ebtables table replacement only. See: https://github.com/linux-audit/audit-kernel/issues/25 See: https://github.com/linux-audit/audit-kernel/issues/35 See: https://github.com/linux-audit/audit-kernel/issues/43 Signed-off-by: Richard Guy Briggs Signed-off-by: Paul Moore --- include/linux/audit.h | 21 +++++++++++++++++++++ kernel/auditsc.c | 24 ++++++++++++++++++++++++ net/bridge/netfilter/ebtables.c | 12 ++++-------- net/netfilter/x_tables.c | 12 +++--------- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/include/linux/audit.h b/include/linux/audit.h index f9ceae57ca8d..5c7f07a9776d 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -94,6 +94,11 @@ struct audit_ntp_data { struct audit_ntp_data {}; #endif +enum audit_nfcfgop { + AUDIT_XT_OP_REGISTER, + AUDIT_XT_OP_REPLACE, +}; + extern int is_audit_feature_set(int which); extern int __init audit_register_class(int class, unsigned *list); @@ -379,6 +384,8 @@ extern void __audit_log_kern_module(char *name); extern void __audit_fanotify(unsigned int response); extern void __audit_tk_injoffset(struct timespec64 offset); extern void __audit_ntp_log(const struct audit_ntp_data *ad); +extern void __audit_log_nfcfg(const char *name, u8 af, unsigned int nentries, + enum audit_nfcfgop op); static inline void audit_ipc_obj(struct kern_ipc_perm *ipcp) { @@ -514,6 +521,14 @@ static inline void audit_ntp_log(const struct audit_ntp_data *ad) __audit_ntp_log(ad); } +static inline void audit_log_nfcfg(const char *name, u8 af, + unsigned int nentries, + enum audit_nfcfgop op) +{ + if (audit_enabled) + __audit_log_nfcfg(name, af, nentries, op); +} + extern int audit_n_rules; extern int audit_signals; #else /* CONFIG_AUDITSYSCALL */ @@ -646,6 +661,12 @@ static inline void audit_ntp_log(const struct audit_ntp_data *ad) static inline void audit_ptrace(struct task_struct *t) { } + +static inline void audit_log_nfcfg(const char *name, u8 af, + unsigned int nentries, + enum audit_nfcfgop op) +{ } + #define audit_n_rules 0 #define audit_signals 0 #endif /* CONFIG_AUDITSYSCALL */ diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 814406a35db1..705beac0ce29 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -130,6 +130,16 @@ struct audit_tree_refs { struct audit_chunk *c[31]; }; +struct audit_nfcfgop_tab { + enum audit_nfcfgop op; + const char *s; +}; + +const struct audit_nfcfgop_tab audit_nfcfgs[] = { + { AUDIT_XT_OP_REGISTER, "register" }, + { AUDIT_XT_OP_REPLACE, "replace" }, +}; + static int audit_match_perm(struct audit_context *ctx, int mask) { unsigned n; @@ -2542,6 +2552,20 @@ void __audit_ntp_log(const struct audit_ntp_data *ad) audit_log_ntp_val(ad, "adjust", AUDIT_NTP_ADJUST); } +void __audit_log_nfcfg(const char *name, u8 af, unsigned int nentries, + enum audit_nfcfgop op) +{ + struct audit_buffer *ab; + + ab = audit_log_start(audit_context(), GFP_KERNEL, AUDIT_NETFILTER_CFG); + if (!ab) + return; + audit_log_format(ab, "table=%s family=%u entries=%u op=%s", + name, af, nentries, audit_nfcfgs[op].s); + audit_log_end(ab); +} +EXPORT_SYMBOL_GPL(__audit_log_nfcfg); + static void audit_log_task(struct audit_buffer *ab) { kuid_t auid, uid; diff --git a/net/bridge/netfilter/ebtables.c b/net/bridge/netfilter/ebtables.c index 78db58c7aec2..0a148e68b6e1 100644 --- a/net/bridge/netfilter/ebtables.c +++ b/net/bridge/netfilter/ebtables.c @@ -1046,14 +1046,8 @@ static int do_replace_finish(struct net *net, struct ebt_replace *repl, vfree(table); vfree(counterstmp); -#ifdef CONFIG_AUDIT - if (audit_enabled) { - audit_log(audit_context(), GFP_KERNEL, - AUDIT_NETFILTER_CFG, - "table=%s family=%u entries=%u", - repl->name, AF_BRIDGE, repl->nentries); - } -#endif + audit_log_nfcfg(repl->name, AF_BRIDGE, repl->nentries, + AUDIT_XT_OP_REPLACE); return ret; free_unlock: @@ -1223,6 +1217,8 @@ int ebt_register_table(struct net *net, const struct ebt_table *input_table, *res = NULL; } + audit_log_nfcfg(repl->name, AF_BRIDGE, repl->nentries, + AUDIT_XT_OP_REGISTER); return ret; free_unlock: mutex_unlock(&ebt_mutex); diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c index cd2b034eef59..8f8c5dbf603d 100644 --- a/net/netfilter/x_tables.c +++ b/net/netfilter/x_tables.c @@ -1408,15 +1408,9 @@ xt_replace_table(struct xt_table *table, } } -#ifdef CONFIG_AUDIT - if (audit_enabled) { - audit_log(audit_context(), GFP_KERNEL, - AUDIT_NETFILTER_CFG, - "table=%s family=%u entries=%u", - table->name, table->af, private->number); - } -#endif - + audit_log_nfcfg(table->name, table->af, private->number, + !private->number ? AUDIT_XT_OP_REGISTER : + AUDIT_XT_OP_REPLACE); return private; } EXPORT_SYMBOL_GPL(xt_replace_table); From a45d88530b2552ad5ea0da18861600b4ecc9d0c7 Mon Sep 17 00:00:00 2001 From: Richard Guy Briggs Date: Wed, 22 Apr 2020 17:39:29 -0400 Subject: [PATCH 0162/1043] netfilter: add audit table unregister actions Audit the action of unregistering ebtables and x_tables. See: https://github.com/linux-audit/audit-kernel/issues/44 Signed-off-by: Richard Guy Briggs Signed-off-by: Paul Moore --- include/linux/audit.h | 1 + kernel/auditsc.c | 5 +++-- net/bridge/netfilter/ebtables.c | 2 ++ net/netfilter/x_tables.c | 2 ++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/include/linux/audit.h b/include/linux/audit.h index 5c7f07a9776d..1a2351508211 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -97,6 +97,7 @@ struct audit_ntp_data {}; enum audit_nfcfgop { AUDIT_XT_OP_REGISTER, AUDIT_XT_OP_REPLACE, + AUDIT_XT_OP_UNREGISTER, }; extern int is_audit_feature_set(int which); diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 705beac0ce29..d281c18d1771 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -136,8 +136,9 @@ struct audit_nfcfgop_tab { }; const struct audit_nfcfgop_tab audit_nfcfgs[] = { - { AUDIT_XT_OP_REGISTER, "register" }, - { AUDIT_XT_OP_REPLACE, "replace" }, + { AUDIT_XT_OP_REGISTER, "register" }, + { AUDIT_XT_OP_REPLACE, "replace" }, + { AUDIT_XT_OP_UNREGISTER, "unregister" }, }; static int audit_match_perm(struct audit_context *ctx, int mask) diff --git a/net/bridge/netfilter/ebtables.c b/net/bridge/netfilter/ebtables.c index 0a148e68b6e1..4778db5601b0 100644 --- a/net/bridge/netfilter/ebtables.c +++ b/net/bridge/netfilter/ebtables.c @@ -1124,6 +1124,8 @@ static void __ebt_unregister_table(struct net *net, struct ebt_table *table) mutex_lock(&ebt_mutex); list_del(&table->list); mutex_unlock(&ebt_mutex); + audit_log_nfcfg(table->name, AF_BRIDGE, table->private->nentries, + AUDIT_XT_OP_UNREGISTER); EBT_ENTRY_ITERATE(table->private->entries, table->private->entries_size, ebt_cleanup_entry, net, NULL); if (table->private->nentries) diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c index 8f8c5dbf603d..99a468be4a59 100644 --- a/net/netfilter/x_tables.c +++ b/net/netfilter/x_tables.c @@ -1472,6 +1472,8 @@ void *xt_unregister_table(struct xt_table *table) private = table->private; list_del(&table->list); mutex_unlock(&xt[table->af].mutex); + audit_log_nfcfg(table->name, table->af, private->number, + AUDIT_XT_OP_UNREGISTER); kfree(table); return private; From 2c3cc858a68707dd710aefddd737e9f6aa0a16e1 Mon Sep 17 00:00:00 2001 From: Liangliang Huang Date: Wed, 29 Apr 2020 17:04:17 +0800 Subject: [PATCH 0163/1043] MIPS: Loongson64: Switch the order of RS780E and LS7A Sort the members of enum in alphabetical order is better to avoid duplicate mistakes (because the list may be grow very large), so fix it by exchanging the order. Signed-off-by: Liangliang Huang Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mach-loongson64/boot_param.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/include/asm/mach-loongson64/boot_param.h b/arch/mips/include/asm/mach-loongson64/boot_param.h index fc9f14bc5777..f082d87665bc 100644 --- a/arch/mips/include/asm/mach-loongson64/boot_param.h +++ b/arch/mips/include/asm/mach-loongson64/boot_param.h @@ -193,8 +193,8 @@ struct boot_params { }; enum loongson_bridge_type { - RS780E = 1, - LS7A = 2 + LS7A = 1, + RS780E = 2 }; struct loongson_system_configuration { From 59bd128a412702931afaf758af6b640e3528d1b5 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Mon, 13 Apr 2020 17:26:26 +0200 Subject: [PATCH 0164/1043] MIPS: ingenic: DTS: Fix invalid value in #dma-cells The driver requires two cells and not just one. Since these nodes are both disabled as no hardware currently use them, this fix does not really requires a Fixes: tag. Signed-off-by: Paul Cercueil Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ingenic/jz4770.dtsi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/boot/dts/ingenic/jz4770.dtsi b/arch/mips/boot/dts/ingenic/jz4770.dtsi index 0bfb9edff3d0..3805816cea2e 100644 --- a/arch/mips/boot/dts/ingenic/jz4770.dtsi +++ b/arch/mips/boot/dts/ingenic/jz4770.dtsi @@ -222,7 +222,7 @@ reg = <0x13420000 0xC0 0x13420300 0x20>; - #dma-cells = <1>; + #dma-cells = <2>; clocks = <&cgu JZ4770_CLK_DMA>; interrupt-parent = <&intc>; @@ -237,7 +237,7 @@ reg = <0x13420100 0xC0 0x13420400 0x20>; - #dma-cells = <1>; + #dma-cells = <2>; clocks = <&cgu JZ4770_CLK_DMA>; interrupt-parent = <&intc>; From cf2e6b8e6f6f28c186700cb93c0b9b98960705c7 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Mon, 13 Apr 2020 17:26:27 +0200 Subject: [PATCH 0165/1043] MIPS: ingenic: DTS: Respect cell count of common properties If N fields of X cells should be provided, then that's what the devicetree should represent, instead of having one single field of (N*X) cells. Signed-off-by: Paul Cercueil Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ingenic/jz4740.dtsi | 19 +++++++++---------- arch/mips/boot/dts/ingenic/jz4770.dtsi | 12 +++++------- arch/mips/boot/dts/ingenic/jz4780.dtsi | 26 ++++++++++++-------------- arch/mips/boot/dts/ingenic/x1000.dtsi | 9 ++++----- 4 files changed, 30 insertions(+), 36 deletions(-) diff --git a/arch/mips/boot/dts/ingenic/jz4740.dtsi b/arch/mips/boot/dts/ingenic/jz4740.dtsi index a3301bab9231..b56671157bbd 100644 --- a/arch/mips/boot/dts/ingenic/jz4740.dtsi +++ b/arch/mips/boot/dts/ingenic/jz4740.dtsi @@ -55,10 +55,10 @@ #clock-cells = <1>; - clocks = <&cgu JZ4740_CLK_RTC - &cgu JZ4740_CLK_EXT - &cgu JZ4740_CLK_PCLK - &cgu JZ4740_CLK_TCU>; + clocks = <&cgu JZ4740_CLK_RTC>, + <&cgu JZ4740_CLK_EXT>, + <&cgu JZ4740_CLK_PCLK>, + <&cgu JZ4740_CLK_TCU>; clock-names = "rtc", "ext", "pclk", "tcu"; interrupt-controller; @@ -241,10 +241,10 @@ reg = <0x13010000 0x54>; #address-cells = <2>; #size-cells = <1>; - ranges = <1 0 0x18000000 0x4000000 - 2 0 0x14000000 0x4000000 - 3 0 0x0c000000 0x4000000 - 4 0 0x08000000 0x4000000>; + ranges = <1 0 0x18000000 0x4000000>, + <2 0 0x14000000 0x4000000>, + <3 0 0x0c000000 0x4000000>, + <4 0 0x08000000 0x4000000>; clocks = <&cgu JZ4740_CLK_MCLK>; }; @@ -258,8 +258,7 @@ dmac: dma-controller@13020000 { compatible = "ingenic,jz4740-dma"; - reg = <0x13020000 0xbc - 0x13020300 0x14>; + reg = <0x13020000 0xbc>, <0x13020300 0x14>; #dma-cells = <2>; interrupt-parent = <&intc>; diff --git a/arch/mips/boot/dts/ingenic/jz4770.dtsi b/arch/mips/boot/dts/ingenic/jz4770.dtsi index 3805816cea2e..9a25a6f7a48d 100644 --- a/arch/mips/boot/dts/ingenic/jz4770.dtsi +++ b/arch/mips/boot/dts/ingenic/jz4770.dtsi @@ -55,9 +55,9 @@ #clock-cells = <1>; - clocks = <&cgu JZ4770_CLK_RTC - &cgu JZ4770_CLK_EXT - &cgu JZ4770_CLK_PCLK>; + clocks = <&cgu JZ4770_CLK_RTC>, + <&cgu JZ4770_CLK_EXT>, + <&cgu JZ4770_CLK_PCLK>; clock-names = "rtc", "ext", "pclk"; interrupt-controller; @@ -219,8 +219,7 @@ dmac0: dma-controller@13420000 { compatible = "ingenic,jz4770-dma"; - reg = <0x13420000 0xC0 - 0x13420300 0x20>; + reg = <0x13420000 0xC0>, <0x13420300 0x20>; #dma-cells = <2>; @@ -234,8 +233,7 @@ dmac1: dma-controller@13420100 { compatible = "ingenic,jz4770-dma"; - reg = <0x13420100 0xC0 - 0x13420400 0x20>; + reg = <0x13420100 0xC0>, <0x13420400 0x20>; #dma-cells = <2>; diff --git a/arch/mips/boot/dts/ingenic/jz4780.dtsi b/arch/mips/boot/dts/ingenic/jz4780.dtsi index 1c94f6791127..15780fb118cf 100644 --- a/arch/mips/boot/dts/ingenic/jz4780.dtsi +++ b/arch/mips/boot/dts/ingenic/jz4780.dtsi @@ -58,9 +58,9 @@ #clock-cells = <1>; - clocks = <&cgu JZ4780_CLK_RTCLK - &cgu JZ4780_CLK_EXCLK - &cgu JZ4780_CLK_PCLK>; + clocks = <&cgu JZ4780_CLK_RTCLK>, + <&cgu JZ4780_CLK_EXCLK>, + <&cgu JZ4780_CLK_PCLK>; clock-names = "rtc", "ext", "pclk"; interrupt-controller; @@ -196,8 +196,7 @@ gpio-miso = <&gpe 14 0>; gpio-sck = <&gpe 15 0>; gpio-mosi = <&gpe 17 0>; - cs-gpios = <&gpe 16 0 - &gpe 18 0>; + cs-gpios = <&gpe 16 0>, <&gpe 18 0>; spidev@0 { compatible = "spidev"; @@ -362,13 +361,13 @@ reg = <0x13410000 0x10000>; #address-cells = <2>; #size-cells = <1>; - ranges = <0 0 0x13410000 0x10000 - 1 0 0x1b000000 0x1000000 - 2 0 0x1a000000 0x1000000 - 3 0 0x19000000 0x1000000 - 4 0 0x18000000 0x1000000 - 5 0 0x17000000 0x1000000 - 6 0 0x16000000 0x1000000>; + ranges = <0 0 0x13410000 0x10000>, + <1 0 0x1b000000 0x1000000>, + <2 0 0x1a000000 0x1000000>, + <3 0 0x19000000 0x1000000>, + <4 0 0x18000000 0x1000000>, + <5 0 0x17000000 0x1000000>, + <6 0 0x16000000 0x1000000>; clocks = <&cgu JZ4780_CLK_NEMC>; @@ -391,8 +390,7 @@ dma: dma@13420000 { compatible = "ingenic,jz4780-dma"; - reg = <0x13420000 0x400 - 0x13421000 0x40>; + reg = <0x13420000 0x400>, <0x13421000 0x40>; #dma-cells = <2>; interrupt-parent = <&intc>; diff --git a/arch/mips/boot/dts/ingenic/x1000.dtsi b/arch/mips/boot/dts/ingenic/x1000.dtsi index 147f7d5c243a..59a63a0985a8 100644 --- a/arch/mips/boot/dts/ingenic/x1000.dtsi +++ b/arch/mips/boot/dts/ingenic/x1000.dtsi @@ -58,9 +58,9 @@ #clock-cells = <1>; - clocks = <&cgu X1000_CLK_RTCLK - &cgu X1000_CLK_EXCLK - &cgu X1000_CLK_PCLK>; + clocks = <&cgu X1000_CLK_RTCLK>, + <&cgu X1000_CLK_EXCLK>, + <&cgu X1000_CLK_PCLK>; clock-names = "rtc", "ext", "pclk"; interrupt-controller; @@ -239,8 +239,7 @@ pdma: dma-controller@13420000 { compatible = "ingenic,x1000-dma"; - reg = <0x13420000 0x400 - 0x13421000 0x40>; + reg = <0x13420000 0x400>, <0x13421000 0x40>; #dma-cells = <2>; interrupt-parent = <&intc>; From bf40bf5ecf3f96b95c0ed8d6badf0b34df879adf Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Mon, 13 Apr 2020 17:26:28 +0200 Subject: [PATCH 0166/1043] MIPS: ingenic: DTS: Add nodes for the watchdog/PWM/OST Add the TCU nodes to the JZ4780, JZ4770 and JZ4740 devicetree files. Signed-off-by: Paul Cercueil Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ingenic/jz4740.dtsi | 14 +++++++++++ arch/mips/boot/dts/ingenic/jz4770.dtsi | 34 ++++++++++++++++++++++++++ arch/mips/boot/dts/ingenic/jz4780.dtsi | 24 ++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/arch/mips/boot/dts/ingenic/jz4740.dtsi b/arch/mips/boot/dts/ingenic/jz4740.dtsi index b56671157bbd..1520585c235c 100644 --- a/arch/mips/boot/dts/ingenic/jz4740.dtsi +++ b/arch/mips/boot/dts/ingenic/jz4740.dtsi @@ -74,6 +74,20 @@ clocks = <&tcu TCU_CLK_WDT>; clock-names = "wdt"; }; + + pwm: pwm@40 { + compatible = "ingenic,jz4740-pwm"; + reg = <0x40 0x80>; + + #pwm-cells = <3>; + + clocks = <&tcu TCU_CLK_TIMER0>, <&tcu TCU_CLK_TIMER1>, + <&tcu TCU_CLK_TIMER2>, <&tcu TCU_CLK_TIMER3>, + <&tcu TCU_CLK_TIMER4>, <&tcu TCU_CLK_TIMER5>, + <&tcu TCU_CLK_TIMER6>, <&tcu TCU_CLK_TIMER7>; + clock-names = "timer0", "timer1", "timer2", "timer3", + "timer4", "timer5", "timer6", "timer7"; + }; }; rtc_dev: rtc@10003000 { diff --git a/arch/mips/boot/dts/ingenic/jz4770.dtsi b/arch/mips/boot/dts/ingenic/jz4770.dtsi index 9a25a6f7a48d..3024c360c3ca 100644 --- a/arch/mips/boot/dts/ingenic/jz4770.dtsi +++ b/arch/mips/boot/dts/ingenic/jz4770.dtsi @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include / { #address-cells = <1>; @@ -65,6 +66,39 @@ interrupt-parent = <&intc>; interrupts = <27 26 25>; + + watchdog: watchdog@0 { + compatible = "ingenic,jz4770-watchdog", + "ingenic,jz4740-watchdog"; + reg = <0x0 0xc>; + + clocks = <&tcu TCU_CLK_WDT>; + clock-names = "wdt"; + }; + + pwm: pwm@40 { + compatible = "ingenic,jz4770-pwm", "ingenic,jz4740-pwm"; + reg = <0x40 0x80>; + + #pwm-cells = <3>; + + clocks = <&tcu TCU_CLK_TIMER0>, <&tcu TCU_CLK_TIMER1>, + <&tcu TCU_CLK_TIMER2>, <&tcu TCU_CLK_TIMER3>, + <&tcu TCU_CLK_TIMER4>, <&tcu TCU_CLK_TIMER5>, + <&tcu TCU_CLK_TIMER6>, <&tcu TCU_CLK_TIMER7>; + clock-names = "timer0", "timer1", "timer2", "timer3", + "timer4", "timer5", "timer6", "timer7"; + }; + + ost: timer@e0 { + compatible = "ingenic,jz4770-ost"; + reg = <0xe0 0x20>; + + clocks = <&tcu TCU_CLK_OST>; + clock-names = "ost"; + + interrupts = <15>; + }; }; pinctrl: pin-controller@10010000 { diff --git a/arch/mips/boot/dts/ingenic/jz4780.dtsi b/arch/mips/boot/dts/ingenic/jz4780.dtsi index 15780fb118cf..b7f409a7cf5d 100644 --- a/arch/mips/boot/dts/ingenic/jz4780.dtsi +++ b/arch/mips/boot/dts/ingenic/jz4780.dtsi @@ -76,6 +76,30 @@ clocks = <&tcu TCU_CLK_WDT>; clock-names = "wdt"; }; + + pwm: pwm@40 { + compatible = "ingenic,jz4780-pwm", "ingenic,jz4740-pwm"; + reg = <0x40 0x80>; + + #pwm-cells = <3>; + + clocks = <&tcu TCU_CLK_TIMER0>, <&tcu TCU_CLK_TIMER1>, + <&tcu TCU_CLK_TIMER2>, <&tcu TCU_CLK_TIMER3>, + <&tcu TCU_CLK_TIMER4>, <&tcu TCU_CLK_TIMER5>, + <&tcu TCU_CLK_TIMER6>, <&tcu TCU_CLK_TIMER7>; + clock-names = "timer0", "timer1", "timer2", "timer3", + "timer4", "timer5", "timer6", "timer7"; + }; + + ost: timer@e0 { + compatible = "ingenic,jz4780-ost", "ingenic,jz4770-ost"; + reg = <0xe0 0x20>; + + clocks = <&tcu TCU_CLK_OST>; + clock-names = "ost"; + + interrupts = <15>; + }; }; rtc_dev: rtc@10003000 { From 061e35b2c76e9cd230009c055e6a7e20dca51dd0 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Mon, 13 Apr 2020 17:26:29 +0200 Subject: [PATCH 0167/1043] MIPS: ingenic: DTS: Update JZ4770 support Add support for the RTC, AIC, CODEC, MMC 0/1/2, ADC, GPU, LCD, USB OTG, USB PHY controllers. Signed-off-by: Paul Cercueil Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ingenic/jz4770.dtsi | 177 ++++++++++++++++++++++++- 1 file changed, 170 insertions(+), 7 deletions(-) diff --git a/arch/mips/boot/dts/ingenic/jz4770.dtsi b/arch/mips/boot/dts/ingenic/jz4770.dtsi index 3024c360c3ca..fa11ac950499 100644 --- a/arch/mips/boot/dts/ingenic/jz4770.dtsi +++ b/arch/mips/boot/dts/ingenic/jz4770.dtsi @@ -38,13 +38,25 @@ }; cgu: jz4770-cgu@10000000 { - compatible = "ingenic,jz4770-cgu"; + compatible = "ingenic,jz4770-cgu", "simple-mfd"; reg = <0x10000000 0x100>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x10000000 0x100>; clocks = <&ext>, <&osc32k>; clock-names = "ext", "osc32k"; #clock-cells = <1>; + + otg_phy: usb-phy@3c { + compatible = "ingenic,jz4770-phy"; + reg = <0x3c 0x10>; + + clocks = <&cgu JZ4770_CLK_OTG_PHY>; + + #phy-cells = <0>; + }; }; tcu: timer@10002000 { @@ -101,6 +113,14 @@ }; }; + rtc: rtc@10003000 { + compatible = "ingenic,jz4770-rtc", "ingenic,jz4760-rtc"; + reg = <0x10003000 0x40>; + + interrupt-parent = <&intc>; + interrupts = <32>; + }; + pinctrl: pin-controller@10010000 { compatible = "ingenic,jz4770-pinctrl"; reg = <0x10010000 0x600>; @@ -199,6 +219,93 @@ }; }; + aic: audio-controller@10020000 { + compatible = "ingenic,jz4770-i2s"; + reg = <0x10020000 0x94>; + + #sound-dai-cells = <0>; + + clocks = <&cgu JZ4770_CLK_AIC>, <&cgu JZ4770_CLK_I2S>, + <&cgu JZ4770_CLK_EXT>, <&cgu JZ4770_CLK_PLL0>; + clock-names = "aic", "i2s", "ext", "pll half"; + + interrupt-parent = <&intc>; + interrupts = <34>; + + dmas = <&dmac0 25 0xffffffff>, <&dmac0 24 0xffffffff>; + dma-names = "rx", "tx"; + }; + + codec: audio-codec@100200a0 { + compatible = "ingenic,jz4770-codec"; + reg = <0x100200a4 0x8>; + + #sound-dai-cells = <0>; + + clocks = <&cgu JZ4770_CLK_AIC>; + clock-names = "aic"; + }; + + mmc0: mmc@10021000 { + compatible = "ingenic,jz4770-mmc", "ingenic,jz4760-mmc"; + reg = <0x10021000 0x1000>; + + clocks = <&cgu JZ4770_CLK_MMC0>; + clock-names = "mmc"; + + interrupt-parent = <&intc>; + interrupts = <37>; + + dmas = <&dmac1 27 0xffffffff>, <&dmac1 26 0xffffffff>; + dma-names = "rx", "tx"; + + cap-sd-highspeed; + cap-mmc-highspeed; + cap-sdio-irq; + + status = "disabled"; + }; + + mmc1: mmc@10022000 { + compatible = "ingenic,jz4770-mmc", "ingenic,jz4760-mmc"; + reg = <0x10022000 0x1000>; + + clocks = <&cgu JZ4770_CLK_MMC1>; + clock-names = "mmc"; + + interrupt-parent = <&intc>; + interrupts = <36>; + + dmas = <&dmac1 31 0xffffffff>, <&dmac1 30 0xffffffff>; + dma-names = "rx", "tx"; + + cap-sd-highspeed; + cap-mmc-highspeed; + cap-sdio-irq; + + status = "disabled"; + }; + + mmc2: mmc@10023000 { + compatible = "ingenic,jz4770-mmc", "ingenic,jz4760-mmc"; + reg = <0x10023000 0x1000>; + + clocks = <&cgu JZ4770_CLK_MMC2>; + clock-names = "mmc"; + + interrupt-parent = <&intc>; + interrupts = <35>; + + dmas = <&dmac1 37 0xffffffff>, <&dmac1 36 0xffffffff>; + dma-names = "rx", "tx"; + + cap-sd-highspeed; + cap-mmc-highspeed; + cap-sdio-irq; + + status = "disabled"; + }; + uart0: serial@10030000 { compatible = "ingenic,jz4770-uart"; reg = <0x10030000 0x100>; @@ -251,6 +358,43 @@ status = "disabled"; }; + adc: adc@10070000 { + compatible = "ingenic,jz4770-adc"; + reg = <0x10070000 0x30>; + + #io-channel-cells = <1>; + + clocks = <&cgu JZ4770_CLK_ADC>; + clock-names = "adc"; + + interrupt-parent = <&intc>; + interrupts = <18>; + }; + + gpu: gpu@13040000 { + compatible = "vivante,gc"; + reg = <0x13040000 0x10000>; + + clocks = <&cgu JZ4770_CLK_GPU>, + <&cgu JZ4770_CLK_GPU>, + <&cgu JZ4770_CLK_GPU>; + clock-names = "bus", "core", "shader"; + + interrupt-parent = <&intc>; + interrupts = <6>; + }; + + lcd: lcd-controller@13050000 { + compatible = "ingenic,jz4770-lcd"; + reg = <0x13050000 0x300>; + + interrupt-parent = <&intc>; + interrupts = <31>; + + clocks = <&cgu JZ4770_CLK_LPCLK_MUX>; + clock-names = "lcd_pclk"; + }; + dmac0: dma-controller@13420000 { compatible = "ingenic,jz4770-dma"; reg = <0x13420000 0xC0>, <0x13420300 0x20>; @@ -260,9 +404,6 @@ clocks = <&cgu JZ4770_CLK_DMA>; interrupt-parent = <&intc>; interrupts = <24>; - - /* Disable dmac0 until we have something that uses it */ - status = "disabled"; }; dmac1: dma-controller@13420100 { @@ -274,9 +415,6 @@ clocks = <&cgu JZ4770_CLK_DMA>; interrupt-parent = <&intc>; interrupts = <23>; - - /* Disable dmac1 until we have something that uses it */ - status = "disabled"; }; uhc: uhc@13430000 { @@ -292,4 +430,29 @@ status = "disabled"; }; + + usb_otg: usb@13440000 { + compatible = "ingenic,jz4770-musb"; + reg = <0x13440000 0x10000>; + + clocks = <&cgu JZ4770_CLK_OTG>; + clock-names = "udc"; + + interrupt-parent = <&intc>; + interrupts = <21>; + interrupt-names = "mc"; + + phys = <&otg_phy>; + + usb-role-switch; + }; + + rom: memory@1fc00000 { + compatible = "mtd-rom"; + probe-type = "map_rom"; + reg = <0x1fc00000 0x2000>; + + bank-width = <4>; + device-width = <1>; + }; }; From b1bfdb660516ad57087dfc8f8698853dfbf5e5fd Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Mon, 13 Apr 2020 17:26:30 +0200 Subject: [PATCH 0168/1043] MIPS: ingenic: DTS: Update GCW0 support Add support for the face buttons, the ACT8600 PMUC, the LCD panel with backlight, the rumble, internal/external SD readers, and other things. Note that the otg-phy node was dropped in the process as it was neither useful nor used, and was inside a non-compliant board "bus". Signed-off-by: Paul Cercueil Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ingenic/gcw0.dts | 501 +++++++++++++++++++++++++++- 1 file changed, 485 insertions(+), 16 deletions(-) diff --git a/arch/mips/boot/dts/ingenic/gcw0.dts b/arch/mips/boot/dts/ingenic/gcw0.dts index f58d239c2058..c2396b5ca175 100644 --- a/arch/mips/boot/dts/ingenic/gcw0.dts +++ b/arch/mips/boot/dts/ingenic/gcw0.dts @@ -4,6 +4,10 @@ #include "jz4770.dtsi" #include +#include +#include +#include + / { compatible = "gcw,zero", "ingenic,jz4770"; model = "GCW Zero"; @@ -19,16 +23,360 @@ stdout-path = "serial2:57600n8"; }; - board { - compatible = "simple-bus"; - #address-cells = <1>; - #size-cells = <1>; - ranges; + vcc: regulator@0 { + compatible = "regulator-fixed"; + regulator-name = "vcc"; - otg_phy: otg-phy { - compatible = "usb-nop-xceiv"; - clocks = <&cgu JZ4770_CLK_OTG_PHY>; - clock-names = "main_clk"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; + + mmc1_power: regulator@1 { + compatible = "regulator-fixed"; + regulator-name = "mmc1_vcc"; + gpio = <&gpe 9 0>; + + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + vin-supply = <&vcc>; + }; + + headphones_amp: analog-amplifier@0 { + compatible = "simple-audio-amplifier"; + enable-gpios = <&gpf 3 GPIO_ACTIVE_LOW>; + enable-delay-ms = <50>; + + VCC-supply = <&ldo5>; + sound-name-prefix = "Headphones Amp"; + }; + + speaker_amp: analog-amplifier@1 { + compatible = "simple-audio-amplifier"; + enable-gpios = <&gpf 20 GPIO_ACTIVE_HIGH>; + + VCC-supply = <&ldo5>; + sound-name-prefix = "Speaker Amp"; + }; + + sound { + compatible = "simple-audio-card"; + + simple-audio-card,name = "gcw0-audio"; + simple-audio-card,format = "i2s"; + + simple-audio-card,widgets = + "Speaker", "Speaker", + "Headphone", "Headphones", + "Line", "FM Radio", + "Microphone", "Built-in Mic"; + simple-audio-card,routing = + "Headphones Amp INL", "LHPOUT", + "Headphones Amp INR", "RHPOUT", + "Headphones", "Headphones Amp OUTL", + "Headphones", "Headphones Amp OUTR", + "Speaker Amp INL", "LOUT", + "Speaker Amp INR", "ROUT", + "Speaker", "Speaker Amp OUTL", + "Speaker", "Speaker Amp OUTR", + "LLINEIN", "FM Radio", + "RLINEIN", "FM Radio", + "Built-in Mic", "MICBIAS", + "MIC1P", "Built-in Mic", + "MIC1N", "Built-in Mic"; + simple-audio-card,pin-switches = "Speaker", "Headphones"; + + simple-audio-card,hp-det-gpio = <&gpf 21 GPIO_ACTIVE_HIGH>; + simple-audio-card,aux-devs = <&speaker_amp>, <&headphones_amp>; + + simple-audio-card,bitclock-master = <&dai_codec>; + simple-audio-card,frame-master = <&dai_codec>; + + dai_cpu: simple-audio-card,cpu { + sound-dai = <&aic>; + }; + + dai_codec: simple-audio-card,codec { + sound-dai = <&codec>; + }; + }; + + rumble { + compatible = "pwm-vibrator"; + pwms = <&pwm 4 2000000 0>; + pwm-names = "enable"; + + pinctrl-names = "default"; + pinctrl-0 = <&pins_pwm4>; + }; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pwm 1 40000 0>; + power-supply = <&vcc>; + + brightness-levels = <0 16 32 48 64 80 96 112 128 + 144 160 176 192 208 224 240 255>; + default-brightness-level = <12>; + + pinctrl-names = "default"; + pinctrl-0 = <&pins_pwm1>; + }; + + gpio-keys { + compatible = "gpio-keys"; + #address-cells = <1>; + #size-cells = <0>; + + autorepeat; + + button@0 { + label = "D-pad up"; + linux,code = ; + linux,can-disable; + gpios = <&gpe 21 GPIO_ACTIVE_LOW>; + }; + + button@1 { + label = "D-pad down"; + linux,code = ; + linux,can-disable; + gpios = <&gpe 25 GPIO_ACTIVE_LOW>; + }; + + button@2 { + label = "D-pad left"; + linux,code = ; + linux,can-disable; + gpios = <&gpe 23 GPIO_ACTIVE_LOW>; + }; + + button@3 { + label = "D-pad right"; + linux,code = ; + linux,can-disable; + gpios = <&gpe 24 GPIO_ACTIVE_LOW>; + }; + + button@4 { + label = "Button A"; + linux,code = ; + linux,can-disable; + gpios = <&gpe 29 GPIO_ACTIVE_LOW>; + }; + + button@5 { + label = "Button B"; + linux,code = ; + linux,can-disable; + gpios = <&gpe 20 GPIO_ACTIVE_LOW>; + }; + + button@6 { + label = "Button Y"; + linux,code = ; + linux,can-disable; + gpios = <&gpe 27 GPIO_ACTIVE_LOW>; + }; + + button@7 { + label = "Button X"; + linux,code = ; + linux,can-disable; + gpios = <&gpe 28 GPIO_ACTIVE_LOW>; + }; + + button@8 { + label = "Left shoulder button"; + linux,code = ; + linux,can-disable; + gpios = <&gpb 20 GPIO_ACTIVE_LOW>; + }; + + button@9 { + label = "Right shoulder button"; + linux,code = ; + linux,can-disable; + gpios = <&gpe 26 GPIO_ACTIVE_LOW>; + }; + + button@10 { + label = "Start button"; + linux,code = ; + linux,can-disable; + gpios = <&gpb 21 GPIO_ACTIVE_LOW>; + }; + + button@11 { + label = "Select button"; + linux,code = ; + linux,can-disable; + /* + * This is the only button that is active high, + * since it doubles as BOOT_SEL1. + */ + gpios = <&gpd 18 GPIO_ACTIVE_HIGH>; + }; + + button@12 { + label = "Power slider"; + linux,code = ; + linux,can-disable; + gpios = <&gpa 30 GPIO_ACTIVE_LOW>; + wakeup-source; + }; + + button@13 { + label = "Power hold"; + linux,code = ; + linux,can-disable; + gpios = <&gpf 11 GPIO_ACTIVE_LOW>; + }; + }; + + i2c3: i2c-controller@3 { + compatible = "i2c-gpio"; + #address-cells = <1>; + #size-cells = <0>; + + sda-gpios = <&gpd 5 GPIO_ACTIVE_HIGH>; + scl-gpios = <&gpd 4 GPIO_ACTIVE_HIGH>; + i2c-gpio,delay-us = <2>; /* 250 kHz */ + + act8600: pmic@5a { + compatible = "active-semi,act8600"; + reg = <0x5a>; + + regulators { + /* USB OTG */ + otg_vbus: SUDCDC_REG4 { + /* + * 5.3V instead of 5.0V to compensate + * for the voltage drop of a diode + * between the regulator and the + * connector. + */ + regulator-min-microvolt = <5300000>; + regulator-max-microvolt = <5300000>; + inl-supply = <&vcc>; + }; + + /* + * When this is off, there is no sound, but also + * no USB networking. + */ + ldo5: LDO5 { + regulator-min-microvolt = <2500000>; + regulator-max-microvolt = <2500000>; + inl-supply = <&vcc>; + }; + + /* LCD panel and FM radio */ + ldo6: LDO6 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + inl-supply = <&vcc>; + }; + + /* ??? */ + LDO7 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + /*regulator-always-on;*/ + inl-supply = <&vcc>; + }; + + /* + * The colors on the LCD are wrong when this is + * off. Which is strange, since the LCD panel + * data sheet only mentions a 3.3V input. + */ + LDO8 { + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + regulator-always-on; + inl-supply = <&vcc>; + }; + + /* RTC fixed 3.3V */ + LDO_REG9 { + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + inl-supply = <&vcc>; + }; + + /* Unused fixed 1.2V */ + LDO_REG10 { + inl-supply = <&vcc>; + }; + }; + }; + }; + + leds { + compatible = "gpio-leds"; + + led { + gpios = <&gpb 30 GPIO_ACTIVE_LOW>; + default-state = "on"; + }; + }; + + spi { + compatible = "spi-gpio"; + #address-cells = <1>; + #size-cells = <0>; + + sck-gpios = <&gpe 15 GPIO_ACTIVE_HIGH>; + mosi-gpios = <&gpe 17 GPIO_ACTIVE_HIGH>; + cs-gpios = <&gpe 16 GPIO_ACTIVE_HIGH>; + num-chipselects = <1>; + + nt39016@0 { + compatible = "kingdisplay,kd035g6-54nt"; + reg = <0>; + + spi-max-frequency = <3125000>; + spi-3wire; + spi-cs-high; + + reset-gpios = <&gpe 2 GPIO_ACTIVE_LOW>; + + backlight = <&backlight>; + power-supply = <&ldo6>; + + port { + panel_input: endpoint { + remote-endpoint = <&panel_output>; + }; + }; + }; + }; + + connector { + compatible = "gpio-usb-b-connector", "usb-b-connector"; + label = "mini-USB"; + type = "mini"; + + /* + * USB OTG is not yet working reliably, the ID detection + * mechanism tends to fry easily for unknown reasons. + * Until this is fixed, disable OTG by not providing the + * ID GPIO to the driver. + */ + //id-gpios = <&gpf 18 GPIO_ACTIVE_LOW>; + + vbus-gpios = <&gpb 5 GPIO_ACTIVE_HIGH>; + vbus-supply = <&otg_vbus>; + + pinctrl-names = "default"; + pinctrl-0 = <&pins_otg>; + + port { + usb_ep: endpoint { + remote-endpoint = <&usb_otg_ep>; + }; }; }; }; @@ -37,24 +385,86 @@ clock-frequency = <12000000>; }; +&pinctrl { + pins_lcd: lcd { + function = "lcd"; + groups = "lcd-24bit"; + }; + + pins_uart2: uart2 { + function = "uart2"; + groups = "uart2-data"; + }; + + pins_mmc0: mmc0 { + function = "mmc0"; + groups = "mmc0-1bit-a", "mmc0-4bit-a"; + }; + + pins_mmc1: mmc1 { + function = "mmc1"; + groups = "mmc1-1bit-d", "mmc1-4bit-d"; + }; + + pins_otg: otg { + otg-vbus-pin { + function = "otg"; + groups = "otg-vbus"; + }; + + vbus-pin { + pins = "PB5"; + bias-disable; + }; + }; + + pins_pwm1: pwm1 { + function = "pwm1"; + groups = "pwm1"; + }; + + pins_pwm4: pwm4 { + function = "pwm4"; + groups = "pwm4"; + }; +}; + &uart2 { + pinctrl-names = "default"; + pinctrl-0 = <&pins_uart2>; + status = "okay"; }; &cgu { - /* Put high-speed peripherals under PLL1, such that we can change the + /* + * Put high-speed peripherals under PLL1, such that we can change the * PLL0 frequency on demand without having to suspend peripherals. * We use a rate of 432 MHz, which is the least common multiple of * 27 MHz (required by TV encoder) and 48 MHz (required by USB host). + * Put the GPU under PLL0 since we want a higher frequency. + * Use the 32 kHz oscillator as the parent of the RTC for a higher + * precision. */ assigned-clocks = <&cgu JZ4770_CLK_PLL1>, - <&cgu JZ4770_CLK_UHC>; + <&cgu JZ4770_CLK_GPU>, + <&cgu JZ4770_CLK_RTC>, + <&cgu JZ4770_CLK_UHC>, + <&cgu JZ4770_CLK_LPCLK_MUX>, + <&cgu JZ4770_CLK_MMC0_MUX>, + <&cgu JZ4770_CLK_MMC1_MUX>; assigned-clock-parents = <0>, + <&cgu JZ4770_CLK_PLL0>, + <&cgu JZ4770_CLK_OSC32K>, + <&cgu JZ4770_CLK_PLL1>, + <&cgu JZ4770_CLK_PLL1>, + <&cgu JZ4770_CLK_PLL1>, <&cgu JZ4770_CLK_PLL1>; assigned-clock-rates = - <432000000>; + <432000000>, + <600000000>; }; &uhc { @@ -63,10 +473,69 @@ }; &tcu { - /* 750 kHz for the system timer and clocksource */ - assigned-clocks = <&tcu TCU_CLK_TIMER0>, <&tcu TCU_CLK_TIMER2>; - assigned-clock-rates = <750000>, <750000>; + /* + * 750 kHz for the system timer and clocksource, 12 MHz for the OST, + * and use RTC as the parent for the watchdog clock + */ + assigned-clocks = <&tcu TCU_CLK_TIMER0>, <&tcu TCU_CLK_TIMER2>, + <&tcu TCU_CLK_OST>, <&tcu TCU_CLK_WDT>; + assigned-clock-parents = <0>, <0>, <0>, <&cgu JZ4770_CLK_RTC>; + assigned-clock-rates = <750000>, <750000>, <12000000>; - /* PWM1 is in use, so reserve channel #2 for the clocksource */ + /* PWM1 is in use, so use channel #2 for the clocksource */ ingenic,pwm-channels-mask = <0xfa>; }; + +&usb_otg { + port { + usb_otg_ep: endpoint { + remote-endpoint = <&usb_ep>; + }; + }; +}; + +&otg_phy { + vcc-supply = <&ldo5>; +}; + +&rtc { + clocks = <&cgu JZ4770_CLK_RTC>; + clock-names = "rtc"; + + system-power-controller; +}; + +&mmc0 { + status = "okay"; + + bus-width = <4>; + max-frequency = <48000000>; + vmmc-supply = <&vcc>; + non-removable; + + pinctrl-names = "default"; + pinctrl-0 = <&pins_mmc0>; +}; + +&mmc1 { + status = "okay"; + + bus-width = <4>; + max-frequency = <48000000>; + cd-gpios = <&gpb 2 GPIO_ACTIVE_LOW>; + vmmc-supply = <&mmc1_power>; + + pinctrl-names = "default"; + pinctrl-0 = <&pins_mmc1>; +}; + +&lcd { + pinctrl-names = "default"; + pinctrl-0 = <&pins_lcd>; + + port { + panel_output: endpoint { + remote-endpoint = <&panel_input>; + }; + }; +}; From 818c2b36383e3ef421b9b435a1215a50fc163f82 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Mon, 13 Apr 2020 17:26:31 +0200 Subject: [PATCH 0169/1043] MIPS: ingenic: CI20: enable OST, PWM drivers in defconfig The OST driver provides a clocksource and sched_clock that are much more accurate than the default ones. The PWM driver allows to use the PWM pins on the external header of the board. Signed-off-by: Paul Cercueil Signed-off-by: Thomas Bogendoerfer --- arch/mips/configs/ci20_defconfig | 62 +++++++++++++++++--------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/arch/mips/configs/ci20_defconfig b/arch/mips/configs/ci20_defconfig index 0db0088bbc1c..e511fe0243a5 100644 --- a/arch/mips/configs/ci20_defconfig +++ b/arch/mips/configs/ci20_defconfig @@ -1,5 +1,4 @@ # CONFIG_LOCALVERSION_AUTO is not set -CONFIG_MODULES=y CONFIG_KERNEL_XZ=y CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y @@ -29,6 +28,7 @@ CONFIG_HIGHMEM=y CONFIG_HZ_100=y # CONFIG_SECCOMP is not set # CONFIG_SUSPEND is not set +CONFIG_MODULES=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set # CONFIG_COMPACTION is not set CONFIG_CMA=y @@ -38,17 +38,12 @@ CONFIG_UNIX=y CONFIG_INET=y CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set CONFIG_DEVTMPFS=y # CONFIG_FW_LOADER is not set # CONFIG_ALLOW_DEV_COREDUMP is not set -CONFIG_DMA_CMA=y -CONFIG_CMA_SIZE_MBYTES=32 CONFIG_MTD=y CONFIG_MTD_RAW_NAND=y CONFIG_MTD_NAND_JZ4780=y @@ -72,9 +67,8 @@ CONFIG_DM9000_FORCE_SIMPLE_PHY_POLL=y # CONFIG_NET_VENDOR_VIA is not set # CONFIG_NET_VENDOR_WIZNET is not set # CONFIG_WLAN is not set -# CONFIG_INPUT_KEYBOARD is not set +CONFIG_KEYBOARD_GPIO=m # CONFIG_INPUT_MOUSE is not set -# CONFIG_SERIO is not set CONFIG_VT_HW_CONSOLE_BINDING=y CONFIG_LEGACY_PTY_COUNT=2 CONFIG_SERIAL_8250=y @@ -89,7 +83,7 @@ CONFIG_I2C_JZ4780=y CONFIG_SPI=y CONFIG_SPI_GPIO=y CONFIG_GPIO_SYSFS=y -CONFIG_KEYBOARD_GPIO=m +CONFIG_POWER_SUPPLY=y # CONFIG_HWMON is not set CONFIG_WATCHDOG=y CONFIG_JZ4740_WDT=y @@ -97,17 +91,45 @@ CONFIG_REGULATOR=y CONFIG_REGULATOR_DEBUG=y CONFIG_REGULATOR_FIXED_VOLTAGE=y CONFIG_REGULATOR_ACT8865=y +CONFIG_RC_CORE=m +CONFIG_LIRC=y +CONFIG_RC_DEVICES=y +CONFIG_IR_GPIO_CIR=m +CONFIG_IR_GPIO_TX=m +CONFIG_MEDIA_SUPPORT=m # CONFIG_VGA_CONSOLE is not set # CONFIG_HID is not set # CONFIG_USB_SUPPORT is not set CONFIG_MMC=y CONFIG_MMC_JZ4740=y +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_TRIGGERS=y +CONFIG_LEDS_TRIGGER_TIMER=y +CONFIG_LEDS_TRIGGER_ONESHOT=y +CONFIG_LEDS_TRIGGER_MTD=y +CONFIG_LEDS_TRIGGER_HEARTBEAT=y +CONFIG_LEDS_TRIGGER_BACKLIGHT=m +CONFIG_LEDS_TRIGGER_CPU=y +CONFIG_LEDS_TRIGGER_ACTIVITY=y +CONFIG_LEDS_TRIGGER_GPIO=y +CONFIG_LEDS_TRIGGER_DEFAULT_ON=y +CONFIG_LEDS_TRIGGER_TRANSIENT=y +CONFIG_LEDS_TRIGGER_CAMERA=m +CONFIG_LEDS_TRIGGER_PANIC=y +CONFIG_LEDS_TRIGGER_NETDEV=y +CONFIG_LEDS_TRIGGER_PATTERN=y +CONFIG_LEDS_TRIGGER_AUDIO=y CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_JZ4740=y CONFIG_DMADEVICES=y CONFIG_DMA_JZ4780=y +CONFIG_INGENIC_OST=y # CONFIG_IOMMU_SUPPORT is not set CONFIG_MEMORY=y +CONFIG_PWM=y +CONFIG_PWM_JZ4740=m CONFIG_EXT4_FS=y # CONFIG_DNOTIFY is not set CONFIG_PROC_KCORE=y @@ -156,11 +178,13 @@ CONFIG_NLS_ISO8859_15=y CONFIG_NLS_KOI8_R=y CONFIG_NLS_KOI8_U=y CONFIG_NLS_UTF8=y +CONFIG_DMA_CMA=y +CONFIG_CMA_SIZE_MBYTES=32 CONFIG_PRINTK_TIME=y CONFIG_DEBUG_INFO=y CONFIG_STRIP_ASM_SYMS=y -CONFIG_DEBUG_FS=y CONFIG_MAGIC_SYSRQ=y +CONFIG_DEBUG_FS=y CONFIG_PANIC_ON_OOPS=y CONFIG_PANIC_TIMEOUT=10 # CONFIG_SCHED_DEBUG is not set @@ -169,21 +193,3 @@ CONFIG_STACKTRACE=y # CONFIG_FTRACE is not set CONFIG_CMDLINE_BOOL=y CONFIG_CMDLINE="earlycon console=ttyS4,115200 clk_ignore_unused" -CONFIG_LEDS_CLASS=y -CONFIG_LEDS_GPIO=y -CONFIG_LEDS_TRIGGERS=y -CONFIG_LEDS_TRIGGER_MTD=y -CONFIG_LEDS_TRIGGER_TIMER=y -CONFIG_LEDS_TRIGGER_ONESHOT=y -CONFIG_LEDS_TRIGGER_ONESHOT=y -CONFIG_LEDS_TRIGGER_HEARTBEAT=y -CONFIG_LEDS_TRIGGER_BACKLIGHT=m -CONFIG_LEDS_TRIGGER_CPU=y -CONFIG_LEDS_TRIGGER_DEFAULT_ON=y -CONFIG_LEDS_TRIGGER_TRANSIENT=y -CONFIG_LEDS_TRIGGER_CAMERA=m -CONFIG_LIRC=y -CONFIG_MEDIA_SUPPORT=m -CONFIG_RC_DEVICES=y -CONFIG_IR_GPIO_CIR=m -CONFIG_IR_GPIO_TX=m From 8827af9427dea9432d5b9c5366c7312e7f647484 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Mon, 13 Apr 2020 17:26:32 +0200 Subject: [PATCH 0170/1043] MIPS: ingenic: GCW0: Update defconfig Enable support for the new hardware that was added in the devicetree. Signed-off-by: Paul Cercueil Signed-off-by: Thomas Bogendoerfer --- arch/mips/configs/gcw0_defconfig | 131 ++++++++++++++++++++++++++++++- 1 file changed, 128 insertions(+), 3 deletions(-) diff --git a/arch/mips/configs/gcw0_defconfig b/arch/mips/configs/gcw0_defconfig index a3e3eb3c5a8b..48131cb47e66 100644 --- a/arch/mips/configs/gcw0_defconfig +++ b/arch/mips/configs/gcw0_defconfig @@ -1,27 +1,152 @@ +CONFIG_DEFAULT_HOSTNAME="gcw0" CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT_VOLUNTARY=y CONFIG_EMBEDDED=y +CONFIG_PROFILING=y CONFIG_MACH_INGENIC=y CONFIG_JZ4770_GCW0=y CONFIG_HIGHMEM=y # CONFIG_SECCOMP is not set -# CONFIG_SUSPEND is not set +CONFIG_MIPS_RAW_APPENDED_DTB=y +CONFIG_MIPS_CMDLINE_DTB_EXTEND=y +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set # CONFIG_BOUNCE is not set CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y +CONFIG_CFG80211=y +CONFIG_MAC80211=m CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_BLK_DEV_LOOP_MIN_COUNT=0 CONFIG_NETDEVICES=y +# CONFIG_ETHERNET is not set +# CONFIG_WLAN_VENDOR_ADMTEK is not set +# CONFIG_WLAN_VENDOR_ATH is not set +# CONFIG_WLAN_VENDOR_ATMEL is not set +# CONFIG_WLAN_VENDOR_BROADCOM is not set +# CONFIG_WLAN_VENDOR_CISCO is not set +# CONFIG_WLAN_VENDOR_INTEL is not set +# CONFIG_WLAN_VENDOR_INTERSIL is not set +# CONFIG_WLAN_VENDOR_MARVELL is not set +# CONFIG_WLAN_VENDOR_MEDIATEK is not set +# CONFIG_WLAN_VENDOR_RALINK is not set +CONFIG_RTL8192CU=m +# CONFIG_RTLWIFI_DEBUG is not set +# CONFIG_WLAN_VENDOR_RSI is not set +# CONFIG_WLAN_VENDOR_ST is not set +# CONFIG_WLAN_VENDOR_TI is not set +# CONFIG_WLAN_VENDOR_ZYDAS is not set +# CONFIG_WLAN_VENDOR_QUANTENNA is not set +CONFIG_INPUT_EVDEV=y +# CONFIG_KEYBOARD_ATKBD is not set +CONFIG_KEYBOARD_GPIO=y +# CONFIG_INPUT_MOUSE is not set +CONFIG_INPUT_JOYSTICK=y +CONFIG_INPUT_MISC=y +CONFIG_INPUT_UINPUT=y +CONFIG_INPUT_PWM_VIBRA=y +# CONFIG_SERIO is not set CONFIG_SERIAL_8250=y # CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set CONFIG_SERIAL_8250_CONSOLE=y CONFIG_SERIAL_8250_INGENIC=y +CONFIG_HW_RANDOM=y +CONFIG_I2C_GPIO=y +CONFIG_SPI=y +CONFIG_SPI_GPIO=y +CONFIG_POWER_SUPPLY=y +# CONFIG_HWMON is not set +CONFIG_WATCHDOG=y +CONFIG_JZ4740_WDT=y +CONFIG_REGULATOR=y +CONFIG_REGULATOR_FIXED_VOLTAGE=y +CONFIG_REGULATOR_ACT8865=y +CONFIG_DRM=y +CONFIG_DRM_FBDEV_OVERALLOC=300 +CONFIG_DRM_PANEL_NOVATEK_NT39016=y +CONFIG_DRM_INGENIC=y +CONFIG_DRM_ETNAVIV=y +CONFIG_BACKLIGHT_CLASS_DEVICE=y +# CONFIG_BACKLIGHT_GENERIC is not set +CONFIG_BACKLIGHT_PWM=y +# CONFIG_VGA_CONSOLE is not set +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_LOGO=y +# CONFIG_LOGO_LINUX_MONO is not set +# CONFIG_LOGO_LINUX_VGA16 is not set +CONFIG_SOUND=y +CONFIG_SND=y +# CONFIG_SND_SUPPORT_OLD_API is not set +# CONFIG_SND_PROC_FS is not set +# CONFIG_SND_DRIVERS is not set +# CONFIG_SND_SPI is not set +# CONFIG_SND_MIPS is not set +# CONFIG_SND_USB is not set +CONFIG_SND_SOC=y +CONFIG_SND_JZ4740_SOC_I2S=y +CONFIG_SND_SOC_JZ4770_CODEC=y +CONFIG_SND_SOC_SIMPLE_AMPLIFIER=y +CONFIG_SND_SIMPLE_CARD=y +CONFIG_USB_CONN_GPIO=y CONFIG_USB=y +CONFIG_USB_OTG=y +CONFIG_USB_OTG_BLACKLIST_HUB=y CONFIG_USB_OHCI_HCD=y CONFIG_USB_OHCI_HCD_PLATFORM=y -CONFIG_NOP_USB_XCEIV=y -CONFIG_TMPFS=y +CONFIG_USB_MUSB_HDRC=y +CONFIG_USB_MUSB_GADGET=y +CONFIG_USB_MUSB_JZ4740=y +CONFIG_USB_INVENTRA_DMA=y +CONFIG_JZ4770_PHY=y +CONFIG_USB_GADGET=y +CONFIG_USB_GADGET_VBUS_DRAW=500 +CONFIG_USB_ETH=y +CONFIG_MMC=y +# CONFIG_PWRSEQ_EMMC is not set +# CONFIG_PWRSEQ_SIMPLE is not set +CONFIG_MMC_JZ4740=y +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_GPIO=y +CONFIG_LEDS_TRIGGERS=y +CONFIG_LEDS_TRIGGER_TIMER=y +CONFIG_LEDS_TRIGGER_DEFAULT_ON=y +CONFIG_LEDS_TRIGGER_PANIC=y +CONFIG_RTC_CLASS=y +# CONFIG_RTC_NVMEM is not set +CONFIG_RTC_DRV_JZ4740=y +CONFIG_DMADEVICES=y +CONFIG_DMA_JZ4780=y +# CONFIG_VIRTIO_MENU is not set +CONFIG_STAGING=y +CONFIG_R8188EU=m +CONFIG_INGENIC_OST=y +# CONFIG_IOMMU_SUPPORT is not set +CONFIG_IIO=y +CONFIG_IIO_BUFFER=y +CONFIG_IIO_BUFFER_CB=y +CONFIG_IIO_KFIFO_BUF=y +CONFIG_MXC6255=m +CONFIG_INGENIC_ADC=y +CONFIG_PWM=y +CONFIG_PWM_JZ4740=y +CONFIG_EXT4_FS=y +CONFIG_VFAT_FS=y +CONFIG_SQUASHFS=y +CONFIG_SQUASHFS_FILE_DIRECT=y +CONFIG_SQUASHFS_DECOMP_MULTI=y +# CONFIG_NETWORK_FILESYSTEMS is not set +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ISO8859_1=y +CONFIG_NLS_UTF8=y +CONFIG_LSM="lockdown,yama,loadpin,safesetid,integrity" +CONFIG_CRYPTO_ECHAINIV=y +CONFIG_FONTS=y +CONFIG_FONT_6x10=y +CONFIG_DEBUG_FS=y From f932449c11dabb4b0588e30f35c084b910483ca5 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Mon, 13 Apr 2020 17:26:33 +0200 Subject: [PATCH 0171/1043] MIPS: ingenic: Drop obsolete code, merge the rest in setup.c Drop a bootload of 10-years-old dirty code, that is not used anymore, as it has been replaced with clean code over the ages. Merge the very few bits left inside setup.c, so that everything is clean and tidy now. Signed-off-by: Paul Cercueil Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mach-jz4740/base.h | 27 ----- arch/mips/include/asm/mach-jz4740/dma.h | 23 ---- arch/mips/include/asm/mach-jz4740/irq.h | 45 +------- arch/mips/include/asm/mach-jz4740/timer.h | 126 ---------------------- arch/mips/jz4740/Makefile | 7 +- arch/mips/jz4740/pm.c | 34 ------ arch/mips/jz4740/prom.c | 19 ---- arch/mips/jz4740/reset.c | 24 ----- arch/mips/jz4740/reset.h | 7 -- arch/mips/jz4740/setup.c | 63 ++++++++++- arch/mips/jz4740/time.c | 17 --- arch/mips/jz4740/timer.c | 42 -------- 12 files changed, 61 insertions(+), 373 deletions(-) delete mode 100644 arch/mips/include/asm/mach-jz4740/base.h delete mode 100644 arch/mips/include/asm/mach-jz4740/dma.h delete mode 100644 arch/mips/include/asm/mach-jz4740/timer.h delete mode 100644 arch/mips/jz4740/pm.c delete mode 100644 arch/mips/jz4740/prom.c delete mode 100644 arch/mips/jz4740/reset.c delete mode 100644 arch/mips/jz4740/reset.h delete mode 100644 arch/mips/jz4740/time.c delete mode 100644 arch/mips/jz4740/timer.c diff --git a/arch/mips/include/asm/mach-jz4740/base.h b/arch/mips/include/asm/mach-jz4740/base.h deleted file mode 100644 index 96b2d6674cdb..000000000000 --- a/arch/mips/include/asm/mach-jz4740/base.h +++ /dev/null @@ -1,27 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ASM_MACH_JZ4740_BASE_H__ -#define __ASM_MACH_JZ4740_BASE_H__ - -#define JZ4740_CPM_BASE_ADDR 0x10000000 -#define JZ4740_INTC_BASE_ADDR 0x10001000 -#define JZ4740_WDT_BASE_ADDR 0x10002000 -#define JZ4740_TCU_BASE_ADDR 0x10002010 -#define JZ4740_RTC_BASE_ADDR 0x10003000 -#define JZ4740_GPIO_BASE_ADDR 0x10010000 -#define JZ4740_AIC_BASE_ADDR 0x10020000 -#define JZ4740_MSC_BASE_ADDR 0x10021000 -#define JZ4740_UART0_BASE_ADDR 0x10030000 -#define JZ4740_UART1_BASE_ADDR 0x10031000 -#define JZ4740_I2C_BASE_ADDR 0x10042000 -#define JZ4740_SSI_BASE_ADDR 0x10043000 -#define JZ4740_SADC_BASE_ADDR 0x10070000 -#define JZ4740_EMC_BASE_ADDR 0x13010000 -#define JZ4740_DMAC_BASE_ADDR 0x13020000 -#define JZ4740_UHC_BASE_ADDR 0x13030000 -#define JZ4740_UDC_BASE_ADDR 0x13040000 -#define JZ4740_LCD_BASE_ADDR 0x13050000 -#define JZ4740_SLCD_BASE_ADDR 0x13050000 -#define JZ4740_CIM_BASE_ADDR 0x13060000 -#define JZ4740_IPU_BASE_ADDR 0x13080000 - -#endif diff --git a/arch/mips/include/asm/mach-jz4740/dma.h b/arch/mips/include/asm/mach-jz4740/dma.h deleted file mode 100644 index e5d2a5311a3a..000000000000 --- a/arch/mips/include/asm/mach-jz4740/dma.h +++ /dev/null @@ -1,23 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2010, Lars-Peter Clausen - * JZ7420/JZ4740 DMA definitions - */ - -#ifndef __ASM_MACH_JZ4740_DMA_H__ -#define __ASM_MACH_JZ4740_DMA_H__ - -enum jz4740_dma_request_type { - JZ4740_DMA_TYPE_AUTO_REQUEST = 8, - JZ4740_DMA_TYPE_UART_TRANSMIT = 20, - JZ4740_DMA_TYPE_UART_RECEIVE = 21, - JZ4740_DMA_TYPE_SPI_TRANSMIT = 22, - JZ4740_DMA_TYPE_SPI_RECEIVE = 23, - JZ4740_DMA_TYPE_MMC_TRANSMIT = 26, - JZ4740_DMA_TYPE_MMC_RECEIVE = 27, - JZ4740_DMA_TYPE_TCU = 28, - JZ4740_DMA_TYPE_SADC = 29, - JZ4740_DMA_TYPE_SLCD = 30, -}; - -#endif /* __ASM_JZ4740_DMA_H__ */ diff --git a/arch/mips/include/asm/mach-jz4740/irq.h b/arch/mips/include/asm/mach-jz4740/irq.h index 09c38eac671a..27c543bd340f 100644 --- a/arch/mips/include/asm/mach-jz4740/irq.h +++ b/arch/mips/include/asm/mach-jz4740/irq.h @@ -8,49 +8,6 @@ #define __ASM_MACH_JZ4740_IRQ_H__ #define MIPS_CPU_IRQ_BASE 0 -#define JZ4740_IRQ_BASE 8 - -#ifdef CONFIG_MACH_JZ4740 -# define NR_INTC_IRQS 32 -#else -# define NR_INTC_IRQS 64 -#endif - -/* 1st-level interrupts */ -#define JZ4740_IRQ(x) (JZ4740_IRQ_BASE + (x)) -#define JZ4740_IRQ_I2C JZ4740_IRQ(1) -#define JZ4740_IRQ_UHC JZ4740_IRQ(3) -#define JZ4740_IRQ_UART1 JZ4740_IRQ(8) -#define JZ4740_IRQ_UART0 JZ4740_IRQ(9) -#define JZ4740_IRQ_SADC JZ4740_IRQ(12) -#define JZ4740_IRQ_MSC JZ4740_IRQ(14) -#define JZ4740_IRQ_RTC JZ4740_IRQ(15) -#define JZ4740_IRQ_SSI JZ4740_IRQ(16) -#define JZ4740_IRQ_CIM JZ4740_IRQ(17) -#define JZ4740_IRQ_AIC JZ4740_IRQ(18) -#define JZ4740_IRQ_ETH JZ4740_IRQ(19) -#define JZ4740_IRQ_DMAC JZ4740_IRQ(20) -#define JZ4740_IRQ_TCU2 JZ4740_IRQ(21) -#define JZ4740_IRQ_TCU1 JZ4740_IRQ(22) -#define JZ4740_IRQ_TCU0 JZ4740_IRQ(23) -#define JZ4740_IRQ_UDC JZ4740_IRQ(24) -#define JZ4740_IRQ_GPIO3 JZ4740_IRQ(25) -#define JZ4740_IRQ_GPIO2 JZ4740_IRQ(26) -#define JZ4740_IRQ_GPIO1 JZ4740_IRQ(27) -#define JZ4740_IRQ_GPIO0 JZ4740_IRQ(28) -#define JZ4740_IRQ_IPU JZ4740_IRQ(29) -#define JZ4740_IRQ_LCD JZ4740_IRQ(30) - -#define JZ4780_IRQ_TCU2 JZ4740_IRQ(25) - -/* 2nd-level interrupts */ -#define JZ4740_IRQ_DMA(x) (JZ4740_IRQ(NR_INTC_IRQS) + (x)) - -#define JZ4740_IRQ_INTC_GPIO(x) (JZ4740_IRQ_GPIO0 - (x)) -#define JZ4740_IRQ_GPIO(x) (JZ4740_IRQ(NR_INTC_IRQS + 16) + (x)) - -#define JZ4740_IRQ_ADC_BASE JZ4740_IRQ(NR_INTC_IRQS + 144) - -#define NR_IRQS (JZ4740_IRQ_ADC_BASE + 6) +#define NR_IRQS 256 #endif diff --git a/arch/mips/include/asm/mach-jz4740/timer.h b/arch/mips/include/asm/mach-jz4740/timer.h deleted file mode 100644 index 8a19cfe5bed7..000000000000 --- a/arch/mips/include/asm/mach-jz4740/timer.h +++ /dev/null @@ -1,126 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2010, Lars-Peter Clausen - * JZ4740 platform timer support - */ - -#ifndef __ASM_MACH_JZ4740_TIMER -#define __ASM_MACH_JZ4740_TIMER - -#define JZ_REG_TIMER_STOP 0x0C -#define JZ_REG_TIMER_STOP_SET 0x1C -#define JZ_REG_TIMER_STOP_CLEAR 0x2C -#define JZ_REG_TIMER_ENABLE 0x00 -#define JZ_REG_TIMER_ENABLE_SET 0x04 -#define JZ_REG_TIMER_ENABLE_CLEAR 0x08 -#define JZ_REG_TIMER_FLAG 0x10 -#define JZ_REG_TIMER_FLAG_SET 0x14 -#define JZ_REG_TIMER_FLAG_CLEAR 0x18 -#define JZ_REG_TIMER_MASK 0x20 -#define JZ_REG_TIMER_MASK_SET 0x24 -#define JZ_REG_TIMER_MASK_CLEAR 0x28 - -#define JZ_REG_TIMER_DFR(x) (((x) * 0x10) + 0x30) -#define JZ_REG_TIMER_DHR(x) (((x) * 0x10) + 0x34) -#define JZ_REG_TIMER_CNT(x) (((x) * 0x10) + 0x38) -#define JZ_REG_TIMER_CTRL(x) (((x) * 0x10) + 0x3C) - -#define JZ_TIMER_IRQ_HALF(x) BIT((x) + 0x10) -#define JZ_TIMER_IRQ_FULL(x) BIT(x) - -#define JZ_TIMER_CTRL_PWM_ABBRUPT_SHUTDOWN BIT(9) -#define JZ_TIMER_CTRL_PWM_ACTIVE_LOW BIT(8) -#define JZ_TIMER_CTRL_PWM_ENABLE BIT(7) -#define JZ_TIMER_CTRL_PRESCALE_MASK 0x1c -#define JZ_TIMER_CTRL_PRESCALE_OFFSET 0x3 -#define JZ_TIMER_CTRL_PRESCALE_1 (0 << 3) -#define JZ_TIMER_CTRL_PRESCALE_4 (1 << 3) -#define JZ_TIMER_CTRL_PRESCALE_16 (2 << 3) -#define JZ_TIMER_CTRL_PRESCALE_64 (3 << 3) -#define JZ_TIMER_CTRL_PRESCALE_256 (4 << 3) -#define JZ_TIMER_CTRL_PRESCALE_1024 (5 << 3) - -#define JZ_TIMER_CTRL_PRESCALER(x) ((x) << JZ_TIMER_CTRL_PRESCALE_OFFSET) - -#define JZ_TIMER_CTRL_SRC_EXT BIT(2) -#define JZ_TIMER_CTRL_SRC_RTC BIT(1) -#define JZ_TIMER_CTRL_SRC_PCLK BIT(0) - -extern void __iomem *jz4740_timer_base; -void __init jz4740_timer_init(void); - -void jz4740_timer_enable_watchdog(void); -void jz4740_timer_disable_watchdog(void); - -static inline void jz4740_timer_stop(unsigned int timer) -{ - writel(BIT(timer), jz4740_timer_base + JZ_REG_TIMER_STOP_SET); -} - -static inline void jz4740_timer_start(unsigned int timer) -{ - writel(BIT(timer), jz4740_timer_base + JZ_REG_TIMER_STOP_CLEAR); -} - -static inline bool jz4740_timer_is_enabled(unsigned int timer) -{ - return readb(jz4740_timer_base + JZ_REG_TIMER_ENABLE) & BIT(timer); -} - -static inline void jz4740_timer_enable(unsigned int timer) -{ - writeb(BIT(timer), jz4740_timer_base + JZ_REG_TIMER_ENABLE_SET); -} - -static inline void jz4740_timer_disable(unsigned int timer) -{ - writeb(BIT(timer), jz4740_timer_base + JZ_REG_TIMER_ENABLE_CLEAR); -} - -static inline void jz4740_timer_set_period(unsigned int timer, uint16_t period) -{ - writew(period, jz4740_timer_base + JZ_REG_TIMER_DFR(timer)); -} - -static inline void jz4740_timer_set_duty(unsigned int timer, uint16_t duty) -{ - writew(duty, jz4740_timer_base + JZ_REG_TIMER_DHR(timer)); -} - -static inline void jz4740_timer_set_count(unsigned int timer, uint16_t count) -{ - writew(count, jz4740_timer_base + JZ_REG_TIMER_CNT(timer)); -} - -static inline uint16_t jz4740_timer_get_count(unsigned int timer) -{ - return readw(jz4740_timer_base + JZ_REG_TIMER_CNT(timer)); -} - -static inline void jz4740_timer_ack_full(unsigned int timer) -{ - writel(JZ_TIMER_IRQ_FULL(timer), jz4740_timer_base + JZ_REG_TIMER_FLAG_CLEAR); -} - -static inline void jz4740_timer_irq_full_enable(unsigned int timer) -{ - writel(JZ_TIMER_IRQ_FULL(timer), jz4740_timer_base + JZ_REG_TIMER_FLAG_CLEAR); - writel(JZ_TIMER_IRQ_FULL(timer), jz4740_timer_base + JZ_REG_TIMER_MASK_CLEAR); -} - -static inline void jz4740_timer_irq_full_disable(unsigned int timer) -{ - writel(JZ_TIMER_IRQ_FULL(timer), jz4740_timer_base + JZ_REG_TIMER_MASK_SET); -} - -static inline void jz4740_timer_set_ctrl(unsigned int timer, uint16_t ctrl) -{ - writew(ctrl, jz4740_timer_base + JZ_REG_TIMER_CTRL(timer)); -} - -static inline uint16_t jz4740_timer_get_ctrl(unsigned int timer) -{ - return readw(jz4740_timer_base + JZ_REG_TIMER_CTRL(timer)); -} - -#endif diff --git a/arch/mips/jz4740/Makefile b/arch/mips/jz4740/Makefile index 6de14c0deb4e..f96c0f5eca44 100644 --- a/arch/mips/jz4740/Makefile +++ b/arch/mips/jz4740/Makefile @@ -4,11 +4,6 @@ # # Object file lists. - -obj-y += prom.o time.o reset.o setup.o timer.o +obj-y += setup.o CFLAGS_setup.o = -I$(src)/../../../scripts/dtc/libfdt - -# PM support - -obj-$(CONFIG_PM) += pm.o diff --git a/arch/mips/jz4740/pm.c b/arch/mips/jz4740/pm.c deleted file mode 100644 index f9b551f01f42..000000000000 --- a/arch/mips/jz4740/pm.c +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2010, Lars-Peter Clausen - * JZ4740 SoC power management support - */ - -#include -#include -#include -#include - -static int jz4740_pm_enter(suspend_state_t state) -{ - __asm__(".set\tmips3\n\t" - "wait\n\t" - ".set\tmips0"); - - - - return 0; -} - -static const struct platform_suspend_ops jz4740_pm_ops = { - .valid = suspend_valid_only_mem, - .enter = jz4740_pm_enter, -}; - -static int __init jz4740_pm_init(void) -{ - suspend_set_ops(&jz4740_pm_ops); - return 0; - -} -late_initcall(jz4740_pm_init); diff --git a/arch/mips/jz4740/prom.c b/arch/mips/jz4740/prom.c deleted file mode 100644 index ff4555c3fb15..000000000000 --- a/arch/mips/jz4740/prom.c +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2010, Lars-Peter Clausen - * JZ4740 SoC prom code - */ - -#include - -#include -#include - -void __init prom_init(void) -{ - fw_init_cmdline(); -} - -void __init prom_free_prom_memory(void) -{ -} diff --git a/arch/mips/jz4740/reset.c b/arch/mips/jz4740/reset.c deleted file mode 100644 index 1f9f02e54085..000000000000 --- a/arch/mips/jz4740/reset.c +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2010, Lars-Peter Clausen - */ - -#include - -#include "reset.h" - -static void jz4740_halt(void) -{ - while (1) { - __asm__(".set push;\n" - ".set mips3;\n" - "wait;\n" - ".set pop;\n" - ); - } -} - -void jz4740_reset_init(void) -{ - _machine_halt = jz4740_halt; -} diff --git a/arch/mips/jz4740/reset.h b/arch/mips/jz4740/reset.h deleted file mode 100644 index 4e8746ee9b61..000000000000 --- a/arch/mips/jz4740/reset.h +++ /dev/null @@ -1,7 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __MIPS_JZ4740_RESET_H__ -#define __MIPS_JZ4740_RESET_H__ - -extern void jz4740_reset_init(void); - -#endif diff --git a/arch/mips/jz4740/setup.c b/arch/mips/jz4740/setup.c index 880c26857aff..81428ddcaa97 100644 --- a/arch/mips/jz4740/setup.c +++ b/arch/mips/jz4740/setup.c @@ -5,17 +5,21 @@ * JZ4740 setup code */ +#include +#include #include #include #include #include #include #include +#include +#include #include +#include #include - -#include "reset.h" +#include #define JZ4740_EMC_BASE_ADDR 0x13010000 @@ -61,8 +65,6 @@ void __init plat_mem_setup(void) int offset; void *dtb; - jz4740_reset_init(); - if (__dtb_start != __dtb_end) dtb = __dtb_start; else @@ -105,3 +107,56 @@ void __init arch_init_irq(void) { irqchip_init(); } + +void __init plat_time_init(void) +{ + of_clk_init(NULL); + timer_probe(); +} + +void __init prom_init(void) +{ + fw_init_cmdline(); +} + +void __init prom_free_prom_memory(void) +{ +} + +static void jz4740_wait_instr(void) +{ + __asm__(".set push;\n" + ".set mips3;\n" + "wait;\n" + ".set pop;\n" + ); +} + +static void jz4740_halt(void) +{ + for (;;) + jz4740_wait_instr(); +} + +static int __maybe_unused jz4740_pm_enter(suspend_state_t state) +{ + jz4740_wait_instr(); + + return 0; +} + +static const struct platform_suspend_ops jz4740_pm_ops __maybe_unused = { + .valid = suspend_valid_only_mem, + .enter = jz4740_pm_enter, +}; + +static int __init jz4740_pm_init(void) +{ + if (IS_ENABLED(CONFIG_PM_SLEEP)) + suspend_set_ops(&jz4740_pm_ops); + _machine_halt = jz4740_halt; + + return 0; + +} +late_initcall(jz4740_pm_init); diff --git a/arch/mips/jz4740/time.c b/arch/mips/jz4740/time.c deleted file mode 100644 index 605a84a250bf..000000000000 --- a/arch/mips/jz4740/time.c +++ /dev/null @@ -1,17 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2010, Lars-Peter Clausen - * JZ4740 platform time support - */ - -#include -#include - -#include - -void __init plat_time_init(void) -{ - of_clk_init(NULL); - jz4740_timer_init(); - timer_probe(); -} diff --git a/arch/mips/jz4740/timer.c b/arch/mips/jz4740/timer.c deleted file mode 100644 index 5c9f82de6a82..000000000000 --- a/arch/mips/jz4740/timer.c +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2010, Lars-Peter Clausen - * JZ4740 platform timer support - */ - -#include -#include -#include -#include - -#include -#include - -void __iomem *jz4740_timer_base; -EXPORT_SYMBOL_GPL(jz4740_timer_base); - -void jz4740_timer_enable_watchdog(void) -{ - writel(BIT(16), jz4740_timer_base + JZ_REG_TIMER_STOP_CLEAR); -} -EXPORT_SYMBOL_GPL(jz4740_timer_enable_watchdog); - -void jz4740_timer_disable_watchdog(void) -{ - writel(BIT(16), jz4740_timer_base + JZ_REG_TIMER_STOP_SET); -} -EXPORT_SYMBOL_GPL(jz4740_timer_disable_watchdog); - -void __init jz4740_timer_init(void) -{ - jz4740_timer_base = ioremap(JZ4740_TCU_BASE_ADDR, 0x100); - - if (!jz4740_timer_base) - panic("Failed to ioremap timer registers"); - - /* Disable all timer clocks except for those used as system timers */ - writel(0x000100fc, jz4740_timer_base + JZ_REG_TIMER_STOP_SET); - - /* Timer irqs are unmasked by default, mask them */ - writel(0x00ff00ff, jz4740_timer_base + JZ_REG_TIMER_MASK_SET); -} From 44220fd84f3fa25208a441ae7a2cf0cf44699655 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Thu, 30 Apr 2020 20:36:24 +0800 Subject: [PATCH 0172/1043] MIPS: tools: Show result for loongson3-llsc-check It is better to show the result before loongson3-llsc-check exit, otherwise we can see nothing if the return status is EXIT_SUCCESS, it seems confusing. E.g. without this patch: [loongson@localhost tools]$ ./loongson3-llsc-check ../../../vmlinux [loongson@localhost tools]$ With this patch: [loongson@localhost tools]$ ./loongson3-llsc-check ../../../vmlinux loongson3-llsc-check returns success [loongson@localhost tools]$ Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/tools/loongson3-llsc-check.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/mips/tools/loongson3-llsc-check.c b/arch/mips/tools/loongson3-llsc-check.c index 0ebddd0ae46f..facd016dc51d 100644 --- a/arch/mips/tools/loongson3-llsc-check.c +++ b/arch/mips/tools/loongson3-llsc-check.c @@ -303,5 +303,7 @@ out_munmap: out_close: close(vmlinux_fd); out_ret: + fprintf(stdout, "loongson3-llsc-check %s\n", + status ? "returns failure" : "returns success"); return status; } From e9dfbaaeef1c9fee3f3a898defc4562db20c2edf Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Thu, 30 Apr 2020 17:45:16 +0800 Subject: [PATCH 0173/1043] MIPS: perf: Add hardware perf events support for new Loongson-3 New Loongson-3 means Loongson-3A R2 (Loongson-3A2000) and newer CPUs. Loongson-3 processors have three types of PMU types (so there are three event maps): Loongson-3A1000/Loonngson-3B1000/Loongson-3B1500 is Type-1, Loongson-3A2000/Loongson-3A3000 is Type-2, Loongson-3A4000+ is Type-3. Signed-off-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- .../mach-loongson64/cpu-feature-overrides.h | 1 + arch/mips/kernel/perf_event_mipsxx.c | 375 ++++++++++++++++-- 2 files changed, 342 insertions(+), 34 deletions(-) diff --git a/arch/mips/include/asm/mach-loongson64/cpu-feature-overrides.h b/arch/mips/include/asm/mach-loongson64/cpu-feature-overrides.h index 4fab38c743dd..b6e9c99b85a5 100644 --- a/arch/mips/include/asm/mach-loongson64/cpu-feature-overrides.h +++ b/arch/mips/include/asm/mach-loongson64/cpu-feature-overrides.h @@ -48,5 +48,6 @@ #define cpu_hwrena_impl_bits 0xc0000000 #define cpu_has_mac2008_only 1 #define cpu_has_mips_r2_exec_hazard 0 +#define cpu_has_perf_cntr_intr_bit 0 #endif /* __ASM_MACH_LOONGSON64_CPU_FEATURE_OVERRIDES_H */ diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c index 128fc9999c56..a14974ca6d13 100644 --- a/arch/mips/kernel/perf_event_mipsxx.c +++ b/arch/mips/kernel/perf_event_mipsxx.c @@ -90,6 +90,7 @@ struct mips_pmu { unsigned int num_counters; }; +static int counter_bits; static struct mips_pmu mipspmu; #define M_PERFCTL_EVENT(event) (((event) << MIPS_PERFCTRL_EVENT_S) & \ @@ -118,6 +119,7 @@ static struct mips_pmu mipspmu; #define M_PERFCTL_CONFIG_MASK 0x1f #endif +#define CNTR_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1)) #ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS static DEFINE_RWLOCK(pmuint_rwlock); @@ -154,6 +156,31 @@ static void pause_local_counters(void); static irqreturn_t mipsxx_pmu_handle_irq(int, void *); static int mipsxx_pmu_handle_shared_irq(void); +/* 0: Not Loongson-3 + * 1: Loongson-3A1000/3B1000/3B1500 + * 2: Loongson-3A2000/3A3000 + * 3: Loongson-3A4000+ + */ + +#define LOONGSON_PMU_TYPE0 0 +#define LOONGSON_PMU_TYPE1 1 +#define LOONGSON_PMU_TYPE2 2 +#define LOONGSON_PMU_TYPE3 3 + +static inline int get_loongson3_pmu_type(void) +{ + if (boot_cpu_type() != CPU_LOONGSON64) + return LOONGSON_PMU_TYPE0; + if ((boot_cpu_data.processor_id & PRID_COMP_MASK) == PRID_COMP_LEGACY) + return LOONGSON_PMU_TYPE1; + if ((boot_cpu_data.processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64C) + return LOONGSON_PMU_TYPE2; + if ((boot_cpu_data.processor_id & PRID_IMP_MASK) == PRID_IMP_LOONGSON_64G) + return LOONGSON_PMU_TYPE3; + + return LOONGSON_PMU_TYPE0; +} + static unsigned int mipsxx_pmu_swizzle_perf_idx(unsigned int idx) { if (vpe_id() == 1) @@ -186,17 +213,18 @@ static u64 mipsxx_pmu_read_counter(unsigned int idx) static u64 mipsxx_pmu_read_counter_64(unsigned int idx) { + u64 mask = CNTR_BIT_MASK(counter_bits); idx = mipsxx_pmu_swizzle_perf_idx(idx); switch (idx) { case 0: - return read_c0_perfcntr0_64(); + return read_c0_perfcntr0_64() & mask; case 1: - return read_c0_perfcntr1_64(); + return read_c0_perfcntr1_64() & mask; case 2: - return read_c0_perfcntr2_64(); + return read_c0_perfcntr2_64() & mask; case 3: - return read_c0_perfcntr3_64(); + return read_c0_perfcntr3_64() & mask; default: WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); return 0; @@ -225,6 +253,7 @@ static void mipsxx_pmu_write_counter(unsigned int idx, u64 val) static void mipsxx_pmu_write_counter_64(unsigned int idx, u64 val) { + val &= CNTR_BIT_MASK(counter_bits); idx = mipsxx_pmu_swizzle_perf_idx(idx); switch (idx) { @@ -286,12 +315,16 @@ static int mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc) { int i; + unsigned long cntr_mask; /* * We only need to care the counter mask. The range has been * checked definitely. */ - unsigned long cntr_mask = (hwc->event_base >> 8) & 0xffff; + if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2) + cntr_mask = (hwc->event_base >> 10) & 0xffff; + else + cntr_mask = (hwc->event_base >> 8) & 0xffff; for (i = mipspmu.num_counters - 1; i >= 0; i--) { /* @@ -320,10 +353,16 @@ static void mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx) WARN_ON(idx < 0 || idx >= mipspmu.num_counters); - cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) | - (evt->config_base & M_PERFCTL_CONFIG_MASK) | - /* Make sure interrupt enabled. */ - MIPS_PERFCTRL_IE; + if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2) + cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0x3ff) | + (evt->config_base & M_PERFCTL_CONFIG_MASK) | + /* Make sure interrupt enabled. */ + MIPS_PERFCTRL_IE; + else + cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) | + (evt->config_base & M_PERFCTL_CONFIG_MASK) | + /* Make sure interrupt enabled. */ + MIPS_PERFCTRL_IE; if (IS_ENABLED(CONFIG_CPU_BMIPS5000)) { /* enable the counter for the calling thread */ @@ -396,6 +435,10 @@ static int mipspmu_event_set_period(struct perf_event *event, local64_set(&hwc->prev_count, mipspmu.overflow - left); + if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2) + mipsxx_pmu_write_control(idx, + M_PERFCTL_EVENT(hwc->event_base & 0x3ff)); + mipspmu.write_counter(idx, mipspmu.overflow - left); perf_event_update_userpage(event); @@ -667,8 +710,14 @@ static unsigned int mipspmu_perf_event_encode(const struct mips_perf_event *pev) (pev->event_id & 0xff); else #endif /* CONFIG_MIPS_MT_SMP */ - return ((pev->cntr_mask & 0xffff00) | - (pev->event_id & 0xff)); + { + if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2) + return (pev->cntr_mask & 0xfffc00) | + (pev->event_id & 0x3ff); + else + return (pev->cntr_mask & 0xffff00) | + (pev->event_id & 0xff); + } } static const struct mips_perf_event *mipspmu_map_general_event(int idx) @@ -783,26 +832,104 @@ static int n_counters(void) return counters; } -static void reset_counters(void *arg) +static void loongson3_reset_counters(void *arg) { int counters = (int)(long)arg; + switch (counters) { case 4: mipsxx_pmu_write_control(3, 0); mipspmu.write_counter(3, 0); - /* fall through */ + mipsxx_pmu_write_control(3, 127<<5); + mipspmu.write_counter(3, 0); + mipsxx_pmu_write_control(3, 191<<5); + mipspmu.write_counter(3, 0); + mipsxx_pmu_write_control(3, 255<<5); + mipspmu.write_counter(3, 0); + mipsxx_pmu_write_control(3, 319<<5); + mipspmu.write_counter(3, 0); + mipsxx_pmu_write_control(3, 383<<5); + mipspmu.write_counter(3, 0); + mipsxx_pmu_write_control(3, 575<<5); + mipspmu.write_counter(3, 0); + fallthrough; case 3: mipsxx_pmu_write_control(2, 0); mipspmu.write_counter(2, 0); - /* fall through */ + mipsxx_pmu_write_control(2, 127<<5); + mipspmu.write_counter(2, 0); + mipsxx_pmu_write_control(2, 191<<5); + mipspmu.write_counter(2, 0); + mipsxx_pmu_write_control(2, 255<<5); + mipspmu.write_counter(2, 0); + mipsxx_pmu_write_control(2, 319<<5); + mipspmu.write_counter(2, 0); + mipsxx_pmu_write_control(2, 383<<5); + mipspmu.write_counter(2, 0); + mipsxx_pmu_write_control(2, 575<<5); + mipspmu.write_counter(2, 0); + fallthrough; case 2: mipsxx_pmu_write_control(1, 0); mipspmu.write_counter(1, 0); - /* fall through */ + mipsxx_pmu_write_control(1, 127<<5); + mipspmu.write_counter(1, 0); + mipsxx_pmu_write_control(1, 191<<5); + mipspmu.write_counter(1, 0); + mipsxx_pmu_write_control(1, 255<<5); + mipspmu.write_counter(1, 0); + mipsxx_pmu_write_control(1, 319<<5); + mipspmu.write_counter(1, 0); + mipsxx_pmu_write_control(1, 383<<5); + mipspmu.write_counter(1, 0); + mipsxx_pmu_write_control(1, 575<<5); + mipspmu.write_counter(1, 0); + fallthrough; case 1: mipsxx_pmu_write_control(0, 0); mipspmu.write_counter(0, 0); - /* fall through */ + mipsxx_pmu_write_control(0, 127<<5); + mipspmu.write_counter(0, 0); + mipsxx_pmu_write_control(0, 191<<5); + mipspmu.write_counter(0, 0); + mipsxx_pmu_write_control(0, 255<<5); + mipspmu.write_counter(0, 0); + mipsxx_pmu_write_control(0, 319<<5); + mipspmu.write_counter(0, 0); + mipsxx_pmu_write_control(0, 383<<5); + mipspmu.write_counter(0, 0); + mipsxx_pmu_write_control(0, 575<<5); + mipspmu.write_counter(0, 0); + fallthrough; + } +} + +static void reset_counters(void *arg) +{ + int counters = (int)(long)arg; + + if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2) { + loongson3_reset_counters(arg); + return; + } + + switch (counters) { + case 4: + mipsxx_pmu_write_control(3, 0); + mipspmu.write_counter(3, 0); + fallthrough; + case 3: + mipsxx_pmu_write_control(2, 0); + mipspmu.write_counter(2, 0); + fallthrough; + case 2: + mipsxx_pmu_write_control(1, 0); + mipspmu.write_counter(1, 0); + fallthrough; + case 1: + mipsxx_pmu_write_control(0, 0); + mipspmu.write_counter(0, 0); + fallthrough; } } @@ -834,13 +961,30 @@ static const struct mips_perf_event i6x00_event_map[PERF_COUNT_HW_MAX] = { [PERF_COUNT_HW_BRANCH_MISSES] = { 0x16, CNTR_EVEN | CNTR_ODD }, }; -static const struct mips_perf_event loongson3_event_map[PERF_COUNT_HW_MAX] = { +static const struct mips_perf_event loongson3_event_map1[PERF_COUNT_HW_MAX] = { [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN }, [PERF_COUNT_HW_INSTRUCTIONS] = { 0x00, CNTR_ODD }, [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x01, CNTR_EVEN }, [PERF_COUNT_HW_BRANCH_MISSES] = { 0x01, CNTR_ODD }, }; +static const struct mips_perf_event loongson3_event_map2[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = { 0x80, CNTR_ALL }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x81, CNTR_ALL }, + [PERF_COUNT_HW_CACHE_MISSES] = { 0x18, CNTR_ALL }, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x94, CNTR_ALL }, + [PERF_COUNT_HW_BRANCH_MISSES] = { 0x9c, CNTR_ALL }, +}; + +static const struct mips_perf_event loongson3_event_map3[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_ALL }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_ALL }, + [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x1c, CNTR_ALL }, + [PERF_COUNT_HW_CACHE_MISSES] = { 0x1d, CNTR_ALL }, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02, CNTR_ALL }, + [PERF_COUNT_HW_BRANCH_MISSES] = { 0x08, CNTR_ALL }, +}; + static const struct mips_perf_event octeon_event_map[PERF_COUNT_HW_MAX] = { [PERF_COUNT_HW_CPU_CYCLES] = { 0x01, CNTR_ALL }, [PERF_COUNT_HW_INSTRUCTIONS] = { 0x03, CNTR_ALL }, @@ -1064,7 +1208,7 @@ static const struct mips_perf_event i6x00_cache_map }, }; -static const struct mips_perf_event loongson3_cache_map +static const struct mips_perf_event loongson3_cache_map1 [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX] = { @@ -1109,12 +1253,127 @@ static const struct mips_perf_event loongson3_cache_map [C(BPU)] = { /* Using the same code for *HW_BRANCH* */ [C(OP_READ)] = { - [C(RESULT_ACCESS)] = { 0x02, CNTR_EVEN }, - [C(RESULT_MISS)] = { 0x02, CNTR_ODD }, + [C(RESULT_ACCESS)] = { 0x01, CNTR_EVEN }, + [C(RESULT_MISS)] = { 0x01, CNTR_ODD }, }, [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = { 0x02, CNTR_EVEN }, - [C(RESULT_MISS)] = { 0x02, CNTR_ODD }, + [C(RESULT_ACCESS)] = { 0x01, CNTR_EVEN }, + [C(RESULT_MISS)] = { 0x01, CNTR_ODD }, + }, +}, +}; + +static const struct mips_perf_event loongson3_cache_map2 + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { +[C(L1D)] = { + /* + * Like some other architectures (e.g. ARM), the performance + * counters don't differentiate between read and write + * accesses/misses, so this isn't strictly correct, but it's the + * best we can do. Writes and reads get combined. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x156, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x155, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x153, CNTR_ALL }, + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x18, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_MISS)] = { 0x18, CNTR_ALL }, + }, +}, +[C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x1b6, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x1b7, CNTR_ALL }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { 0x1bf, CNTR_ALL }, + }, +}, +[C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x92, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_MISS)] = { 0x92, CNTR_ALL }, + }, +}, +[C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x1a, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_MISS)] = { 0x1a, CNTR_ALL }, + }, +}, +[C(BPU)] = { + /* Using the same code for *HW_BRANCH* */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x94, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x9c, CNTR_ALL }, + }, +}, +}; + +static const struct mips_perf_event loongson3_cache_map3 + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { +[C(L1D)] = { + /* + * Like some other architectures (e.g. ARM), the performance + * counters don't differentiate between read and write + * accesses/misses, so this isn't strictly correct, but it's the + * best we can do. Writes and reads get combined. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x1e, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x1f, CNTR_ALL }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { 0xaa, CNTR_ALL }, + [C(RESULT_MISS)] = { 0xa9, CNTR_ALL }, + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x1c, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x1d, CNTR_ALL }, + }, +}, +[C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x2e, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x2f, CNTR_ALL }, + }, +}, +[C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x14, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x1b, CNTR_ALL }, + }, +}, +[C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x1a, CNTR_ALL }, + }, +}, +[C(BPU)] = { + /* Using the same code for *HW_BRANCH* */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x02, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x08, CNTR_ALL }, }, }, }; @@ -1178,7 +1437,6 @@ static const struct mips_perf_event bmips5000_cache_map }, }; - static const struct mips_perf_event octeon_cache_map [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] @@ -1512,6 +1770,7 @@ static irqreturn_t mipsxx_pmu_handle_irq(int irq, void *dev) static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config) { /* currently most cores have 7-bit event numbers */ + int pmu_type; unsigned int raw_id = config & 0xff; unsigned int base_id = raw_id & 0x7f; @@ -1624,8 +1883,33 @@ static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config) raw_id > 127 ? CNTR_ODD : CNTR_EVEN; break; case CPU_LOONGSON64: - raw_event.cntr_mask = raw_id > 127 ? CNTR_ODD : CNTR_EVEN; - break; + pmu_type = get_loongson3_pmu_type(); + + switch (pmu_type) { + case LOONGSON_PMU_TYPE1: + raw_event.cntr_mask = + raw_id > 127 ? CNTR_ODD : CNTR_EVEN; + break; + case LOONGSON_PMU_TYPE2: + base_id = config & 0x3ff; + raw_event.cntr_mask = CNTR_ALL; + + if ((base_id >= 1 && base_id < 28) || + (base_id >= 64 && base_id < 90) || + (base_id >= 128 && base_id < 164) || + (base_id >= 192 && base_id < 200) || + (base_id >= 256 && base_id < 274) || + (base_id >= 320 && base_id < 358) || + (base_id >= 384 && base_id < 574)) + break; + + return ERR_PTR(-EOPNOTSUPP); + case LOONGSON_PMU_TYPE3: + base_id = raw_id; + raw_event.cntr_mask = CNTR_ALL; + break; + } + break; } raw_event.event_id = base_id; @@ -1683,8 +1967,7 @@ static const struct mips_perf_event *xlp_pmu_map_raw_event(u64 config) static int __init init_hw_perf_events(void) { - int counters, irq; - int counter_bits; + int counters, irq, pmu_type; pr_info("Performance counters: "); @@ -1771,8 +2054,25 @@ init_hw_perf_events(void) break; case CPU_LOONGSON64: mipspmu.name = "mips/loongson3"; - mipspmu.general_event_map = &loongson3_event_map; - mipspmu.cache_event_map = &loongson3_cache_map; + pmu_type = get_loongson3_pmu_type(); + + switch (pmu_type) { + case LOONGSON_PMU_TYPE1: + counters = 2; + mipspmu.general_event_map = &loongson3_event_map1; + mipspmu.cache_event_map = &loongson3_cache_map1; + break; + case LOONGSON_PMU_TYPE2: + counters = 4; + mipspmu.general_event_map = &loongson3_event_map2; + mipspmu.cache_event_map = &loongson3_cache_map2; + break; + case LOONGSON_PMU_TYPE3: + counters = 4; + mipspmu.general_event_map = &loongson3_event_map3; + mipspmu.cache_event_map = &loongson3_cache_map3; + break; + } break; case CPU_CAVIUM_OCTEON: case CPU_CAVIUM_OCTEON_PLUS: @@ -1803,19 +2103,26 @@ init_hw_perf_events(void) mipspmu.irq = irq; if (read_c0_perfctrl0() & MIPS_PERFCTRL_W) { - mipspmu.max_period = (1ULL << 63) - 1; - mipspmu.valid_count = (1ULL << 63) - 1; - mipspmu.overflow = 1ULL << 63; + if (get_loongson3_pmu_type() == LOONGSON_PMU_TYPE2) { + counter_bits = 48; + mipspmu.max_period = (1ULL << 47) - 1; + mipspmu.valid_count = (1ULL << 47) - 1; + mipspmu.overflow = 1ULL << 47; + } else { + counter_bits = 64; + mipspmu.max_period = (1ULL << 63) - 1; + mipspmu.valid_count = (1ULL << 63) - 1; + mipspmu.overflow = 1ULL << 63; + } mipspmu.read_counter = mipsxx_pmu_read_counter_64; mipspmu.write_counter = mipsxx_pmu_write_counter_64; - counter_bits = 64; } else { + counter_bits = 32; mipspmu.max_period = (1ULL << 31) - 1; mipspmu.valid_count = (1ULL << 31) - 1; mipspmu.overflow = 1ULL << 31; mipspmu.read_counter = mipsxx_pmu_read_counter; mipspmu.write_counter = mipsxx_pmu_write_counter; - counter_bits = 32; } on_each_cpu(reset_counters, (void *)(long)counters, 1); From da1bd29742b185c00a1737ba955aa3e75659be2b Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Thu, 30 Apr 2020 11:18:35 +0800 Subject: [PATCH 0174/1043] MIPS: Loongson64: Probe CPU features via CPUCFG CPUCFG is a Loongson self-defined instruction used to mark CPU features for Loongson processors started from Loongson-3A4000. Slightly adjust cpu_probe_loongson function as well. Remove features that already probed via decode_configs in processor's PRID case and add a comment about TLBINV. Signed-off-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/cpu-probe.c | 45 +++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index 6de14b527c68..1736c173b242 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -1932,8 +1932,35 @@ platform: } } +#ifdef CONFIG_CPU_LOONGSON64 +#include + +static inline void decode_cpucfg(struct cpuinfo_mips *c) +{ + u32 cfg1 = read_cpucfg(LOONGSON_CFG1); + u32 cfg2 = read_cpucfg(LOONGSON_CFG2); + u32 cfg3 = read_cpucfg(LOONGSON_CFG3); + + if (cfg1 & LOONGSON_CFG1_MMI) + c->ases |= MIPS_ASE_LOONGSON_MMI; + + if (cfg2 & LOONGSON_CFG2_LEXT1) + c->ases |= MIPS_ASE_LOONGSON_EXT; + + if (cfg2 & LOONGSON_CFG2_LEXT2) + c->ases |= MIPS_ASE_LOONGSON_EXT2; + + if (cfg2 & LOONGSON_CFG2_LSPW) + c->options |= MIPS_CPU_LDPTE; + + if (cfg3 & LOONGSON_CFG3_LCAMP) + c->ases |= MIPS_ASE_LOONGSON_CAM; +} + static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) { + decode_configs(c); + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_LOONGSON_64R: /* Loongson-64 Reduced */ switch (c->processor_id & PRID_REV_MASK) { @@ -1947,7 +1974,6 @@ static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) set_isa(c, MIPS_CPU_ISA_M64R2); break; } - decode_configs(c); c->writecombine = _CACHE_UNCACHED_ACCELERATED; c->ases |= (MIPS_ASE_LOONGSON_MMI | MIPS_ASE_LOONGSON_EXT | MIPS_ASE_LOONGSON_EXT2); @@ -1969,9 +1995,12 @@ static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) set_isa(c, MIPS_CPU_ISA_M64R2); break; } - - decode_configs(c); - c->options |= MIPS_CPU_FTLB | MIPS_CPU_TLBINV | MIPS_CPU_LDPTE; + /* + * Loongson-3 Classic did not implement MIPS standard TLBINV + * but implemented TLBINVF and EHINV. As currently we're only + * using these two features, enable MIPS_CPU_TLBINV as well. + */ + c->options |= MIPS_CPU_TLBINV | MIPS_CPU_LDPTE; c->writecombine = _CACHE_UNCACHED_ACCELERATED; c->ases |= (MIPS_ASE_LOONGSON_MMI | MIPS_ASE_LOONGSON_CAM | MIPS_ASE_LOONGSON_EXT | MIPS_ASE_LOONGSON_EXT2); @@ -1981,17 +2010,17 @@ static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) __cpu_name[cpu] = "ICT Loongson-3"; set_elf_platform(cpu, "loongson3a"); set_isa(c, MIPS_CPU_ISA_M64R2); - decode_configs(c); - c->options |= MIPS_CPU_FTLB | MIPS_CPU_TLBINV | MIPS_CPU_LDPTE; + decode_cpucfg(c); c->writecombine = _CACHE_UNCACHED_ACCELERATED; - c->ases |= (MIPS_ASE_LOONGSON_MMI | MIPS_ASE_LOONGSON_CAM | - MIPS_ASE_LOONGSON_EXT | MIPS_ASE_LOONGSON_EXT2); break; default: panic("Unknown Loongson Processor ID!"); break; } } +#else +static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) { } +#endif static inline void cpu_probe_ingenic(struct cpuinfo_mips *c, unsigned int cpu) { From f080d93e1d419099a99d7473ed532289ca8dc717 Mon Sep 17 00:00:00 2001 From: Xie XiuQi Date: Tue, 14 Apr 2020 20:57:21 +0800 Subject: [PATCH 0175/1043] sched/debug: Fix trival print_task() format Ensure leave one space between state and task name. w/o patch: runnable tasks: S task PID tree-key switches prio wait Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200414125721.195801-1-xiexiuqi@huawei.com --- kernel/sched/debug.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index a562df57a86e..b3ac1c10b032 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -437,7 +437,7 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) else SEQ_printf(m, " %c", task_state_to_char(p)); - SEQ_printf(m, "%15s %5d %9Ld.%06ld %9Ld %5d ", + SEQ_printf(m, " %15s %5d %9Ld.%06ld %9Ld %5d ", p->comm, task_pid_nr(p), SPLIT_NS(p->se.vruntime), (long long)(p->nvcsw + p->nivcsw), @@ -464,10 +464,10 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) SEQ_printf(m, "\n"); SEQ_printf(m, "runnable tasks:\n"); - SEQ_printf(m, " S task PID tree-key switches prio" + SEQ_printf(m, " S task PID tree-key switches prio" " wait-time sum-exec sum-sleep\n"); SEQ_printf(m, "-------------------------------------------------------" - "----------------------------------------------------\n"); + "------------------------------------------------------\n"); rcu_read_lock(); for_each_process_thread(g, p) { From e98fa02c4f2ea4991dae422ac7e34d102d2f0599 Mon Sep 17 00:00:00 2001 From: Paul Turner Date: Fri, 10 Apr 2020 15:52:07 -0700 Subject: [PATCH 0176/1043] sched/fair: Eliminate bandwidth race between throttling and distribution There is a race window in which an entity begins throttling before quota is added to the pool, but does not finish throttling until after we have finished with distribute_cfs_runtime(). This entity is not observed by distribute_cfs_runtime() because it was not on the throttled list at the time that distribution was running. This race manifests as rare period-length statlls for such entities. Rather than heavy-weight the synchronization with the progress of distribution, we can fix this by aborting throttling if bandwidth has become available. Otherwise, we immediately add the entity to the throttled list so that it can be observed by a subsequent distribution. Additionally, we can remove the case of adding the throttled entity to the head of the throttled list, and simply always add to the tail. Thanks to 26a8b12747c97, distribute_cfs_runtime() no longer holds onto its own pool of runtime. This means that if we do hit the !assign and distribute_running case, we know that distribution is about to end. Signed-off-by: Paul Turner Signed-off-by: Ben Segall Signed-off-by: Josh Don Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Phil Auld Link: https://lkml.kernel.org/r/20200410225208.109717-2-joshdon@google.com --- kernel/sched/fair.c | 79 +++++++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 02f323b85b6d..0c13a41bde81 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4588,16 +4588,16 @@ static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) } /* returns 0 on failure to allocate runtime */ -static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) +static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b, + struct cfs_rq *cfs_rq, u64 target_runtime) { - struct task_group *tg = cfs_rq->tg; - struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg); - u64 amount = 0, min_amount; + u64 min_amount, amount = 0; + + lockdep_assert_held(&cfs_b->lock); /* note: this is a positive sum as runtime_remaining <= 0 */ - min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining; + min_amount = target_runtime - cfs_rq->runtime_remaining; - raw_spin_lock(&cfs_b->lock); if (cfs_b->quota == RUNTIME_INF) amount = min_amount; else { @@ -4609,13 +4609,25 @@ static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) cfs_b->idle = 0; } } - raw_spin_unlock(&cfs_b->lock); cfs_rq->runtime_remaining += amount; return cfs_rq->runtime_remaining > 0; } +/* returns 0 on failure to allocate runtime */ +static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) +{ + struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); + int ret; + + raw_spin_lock(&cfs_b->lock); + ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice()); + raw_spin_unlock(&cfs_b->lock); + + return ret; +} + static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) { /* dock delta_exec before expiring quota (as it could span periods) */ @@ -4704,13 +4716,33 @@ static int tg_throttle_down(struct task_group *tg, void *data) return 0; } -static void throttle_cfs_rq(struct cfs_rq *cfs_rq) +static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) { struct rq *rq = rq_of(cfs_rq); struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg); struct sched_entity *se; long task_delta, idle_task_delta, dequeue = 1; - bool empty; + + raw_spin_lock(&cfs_b->lock); + /* This will start the period timer if necessary */ + if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) { + /* + * We have raced with bandwidth becoming available, and if we + * actually throttled the timer might not unthrottle us for an + * entire period. We additionally needed to make sure that any + * subsequent check_cfs_rq_runtime calls agree not to throttle + * us, as we may commit to do cfs put_prev+pick_next, so we ask + * for 1ns of runtime rather than just check cfs_b. + */ + dequeue = 0; + } else { + list_add_tail_rcu(&cfs_rq->throttled_list, + &cfs_b->throttled_cfs_rq); + } + raw_spin_unlock(&cfs_b->lock); + + if (!dequeue) + return false; /* Throttle no longer required. */ se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))]; @@ -4744,29 +4776,13 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) if (!se) sub_nr_running(rq, task_delta); + /* + * Note: distribution will already see us throttled via the + * throttled-list. rq->lock protects completion. + */ cfs_rq->throttled = 1; cfs_rq->throttled_clock = rq_clock(rq); - raw_spin_lock(&cfs_b->lock); - empty = list_empty(&cfs_b->throttled_cfs_rq); - - /* - * Add to the _head_ of the list, so that an already-started - * distribute_cfs_runtime will not see us. If disribute_cfs_runtime is - * not running add to the tail so that later runqueues don't get starved. - */ - if (cfs_b->distribute_running) - list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); - else - list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq); - - /* - * If we're the first throttled task, make sure the bandwidth - * timer is running. - */ - if (empty) - start_cfs_bandwidth(cfs_b); - - raw_spin_unlock(&cfs_b->lock); + return true; } void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) @@ -5121,8 +5137,7 @@ static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) if (cfs_rq_throttled(cfs_rq)) return true; - throttle_cfs_rq(cfs_rq); - return true; + return throttle_cfs_rq(cfs_rq); } static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) From ab93a4bc955b3980c699430bc0b633f0d8b607be Mon Sep 17 00:00:00 2001 From: Josh Don Date: Fri, 10 Apr 2020 15:52:08 -0700 Subject: [PATCH 0177/1043] sched/fair: Remove distribute_running from CFS bandwidth This is mostly a revert of commit: baa9be4ffb55 ("sched/fair: Fix throttle_list starvation with low CFS quota") The primary use of distribute_running was to determine whether to add throttled entities to the head or the tail of the throttled list. Now that we always add to the tail, we can remove this field. The other use of distribute_running is in the slack_timer, so that we don't start a distribution while one is already running. However, even in the event that this race occurs, it is fine to have two distributions running (especially now that distribute grabs the cfs_b->lock to determine remaining quota before assigning). Signed-off-by: Josh Don Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Phil Auld Tested-by: Phil Auld Link: https://lkml.kernel.org/r/20200410225208.109717-3-joshdon@google.com --- kernel/sched/fair.c | 13 +------------ kernel/sched/sched.h | 1 - 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 0c13a41bde81..3d6ce751abb6 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4931,14 +4931,12 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, u /* * This check is repeated as we release cfs_b->lock while we unthrottle. */ - while (throttled && cfs_b->runtime > 0 && !cfs_b->distribute_running) { - cfs_b->distribute_running = 1; + while (throttled && cfs_b->runtime > 0) { raw_spin_unlock_irqrestore(&cfs_b->lock, flags); /* we can't nest cfs_b->lock while distributing bandwidth */ distribute_cfs_runtime(cfs_b); raw_spin_lock_irqsave(&cfs_b->lock, flags); - cfs_b->distribute_running = 0; throttled = !list_empty(&cfs_b->throttled_cfs_rq); } @@ -5052,10 +5050,6 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) /* confirm we're still not at a refresh boundary */ raw_spin_lock_irqsave(&cfs_b->lock, flags); cfs_b->slack_started = false; - if (cfs_b->distribute_running) { - raw_spin_unlock_irqrestore(&cfs_b->lock, flags); - return; - } if (runtime_refresh_within(cfs_b, min_bandwidth_expiration)) { raw_spin_unlock_irqrestore(&cfs_b->lock, flags); @@ -5065,9 +5059,6 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice) runtime = cfs_b->runtime; - if (runtime) - cfs_b->distribute_running = 1; - raw_spin_unlock_irqrestore(&cfs_b->lock, flags); if (!runtime) @@ -5076,7 +5067,6 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) distribute_cfs_runtime(cfs_b); raw_spin_lock_irqsave(&cfs_b->lock, flags); - cfs_b->distribute_running = 0; raw_spin_unlock_irqrestore(&cfs_b->lock, flags); } @@ -5218,7 +5208,6 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) cfs_b->period_timer.function = sched_cfs_period_timer; hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); cfs_b->slack_timer.function = sched_cfs_slack_timer; - cfs_b->distribute_running = 0; cfs_b->slack_started = false; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index db3a57675ccf..7198683b2869 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -349,7 +349,6 @@ struct cfs_bandwidth { u8 idle; u8 period_active; - u8 distribute_running; u8 slack_started; struct hrtimer period_timer; struct hrtimer slack_timer; From 64297f2b03cc7a6d0184a435f1b296beca1bedd1 Mon Sep 17 00:00:00 2001 From: Peng Wang Date: Sat, 11 Apr 2020 17:20:20 +0800 Subject: [PATCH 0178/1043] sched/fair: Simplify the code of should_we_balance() We only consider group_balance_cpu() after there is no idle cpu. So, just do comparison before return at these two cases. Signed-off-by: Peng Wang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Valentin Schneider Reviewed-by: Vincent Guittot Link: https://lkml.kernel.org/r/245c792f0e580b3ca342ad61257f4c066ee0f84f.1586594833.git.rocking@linux.alibaba.com --- kernel/sched/fair.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 3d6ce751abb6..63419c6d9641 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9413,7 +9413,7 @@ static int active_load_balance_cpu_stop(void *data); static int should_we_balance(struct lb_env *env) { struct sched_group *sg = env->sd->groups; - int cpu, balance_cpu = -1; + int cpu; /* * Ensure the balancing environment is consistent; can happen @@ -9434,18 +9434,12 @@ static int should_we_balance(struct lb_env *env) if (!idle_cpu(cpu)) continue; - balance_cpu = cpu; - break; + /* Are we the first idle CPU? */ + return cpu == env->dst_cpu; } - if (balance_cpu == -1) - balance_cpu = group_balance_cpu(sg); - - /* - * First idle CPU or the first CPU(busiest) in this sched group - * is eligible for doing load balancing at this and above domains. - */ - return balance_cpu == env->dst_cpu; + /* Are we the first CPU of this group ? */ + return group_balance_cpu(sg) == env->dst_cpu; } /* From 586b58cac8b4683eb58a1446fbc399de18974e40 Mon Sep 17 00:00:00 2001 From: Jann Horn Date: Thu, 5 Mar 2020 23:06:57 +0100 Subject: [PATCH 0179/1043] exit: Move preemption fixup up, move blocking operations down With CONFIG_DEBUG_ATOMIC_SLEEP=y and CONFIG_CGROUPS=y, kernel oopses in non-preemptible context look untidy; after the main oops, the kernel prints a "sleeping function called from invalid context" report because exit_signals() -> cgroup_threadgroup_change_begin() -> percpu_down_read() can sleep, and that happens before the preempt_count_set(PREEMPT_ENABLED) fixup. It looks like the same thing applies to profile_task_exit() and kcov_task_exit(). Fix it by moving the preemption fixup up and the calls to profile_task_exit() and kcov_task_exit() down. Fixes: 1dc0fffc48af ("sched/core: Robustify preemption leak checks") Signed-off-by: Jann Horn Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200305220657.46800-1-jannh@google.com --- kernel/exit.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/kernel/exit.c b/kernel/exit.c index ce2a75bc0ade..d56fe51bdf07 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -708,8 +708,12 @@ void __noreturn do_exit(long code) struct task_struct *tsk = current; int group_dead; - profile_task_exit(tsk); - kcov_task_exit(tsk); + /* + * We can get here from a kernel oops, sometimes with preemption off. + * Start by checking for critical errors. + * Then fix up important state like USER_DS and preemption. + * Then do everything else. + */ WARN_ON(blk_needs_flush_plug(tsk)); @@ -727,6 +731,16 @@ void __noreturn do_exit(long code) */ set_fs(USER_DS); + if (unlikely(in_atomic())) { + pr_info("note: %s[%d] exited with preempt_count %d\n", + current->comm, task_pid_nr(current), + preempt_count()); + preempt_count_set(PREEMPT_ENABLED); + } + + profile_task_exit(tsk); + kcov_task_exit(tsk); + ptrace_event(PTRACE_EVENT_EXIT, code); validate_creds_for_do_exit(tsk); @@ -744,13 +758,6 @@ void __noreturn do_exit(long code) exit_signals(tsk); /* sets PF_EXITING */ - if (unlikely(in_atomic())) { - pr_info("note: %s[%d] exited with preempt_count %d\n", - current->comm, task_pid_nr(current), - preempt_count()); - preempt_count_set(PREEMPT_ENABLED); - } - /* sync mm's RSS info before statistics gathering */ if (tsk->mm) sync_mm_rss(tsk->mm); From 45da27732b0b9b7a04696653065d5e6037bc5ac0 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Wed, 15 Apr 2020 22:05:04 +0100 Subject: [PATCH 0180/1043] sched/fair: find_idlest_group(): Remove unused sd_flag parameter The last use of that parameter was removed by commit 57abff067a08 ("sched/fair: Rework find_idlest_group()") Get rid of the parameter. Signed-off-by: Valentin Schneider Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Dietmar Eggemann Reviewed-by: Vincent Guittot Link: https://lkml.kernel.org/r/20200415210512.805-2-valentin.schneider@arm.com --- kernel/sched/fair.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 63419c6d9641..617ca44ed61b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5825,8 +5825,7 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p, } static struct sched_group * -find_idlest_group(struct sched_domain *sd, struct task_struct *p, - int this_cpu, int sd_flag); +find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu); /* * find_idlest_group_cpu - find the idlest CPU among the CPUs in the group. @@ -5909,7 +5908,7 @@ static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p continue; } - group = find_idlest_group(sd, p, cpu, sd_flag); + group = find_idlest_group(sd, p, cpu); if (!group) { sd = sd->child; continue; @@ -8681,8 +8680,7 @@ static bool update_pick_idlest(struct sched_group *idlest, * Assumes p is allowed on at least one CPU in sd. */ static struct sched_group * -find_idlest_group(struct sched_domain *sd, struct task_struct *p, - int this_cpu, int sd_flag) +find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) { struct sched_group *idlest = NULL, *local = NULL, *group = sd->groups; struct sg_lb_stats local_sgs, tmp_sgs; From 9818427c6270a9ce8c52c8621026fe9cebae0f92 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Wed, 15 Apr 2020 22:05:05 +0100 Subject: [PATCH 0181/1043] sched/debug: Make sd->flags sysctl read-only Writing to the sysctl of a sched_domain->flags directly updates the value of the field, and goes nowhere near update_top_cache_domain(). This means that the cached domain pointers can end up containing stale data (e.g. the domain pointed to doesn't have the relevant flag set anymore). Explicit domain walks that check for flags will be affected by the write, but this won't be in sync with the cached pointers which will still point to the domains that were cached at the last sched_domain build. In other words, writing to this interface is playing a dangerous game. It could be made to trigger an update of the cached sched_domain pointers when written to, but this does not seem to be worth the trouble. Make it read-only. Signed-off-by: Valentin Schneider Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200415210512.805-3-valentin.schneider@arm.com --- kernel/sched/debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index b3ac1c10b032..c6cc02a6aad0 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -258,7 +258,7 @@ sd_alloc_ctl_domain_table(struct sched_domain *sd) set_table_entry(&table[2], "busy_factor", &sd->busy_factor, sizeof(int), 0644, proc_dointvec_minmax); set_table_entry(&table[3], "imbalance_pct", &sd->imbalance_pct, sizeof(int), 0644, proc_dointvec_minmax); set_table_entry(&table[4], "cache_nice_tries", &sd->cache_nice_tries, sizeof(int), 0644, proc_dointvec_minmax); - set_table_entry(&table[5], "flags", &sd->flags, sizeof(int), 0644, proc_dointvec_minmax); + set_table_entry(&table[5], "flags", &sd->flags, sizeof(int), 0444, proc_dointvec_minmax); set_table_entry(&table[6], "max_newidle_lb_cost", &sd->max_newidle_lb_cost, sizeof(long), 0644, proc_doulongvec_minmax); set_table_entry(&table[7], "name", sd->name, CORENAME_MAX_SIZE, 0444, proc_dostring); /* &table[8] is terminator */ From e669ac8ab952df2f07dee1e1efbf40647d6de332 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Wed, 15 Apr 2020 22:05:06 +0100 Subject: [PATCH 0182/1043] sched: Remove checks against SD_LOAD_BALANCE The SD_LOAD_BALANCE flag is set unconditionally for all domains in sd_init(). By making the sched_domain->flags syctl interface read-only, we have removed the last piece of code that could clear that flag - as such, it will now be always present. Rather than to keep carrying it along, we can work towards getting rid of it entirely. cpusets don't need it because they can make CPUs be attached to the NULL domain (e.g. cpuset with sched_load_balance=0), or to a partitioned root_domain, i.e. a sched_domain hierarchy that doesn't span the entire system (e.g. root cpuset with sched_load_balance=0 and sibling cpusets with sched_load_balance=1). isolcpus apply the same "trick": isolated CPUs are explicitly taken out of the sched_domain rebuild (using housekeeping_cpumask()), so they get the NULL domain treatment as well. Remove the checks against SD_LOAD_BALANCE. Signed-off-by: Valentin Schneider Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200415210512.805-4-valentin.schneider@arm.com --- kernel/sched/fair.c | 14 ++------------ kernel/sched/topology.c | 28 +++++++++------------------- 2 files changed, 11 insertions(+), 31 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 617ca44ed61b..4b959c0a7d7b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6649,9 +6649,6 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_f rcu_read_lock(); for_each_domain(cpu, tmp) { - if (!(tmp->flags & SD_LOAD_BALANCE)) - break; - /* * If both 'cpu' and 'prev_cpu' are part of this domain, * cpu is a valid SD_WAKE_AFFINE target. @@ -9790,9 +9787,8 @@ static int active_load_balance_cpu_stop(void *data) /* Search for an sd spanning us and the target CPU. */ rcu_read_lock(); for_each_domain(target_cpu, sd) { - if ((sd->flags & SD_LOAD_BALANCE) && - cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) - break; + if (cpumask_test_cpu(busiest_cpu, sched_domain_span(sd))) + break; } if (likely(sd)) { @@ -9881,9 +9877,6 @@ static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle) } max_cost += sd->max_newidle_lb_cost; - if (!(sd->flags & SD_LOAD_BALANCE)) - continue; - /* * Stop the load balance at this level. There is another * CPU in our sched group which is doing load balancing more @@ -10472,9 +10465,6 @@ int newidle_balance(struct rq *this_rq, struct rq_flags *rf) int continue_balancing = 1; u64 t0, domain_cost; - if (!(sd->flags & SD_LOAD_BALANCE)) - continue; - if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost) { update_next_balance(sd, &next_balance); break; diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 8344757bba6e..a9dc34a0ebc1 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -33,14 +33,6 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, cpumask_clear(groupmask); printk(KERN_DEBUG "%*s domain-%d: ", level, "", level); - - if (!(sd->flags & SD_LOAD_BALANCE)) { - printk("does not load-balance\n"); - if (sd->parent) - printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain has parent"); - return -1; - } - printk(KERN_CONT "span=%*pbl level=%s\n", cpumask_pr_args(sched_domain_span(sd)), sd->name); @@ -151,8 +143,7 @@ static int sd_degenerate(struct sched_domain *sd) return 1; /* Following flags need at least 2 groups */ - if (sd->flags & (SD_LOAD_BALANCE | - SD_BALANCE_NEWIDLE | + if (sd->flags & (SD_BALANCE_NEWIDLE | SD_BALANCE_FORK | SD_BALANCE_EXEC | SD_SHARE_CPUCAPACITY | @@ -183,15 +174,14 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) /* Flags needing groups don't count if only 1 group in parent */ if (parent->groups == parent->groups->next) { - pflags &= ~(SD_LOAD_BALANCE | - SD_BALANCE_NEWIDLE | - SD_BALANCE_FORK | - SD_BALANCE_EXEC | - SD_ASYM_CPUCAPACITY | - SD_SHARE_CPUCAPACITY | - SD_SHARE_PKG_RESOURCES | - SD_PREFER_SIBLING | - SD_SHARE_POWERDOMAIN); + pflags &= ~(SD_BALANCE_NEWIDLE | + SD_BALANCE_FORK | + SD_BALANCE_EXEC | + SD_ASYM_CPUCAPACITY | + SD_SHARE_CPUCAPACITY | + SD_SHARE_PKG_RESOURCES | + SD_PREFER_SIBLING | + SD_SHARE_POWERDOMAIN); if (nr_node_ids == 1) pflags &= ~SD_SERIALIZE; } From 36c5bdc4387056af3840adb4478c752faeb9d15e Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Wed, 15 Apr 2020 22:05:07 +0100 Subject: [PATCH 0183/1043] sched/topology: Kill SD_LOAD_BALANCE That flag is set unconditionally in sd_init(), and no one checks for it anymore. Remove it. Signed-off-by: Valentin Schneider Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200415210512.805-5-valentin.schneider@arm.com --- include/linux/sched/topology.h | 29 ++++++++++++++--------------- kernel/sched/topology.c | 3 +-- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 95253ad792b0..fb11091129b3 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -11,21 +11,20 @@ */ #ifdef CONFIG_SMP -#define SD_LOAD_BALANCE 0x0001 /* Do load balancing on this domain. */ -#define SD_BALANCE_NEWIDLE 0x0002 /* Balance when about to become idle */ -#define SD_BALANCE_EXEC 0x0004 /* Balance on exec */ -#define SD_BALANCE_FORK 0x0008 /* Balance on fork, clone */ -#define SD_BALANCE_WAKE 0x0010 /* Balance on wakeup */ -#define SD_WAKE_AFFINE 0x0020 /* Wake task to waking CPU */ -#define SD_ASYM_CPUCAPACITY 0x0040 /* Domain members have different CPU capacities */ -#define SD_SHARE_CPUCAPACITY 0x0080 /* Domain members share CPU capacity */ -#define SD_SHARE_POWERDOMAIN 0x0100 /* Domain members share power domain */ -#define SD_SHARE_PKG_RESOURCES 0x0200 /* Domain members share CPU pkg resources */ -#define SD_SERIALIZE 0x0400 /* Only a single load balancing instance */ -#define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */ -#define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */ -#define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */ -#define SD_NUMA 0x4000 /* cross-node balancing */ +#define SD_BALANCE_NEWIDLE 0x0001 /* Balance when about to become idle */ +#define SD_BALANCE_EXEC 0x0002 /* Balance on exec */ +#define SD_BALANCE_FORK 0x0004 /* Balance on fork, clone */ +#define SD_BALANCE_WAKE 0x0008 /* Balance on wakeup */ +#define SD_WAKE_AFFINE 0x0010 /* Wake task to waking CPU */ +#define SD_ASYM_CPUCAPACITY 0x0020 /* Domain members have different CPU capacities */ +#define SD_SHARE_CPUCAPACITY 0x0040 /* Domain members share CPU capacity */ +#define SD_SHARE_POWERDOMAIN 0x0080 /* Domain members share power domain */ +#define SD_SHARE_PKG_RESOURCES 0x0100 /* Domain members share CPU pkg resources */ +#define SD_SERIALIZE 0x0200 /* Only a single load balancing instance */ +#define SD_ASYM_PACKING 0x0400 /* Place busy groups earlier in the domain */ +#define SD_PREFER_SIBLING 0x0800 /* Prefer to place tasks in a sibling domain */ +#define SD_OVERLAP 0x1000 /* sched_domains of this level overlap */ +#define SD_NUMA 0x2000 /* cross-node balancing */ #ifdef CONFIG_SCHED_SMT static inline int cpu_smt_flags(void) diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index a9dc34a0ebc1..1d7b446fac7d 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -1341,8 +1341,7 @@ sd_init(struct sched_domain_topology_level *tl, .cache_nice_tries = 0, - .flags = 1*SD_LOAD_BALANCE - | 1*SD_BALANCE_NEWIDLE + .flags = 1*SD_BALANCE_NEWIDLE | 1*SD_BALANCE_EXEC | 1*SD_BALANCE_FORK | 0*SD_BALANCE_WAKE From d91cecc156620ec75d94c55369509c807c3d07e6 Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Tue, 21 Apr 2020 18:50:34 +0800 Subject: [PATCH 0184/1043] sched: Make newidle_balance() static again After Commit 6e2df0581f56 ("sched: Fix pick_next_task() vs 'change' pattern race"), there is no need to expose newidle_balance() as it is only used within fair.c file. Change this function back to static again. No functional change. Reported-by: kbuild test robot Suggested-by: Peter Zijlstra Signed-off-by: Chen Yu Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/83cd3030b031ca5d646cd5e225be10e7a0fdd8f5.1587464698.git.yu.c.chen@intel.com --- kernel/sched/fair.c | 6 ++++-- kernel/sched/sched.h | 4 ---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 4b959c0a7d7b..c0216ef33c0c 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -3873,6 +3873,8 @@ static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) return cfs_rq->avg.load_avg; } +static int newidle_balance(struct rq *this_rq, struct rq_flags *rf); + static inline unsigned long task_util(struct task_struct *p) { return READ_ONCE(p->se.avg.util_avg); @@ -4054,7 +4056,7 @@ attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} static inline void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} -static inline int idle_balance(struct rq *rq, struct rq_flags *rf) +static inline int newidle_balance(struct rq *rq, struct rq_flags *rf) { return 0; } @@ -10414,7 +10416,7 @@ static inline void nohz_newidle_balance(struct rq *this_rq) { } * 0 - failed, no new tasks * > 0 - success, new (fair) tasks present */ -int newidle_balance(struct rq *this_rq, struct rq_flags *rf) +static int newidle_balance(struct rq *this_rq, struct rq_flags *rf) { unsigned long next_balance = jiffies + HZ; int this_cpu = this_rq->cpu; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 7198683b2869..978c6fac8cb8 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1503,14 +1503,10 @@ static inline void unregister_sched_domain_sysctl(void) } #endif -extern int newidle_balance(struct rq *this_rq, struct rq_flags *rf); - #else static inline void sched_ttwu_pending(void) { } -static inline int newidle_balance(struct rq *this_rq, struct rq_flags *rf) { return 0; } - #endif /* CONFIG_SMP */ #include "stats.h" From 457d1f465778ccb5f14f7d7a62245e41d12a3804 Mon Sep 17 00:00:00 2001 From: Chen Yu Date: Tue, 21 Apr 2020 18:50:43 +0800 Subject: [PATCH 0185/1043] sched: Extract the task putting code from pick_next_task() Introduce a new function put_prev_task_balance() to do the balance when necessary, and then put previous task back to the run queue. This function is extracted from pick_next_task() to prepare for future usage by other type of task picking logic. No functional change. Suggested-by: Peter Zijlstra Signed-off-by: Chen Yu Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Valentin Schneider Reviewed-by: Vincent Guittot Reviewed-by: Steven Rostedt (VMware) Link: https://lkml.kernel.org/r/5a99860cf66293db58a397d6248bcb2eee326776.1587464698.git.yu.c.chen@intel.com --- kernel/sched/core.c | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 9a2fbf98fd6f..2e6ba9e301f0 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3899,6 +3899,28 @@ static inline void schedule_debug(struct task_struct *prev, bool preempt) schedstat_inc(this_rq()->sched_count); } +static void put_prev_task_balance(struct rq *rq, struct task_struct *prev, + struct rq_flags *rf) +{ +#ifdef CONFIG_SMP + const struct sched_class *class; + /* + * We must do the balancing pass before put_prev_task(), such + * that when we release the rq->lock the task is in the same + * state as before we took rq->lock. + * + * We can terminate the balance pass as soon as we know there is + * a runnable task of @class priority or higher. + */ + for_class_range(class, prev->sched_class, &idle_sched_class) { + if (class->balance(rq, prev, rf)) + break; + } +#endif + + put_prev_task(rq, prev); +} + /* * Pick up the highest-prio task: */ @@ -3932,22 +3954,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) } restart: -#ifdef CONFIG_SMP - /* - * We must do the balancing pass before put_next_task(), such - * that when we release the rq->lock the task is in the same - * state as before we took rq->lock. - * - * We can terminate the balance pass as soon as we know there is - * a runnable task of @class priority or higher. - */ - for_class_range(class, prev->sched_class, &idle_sched_class) { - if (class->balance(rq, prev, rf)) - break; - } -#endif - - put_prev_task(rq, prev); + put_prev_task_balance(rq, prev, rf); for_each_class(class) { p = class->pick_next_task(rq); From 5a6d6a6ccb5f48ca8cf7c6d64ff83fd9c7999390 Mon Sep 17 00:00:00 2001 From: Huaixin Chang Date: Mon, 20 Apr 2020 10:44:21 +0800 Subject: [PATCH 0186/1043] sched/fair: Refill bandwidth before scaling In order to prevent possible hardlockup of sched_cfs_period_timer() loop, loop count is introduced to denote whether to scale quota and period or not. However, scale is done between forwarding period timer and refilling cfs bandwidth runtime, which means that period timer is forwarded with old "period" while runtime is refilled with scaled "quota". Move do_sched_cfs_period_timer() before scaling to solve this. Fixes: 2e8e19226398 ("sched/fair: Limit sched_cfs_period_timer() loop to avoid hard lockup") Signed-off-by: Huaixin Chang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Ben Segall Reviewed-by: Phil Auld Link: https://lkml.kernel.org/r/20200420024421.22442-3-changhuaixin@linux.alibaba.com --- kernel/sched/fair.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c0216ef33c0c..fac5b2f85247 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5159,6 +5159,8 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) if (!overrun) break; + idle = do_sched_cfs_period_timer(cfs_b, overrun, flags); + if (++count > 3) { u64 new, old = ktime_to_ns(cfs_b->period); @@ -5188,8 +5190,6 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) /* reset count so we don't come right back in here */ count = 0; } - - idle = do_sched_cfs_period_timer(cfs_b, overrun, flags); } if (idle) cfs_b->period_active = 0; From f38f12d1e0811c0ee59260b2bdadedf99e16c4af Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Mon, 6 Apr 2020 15:47:50 +0800 Subject: [PATCH 0187/1043] sched/fair: Mark sched_init_granularity __init Function sched_init_granularity() is only called from __init functions, so mark it __init as well. Signed-off-by: Muchun Song Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Steven Rostedt (VMware) Link: https://lkml.kernel.org/r/20200406074750.56533-1-songmuchun@bytedance.com --- kernel/sched/fair.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index fac5b2f85247..cd7fd7e2b579 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -191,7 +191,7 @@ static void update_sysctl(void) #undef SET_SYSCTL } -void sched_init_granularity(void) +void __init sched_init_granularity(void) { update_sysctl(); } From bf2c59fce4074e55d622089b34be3a6bc95484fb Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 1 Apr 2020 17:40:33 -0400 Subject: [PATCH 0188/1043] sched/core: Fix illegal RCU from offline CPUs In the CPU-offline process, it calls mmdrop() after idle entry and the subsequent call to cpuhp_report_idle_dead(). Once execution passes the call to rcu_report_dead(), RCU is ignoring the CPU, which results in lockdep complaining when mmdrop() uses RCU from either memcg or debugobjects below. Fix it by cleaning up the active_mm state from BP instead. Every arch which has CONFIG_HOTPLUG_CPU should have already called idle_task_exit() from AP. The only exception is parisc because it switches them to &init_mm unconditionally (see smp_boot_one_cpu() and smp_cpu_init()), but the patch will still work there because it calls mmgrab(&init_mm) in smp_cpu_init() and then should call mmdrop(&init_mm) in finish_cpu(). WARNING: suspicious RCU usage ----------------------------- kernel/workqueue.c:710 RCU or wq_pool_mutex should be held! other info that might help us debug this: RCU used illegally from offline CPU! Call Trace: dump_stack+0xf4/0x164 (unreliable) lockdep_rcu_suspicious+0x140/0x164 get_work_pool+0x110/0x150 __queue_work+0x1bc/0xca0 queue_work_on+0x114/0x120 css_release+0x9c/0xc0 percpu_ref_put_many+0x204/0x230 free_pcp_prepare+0x264/0x570 free_unref_page+0x38/0xf0 __mmdrop+0x21c/0x2c0 idle_task_exit+0x170/0x1b0 pnv_smp_cpu_kill_self+0x38/0x2e0 cpu_die+0x48/0x64 arch_cpu_idle_dead+0x30/0x50 do_idle+0x2f4/0x470 cpu_startup_entry+0x38/0x40 start_secondary+0x7a8/0xa80 start_secondary_resume+0x10/0x14 Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Qian Cai Signed-off-by: Peter Zijlstra (Intel) Acked-by: Michael Ellerman (powerpc) Link: https://lkml.kernel.org/r/20200401214033.8448-1-cai@lca.pw --- arch/powerpc/platforms/powernv/smp.c | 1 - include/linux/sched/mm.h | 2 ++ kernel/cpu.c | 18 +++++++++++++++++- kernel/sched/core.c | 5 +++-- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c index 13e251699346..b2ba3e95bda7 100644 --- a/arch/powerpc/platforms/powernv/smp.c +++ b/arch/powerpc/platforms/powernv/smp.c @@ -167,7 +167,6 @@ static void pnv_smp_cpu_kill_self(void) /* Standard hot unplug procedure */ idle_task_exit(); - current->active_mm = NULL; /* for sanity */ cpu = smp_processor_id(); DBG("CPU%d offline\n", cpu); generic_set_cpu_dead(cpu); diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index c49257a3b510..a132d875d351 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -49,6 +49,8 @@ static inline void mmdrop(struct mm_struct *mm) __mmdrop(mm); } +void mmdrop(struct mm_struct *mm); + /* * This has to be called after a get_task_mm()/mmget_not_zero() * followed by taking the mmap_sem for writing before modifying the diff --git a/kernel/cpu.c b/kernel/cpu.c index 2371292f30b0..244d30544377 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -3,6 +3,7 @@ * * This code is licenced under the GPL. */ +#include #include #include #include @@ -564,6 +565,21 @@ static int bringup_cpu(unsigned int cpu) return bringup_wait_for_ap(cpu); } +static int finish_cpu(unsigned int cpu) +{ + struct task_struct *idle = idle_thread_get(cpu); + struct mm_struct *mm = idle->active_mm; + + /* + * idle_task_exit() will have switched to &init_mm, now + * clean up any remaining active_mm state. + */ + if (mm != &init_mm) + idle->active_mm = &init_mm; + mmdrop(mm); + return 0; +} + /* * Hotplug state machine related functions */ @@ -1549,7 +1565,7 @@ static struct cpuhp_step cpuhp_hp_states[] = { [CPUHP_BRINGUP_CPU] = { .name = "cpu:bringup", .startup.single = bringup_cpu, - .teardown.single = NULL, + .teardown.single = finish_cpu, .cant_stop = true, }, /* Final state before CPU kills itself */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 2e6ba9e301f0..f6ae2629f4e1 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6197,13 +6197,14 @@ void idle_task_exit(void) struct mm_struct *mm = current->active_mm; BUG_ON(cpu_online(smp_processor_id())); + BUG_ON(current != this_rq()->idle); if (mm != &init_mm) { switch_mm(mm, &init_mm, current); - current->active_mm = &init_mm; finish_arch_post_lock_switch(); } - mmdrop(mm); + + /* finish_cpu(), as ran on the BP, will clean up the active_mm state */ } /* From 17c891ab349138e8d8a59ca2700f42ce8af96f4e Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Tue, 21 Apr 2020 22:41:23 +0800 Subject: [PATCH 0189/1043] sched/fair: Use __this_cpu_read() in wake_wide() The code is executed with preemption(and interrupts) disabled, so it's safe to use __this_cpu_write(). Signed-off-by: Muchun Song Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200421144123.33580-1-songmuchun@bytedance.com --- kernel/sched/fair.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index cd7fd7e2b579..46b7bd41573f 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5718,7 +5718,7 @@ static int wake_wide(struct task_struct *p) { unsigned int master = current->wakee_flips; unsigned int slave = p->wakee_flips; - int factor = this_cpu_read(sd_llc_size); + int factor = __this_cpu_read(sd_llc_size); if (master < slave) swap(master, slave); From 12ac6782a40ad7636b6ef45680741825b64ab221 Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Tue, 21 Apr 2020 21:07:39 -0700 Subject: [PATCH 0190/1043] sched/swait: Reword some of the main description With both the increased use of swait and kvm no longer using it, we can reword some of the comments. While removing Linus' valid rant, I've also cared to explicitly mention that swait is very different than regular wait. In addition it is mentioned against using swait in favor of the regular flavor. Signed-off-by: Davidlohr Bueso Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200422040739.18601-6-dave@stgolabs.net --- include/linux/swait.h | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/include/linux/swait.h b/include/linux/swait.h index 73e06e9986d4..6a8c22b8c2a5 100644 --- a/include/linux/swait.h +++ b/include/linux/swait.h @@ -9,23 +9,10 @@ #include /* - * BROKEN wait-queues. - * - * These "simple" wait-queues are broken garbage, and should never be - * used. The comments below claim that they are "similar" to regular - * wait-queues, but the semantics are actually completely different, and - * every single user we have ever had has been buggy (or pointless). - * - * A "swake_up_one()" only wakes up _one_ waiter, which is not at all what - * "wake_up()" does, and has led to problems. In other cases, it has - * been fine, because there's only ever one waiter (kvm), but in that - * case gthe whole "simple" wait-queue is just pointless to begin with, - * since there is no "queue". Use "wake_up_process()" with a direct - * pointer instead. - * - * While these are very similar to regular wait queues (wait.h) the most - * important difference is that the simple waitqueue allows for deterministic - * behaviour -- IOW it has strictly bounded IRQ and lock hold times. + * Simple waitqueues are semantically very different to regular wait queues + * (wait.h). The most important difference is that the simple waitqueue allows + * for deterministic behaviour -- IOW it has strictly bounded IRQ and lock hold + * times. * * Mainly, this is accomplished by two things. Firstly not allowing swake_up_all * from IRQ disabled, and dropping the lock upon every wakeup, giving a higher @@ -39,7 +26,7 @@ * sleeper state. * * - the !exclusive mode; because that leads to O(n) wakeups, everything is - * exclusive. + * exclusive. As such swake_up_one will only ever awake _one_ waiter. * * - custom wake callback functions; because you cannot give any guarantees * about random code. This also allows swait to be used in RT, such that From b1d1779e5ef7a60b192b61fd97201f322e1e9303 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Thu, 23 Apr 2020 21:44:43 +0000 Subject: [PATCH 0191/1043] sched/core: Simplify sched_init() Currently root_task_group.shares and cfs_bandwidth are initialized for each online cpu, which not necessary. Let's take it out to do it only once. Signed-off-by: Wei Yang Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200423214443.29994-1-richard.weiyang@gmail.com --- kernel/sched/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index f6ae2629f4e1..b58efb1156eb 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -6597,6 +6597,8 @@ void __init sched_init(void) root_task_group.cfs_rq = (struct cfs_rq **)ptr; ptr += nr_cpu_ids * sizeof(void **); + root_task_group.shares = ROOT_TASK_GROUP_LOAD; + init_cfs_bandwidth(&root_task_group.cfs_bandwidth); #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_RT_GROUP_SCHED root_task_group.rt_se = (struct sched_rt_entity **)ptr; @@ -6649,7 +6651,6 @@ void __init sched_init(void) init_rt_rq(&rq->rt); init_dl_rq(&rq->dl); #ifdef CONFIG_FAIR_GROUP_SCHED - root_task_group.shares = ROOT_TASK_GROUP_LOAD; INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); rq->tmp_alone_branch = &rq->leaf_cfs_rq_list; /* @@ -6671,7 +6672,6 @@ void __init sched_init(void) * We achieve this by letting root_task_group's tasks sit * directly in rq->cfs (i.e root_task_group->se[] = NULL). */ - init_cfs_bandwidth(&root_task_group.cfs_bandwidth); init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); #endif /* CONFIG_FAIR_GROUP_SCHED */ From db9ff6ecf6efa6ffc45bfd5dc8d5708cfe5e89cb Mon Sep 17 00:00:00 2001 From: Zheng Bin Date: Wed, 29 Apr 2020 17:26:48 +0800 Subject: [PATCH 0192/1043] audit: make symbol 'audit_nfcfgs' static Fix sparse warnings: kernel/auditsc.c:138:32: warning: symbol 'audit_nfcfgs' was not declared. Should it be static? Reported-by: Hulk Robot Signed-off-by: Zheng Bin Signed-off-by: Paul Moore --- kernel/auditsc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/auditsc.c b/kernel/auditsc.c index d281c18d1771..cfe3486e5f31 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -135,7 +135,7 @@ struct audit_nfcfgop_tab { const char *s; }; -const struct audit_nfcfgop_tab audit_nfcfgs[] = { +static const struct audit_nfcfgop_tab audit_nfcfgs[] = { { AUDIT_XT_OP_REGISTER, "register" }, { AUDIT_XT_OP_REPLACE, "replace" }, { AUDIT_XT_OP_UNREGISTER, "unregister" }, From 4c09f8b6913a779ca0c70ea8058bf21537eebb3b Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Wed, 29 Apr 2020 07:30:53 +0000 Subject: [PATCH 0193/1043] selinux: fix error return code in policydb_read() Fix to return negative error code -ENOMEM from the kvcalloc() error handling case instead of 0, as done elsewhere in this function. Fixes: acdf52d97f82 ("selinux: convert to kvmalloc") Signed-off-by: Wei Yongjun Signed-off-by: Paul Moore --- security/selinux/ss/policydb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index 1c0041576643..a42369dd96a9 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -2638,6 +2638,7 @@ int policydb_read(struct policydb *p, void *fp) if (rc) goto bad; + rc = -ENOMEM; p->type_attr_map_array = kvcalloc(p->p_types.nprim, sizeof(*p->type_attr_map_array), GFP_KERNEL); From 3348bd33e8cf8a17138e8ce716ae474ec5d7001e Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Tue, 28 Apr 2020 14:55:11 +0200 Subject: [PATCH 0194/1043] selinux: simplify range_write() No need to traverse the hashtab to count its elements, hashtab already tracks it for us. Signed-off-by: Ondrej Mosnacek Signed-off-by: Paul Moore --- security/selinux/ss/policydb.c | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index a42369dd96a9..8a287a7afd9f 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -3405,14 +3405,6 @@ static int genfs_write(struct policydb *p, void *fp) return 0; } -static int hashtab_cnt(void *key, void *data, void *ptr) -{ - int *cnt = ptr; - *cnt = *cnt + 1; - - return 0; -} - static int range_write_helper(void *key, void *data, void *ptr) { __le32 buf[2]; @@ -3444,19 +3436,13 @@ static int range_write_helper(void *key, void *data, void *ptr) static int range_write(struct policydb *p, void *fp) { __le32 buf[1]; - int rc, nel; + int rc; struct policy_data pd; pd.p = p; pd.fp = fp; - /* count the number of entries in the hashtab */ - nel = 0; - rc = hashtab_map(p->range_tr, hashtab_cnt, &nel); - if (rc) - return rc; - - buf[0] = cpu_to_le32(nel); + buf[0] = cpu_to_le32(p->range_tr->nel); rc = put_entry(buf, sizeof(u32), 1, fp); if (rc) return rc; From 46619b44e431d85d64a8dfcb7166d0ae098544c8 Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Fri, 1 May 2020 21:51:11 +0200 Subject: [PATCH 0195/1043] selinux: fix return value on error in policydb_read() The value of rc is still zero from the last assignment when the error path is taken. Fix it by setting it to -ENOMEM before the hashtab_create() call. Reported-by: Dan Carpenter Fixes: e67b2ec9f617 ("selinux: store role transitions in a hash table") Signed-off-by: Ondrej Mosnacek Signed-off-by: Paul Moore --- security/selinux/ss/policydb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index 8a287a7afd9f..76358c9de129 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -2540,6 +2540,7 @@ int policydb_read(struct policydb *p, void *fp) goto bad; nel = le32_to_cpu(buf[0]); + rc = -ENOMEM; p->role_tr = hashtab_create(role_trans_hash, role_trans_cmp, nel); if (!p->role_tr) goto bad; From 03414a49ad5f3c56988c36d2070e402ffa17feaf Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Tue, 28 Apr 2020 14:55:12 +0200 Subject: [PATCH 0196/1043] selinux: do not allocate hashtabs dynamically It is simpler to allocate them statically in the corresponding structure, avoiding unnecessary kmalloc() calls and pointer dereferencing. Signed-off-by: Ondrej Mosnacek [PM: manual merging required in policydb.c] Signed-off-by: Paul Moore --- security/selinux/ss/hashtab.c | 51 ++++--------- security/selinux/ss/hashtab.h | 13 ++-- security/selinux/ss/mls.c | 14 ++-- security/selinux/ss/policydb.c | 127 ++++++++++++++++----------------- security/selinux/ss/policydb.h | 6 +- security/selinux/ss/services.c | 44 ++++++------ security/selinux/ss/symtab.c | 5 +- security/selinux/ss/symtab.h | 2 +- 8 files changed, 116 insertions(+), 146 deletions(-) diff --git a/security/selinux/ss/hashtab.c b/security/selinux/ss/hashtab.c index 883f19d32c28..5ee868116d70 100644 --- a/security/selinux/ss/hashtab.c +++ b/security/selinux/ss/hashtab.c @@ -29,34 +29,21 @@ static u32 hashtab_compute_size(u32 nel) return nel == 0 ? 0 : roundup_pow_of_two(nel); } -struct hashtab *hashtab_create(u32 (*hash_value)(struct hashtab *h, const void *key), - int (*keycmp)(struct hashtab *h, const void *key1, const void *key2), - u32 nel_hint) +int hashtab_init(struct hashtab *h, + u32 (*hash_value)(struct hashtab *h, const void *key), + int (*keycmp)(struct hashtab *h, const void *key1, + const void *key2), + u32 nel_hint) { - struct hashtab *p; - u32 i, size = hashtab_compute_size(nel_hint); + h->size = hashtab_compute_size(nel_hint); + h->nel = 0; + h->hash_value = hash_value; + h->keycmp = keycmp; + if (!h->size) + return 0; - p = kzalloc(sizeof(*p), GFP_KERNEL); - if (!p) - return p; - - p->size = size; - p->nel = 0; - p->hash_value = hash_value; - p->keycmp = keycmp; - if (!size) - return p; - - p->htable = kmalloc_array(size, sizeof(*p->htable), GFP_KERNEL); - if (!p->htable) { - kfree(p); - return NULL; - } - - for (i = 0; i < size; i++) - p->htable[i] = NULL; - - return p; + h->htable = kcalloc(h->size, sizeof(*h->htable), GFP_KERNEL); + return h->htable ? 0 : -ENOMEM; } int hashtab_insert(struct hashtab *h, void *key, void *datum) @@ -66,7 +53,7 @@ int hashtab_insert(struct hashtab *h, void *key, void *datum) cond_resched(); - if (!h || !h->size || h->nel == HASHTAB_MAX_NODES) + if (!h->size || h->nel == HASHTAB_MAX_NODES) return -EINVAL; hvalue = h->hash_value(h, key); @@ -102,7 +89,7 @@ void *hashtab_search(struct hashtab *h, const void *key) u32 hvalue; struct hashtab_node *cur; - if (!h || !h->size) + if (!h->size) return NULL; hvalue = h->hash_value(h, key); @@ -121,9 +108,6 @@ void hashtab_destroy(struct hashtab *h) u32 i; struct hashtab_node *cur, *temp; - if (!h) - return; - for (i = 0; i < h->size; i++) { cur = h->htable[i]; while (cur) { @@ -136,8 +120,6 @@ void hashtab_destroy(struct hashtab *h) kfree(h->htable); h->htable = NULL; - - kfree(h); } int hashtab_map(struct hashtab *h, @@ -148,9 +130,6 @@ int hashtab_map(struct hashtab *h, int ret; struct hashtab_node *cur; - if (!h) - return 0; - for (i = 0; i < h->size; i++) { cur = h->htable[i]; while (cur) { diff --git a/security/selinux/ss/hashtab.h b/security/selinux/ss/hashtab.h index dde54d9ff01c..31c11511fe10 100644 --- a/security/selinux/ss/hashtab.h +++ b/security/selinux/ss/hashtab.h @@ -35,14 +35,15 @@ struct hashtab_info { }; /* - * Creates a new hash table with the specified characteristics. + * Initializes a new hash table with the specified characteristics. * - * Returns NULL if insufficent space is available or - * the new hash table otherwise. + * Returns -ENOMEM if insufficient space is available or 0 otherwise. */ -struct hashtab *hashtab_create(u32 (*hash_value)(struct hashtab *h, const void *key), - int (*keycmp)(struct hashtab *h, const void *key1, const void *key2), - u32 nel_hint); +int hashtab_init(struct hashtab *h, + u32 (*hash_value)(struct hashtab *h, const void *key), + int (*keycmp)(struct hashtab *h, const void *key1, + const void *key2), + u32 nel_hint); /* * Inserts the specified (key, datum) pair into the specified hash table. diff --git a/security/selinux/ss/mls.c b/security/selinux/ss/mls.c index 6a5d7d08933d..cd8734f25b39 100644 --- a/security/selinux/ss/mls.c +++ b/security/selinux/ss/mls.c @@ -165,7 +165,7 @@ int mls_level_isvalid(struct policydb *p, struct mls_level *l) if (!l->sens || l->sens > p->p_levels.nprim) return 0; - levdatum = hashtab_search(p->p_levels.table, + levdatum = hashtab_search(&p->p_levels.table, sym_name(p, SYM_LEVELS, l->sens - 1)); if (!levdatum) return 0; @@ -293,7 +293,7 @@ int mls_context_to_sid(struct policydb *pol, *(next_cat++) = '\0'; /* Parse sensitivity. */ - levdatum = hashtab_search(pol->p_levels.table, sensitivity); + levdatum = hashtab_search(&pol->p_levels.table, sensitivity); if (!levdatum) return -EINVAL; context->range.level[l].sens = levdatum->level->sens; @@ -312,7 +312,7 @@ int mls_context_to_sid(struct policydb *pol, *rngptr++ = '\0'; } - catdatum = hashtab_search(pol->p_cats.table, cur_cat); + catdatum = hashtab_search(&pol->p_cats.table, cur_cat); if (!catdatum) return -EINVAL; @@ -325,7 +325,7 @@ int mls_context_to_sid(struct policydb *pol, if (rngptr == NULL) continue; - rngdatum = hashtab_search(pol->p_cats.table, rngptr); + rngdatum = hashtab_search(&pol->p_cats.table, rngptr); if (!rngdatum) return -EINVAL; @@ -458,7 +458,7 @@ int mls_convert_context(struct policydb *oldp, return 0; for (l = 0; l < 2; l++) { - levdatum = hashtab_search(newp->p_levels.table, + levdatum = hashtab_search(&newp->p_levels.table, sym_name(oldp, SYM_LEVELS, oldc->range.level[l].sens - 1)); @@ -470,7 +470,7 @@ int mls_convert_context(struct policydb *oldp, node, i) { int rc; - catdatum = hashtab_search(newp->p_cats.table, + catdatum = hashtab_search(&newp->p_cats.table, sym_name(oldp, SYM_CATS, i)); if (!catdatum) return -EINVAL; @@ -506,7 +506,7 @@ int mls_compute_sid(struct policydb *p, rtr.source_type = scontext->type; rtr.target_type = tcontext->type; rtr.target_class = tclass; - r = hashtab_search(p->range_tr, &rtr); + r = hashtab_search(&p->range_tr, &rtr); if (r) return mls_range_set(newcontext, r); diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index 76358c9de129..a30cad18931b 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -195,8 +195,8 @@ static int common_destroy(void *key, void *datum, void *p) kfree(key); if (datum) { comdatum = datum; - hashtab_map(comdatum->permissions.table, perm_destroy, NULL); - hashtab_destroy(comdatum->permissions.table); + hashtab_map(&comdatum->permissions.table, perm_destroy, NULL); + hashtab_destroy(&comdatum->permissions.table); } kfree(datum); return 0; @@ -224,8 +224,8 @@ static int cls_destroy(void *key, void *datum, void *p) kfree(key); if (datum) { cladatum = datum; - hashtab_map(cladatum->permissions.table, perm_destroy, NULL); - hashtab_destroy(cladatum->permissions.table); + hashtab_map(&cladatum->permissions.table, perm_destroy, NULL); + hashtab_destroy(&cladatum->permissions.table); constraint = cladatum->constraints; while (constraint) { e = constraint->expr; @@ -400,7 +400,7 @@ static int roles_init(struct policydb *p) if (!key) goto out; - rc = hashtab_insert(p->p_roles.table, key, role); + rc = hashtab_insert(&p->p_roles.table, key, role); if (rc) goto out; @@ -668,7 +668,7 @@ static void symtab_hash_eval(struct symtab *s) int i; for (i = 0; i < SYM_NUM; i++) - hash_eval(s[i].table, symtab_name[i]); + hash_eval(&s[i].table, symtab_name[i]); } #else @@ -739,7 +739,7 @@ static int policydb_index(struct policydb *p) if (!p->sym_val_to_name[i]) return -ENOMEM; - rc = hashtab_map(p->symtab[i].table, index_f[i], p); + rc = hashtab_map(&p->symtab[i].table, index_f[i], p); if (rc) goto out; } @@ -760,8 +760,8 @@ void policydb_destroy(struct policydb *p) for (i = 0; i < SYM_NUM; i++) { cond_resched(); - hashtab_map(p->symtab[i].table, destroy_f[i], NULL); - hashtab_destroy(p->symtab[i].table); + hashtab_map(&p->symtab[i].table, destroy_f[i], NULL); + hashtab_destroy(&p->symtab[i].table); } for (i = 0; i < SYM_NUM; i++) @@ -803,8 +803,8 @@ void policydb_destroy(struct policydb *p) cond_policydb_destroy(p); - hashtab_map(p->role_tr, role_tr_destroy, NULL); - hashtab_destroy(p->role_tr); + hashtab_map(&p->role_tr, role_tr_destroy, NULL); + hashtab_destroy(&p->role_tr); for (ra = p->role_allow; ra; ra = ra->next) { cond_resched(); @@ -813,11 +813,11 @@ void policydb_destroy(struct policydb *p) } kfree(lra); - hashtab_map(p->filename_trans, filenametr_destroy, NULL); - hashtab_destroy(p->filename_trans); + hashtab_map(&p->filename_trans, filenametr_destroy, NULL); + hashtab_destroy(&p->filename_trans); - hashtab_map(p->range_tr, range_tr_destroy, NULL); - hashtab_destroy(p->range_tr); + hashtab_map(&p->range_tr, range_tr_destroy, NULL); + hashtab_destroy(&p->range_tr); if (p->type_attr_map_array) { for (i = 0; i < p->p_types.nprim; i++) @@ -1128,7 +1128,7 @@ static int common_read(struct policydb *p, struct hashtab *h, void *fp) goto bad; for (i = 0; i < nel; i++) { - rc = perm_read(p, comdatum->permissions.table, fp); + rc = perm_read(p, &comdatum->permissions.table, fp); if (rc) goto bad; } @@ -1300,7 +1300,8 @@ static int class_read(struct policydb *p, struct hashtab *h, void *fp) goto bad; rc = -EINVAL; - cladatum->comdatum = hashtab_search(p->p_commons.table, cladatum->comkey); + cladatum->comdatum = hashtab_search(&p->p_commons.table, + cladatum->comkey); if (!cladatum->comdatum) { pr_err("SELinux: unknown common %s\n", cladatum->comkey); @@ -1308,7 +1309,7 @@ static int class_read(struct policydb *p, struct hashtab *h, void *fp) } } for (i = 0; i < nel; i++) { - rc = perm_read(p, cladatum->permissions.table, fp); + rc = perm_read(p, &cladatum->permissions.table, fp); if (rc) goto bad; } @@ -1731,18 +1732,15 @@ static int policydb_bounds_sanity_check(struct policydb *p) if (p->policyvers < POLICYDB_VERSION_BOUNDARY) return 0; - rc = hashtab_map(p->p_users.table, - user_bounds_sanity_check, p); + rc = hashtab_map(&p->p_users.table, user_bounds_sanity_check, p); if (rc) return rc; - rc = hashtab_map(p->p_roles.table, - role_bounds_sanity_check, p); + rc = hashtab_map(&p->p_roles.table, role_bounds_sanity_check, p); if (rc) return rc; - rc = hashtab_map(p->p_types.table, - type_bounds_sanity_check, p); + rc = hashtab_map(&p->p_types.table, type_bounds_sanity_check, p); if (rc) return rc; @@ -1753,7 +1751,7 @@ u16 string_to_security_class(struct policydb *p, const char *name) { struct class_datum *cladatum; - cladatum = hashtab_search(p->p_classes.table, name); + cladatum = hashtab_search(&p->p_classes.table, name); if (!cladatum) return 0; @@ -1772,11 +1770,9 @@ u32 string_to_av_perm(struct policydb *p, u16 tclass, const char *name) cladatum = p->class_val_to_struct[tclass-1]; comdatum = cladatum->comdatum; if (comdatum) - perdatum = hashtab_search(comdatum->permissions.table, - name); + perdatum = hashtab_search(&comdatum->permissions.table, name); if (!perdatum) - perdatum = hashtab_search(cladatum->permissions.table, - name); + perdatum = hashtab_search(&cladatum->permissions.table, name); if (!perdatum) return 0; @@ -1800,9 +1796,9 @@ static int range_read(struct policydb *p, void *fp) nel = le32_to_cpu(buf[0]); - p->range_tr = hashtab_create(rangetr_hash, rangetr_cmp, nel); - if (!p->range_tr) - return -ENOMEM; + rc = hashtab_init(&p->range_tr, rangetr_hash, rangetr_cmp, nel); + if (rc) + return rc; for (i = 0; i < nel; i++) { rc = -ENOMEM; @@ -1845,14 +1841,14 @@ static int range_read(struct policydb *p, void *fp) goto out; } - rc = hashtab_insert(p->range_tr, rt, r); + rc = hashtab_insert(&p->range_tr, rt, r); if (rc) goto out; rt = NULL; r = NULL; } - hash_eval(p->range_tr, "rangetr"); + hash_eval(&p->range_tr, "rangetr"); rc = 0; out: kfree(rt); @@ -1892,7 +1888,7 @@ static int filename_trans_read_helper_compat(struct policydb *p, void *fp) otype = le32_to_cpu(buf[3]); last = NULL; - datum = hashtab_search(p->filename_trans, &key); + datum = hashtab_search(&p->filename_trans, &key); while (datum) { if (unlikely(ebitmap_get_bit(&datum->stypes, stype - 1))) { /* conflicting/duplicate rules are ignored */ @@ -1922,7 +1918,7 @@ static int filename_trans_read_helper_compat(struct policydb *p, void *fp) if (!ft) goto out; - rc = hashtab_insert(p->filename_trans, ft, datum); + rc = hashtab_insert(&p->filename_trans, ft, datum); if (rc) goto out; name = NULL; @@ -2010,7 +2006,7 @@ static int filename_trans_read_helper(struct policydb *p, void *fp) ft->tclass = tclass; ft->name = name; - rc = hashtab_insert(p->filename_trans, ft, first); + rc = hashtab_insert(&p->filename_trans, ft, first); if (rc == -EEXIST) pr_err("SELinux: Duplicate filename transition key\n"); if (rc) @@ -2047,10 +2043,11 @@ static int filename_trans_read(struct policydb *p, void *fp) if (p->policyvers < POLICYDB_VERSION_COMP_FTRANS) { p->compat_filename_trans_count = nel; - p->filename_trans = hashtab_create(filenametr_hash, - filenametr_cmp, (1 << 11)); - if (!p->filename_trans) - return -ENOMEM; + + rc = hashtab_init(&p->filename_trans, filenametr_hash, + filenametr_cmp, (1 << 11)); + if (rc) + return rc; for (i = 0; i < nel; i++) { rc = filename_trans_read_helper_compat(p, fp); @@ -2058,10 +2055,10 @@ static int filename_trans_read(struct policydb *p, void *fp) return rc; } } else { - p->filename_trans = hashtab_create(filenametr_hash, - filenametr_cmp, nel); - if (!p->filename_trans) - return -ENOMEM; + rc = hashtab_init(&p->filename_trans, filenametr_hash, + filenametr_cmp, nel); + if (rc) + return rc; for (i = 0; i < nel; i++) { rc = filename_trans_read_helper(p, fp); @@ -2069,7 +2066,7 @@ static int filename_trans_read(struct policydb *p, void *fp) return rc; } } - hash_eval(p->filename_trans, "filenametr"); + hash_eval(&p->filename_trans, "filenametr"); return 0; } @@ -2512,7 +2509,7 @@ int policydb_read(struct policydb *p, void *fp) } for (j = 0; j < nel; j++) { - rc = read_f[i](p, p->symtab[i].table, fp); + rc = read_f[i](p, &p->symtab[i].table, fp); if (rc) goto bad; } @@ -2540,9 +2537,8 @@ int policydb_read(struct policydb *p, void *fp) goto bad; nel = le32_to_cpu(buf[0]); - rc = -ENOMEM; - p->role_tr = hashtab_create(role_trans_hash, role_trans_cmp, nel); - if (!p->role_tr) + rc = hashtab_init(&p->role_tr, role_trans_hash, role_trans_cmp, nel); + if (rc) goto bad; for (i = 0; i < nel; i++) { rc = -ENOMEM; @@ -2578,7 +2574,7 @@ int policydb_read(struct policydb *p, void *fp) !policydb_role_isvalid(p, rtd->new_role)) goto bad; - rc = hashtab_insert(p->role_tr, rtk, rtd); + rc = hashtab_insert(&p->role_tr, rtk, rtd); if (rc) goto bad; @@ -2822,12 +2818,12 @@ static int role_trans_write(struct policydb *p, void *fp) __le32 buf[1]; int rc; - buf[0] = cpu_to_le32(p->role_tr->nel); + buf[0] = cpu_to_le32(p->role_tr.nel); rc = put_entry(buf, sizeof(u32), 1, fp); if (rc) return rc; - return hashtab_map(p->role_tr, role_trans_write_one, &pd); + return hashtab_map(&p->role_tr, role_trans_write_one, &pd); } static int role_allow_write(struct role_allow *r, void *fp) @@ -2921,7 +2917,7 @@ static int common_write(void *vkey, void *datum, void *ptr) buf[0] = cpu_to_le32(len); buf[1] = cpu_to_le32(comdatum->value); buf[2] = cpu_to_le32(comdatum->permissions.nprim); - buf[3] = cpu_to_le32(comdatum->permissions.table->nel); + buf[3] = cpu_to_le32(comdatum->permissions.table.nel); rc = put_entry(buf, sizeof(u32), 4, fp); if (rc) return rc; @@ -2930,7 +2926,7 @@ static int common_write(void *vkey, void *datum, void *ptr) if (rc) return rc; - rc = hashtab_map(comdatum->permissions.table, perm_write, fp); + rc = hashtab_map(&comdatum->permissions.table, perm_write, fp); if (rc) return rc; @@ -3029,10 +3025,7 @@ static int class_write(void *vkey, void *datum, void *ptr) buf[1] = cpu_to_le32(len2); buf[2] = cpu_to_le32(cladatum->value); buf[3] = cpu_to_le32(cladatum->permissions.nprim); - if (cladatum->permissions.table) - buf[4] = cpu_to_le32(cladatum->permissions.table->nel); - else - buf[4] = 0; + buf[4] = cpu_to_le32(cladatum->permissions.table.nel); buf[5] = cpu_to_le32(ncons); rc = put_entry(buf, sizeof(u32), 6, fp); if (rc) @@ -3048,7 +3041,7 @@ static int class_write(void *vkey, void *datum, void *ptr) return rc; } - rc = hashtab_map(cladatum->permissions.table, perm_write, fp); + rc = hashtab_map(&cladatum->permissions.table, perm_write, fp); if (rc) return rc; @@ -3443,13 +3436,13 @@ static int range_write(struct policydb *p, void *fp) pd.p = p; pd.fp = fp; - buf[0] = cpu_to_le32(p->range_tr->nel); + buf[0] = cpu_to_le32(p->range_tr.nel); rc = put_entry(buf, sizeof(u32), 1, fp); if (rc) return rc; /* actually write all of the entries */ - rc = hashtab_map(p->range_tr, range_write_helper, &pd); + rc = hashtab_map(&p->range_tr, range_write_helper, &pd); if (rc) return rc; @@ -3556,15 +3549,15 @@ static int filename_trans_write(struct policydb *p, void *fp) if (rc) return rc; - rc = hashtab_map(p->filename_trans, + rc = hashtab_map(&p->filename_trans, filename_write_helper_compat, fp); } else { - buf[0] = cpu_to_le32(p->filename_trans->nel); + buf[0] = cpu_to_le32(p->filename_trans.nel); rc = put_entry(buf, sizeof(u32), 1, fp); if (rc) return rc; - rc = hashtab_map(p->filename_trans, filename_write_helper, fp); + rc = hashtab_map(&p->filename_trans, filename_write_helper, fp); } return rc; } @@ -3653,12 +3646,12 @@ int policydb_write(struct policydb *p, void *fp) pd.p = p; buf[0] = cpu_to_le32(p->symtab[i].nprim); - buf[1] = cpu_to_le32(p->symtab[i].table->nel); + buf[1] = cpu_to_le32(p->symtab[i].table.nel); rc = put_entry(buf, sizeof(u32), 2, fp); if (rc) return rc; - rc = hashtab_map(p->symtab[i].table, write_f[i], &pd); + rc = hashtab_map(&p->symtab[i].table, write_f[i], &pd); if (rc) return rc; } diff --git a/security/selinux/ss/policydb.h b/security/selinux/ss/policydb.h index 35dc6aa7904d..9591c9587cb6 100644 --- a/security/selinux/ss/policydb.h +++ b/security/selinux/ss/policydb.h @@ -263,13 +263,13 @@ struct policydb { struct avtab te_avtab; /* role transitions */ - struct hashtab *role_tr; + struct hashtab role_tr; /* file transitions with the last path component */ /* quickly exclude lookups when parent ttype has no rules */ struct ebitmap filename_trans_ttypes; /* actual set of filename_trans rules */ - struct hashtab *filename_trans; + struct hashtab filename_trans; /* only used if policyvers < POLICYDB_VERSION_COMP_FTRANS */ u32 compat_filename_trans_count; @@ -294,7 +294,7 @@ struct policydb { struct genfs *genfs; /* range transitions table (range_trans_key -> mls_range) */ - struct hashtab *range_tr; + struct hashtab range_tr; /* type -> attribute reverse mapping */ struct ebitmap *type_attr_map_array; diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index b49a336b1e6e..313919bd42f8 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -482,11 +482,11 @@ static void security_dump_masked_av(struct policydb *policydb, /* init permission_names */ if (common_dat && - hashtab_map(common_dat->permissions.table, + hashtab_map(&common_dat->permissions.table, dump_masked_av_helper, permission_names) < 0) goto out; - if (hashtab_map(tclass_dat->permissions.table, + if (hashtab_map(&tclass_dat->permissions.table, dump_masked_av_helper, permission_names) < 0) goto out; @@ -1441,7 +1441,7 @@ static int string_to_context_struct(struct policydb *pol, *p++ = 0; - usrdatum = hashtab_search(pol->p_users.table, scontextp); + usrdatum = hashtab_search(&pol->p_users.table, scontextp); if (!usrdatum) goto out; @@ -1457,7 +1457,7 @@ static int string_to_context_struct(struct policydb *pol, *p++ = 0; - role = hashtab_search(pol->p_roles.table, scontextp); + role = hashtab_search(&pol->p_roles.table, scontextp); if (!role) goto out; ctx->role = role->value; @@ -1469,7 +1469,7 @@ static int string_to_context_struct(struct policydb *pol, oldc = *p; *p++ = 0; - typdatum = hashtab_search(pol->p_types.table, scontextp); + typdatum = hashtab_search(&pol->p_types.table, scontextp); if (!typdatum || typdatum->attribute) goto out; @@ -1671,7 +1671,7 @@ static void filename_compute_type(struct policydb *policydb, ft.tclass = tclass; ft.name = objname; - datum = hashtab_search(policydb->filename_trans, &ft); + datum = hashtab_search(&policydb->filename_trans, &ft); while (datum) { if (ebitmap_get_bit(&datum->stypes, stype - 1)) { newcontext->type = datum->otype; @@ -1834,7 +1834,7 @@ static int security_compute_sid(struct selinux_state *state, .tclass = tclass, }; - rtd = hashtab_search(policydb->role_tr, &rtk); + rtd = hashtab_search(&policydb->role_tr, &rtk); if (rtd) newcontext.role = rtd->new_role; } @@ -2024,7 +2024,7 @@ static int convert_context(struct context *oldc, struct context *newc, void *p) /* Convert the user. */ rc = -EINVAL; - usrdatum = hashtab_search(args->newp->p_users.table, + usrdatum = hashtab_search(&args->newp->p_users.table, sym_name(args->oldp, SYM_USERS, oldc->user - 1)); if (!usrdatum) @@ -2033,7 +2033,7 @@ static int convert_context(struct context *oldc, struct context *newc, void *p) /* Convert the role. */ rc = -EINVAL; - role = hashtab_search(args->newp->p_roles.table, + role = hashtab_search(&args->newp->p_roles.table, sym_name(args->oldp, SYM_ROLES, oldc->role - 1)); if (!role) goto bad; @@ -2041,7 +2041,7 @@ static int convert_context(struct context *oldc, struct context *newc, void *p) /* Convert the type. */ rc = -EINVAL; - typdatum = hashtab_search(args->newp->p_types.table, + typdatum = hashtab_search(&args->newp->p_types.table, sym_name(args->oldp, SYM_TYPES, oldc->type - 1)); if (!typdatum) @@ -2623,7 +2623,7 @@ int security_get_user_sids(struct selinux_state *state, goto out_unlock; rc = -EINVAL; - user = hashtab_search(policydb->p_users.table, username); + user = hashtab_search(&policydb->p_users.table, username); if (!user) goto out_unlock; @@ -2975,7 +2975,7 @@ static int security_preserve_bools(struct selinux_state *state, if (rc) goto out; for (i = 0; i < nbools; i++) { - booldatum = hashtab_search(policydb->p_bools.table, bnames[i]); + booldatum = hashtab_search(&policydb->p_bools.table, bnames[i]); if (booldatum) booldatum->state = bvalues[i]; } @@ -3189,8 +3189,8 @@ int security_get_classes(struct selinux_state *state, if (!*classes) goto out; - rc = hashtab_map(policydb->p_classes.table, get_classes_callback, - *classes); + rc = hashtab_map(&policydb->p_classes.table, get_classes_callback, + *classes); if (rc) { int i; for (i = 0; i < *nclasses; i++) @@ -3226,7 +3226,7 @@ int security_get_permissions(struct selinux_state *state, read_lock(&state->ss->policy_rwlock); rc = -EINVAL; - match = hashtab_search(policydb->p_classes.table, class); + match = hashtab_search(&policydb->p_classes.table, class); if (!match) { pr_err("SELinux: %s: unrecognized class %s\n", __func__, class); @@ -3240,14 +3240,14 @@ int security_get_permissions(struct selinux_state *state, goto out; if (match->comdatum) { - rc = hashtab_map(match->comdatum->permissions.table, - get_permissions_callback, *perms); + rc = hashtab_map(&match->comdatum->permissions.table, + get_permissions_callback, *perms); if (rc) goto err; } - rc = hashtab_map(match->permissions.table, get_permissions_callback, - *perms); + rc = hashtab_map(&match->permissions.table, get_permissions_callback, + *perms); if (rc) goto err; @@ -3365,7 +3365,7 @@ int selinux_audit_rule_init(u32 field, u32 op, char *rulestr, void **vrule) case AUDIT_SUBJ_USER: case AUDIT_OBJ_USER: rc = -EINVAL; - userdatum = hashtab_search(policydb->p_users.table, rulestr); + userdatum = hashtab_search(&policydb->p_users.table, rulestr); if (!userdatum) goto out; tmprule->au_ctxt.user = userdatum->value; @@ -3373,7 +3373,7 @@ int selinux_audit_rule_init(u32 field, u32 op, char *rulestr, void **vrule) case AUDIT_SUBJ_ROLE: case AUDIT_OBJ_ROLE: rc = -EINVAL; - roledatum = hashtab_search(policydb->p_roles.table, rulestr); + roledatum = hashtab_search(&policydb->p_roles.table, rulestr); if (!roledatum) goto out; tmprule->au_ctxt.role = roledatum->value; @@ -3381,7 +3381,7 @@ int selinux_audit_rule_init(u32 field, u32 op, char *rulestr, void **vrule) case AUDIT_SUBJ_TYPE: case AUDIT_OBJ_TYPE: rc = -EINVAL; - typedatum = hashtab_search(policydb->p_types.table, rulestr); + typedatum = hashtab_search(&policydb->p_types.table, rulestr); if (!typedatum) goto out; tmprule->au_ctxt.type = typedatum->value; diff --git a/security/selinux/ss/symtab.c b/security/selinux/ss/symtab.c index dc2ce94165d3..92d7a948070e 100644 --- a/security/selinux/ss/symtab.c +++ b/security/selinux/ss/symtab.c @@ -35,10 +35,7 @@ static int symcmp(struct hashtab *h, const void *key1, const void *key2) int symtab_init(struct symtab *s, unsigned int size) { - s->table = hashtab_create(symhash, symcmp, size); - if (!s->table) - return -ENOMEM; s->nprim = 0; - return 0; + return hashtab_init(&s->table, symhash, symcmp, size); } diff --git a/security/selinux/ss/symtab.h b/security/selinux/ss/symtab.h index d75fcafe7281..f145301b9d9f 100644 --- a/security/selinux/ss/symtab.h +++ b/security/selinux/ss/symtab.h @@ -13,7 +13,7 @@ #include "hashtab.h" struct symtab { - struct hashtab *table; /* hash table (keyed on a string) */ + struct hashtab table; /* hash table (keyed on a string) */ u32 nprim; /* number of primary names in table */ }; From 3aed240e577ea9f5b070358766d46a0e285f0b9e Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Fri, 1 May 2020 00:48:02 +0800 Subject: [PATCH 0197/1043] MIPS: Loongson64: Correct TLB type for Loongson-3 Classic Huacai just informed me that some early Loongson-3A2000 had wrong TLB type in Config0 register. That means we have to correct it via PRID. It looks like I shoudn't drop MIPS_CPU_FTLB flag in PRID case for Loongson-3 Classic. Fixes: da1bd29742b1 ("MIPS: Loongson64: Probe CPU features via CPUCFG") Signed-off-by: Jiaxun Yang Reported-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/cpu-probe.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index 1736c173b242..ca2e6f1af4fe 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -1999,8 +1999,11 @@ static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) * Loongson-3 Classic did not implement MIPS standard TLBINV * but implemented TLBINVF and EHINV. As currently we're only * using these two features, enable MIPS_CPU_TLBINV as well. + * + * Also some early Loongson-3A2000 had wrong TLB type in Config + * register, we correct it here. */ - c->options |= MIPS_CPU_TLBINV | MIPS_CPU_LDPTE; + c->options |= MIPS_CPU_FTLB | MIPS_CPU_TLBINV | MIPS_CPU_LDPTE; c->writecombine = _CACHE_UNCACHED_ACCELERATED; c->ases |= (MIPS_ASE_LOONGSON_MMI | MIPS_ASE_LOONGSON_CAM | MIPS_ASE_LOONGSON_EXT | MIPS_ASE_LOONGSON_EXT2); From 9d139131e973dac0ef6207a161b228a7fcad8180 Mon Sep 17 00:00:00 2001 From: Bin Meng Date: Fri, 1 May 2020 21:25:49 -0700 Subject: [PATCH 0198/1043] mips: Drop CONFIG_MTD_M25P80 in various defconfig files Drop CONFIG_MTD_M25P80 that was removed in commit b35b9a10362d ("mtd: spi-nor: Move m25p80 code in spi-nor.c") Signed-off-by: Bin Meng Signed-off-by: Thomas Bogendoerfer --- arch/mips/configs/ath79_defconfig | 1 - arch/mips/configs/db1xxx_defconfig | 1 - arch/mips/configs/generic/board-ocelot.config | 1 - arch/mips/configs/pistachio_defconfig | 1 - arch/mips/configs/rt305x_defconfig | 1 - 5 files changed, 5 deletions(-) diff --git a/arch/mips/configs/ath79_defconfig b/arch/mips/configs/ath79_defconfig index 3d14d67dc746..96622a2ad333 100644 --- a/arch/mips/configs/ath79_defconfig +++ b/arch/mips/configs/ath79_defconfig @@ -46,7 +46,6 @@ CONFIG_MTD_JEDECPROBE=y CONFIG_MTD_CFI_AMDSTD=y CONFIG_MTD_COMPLEX_MAPPINGS=y CONFIG_MTD_PHYSMAP=y -CONFIG_MTD_M25P80=y CONFIG_MTD_SPI_NOR=y CONFIG_NETDEVICES=y CONFIG_ATH9K=m diff --git a/arch/mips/configs/db1xxx_defconfig b/arch/mips/configs/db1xxx_defconfig index e6f3e8e3da39..b8bd66300996 100644 --- a/arch/mips/configs/db1xxx_defconfig +++ b/arch/mips/configs/db1xxx_defconfig @@ -92,7 +92,6 @@ CONFIG_MTD_CFI=y CONFIG_MTD_CFI_ADV_OPTIONS=y CONFIG_MTD_CFI_AMDSTD=y CONFIG_MTD_PHYSMAP=y -CONFIG_MTD_M25P80=y CONFIG_MTD_SST25L=y CONFIG_MTD_RAW_NAND=y CONFIG_MTD_NAND_ECC_SW_BCH=y diff --git a/arch/mips/configs/generic/board-ocelot.config b/arch/mips/configs/generic/board-ocelot.config index 7626f2a75b03..510709565404 100644 --- a/arch/mips/configs/generic/board-ocelot.config +++ b/arch/mips/configs/generic/board-ocelot.config @@ -9,7 +9,6 @@ CONFIG_GENERIC_PHY=y CONFIG_MTD=y CONFIG_MTD_CMDLINE_PARTS=y CONFIG_MTD_BLOCK=y -CONFIG_MTD_M25P80=y CONFIG_MTD_RAW_NAND=y CONFIG_MTD_NAND_PLATFORM=y CONFIG_MTD_SPI_NOR=y diff --git a/arch/mips/configs/pistachio_defconfig b/arch/mips/configs/pistachio_defconfig index 24e07180c57d..b9adf15ebbec 100644 --- a/arch/mips/configs/pistachio_defconfig +++ b/arch/mips/configs/pistachio_defconfig @@ -127,7 +127,6 @@ CONFIG_DEBUG_DEVRES=y CONFIG_CONNECTOR=y CONFIG_MTD=y CONFIG_MTD_BLOCK=y -CONFIG_MTD_M25P80=y CONFIG_MTD_SPI_NOR=y CONFIG_MTD_UBI=y CONFIG_MTD_UBI_BLOCK=y diff --git a/arch/mips/configs/rt305x_defconfig b/arch/mips/configs/rt305x_defconfig index 8c2ead53007a..fec5851c164b 100644 --- a/arch/mips/configs/rt305x_defconfig +++ b/arch/mips/configs/rt305x_defconfig @@ -76,7 +76,6 @@ CONFIG_MTD_CFI_AMDSTD=y CONFIG_MTD_COMPLEX_MAPPINGS=y CONFIG_MTD_PHYSMAP=y CONFIG_MTD_PHYSMAP_OF=y -CONFIG_MTD_M25P80=y CONFIG_MTD_SPI_NOR=y CONFIG_EEPROM_93CX6=m CONFIG_SCSI=y From 7cc8f2d5aca162f4e2ea07e53d3123f30a7b2582 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Sat, 2 May 2020 18:46:24 +0800 Subject: [PATCH 0199/1043] MIPS: perf: Remove unnecessary "fallthrough" pseudo keywords The last branch of switch-case doesn't need a "fallthrough" pseudo keyword, and it will cause errors when building a kernel with -Werror: arch/mips/kernel/perf_event_mipsxx.c: In function 'reset_counters': include/linux/compiler_attributes.h:200:41: error: attribute 'fallthrough' not preceding a case label or default label [-Werror] 200 | # define fallthrough __attribute__((__fallthrough__)) | ^~~~~~~~~~~~~ >> arch/mips/kernel/perf_event_mipsxx.c:932:3: note: in expansion of macro 'fallthrough' 932 | fallthrough; | ^~~~~~~~~~~ arch/mips/kernel/perf_event_mipsxx.c: In function 'loongson3_reset_counters': include/linux/compiler_attributes.h:200:41: error: attribute 'fallthrough' not preceding a case label or default label [-Werror] 200 | # define fallthrough __attribute__((__fallthrough__)) | ^~~~~~~~~~~~~ arch/mips/kernel/perf_event_mipsxx.c:903:3: note: in expansion of macro 'fallthrough' 903 | fallthrough; | ^~~~~~~~~~~ cc1: all warnings being treated as errors Fix it by removing unnecessary "fallthrough" pseudo keywords. Fixes: e9dfbaaeef1c9fe ("MIPS: perf: Add hardware perf events support for new Loongson-3") Reported-by: kbuild test robot Signed-off-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/perf_event_mipsxx.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c index a14974ca6d13..efce5defcc5c 100644 --- a/arch/mips/kernel/perf_event_mipsxx.c +++ b/arch/mips/kernel/perf_event_mipsxx.c @@ -900,7 +900,7 @@ static void loongson3_reset_counters(void *arg) mipspmu.write_counter(0, 0); mipsxx_pmu_write_control(0, 575<<5); mipspmu.write_counter(0, 0); - fallthrough; + break; } } @@ -929,7 +929,7 @@ static void reset_counters(void *arg) case 1: mipsxx_pmu_write_control(0, 0); mipspmu.write_counter(0, 0); - fallthrough; + break; } } From 3a06c204fbc83de2a7eb6fc2f4548f813faed454 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Sun, 3 May 2020 08:05:02 +0800 Subject: [PATCH 0200/1043] MIPS: tools: Move "returns" after "loongson3-llsc-check" Just move "returns" after "loongson3-llsc-check", no function changes. Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/tools/loongson3-llsc-check.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/tools/loongson3-llsc-check.c b/arch/mips/tools/loongson3-llsc-check.c index facd016dc51d..bdbc7b4324ec 100644 --- a/arch/mips/tools/loongson3-llsc-check.c +++ b/arch/mips/tools/loongson3-llsc-check.c @@ -303,7 +303,7 @@ out_munmap: out_close: close(vmlinux_fd); out_ret: - fprintf(stdout, "loongson3-llsc-check %s\n", - status ? "returns failure" : "returns success"); + fprintf(stdout, "loongson3-llsc-check returns %s\n", + status ? "failure" : "success"); return status; } From efbe3c2493d2f7a1e1a753780fe727b34709ebd2 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Thu, 30 Apr 2020 07:41:33 -0700 Subject: [PATCH 0201/1043] fs: Remove unneeded IS_DAX() check in io_is_direct() Remove the check because DAX now has it's own read/write methods and file systems which support DAX check IS_DAX() prior to IOCB_DIRECT on their own. Therefore, it does not matter if the file state is DAX when the iocb flags are created. Also remove io_is_direct() as it is just a simple flag check. Reviewed-by: Dave Chinner Reviewed-by: Jan Kara Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Signed-off-by: Ira Weiny Signed-off-by: Darrick J. Wong --- drivers/block/loop.c | 6 +++--- include/linux/fs.h | 7 +------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index da693e6a834e..14372df0f354 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -634,8 +634,8 @@ static int do_req_filebacked(struct loop_device *lo, struct request *rq) static inline void loop_update_dio(struct loop_device *lo) { - __loop_update_dio(lo, io_is_direct(lo->lo_backing_file) | - lo->use_dio); + __loop_update_dio(lo, (lo->lo_backing_file->f_flags & O_DIRECT) | + lo->use_dio); } static void loop_reread_partitions(struct loop_device *lo, @@ -1028,7 +1028,7 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode, if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync) blk_queue_write_cache(lo->lo_queue, true, false); - if (io_is_direct(lo->lo_backing_file) && inode->i_sb->s_bdev) { + if ((lo->lo_backing_file->f_flags & O_DIRECT) && inode->i_sb->s_bdev) { /* In case of direct I/O, match underlying block size */ unsigned short bsize = bdev_logical_block_size( inode->i_sb->s_bdev); diff --git a/include/linux/fs.h b/include/linux/fs.h index 4f6f59b4f22a..a87cc5845a02 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3394,11 +3394,6 @@ extern void setattr_copy(struct inode *inode, const struct iattr *attr); extern int file_update_time(struct file *file); -static inline bool io_is_direct(struct file *filp) -{ - return (filp->f_flags & O_DIRECT) || IS_DAX(filp->f_mapping->host); -} - static inline bool vma_is_dax(const struct vm_area_struct *vma) { return vma->vm_file && IS_DAX(vma->vm_file->f_mapping->host); @@ -3423,7 +3418,7 @@ static inline int iocb_flags(struct file *file) int res = 0; if (file->f_flags & O_APPEND) res |= IOCB_APPEND; - if (io_is_direct(file)) + if (file->f_flags & O_DIRECT) res |= IOCB_DIRECT; if ((file->f_flags & O_DSYNC) || IS_SYNC(file->f_mapping->host)) res |= IOCB_DSYNC; From 712b2698e4c024b561694cbcc1abba13eb0fd9ce Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Thu, 30 Apr 2020 07:41:34 -0700 Subject: [PATCH 0202/1043] fs/stat: Define DAX statx attribute In order for users to determine if a file is currently operating in DAX state (effective DAX). Define a statx attribute value and set that attribute if the effective DAX flag is set. To go along with this we propose the following addition to the statx man page: STATX_ATTR_DAX The file is in the DAX (cpu direct access) state. DAX state attempts to minimize software cache effects for both I/O and memory mappings of this file. It requires a file system which has been configured to support DAX. DAX generally assumes all accesses are via cpu load / store instructions which can minimize overhead for small accesses, but may adversely affect cpu utilization for large transfers. File I/O is done directly to/from user-space buffers and memory mapped I/O may be performed with direct memory mappings that bypass kernel page cache. While the DAX property tends to result in data being transferred synchronously, it does not give the same guarantees of O_SYNC where data and the necessary metadata are transferred together. A DAX file may support being mapped with the MAP_SYNC flag, which enables a program to use CPU cache flush instructions to persist CPU store operations without an explicit fsync(2). See mmap(2) for more information. Reviewed-by: Dave Chinner Reviewed-by: Jan Kara Reviewed-by: Darrick J. Wong Signed-off-by: Ira Weiny Signed-off-by: Darrick J. Wong --- fs/stat.c | 3 +++ include/uapi/linux/stat.h | 1 + 2 files changed, 4 insertions(+) diff --git a/fs/stat.c b/fs/stat.c index 030008796479..894699c74dde 100644 --- a/fs/stat.c +++ b/fs/stat.c @@ -79,6 +79,9 @@ int vfs_getattr_nosec(const struct path *path, struct kstat *stat, if (IS_AUTOMOUNT(inode)) stat->attributes |= STATX_ATTR_AUTOMOUNT; + if (IS_DAX(inode)) + stat->attributes |= STATX_ATTR_DAX; + if (inode->i_op->getattr) return inode->i_op->getattr(path, stat, request_mask, query_flags); diff --git a/include/uapi/linux/stat.h b/include/uapi/linux/stat.h index ad80a5c885d5..e5f9d5517f6b 100644 --- a/include/uapi/linux/stat.h +++ b/include/uapi/linux/stat.h @@ -169,6 +169,7 @@ struct statx { #define STATX_ATTR_ENCRYPTED 0x00000800 /* [I] File requires key to decrypt in fs */ #define STATX_ATTR_AUTOMOUNT 0x00001000 /* Dir: Automount trigger */ #define STATX_ATTR_VERITY 0x00100000 /* [I] Verity protected file */ +#define STATX_ATTR_DAX 0x00002000 /* [I] File is DAX */ #endif /* _UAPI_LINUX_STAT_H */ From 83d9088659e8f113741bb197324bd9554d159657 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Thu, 30 Apr 2020 07:41:34 -0700 Subject: [PATCH 0203/1043] Documentation/dax: Update Usage section Update the Usage section to reflect the new individual dax selection functionality. Signed-off-by: Ira Weiny Acked-by: Randy Dunlap Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- Documentation/filesystems/dax.txt | 142 +++++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 3 deletions(-) diff --git a/Documentation/filesystems/dax.txt b/Documentation/filesystems/dax.txt index 679729442fd2..735fb4b54117 100644 --- a/Documentation/filesystems/dax.txt +++ b/Documentation/filesystems/dax.txt @@ -20,8 +20,144 @@ Usage If you have a block device which supports DAX, you can make a filesystem on it as usual. The DAX code currently only supports files with a block size equal to your kernel's PAGE_SIZE, so you may need to specify a block -size when creating the filesystem. When mounting it, use the "-o dax" -option on the command line or add 'dax' to the options in /etc/fstab. +size when creating the filesystem. + +Currently 3 filesystems support DAX: ext2, ext4 and xfs. Enabling DAX on them +is different. + +Enabling DAX on ext4 and ext2 +----------------------------- + +When mounting the filesystem, use the "-o dax" option on the command line or +add 'dax' to the options in /etc/fstab. This works to enable DAX on all files +within the filesystem. It is equivalent to the '-o dax=always' behavior below. + + +Enabling DAX on xfs +------------------- + +Summary +------- + + 1. There exists an in-kernel file access mode flag S_DAX that corresponds to + the statx flag STATX_ATTR_DAX. See the manpage for statx(2) for details + about this access mode. + + 2. There exists a persistent flag FS_XFLAG_DAX that can be applied to regular + files and directories. This advisory flag can be set or cleared at any + time, but doing so does not immediately affect the S_DAX state. + + 3. If the persistent FS_XFLAG_DAX flag is set on a directory, this flag will + be inherited by all regular files and subdirectories that are subsequently + created in this directory. Files and subdirectories that exist at the time + this flag is set or cleared on the parent directory are not modified by + this modification of the parent directory. + + 4. There exist dax mount options which can override FS_XFLAG_DAX in the + setting of the S_DAX flag. Given underlying storage which supports DAX the + following hold: + + "-o dax=inode" means "follow FS_XFLAG_DAX" and is the default. + + "-o dax=never" means "never set S_DAX, ignore FS_XFLAG_DAX." + + "-o dax=always" means "always set S_DAX ignore FS_XFLAG_DAX." + + "-o dax" is a legacy option which is an alias for "dax=always". + This may be removed in the future so "-o dax=always" is + the preferred method for specifying this behavior. + + NOTE: Modifications to and the inheritance behavior of FS_XFLAG_DAX remain + the same even when the filesystem is mounted with a dax option. However, + in-core inode state (S_DAX) will be overridden until the filesystem is + remounted with dax=inode and the inode is evicted from kernel memory. + + 5. The S_DAX policy can be changed via: + + a) Setting the parent directory FS_XFLAG_DAX as needed before files are + created + + b) Setting the appropriate dax="foo" mount option + + c) Changing the FS_XFLAG_DAX flag on existing regular files and + directories. This has runtime constraints and limitations that are + described in 6) below. + + 6. When changing the S_DAX policy via toggling the persistent FS_XFLAG_DAX flag, + the change in behaviour for existing regular files may not occur + immediately. If the change must take effect immediately, the administrator + needs to: + + a) stop the application so there are no active references to the data set + the policy change will affect + + b) evict the data set from kernel caches so it will be re-instantiated when + the application is restarted. This can be achieved by: + + i. drop-caches + ii. a filesystem unmount and mount cycle + iii. a system reboot + + +Details +------- + +There are 2 per-file dax flags. One is a persistent inode setting (FS_XFLAG_DAX) +and the other is a volatile flag indicating the active state of the feature +(S_DAX). + +FS_XFLAG_DAX is preserved within the filesystem. This persistent config +setting can be set, cleared and/or queried using the FS_IOC_FS[GS]ETXATTR ioctl +(see ioctl_xfs_fsgetxattr(2)) or an utility such as 'xfs_io'. + +New files and directories automatically inherit FS_XFLAG_DAX from +their parent directory _when_ _created_. Therefore, setting FS_XFLAG_DAX at +directory creation time can be used to set a default behavior for an entire +sub-tree. + +To clarify inheritance, here are 3 examples: + +Example A: + +mkdir -p a/b/c +xfs_io -c 'chattr +x' a +mkdir a/b/c/d +mkdir a/e + + dax: a,e + no dax: b,c,d + +Example B: + +mkdir a +xfs_io -c 'chattr +x' a +mkdir -p a/b/c/d + + dax: a,b,c,d + no dax: + +Example C: + +mkdir -p a/b/c +xfs_io -c 'chattr +x' c +mkdir a/b/c/d + + dax: c,d + no dax: a,b + + +The current enabled state (S_DAX) is set when a file inode is instantiated in +memory by the kernel. It is set based on the underlying media support, the +value of FS_XFLAG_DAX and the filesystem's dax mount option. + +statx can be used to query S_DAX. NOTE that only regular files will ever have +S_DAX set and therefore statx will never indicate that S_DAX is set on +directories. + +Setting the FS_XFLAG_DAX flag (specifically or through inheritance) occurs even +if the underlying media does not support dax and/or the filesystem is +overridden with a mount option. + Implementation Tips for Block Driver Writers @@ -94,7 +230,7 @@ sysadmins have an option to restore the lost data from a prior backup/inbuilt redundancy in the following ways: 1. Delete the affected file, and restore from a backup (sysadmin route): - This will free the file system blocks that were being used by the file, + This will free the filesystem blocks that were being used by the file, and the next time they're allocated, they will be zeroed first, which happens through the driver, and will clear bad sectors. From 0d2d35a33ea77f34d534106e0b18a5797d3bbcf7 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Tue, 21 Apr 2020 14:16:52 -0700 Subject: [PATCH 0204/1043] xfs: report unrecognized log item type codes during recovery When we're sorting recovered log items ahead of recovering them and encounter a log item of unknown type, actually print the type code when we're rejecting the whole transaction to aid in debugging. Signed-off-by: Darrick J. Wong Reviewed-by: Brian Foster Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_log_recover.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 11c3502b07b1..5f803083ddc3 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -1887,8 +1887,8 @@ xlog_recover_reorder_trans( break; default: xfs_warn(log->l_mp, - "%s: unrecognized type of log operation", - __func__); + "%s: unrecognized type of log operation (%d)", + __func__, ITEM_TYPE(item)); ASSERT(0); /* * return the remaining items back to the transaction From c140735bbb65daa89275a6b87f120c5feca99d6a Mon Sep 17 00:00:00 2001 From: Kaixu Xia Date: Wed, 22 Apr 2020 21:54:27 -0700 Subject: [PATCH 0205/1043] xfs: trace quota allocations for all quota types The trace event xfs_dquot_dqalloc does not depend on the value uq, so remove the condition, and trace quota allocations for all quota types. Signed-off-by: Kaixu Xia Reviewed-by: Chaitanya Kulkarni Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_qm.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index c225691fad15..6678baab37de 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -1730,8 +1730,7 @@ xfs_qm_vop_dqalloc( pq = xfs_qm_dqhold(ip->i_pdquot); } } - if (uq) - trace_xfs_dquot_dqalloc(ip); + trace_xfs_dquot_dqalloc(ip); xfs_iunlock(ip, lockflags); if (O_udqpp) From d51bafe0d227e9fef1b0ab4cc1a424d8e2b59218 Mon Sep 17 00:00:00 2001 From: Kaixu Xia Date: Wed, 22 Apr 2020 21:54:28 -0700 Subject: [PATCH 0206/1043] xfs: combine two if statements with same condition The two if statements have same condition, and the mask value does not change in xfs_setattr_nonsize(), so combine them. Signed-off-by: Kaixu Xia Reviewed-by: Chaitanya Kulkarni Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_iops.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index f7a99b3bbcf7..e34814590453 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -738,12 +738,7 @@ xfs_setattr_nonsize( if (error) /* out of quota */ goto out_cancel; } - } - /* - * Change file ownership. Must be the owner or privileged. - */ - if (mask & (ATTR_UID|ATTR_GID)) { /* * CAP_FSETID overrides the following restrictions: * From fb353ff19d34e9b0ee8ba2b25d78aeab0436f479 Mon Sep 17 00:00:00 2001 From: Kaixu Xia Date: Wed, 22 Apr 2020 21:54:28 -0700 Subject: [PATCH 0207/1043] xfs: reserve quota inode transaction space only when needed We share an inode between gquota and pquota with the older superblock that doesn't have separate pquotino, and for the need_alloc == false case we don't need to call xfs_dir_ialloc() function, so add the check if reserved free disk blocks is needed. Signed-off-by: Kaixu Xia Reviewed-by: Brian Foster Reviewed-by: Dave Chinner Reviewed-by: Eric Sandeen Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_qm.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index 6678baab37de..b684b0410a52 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -780,7 +780,8 @@ xfs_qm_qino_alloc( } error = xfs_trans_alloc(mp, &M_RES(mp)->tr_create, - XFS_QM_QINOCREATE_SPACE_RES(mp), 0, 0, &tp); + need_alloc ? XFS_QM_QINOCREATE_SPACE_RES(mp) : 0, + 0, 0, &tp); if (error) return error; From ea1c90403d5d38bd26c4c5a72627a8f0f87295ff Mon Sep 17 00:00:00 2001 From: Kaixu Xia Date: Wed, 22 Apr 2020 21:54:29 -0700 Subject: [PATCH 0208/1043] xfs: remove unnecessary variable udqp from xfs_ioctl_setattr The initial value of variable udqp is NULL, and we only set the flag XFS_QMOPT_PQUOTA in xfs_qm_vop_dqalloc() function, so only the pdqp value is initialized and the udqp value is still NULL. Since the udqp value is NULL in the rest part of xfs_ioctl_setattr() function, it is meaningless and do nothing. So remove it from xfs_ioctl_setattr(). Signed-off-by: Kaixu Xia Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_ioctl.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 309958186d33..91936ed5e334 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -1513,7 +1513,6 @@ xfs_ioctl_setattr( struct fsxattr old_fa; struct xfs_mount *mp = ip->i_mount; struct xfs_trans *tp; - struct xfs_dquot *udqp = NULL; struct xfs_dquot *pdqp = NULL; struct xfs_dquot *olddquot = NULL; int code; @@ -1536,7 +1535,7 @@ xfs_ioctl_setattr( if (XFS_IS_QUOTA_ON(mp)) { code = xfs_qm_vop_dqalloc(ip, VFS_I(ip)->i_uid, VFS_I(ip)->i_gid, fa->fsx_projid, - XFS_QMOPT_PQUOTA, &udqp, NULL, &pdqp); + XFS_QMOPT_PQUOTA, NULL, NULL, &pdqp); if (code) return code; } @@ -1560,7 +1559,7 @@ xfs_ioctl_setattr( if (XFS_IS_QUOTA_RUNNING(mp) && XFS_IS_PQUOTA_ON(mp) && ip->i_d.di_projid != fa->fsx_projid) { - code = xfs_qm_vop_chown_reserve(tp, ip, udqp, NULL, pdqp, + code = xfs_qm_vop_chown_reserve(tp, ip, NULL, NULL, pdqp, capable(CAP_FOWNER) ? XFS_QMOPT_FORCE_RES : 0); if (code) /* out of quota */ goto error_trans_cancel; @@ -1626,7 +1625,6 @@ xfs_ioctl_setattr( * Release any dquot(s) the inode had kept before chown. */ xfs_qm_dqrele(olddquot); - xfs_qm_dqrele(udqp); xfs_qm_dqrele(pdqp); return code; @@ -1634,7 +1632,6 @@ xfs_ioctl_setattr( error_trans_cancel: xfs_trans_cancel(tp); error_free_dquots: - xfs_qm_dqrele(udqp); xfs_qm_dqrele(pdqp); return code; } From 7994aae8516aa35ca26dba64cf8d6938e93c2265 Mon Sep 17 00:00:00 2001 From: Kaixu Xia Date: Wed, 22 Apr 2020 21:54:29 -0700 Subject: [PATCH 0209/1043] xfs: remove unnecessary assertion from xfs_qm_vop_create_dqattach The check XFS_IS_QUOTA_RUNNING() has been done when enter the xfs_qm_vop_create_dqattach() function, it will return directly if the result is false, so the followed XFS_IS_QUOTA_RUNNING() assertion is unnecessary. If we truly care about this, the check also can be added to the condition of next if statements. Signed-off-by: Kaixu Xia Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_qm.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index b684b0410a52..fc93f88a9926 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -1932,7 +1932,6 @@ xfs_qm_vop_create_dqattach( return; ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL)); - ASSERT(XFS_IS_QUOTA_RUNNING(mp)); if (udqp && XFS_IS_UQUOTA_ON(mp)) { ASSERT(ip->i_udquot == NULL); From cd59455980f94ea4e9a5f84a7c326d42a4d84a78 Mon Sep 17 00:00:00 2001 From: Kaixu Xia Date: Wed, 22 Apr 2020 21:54:30 -0700 Subject: [PATCH 0210/1043] xfs: simplify the flags setting in xfs_qm_scall_quotaon Simplify the setting of the flags value, and only consider quota enforcement stuff here. Signed-off-by: Kaixu Xia Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_qm_syscalls.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/xfs/xfs_qm_syscalls.c b/fs/xfs/xfs_qm_syscalls.c index 5d5ac65aa1cc..944486f2b287 100644 --- a/fs/xfs/xfs_qm_syscalls.c +++ b/fs/xfs/xfs_qm_syscalls.c @@ -357,11 +357,11 @@ xfs_qm_scall_quotaon( int error; uint qf; - flags &= (XFS_ALL_QUOTA_ACCT | XFS_ALL_QUOTA_ENFD); /* - * Switching on quota accounting must be done at mount time. + * Switching on quota accounting must be done at mount time, + * only consider quota enforcement stuff here. */ - flags &= ~(XFS_ALL_QUOTA_ACCT); + flags &= XFS_ALL_QUOTA_ENFD; if (flags == 0) { xfs_debug(mp, "%s: zero flags, m_qflags=%x", From 57fd2d8f61a2bc4d7b465588ca1a2217cd94076c Mon Sep 17 00:00:00 2001 From: Kaixu Xia Date: Wed, 22 Apr 2020 21:54:31 -0700 Subject: [PATCH 0211/1043] xfs: remove unnecessary check of the variable resblks in xfs_symlink Since the "no-allocation" reservations has been removed, the resblks value should be larger than zero, so remove the unnecessary check. Signed-off-by: Kaixu Xia Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_symlink.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index 13fb4b919648..973441992b08 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -243,8 +243,7 @@ xfs_symlink( */ xfs_qm_vop_create_dqattach(tp, ip, udqp, gdqp, pdqp); - if (resblks) - resblks -= XFS_IALLOC_SPACE_RES(mp); + resblks -= XFS_IALLOC_SPACE_RES(mp); /* * If the symlink will fit into the inode, write it inline. */ @@ -265,8 +264,7 @@ xfs_symlink( if (error) goto out_trans_cancel; - if (resblks) - resblks -= fs_blocks; + resblks -= fs_blocks; ip->i_d.di_size = pathlen; xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); From 166405f6b53b7d7eecb7939aa4a79bc7c1e0ed68 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 22 Apr 2020 21:54:30 -0700 Subject: [PATCH 0212/1043] xfs: stop CONFIG_XFS_DEBUG from changing compiler flags I ran into a linker warning in XFS that originates from a mismatch between libelf, binutils and objtool when certain files in the kernel are built with "gcc -g": x86_64-linux-ld: fs/xfs/xfs_trace.o: unable to initialize decompress status for section .debug_info After some discussion, nobody could identify why xfs sets this flag here. CONFIG_XFS_DEBUG used to enable lots of unrelated settings, but now its main purpose is to enable extra consistency checks and assertions that are unrelated to the debug info. Remove the Makefile logic to set the flag here. If anyone relies on the debug info, this can simply be enabled again with the global CONFIG_DEBUG_INFO option. Dave Chinner writes: I'm pretty sure it was needed for the original kgdb integration back in the early 2000s. That was when SGI used to patch their XFS dev tree with kgdb and debug symbols were needed by the custom kgdb modules that were ported across from the Irix kernel debugger. ISTR that the early kcrash kernel dump analysis tools (again, originated from the Irix "icrash" kernel dump tools) had custom XFS debug scripts that needed also the debug info to work correctly... Which is a long way of saying "we don't need it anymore" instead of "nobody knows why it was set"... :) Suggested-by: Christoph Hellwig Link: https://lore.kernel.org/lkml/20200409074130.GD21033@infradead.org/ Signed-off-by: Arnd Bergmann Reviewed-by: Brian Foster Reviewed-by: Allison Collins Reviewed-by: Dave Chinner Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index 4f95df476181..ff94fb90a2ee 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -7,8 +7,6 @@ ccflags-y += -I $(srctree)/$(src) # needed for trace events ccflags-y += -I $(srctree)/$(src)/libxfs -ccflags-$(CONFIG_XFS_DEBUG) += -g - obj-$(CONFIG_XFS_FS) += xfs.o # this one should be compiled first, as the tracing macros can easily blow up From ec43f6da31f100696b56e4781b62fb5367a40d71 Mon Sep 17 00:00:00 2001 From: Eric Sandeen Date: Mon, 27 Apr 2020 11:00:42 -0700 Subject: [PATCH 0213/1043] xfs: define printk_once variants for xfs messages There are a couple places where we directly call printk_once() and one of them doesn't follow the standard xfs subsystem printk format as a result. #define printk_once variants to go with our existing printk_ratelimited #defines so we can do one-shot printks in a consistent manner. Signed-off-by: Eric Sandeen Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_message.h | 21 +++++++++++++++++++-- fs/xfs/xfs_mount.c | 7 +++---- fs/xfs/xfs_pnfs.c | 5 ++--- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/fs/xfs/xfs_message.h b/fs/xfs/xfs_message.h index 0b05e10995a0..802a96190d22 100644 --- a/fs/xfs/xfs_message.h +++ b/fs/xfs/xfs_message.h @@ -31,15 +31,27 @@ void xfs_debug(const struct xfs_mount *mp, const char *fmt, ...) } #endif -#define xfs_printk_ratelimited(func, dev, fmt, ...) \ +#define xfs_printk_ratelimited(func, dev, fmt, ...) \ do { \ static DEFINE_RATELIMIT_STATE(_rs, \ DEFAULT_RATELIMIT_INTERVAL, \ DEFAULT_RATELIMIT_BURST); \ if (__ratelimit(&_rs)) \ - func(dev, fmt, ##__VA_ARGS__); \ + func(dev, fmt, ##__VA_ARGS__); \ } while (0) +#define xfs_printk_once(func, dev, fmt, ...) \ +({ \ + static bool __section(.data.once) __print_once; \ + bool __ret_print_once = !__print_once; \ + \ + if (!__print_once) { \ + __print_once = true; \ + func(dev, fmt, ##__VA_ARGS__); \ + } \ + unlikely(__ret_print_once); \ +}) + #define xfs_emerg_ratelimited(dev, fmt, ...) \ xfs_printk_ratelimited(xfs_emerg, dev, fmt, ##__VA_ARGS__) #define xfs_alert_ratelimited(dev, fmt, ...) \ @@ -57,6 +69,11 @@ do { \ #define xfs_debug_ratelimited(dev, fmt, ...) \ xfs_printk_ratelimited(xfs_debug, dev, fmt, ##__VA_ARGS__) +#define xfs_warn_once(dev, fmt, ...) \ + xfs_printk_once(xfs_warn, dev, fmt, ##__VA_ARGS__) +#define xfs_notice_once(dev, fmt, ...) \ + xfs_printk_once(xfs_notice, dev, fmt, ##__VA_ARGS__) + void assfail(struct xfs_mount *mp, char *expr, char *f, int l); void asswarn(struct xfs_mount *mp, char *expr, char *f, int l); diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index c5513e5a226a..bb91f04266b9 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -1300,10 +1300,9 @@ xfs_mod_fdblocks( spin_unlock(&mp->m_sb_lock); return 0; } - printk_once(KERN_WARNING - "Filesystem \"%s\": reserve blocks depleted! " - "Consider increasing reserve pool size.", - mp->m_super->s_id); + xfs_warn_once(mp, +"Reserve blocks depleted! Consider increasing reserve pool size."); + fdblocks_enospc: spin_unlock(&mp->m_sb_lock); return -ENOSPC; diff --git a/fs/xfs/xfs_pnfs.c b/fs/xfs/xfs_pnfs.c index bb3008d390aa..b101feb2aab4 100644 --- a/fs/xfs/xfs_pnfs.c +++ b/fs/xfs/xfs_pnfs.c @@ -58,9 +58,8 @@ xfs_fs_get_uuid( { struct xfs_mount *mp = XFS_M(sb); - printk_once(KERN_NOTICE -"XFS (%s): using experimental pNFS feature, use at your own risk!\n", - mp->m_super->s_id); + xfs_notice_once(mp, +"Using experimental pNFS feature, use at your own risk!"); if (*len < sizeof(uuid_t)) return -EINVAL; From e968350aadf34c86eedd9cb43f6324263845586b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 27 Apr 2020 11:14:59 -0700 Subject: [PATCH 0214/1043] xfs: refactor the buffer cancellation table helpers Replace the somewhat convoluted use of xlog_peek_buffer_cancelled and xlog_check_buffer_cancelled with two obvious helpers: xlog_is_buffer_cancelled, which returns true if there is a buffer in the cancellation table, and xlog_put_buffer_cancelled, which also decrements the reference count of the buffer cancellation table. Both share a little helper to look up the entry. Signed-off-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_log_recover.c | 113 ++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 61 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 5f803083ddc3..b944ff91646d 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -1972,26 +1972,17 @@ xlog_recover_buffer_pass1( return 0; } -/* - * Check to see whether the buffer being recovered has a corresponding - * entry in the buffer cancel record table. If it is, return the cancel - * buffer structure to the caller. - */ -STATIC struct xfs_buf_cancel * -xlog_peek_buffer_cancelled( +static struct xfs_buf_cancel * +xlog_find_buffer_cancelled( struct xlog *log, xfs_daddr_t blkno, - uint len, - unsigned short flags) + uint len) { struct list_head *bucket; struct xfs_buf_cancel *bcp; - if (!log->l_buf_cancel_table) { - /* empty table means no cancelled buffers in the log */ - ASSERT(!(flags & XFS_BLF_CANCEL)); + if (!log->l_buf_cancel_table) return NULL; - } bucket = XLOG_BUF_CANCEL_BUCKET(log, blkno); list_for_each_entry(bcp, bucket, bc_list) { @@ -1999,50 +1990,48 @@ xlog_peek_buffer_cancelled( return bcp; } - /* - * We didn't find a corresponding entry in the table, so return 0 so - * that the buffer is NOT cancelled. - */ - ASSERT(!(flags & XFS_BLF_CANCEL)); return NULL; } /* - * If the buffer is being cancelled then return 1 so that it will be cancelled, - * otherwise return 0. If the buffer is actually a buffer cancel item - * (XFS_BLF_CANCEL is set), then decrement the refcount on the entry in the - * table and remove it from the table if this is the last reference. - * - * We remove the cancel record from the table when we encounter its last - * occurrence in the log so that if the same buffer is re-used again after its - * last cancellation we actually replay the changes made at that point. + * Check if there is and entry for blkno, len in the buffer cancel record table. */ -STATIC int -xlog_check_buffer_cancelled( +static bool +xlog_is_buffer_cancelled( struct xlog *log, xfs_daddr_t blkno, - uint len, - unsigned short flags) + uint len) +{ + return xlog_find_buffer_cancelled(log, blkno, len) != NULL; +} + +/* + * Check if there is and entry for blkno, len in the buffer cancel record table, + * and decremented the reference count on it if there is one. + * + * Remove the cancel record once the refcount hits zero, so that if the same + * buffer is re-used again after its last cancellation we actually replay the + * changes made at that point. + */ +static bool +xlog_put_buffer_cancelled( + struct xlog *log, + xfs_daddr_t blkno, + uint len) { struct xfs_buf_cancel *bcp; - bcp = xlog_peek_buffer_cancelled(log, blkno, len, flags); - if (!bcp) - return 0; - - /* - * We've go a match, so return 1 so that the recovery of this buffer - * is cancelled. If this buffer is actually a buffer cancel log - * item, then decrement the refcount on the one in the table and - * remove it if this is the last reference. - */ - if (flags & XFS_BLF_CANCEL) { - if (--bcp->bc_refcount == 0) { - list_del(&bcp->bc_list); - kmem_free(bcp); - } + bcp = xlog_find_buffer_cancelled(log, blkno, len); + if (!bcp) { + ASSERT(0); + return false; } - return 1; + + if (--bcp->bc_refcount == 0) { + list_del(&bcp->bc_list); + kmem_free(bcp); + } + return true; } /* @@ -2733,10 +2722,15 @@ xlog_recover_buffer_pass2( * In this pass we only want to recover all the buffers which have * not been cancelled and are not cancellation buffers themselves. */ - if (xlog_check_buffer_cancelled(log, buf_f->blf_blkno, - buf_f->blf_len, buf_f->blf_flags)) { - trace_xfs_log_recover_buf_cancel(log, buf_f); - return 0; + if (buf_f->blf_flags & XFS_BLF_CANCEL) { + if (xlog_put_buffer_cancelled(log, buf_f->blf_blkno, + buf_f->blf_len)) + goto cancelled; + } else { + + if (xlog_is_buffer_cancelled(log, buf_f->blf_blkno, + buf_f->blf_len)) + goto cancelled; } trace_xfs_log_recover_buf_recover(log, buf_f); @@ -2820,6 +2814,9 @@ xlog_recover_buffer_pass2( out_release: xfs_buf_relse(bp); return error; +cancelled: + trace_xfs_log_recover_buf_cancel(log, buf_f); + return 0; } /* @@ -2937,8 +2934,7 @@ xlog_recover_inode_pass2( * Inode buffers can be freed, look out for it, * and do not replay the inode. */ - if (xlog_check_buffer_cancelled(log, in_f->ilf_blkno, - in_f->ilf_len, 0)) { + if (xlog_is_buffer_cancelled(log, in_f->ilf_blkno, in_f->ilf_len)) { error = 0; trace_xfs_log_recover_inode_cancel(log, in_f); goto error; @@ -3840,7 +3836,7 @@ xlog_recover_do_icreate_pass2( daddr = XFS_AGB_TO_DADDR(mp, agno, agbno + i * igeo->blocks_per_cluster); - if (xlog_check_buffer_cancelled(log, daddr, bb_per_cluster, 0)) + if (xlog_is_buffer_cancelled(log, daddr, bb_per_cluster)) cancel_count++; } @@ -3876,11 +3872,8 @@ xlog_recover_buffer_ra_pass2( struct xfs_buf_log_format *buf_f = item->ri_buf[0].i_addr; struct xfs_mount *mp = log->l_mp; - if (xlog_peek_buffer_cancelled(log, buf_f->blf_blkno, - buf_f->blf_len, buf_f->blf_flags)) { + if (xlog_is_buffer_cancelled(log, buf_f->blf_blkno, buf_f->blf_len)) return; - } - xfs_buf_readahead(mp->m_ddev_targp, buf_f->blf_blkno, buf_f->blf_len, NULL); } @@ -3905,9 +3898,8 @@ xlog_recover_inode_ra_pass2( return; } - if (xlog_peek_buffer_cancelled(log, ilfp->ilf_blkno, ilfp->ilf_len, 0)) + if (xlog_is_buffer_cancelled(log, ilfp->ilf_blkno, ilfp->ilf_len)) return; - xfs_buf_readahead(mp->m_ddev_targp, ilfp->ilf_blkno, ilfp->ilf_len, &xfs_inode_buf_ra_ops); } @@ -3943,9 +3935,8 @@ xlog_recover_dquot_ra_pass2( ASSERT(dq_f->qlf_len == 1); len = XFS_FSB_TO_BB(mp, dq_f->qlf_len); - if (xlog_peek_buffer_cancelled(log, dq_f->qlf_blkno, len, 0)) + if (xlog_is_buffer_cancelled(log, dq_f->qlf_blkno, len)) return; - xfs_buf_readahead(mp->m_ddev_targp, dq_f->qlf_blkno, len, &xfs_dquot_buf_ra_ops); } From 5ce70b770d163b0c7697dd23420cbaee070b1cd0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 27 Apr 2020 11:14:59 -0700 Subject: [PATCH 0215/1043] xfs: rename inode_list xlog_recover_reorder_trans This list contains pretty much everything that is not a buffer. The comment calls it item_list, which is a much better name than inode list, so switch the actual variable name to that as well. Signed-off-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_log_recover.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index b944ff91646d..5a4bab695439 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -1847,7 +1847,7 @@ xlog_recover_reorder_trans( LIST_HEAD(cancel_list); LIST_HEAD(buffer_list); LIST_HEAD(inode_buffer_list); - LIST_HEAD(inode_list); + LIST_HEAD(item_list); list_splice_init(&trans->r_itemq, &sort_list); list_for_each_entry_safe(item, n, &sort_list, ri_list) { @@ -1883,7 +1883,7 @@ xlog_recover_reorder_trans( case XFS_LI_BUD: trace_xfs_log_recover_item_reorder_tail(log, trans, item, pass); - list_move_tail(&item->ri_list, &inode_list); + list_move_tail(&item->ri_list, &item_list); break; default: xfs_warn(log->l_mp, @@ -1904,8 +1904,8 @@ out: ASSERT(list_empty(&sort_list)); if (!list_empty(&buffer_list)) list_splice(&buffer_list, &trans->r_itemq); - if (!list_empty(&inode_list)) - list_splice_tail(&inode_list, &trans->r_itemq); + if (!list_empty(&item_list)) + list_splice_tail(&item_list, &trans->r_itemq); if (!list_empty(&inode_buffer_list)) list_splice_tail(&inode_buffer_list, &trans->r_itemq); if (!list_empty(&cancel_list)) From 7d4894b4ce070398952de25e893873edc67d3105 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 27 Apr 2020 18:23:17 -0700 Subject: [PATCH 0216/1043] xfs: factor out a xlog_buf_readahead helper Add a little helper to readahead a buffer if it hasn't been cancelled. Signed-off-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_log_recover.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 5a4bab695439..3a865f9aef90 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2034,6 +2034,17 @@ xlog_put_buffer_cancelled( return true; } +static void +xlog_buf_readahead( + struct xlog *log, + xfs_daddr_t blkno, + uint len, + const struct xfs_buf_ops *ops) +{ + if (!xlog_is_buffer_cancelled(log, blkno, len)) + xfs_buf_readahead(log->l_mp->m_ddev_targp, blkno, len, ops); +} + /* * Perform recovery for a buffer full of inodes. In these buffers, the only * data which should be recovered is that which corresponds to the @@ -3870,12 +3881,8 @@ xlog_recover_buffer_ra_pass2( struct xlog_recover_item *item) { struct xfs_buf_log_format *buf_f = item->ri_buf[0].i_addr; - struct xfs_mount *mp = log->l_mp; - if (xlog_is_buffer_cancelled(log, buf_f->blf_blkno, buf_f->blf_len)) - return; - xfs_buf_readahead(mp->m_ddev_targp, buf_f->blf_blkno, - buf_f->blf_len, NULL); + xlog_buf_readahead(log, buf_f->blf_blkno, buf_f->blf_len, NULL); } STATIC void @@ -3885,7 +3892,6 @@ xlog_recover_inode_ra_pass2( { struct xfs_inode_log_format ilf_buf; struct xfs_inode_log_format *ilfp; - struct xfs_mount *mp = log->l_mp; int error; if (item->ri_buf[0].i_len == sizeof(struct xfs_inode_log_format)) { @@ -3898,10 +3904,8 @@ xlog_recover_inode_ra_pass2( return; } - if (xlog_is_buffer_cancelled(log, ilfp->ilf_blkno, ilfp->ilf_len)) - return; - xfs_buf_readahead(mp->m_ddev_targp, ilfp->ilf_blkno, - ilfp->ilf_len, &xfs_inode_buf_ra_ops); + xlog_buf_readahead(log, ilfp->ilf_blkno, ilfp->ilf_len, + &xfs_inode_buf_ra_ops); } STATIC void @@ -3913,8 +3917,6 @@ xlog_recover_dquot_ra_pass2( struct xfs_disk_dquot *recddq; struct xfs_dq_logformat *dq_f; uint type; - int len; - if (mp->m_qflags == 0) return; @@ -3934,11 +3936,9 @@ xlog_recover_dquot_ra_pass2( ASSERT(dq_f); ASSERT(dq_f->qlf_len == 1); - len = XFS_FSB_TO_BB(mp, dq_f->qlf_len); - if (xlog_is_buffer_cancelled(log, dq_f->qlf_blkno, len)) - return; - xfs_buf_readahead(mp->m_ddev_targp, dq_f->qlf_blkno, len, - &xfs_dquot_buf_ra_ops); + xlog_buf_readahead(log, dq_f->qlf_blkno, + XFS_FSB_TO_BB(mp, dq_f->qlf_len), + &xfs_dquot_buf_ra_ops); } STATIC void From f15ab3f60ef3f25c6139262939388dc5617e2e6f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 27 Apr 2020 18:23:17 -0700 Subject: [PATCH 0217/1043] xfs: simplify xlog_recover_inode_ra_pass2 Don't bother to allocate memory and convert the log item when we only need the block number and the length. Just extract them directly and call xlog_buf_readahead separately in each branch. Signed-off-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_log_recover.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 3a865f9aef90..08c62f5e1ba4 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -3890,22 +3890,17 @@ xlog_recover_inode_ra_pass2( struct xlog *log, struct xlog_recover_item *item) { - struct xfs_inode_log_format ilf_buf; - struct xfs_inode_log_format *ilfp; - int error; - if (item->ri_buf[0].i_len == sizeof(struct xfs_inode_log_format)) { - ilfp = item->ri_buf[0].i_addr; - } else { - ilfp = &ilf_buf; - memset(ilfp, 0, sizeof(*ilfp)); - error = xfs_inode_item_format_convert(&item->ri_buf[0], ilfp); - if (error) - return; - } + struct xfs_inode_log_format *ilfp = item->ri_buf[0].i_addr; - xlog_buf_readahead(log, ilfp->ilf_blkno, ilfp->ilf_len, - &xfs_inode_buf_ra_ops); + xlog_buf_readahead(log, ilfp->ilf_blkno, ilfp->ilf_len, + &xfs_inode_buf_ra_ops); + } else { + struct xfs_inode_log_format_32 *ilfp = item->ri_buf[0].i_addr; + + xlog_buf_readahead(log, ilfp->ilf_blkno, ilfp->ilf_len, + &xfs_inode_buf_ra_ops); + } } STATIC void From 98b69b1285be048b9c811f093ba1fa86e4d8fe87 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 28 Apr 2020 14:39:13 -0700 Subject: [PATCH 0218/1043] xfs: refactor xlog_recover_buffer_pass1 Split out a xlog_add_buffer_cancelled helper which does the low-level manipulation of the buffer cancelation table, and in that helper call xlog_find_buffer_cancelled instead of open coding it. Signed-off-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_log_recover.c | 114 +++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 59 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 08c62f5e1ba4..db47dfc0cada 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -1913,65 +1913,6 @@ out: return error; } -/* - * Build up the table of buf cancel records so that we don't replay - * cancelled data in the second pass. For buffer records that are - * not cancel records, there is nothing to do here so we just return. - * - * If we get a cancel record which is already in the table, this indicates - * that the buffer was cancelled multiple times. In order to ensure - * that during pass 2 we keep the record in the table until we reach its - * last occurrence in the log, we keep a reference count in the cancel - * record in the table to tell us how many times we expect to see this - * record during the second pass. - */ -STATIC int -xlog_recover_buffer_pass1( - struct xlog *log, - struct xlog_recover_item *item) -{ - xfs_buf_log_format_t *buf_f = item->ri_buf[0].i_addr; - struct list_head *bucket; - struct xfs_buf_cancel *bcp; - - if (!xfs_buf_log_check_iovec(&item->ri_buf[0])) { - xfs_err(log->l_mp, "bad buffer log item size (%d)", - item->ri_buf[0].i_len); - return -EFSCORRUPTED; - } - - /* - * If this isn't a cancel buffer item, then just return. - */ - if (!(buf_f->blf_flags & XFS_BLF_CANCEL)) { - trace_xfs_log_recover_buf_not_cancel(log, buf_f); - return 0; - } - - /* - * Insert an xfs_buf_cancel record into the hash table of them. - * If there is already an identical record, bump its reference count. - */ - bucket = XLOG_BUF_CANCEL_BUCKET(log, buf_f->blf_blkno); - list_for_each_entry(bcp, bucket, bc_list) { - if (bcp->bc_blkno == buf_f->blf_blkno && - bcp->bc_len == buf_f->blf_len) { - bcp->bc_refcount++; - trace_xfs_log_recover_buf_cancel_ref_inc(log, buf_f); - return 0; - } - } - - bcp = kmem_alloc(sizeof(struct xfs_buf_cancel), 0); - bcp->bc_blkno = buf_f->blf_blkno; - bcp->bc_len = buf_f->blf_len; - bcp->bc_refcount = 1; - list_add_tail(&bcp->bc_list, bucket); - - trace_xfs_log_recover_buf_cancel_add(log, buf_f); - return 0; -} - static struct xfs_buf_cancel * xlog_find_buffer_cancelled( struct xlog *log, @@ -1993,6 +1934,35 @@ xlog_find_buffer_cancelled( return NULL; } +static bool +xlog_add_buffer_cancelled( + struct xlog *log, + xfs_daddr_t blkno, + uint len) +{ + struct xfs_buf_cancel *bcp; + + /* + * If we find an existing cancel record, this indicates that the buffer + * was cancelled multiple times. To ensure that during pass 2 we keep + * the record in the table until we reach its last occurrence in the + * log, a reference count is kept to tell how many times we expect to + * see this record during the second pass. + */ + bcp = xlog_find_buffer_cancelled(log, blkno, len); + if (bcp) { + bcp->bc_refcount++; + return false; + } + + bcp = kmem_alloc(sizeof(struct xfs_buf_cancel), 0); + bcp->bc_blkno = blkno; + bcp->bc_len = len; + bcp->bc_refcount = 1; + list_add_tail(&bcp->bc_list, XLOG_BUF_CANCEL_BUCKET(log, blkno)); + return true; +} + /* * Check if there is and entry for blkno, len in the buffer cancel record table. */ @@ -2045,6 +2015,32 @@ xlog_buf_readahead( xfs_buf_readahead(log->l_mp->m_ddev_targp, blkno, len, ops); } +/* + * Build up the table of buf cancel records so that we don't replay cancelled + * data in the second pass. + */ +static int +xlog_recover_buffer_pass1( + struct xlog *log, + struct xlog_recover_item *item) +{ + struct xfs_buf_log_format *bf = item->ri_buf[0].i_addr; + + if (!xfs_buf_log_check_iovec(&item->ri_buf[0])) { + xfs_err(log->l_mp, "bad buffer log item size (%d)", + item->ri_buf[0].i_len); + return -EFSCORRUPTED; + } + + if (!(bf->blf_flags & XFS_BLF_CANCEL)) + trace_xfs_log_recover_buf_not_cancel(log, bf); + else if (xlog_add_buffer_cancelled(log, bf->blf_blkno, bf->blf_len)) + trace_xfs_log_recover_buf_cancel_add(log, bf); + else + trace_xfs_log_recover_buf_cancel_ref_inc(log, bf); + return 0; +} + /* * Perform recovery for a buffer full of inodes. In these buffers, the only * data which should be recovered is that which corresponds to the From 82ff450b2d936d778361a1de43eb078cc043c7fe Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 30 Apr 2020 12:52:18 -0700 Subject: [PATCH 0219/1043] xfs: remove the xfs_efi_log_item_t typedef Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_extfree_item.c | 2 +- fs/xfs/xfs_extfree_item.h | 10 +++++----- fs/xfs/xfs_log_recover.c | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index 6ea847f6e298..00309b81607c 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -161,7 +161,7 @@ xfs_efi_init( ASSERT(nextents > 0); if (nextents > XFS_EFI_MAX_FAST_EXTENTS) { - size = (uint)(sizeof(xfs_efi_log_item_t) + + size = (uint)(sizeof(struct xfs_efi_log_item) + ((nextents - 1) * sizeof(xfs_extent_t))); efip = kmem_zalloc(size, 0); } else { diff --git a/fs/xfs/xfs_extfree_item.h b/fs/xfs/xfs_extfree_item.h index 16aaab06d4ec..b9b567f35575 100644 --- a/fs/xfs/xfs_extfree_item.h +++ b/fs/xfs/xfs_extfree_item.h @@ -50,13 +50,13 @@ struct kmem_zone; * of commit failure or log I/O errors. Note that the EFD is not inserted in the * AIL, so at this point both the EFI and EFD are freed. */ -typedef struct xfs_efi_log_item { +struct xfs_efi_log_item { struct xfs_log_item efi_item; atomic_t efi_refcount; atomic_t efi_next_extent; unsigned long efi_flags; /* misc flags */ xfs_efi_log_format_t efi_format; -} xfs_efi_log_item_t; +}; /* * This is the "extent free done" log item. It is used to log @@ -65,7 +65,7 @@ typedef struct xfs_efi_log_item { */ typedef struct xfs_efd_log_item { struct xfs_log_item efd_item; - xfs_efi_log_item_t *efd_efip; + struct xfs_efi_log_item *efd_efip; uint efd_next_extent; xfs_efd_log_format_t efd_format; } xfs_efd_log_item_t; @@ -78,10 +78,10 @@ typedef struct xfs_efd_log_item { extern struct kmem_zone *xfs_efi_zone; extern struct kmem_zone *xfs_efd_zone; -xfs_efi_log_item_t *xfs_efi_init(struct xfs_mount *, uint); +struct xfs_efi_log_item *xfs_efi_init(struct xfs_mount *, uint); int xfs_efi_copy_format(xfs_log_iovec_t *buf, xfs_efi_log_format_t *dst_efi_fmt); -void xfs_efi_item_free(xfs_efi_log_item_t *); +void xfs_efi_item_free(struct xfs_efi_log_item *); void xfs_efi_release(struct xfs_efi_log_item *); int xfs_efi_recover(struct xfs_mount *mp, diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index db47dfc0cada..d0e2dd81de53 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -3368,7 +3368,7 @@ xlog_recover_efd_pass2( struct xlog_recover_item *item) { xfs_efd_log_format_t *efd_formatp; - xfs_efi_log_item_t *efip = NULL; + struct xfs_efi_log_item *efip = NULL; struct xfs_log_item *lip; uint64_t efi_id; struct xfs_ail_cursor cur; @@ -3389,7 +3389,7 @@ xlog_recover_efd_pass2( lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); while (lip != NULL) { if (lip->li_type == XFS_LI_EFI) { - efip = (xfs_efi_log_item_t *)lip; + efip = (struct xfs_efi_log_item *)lip; if (efip->efi_format.efi_id == efi_id) { /* * Drop the EFD reference to the EFI. This From c84e819090f39e96e4d432c9047a50d2424f99e0 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 30 Apr 2020 12:52:19 -0700 Subject: [PATCH 0220/1043] xfs: remove the xfs_efd_log_item_t typedef Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_extfree_item.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/xfs/xfs_extfree_item.h b/fs/xfs/xfs_extfree_item.h index b9b567f35575..a2a736a77fa9 100644 --- a/fs/xfs/xfs_extfree_item.h +++ b/fs/xfs/xfs_extfree_item.h @@ -63,12 +63,12 @@ struct xfs_efi_log_item { * the fact that some extents earlier mentioned in an efi item * have been freed. */ -typedef struct xfs_efd_log_item { +struct xfs_efd_log_item { struct xfs_log_item efd_item; struct xfs_efi_log_item *efd_efip; uint efd_next_extent; xfs_efd_log_format_t efd_format; -} xfs_efd_log_item_t; +}; /* * Max number of extents in fast allocation path. From fd9cbe51215198ccffa64169c98eae35b0916088 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 30 Apr 2020 12:52:19 -0700 Subject: [PATCH 0221/1043] xfs: remove the xfs_inode_log_item_t typedef Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_fork.c | 2 +- fs/xfs/libxfs/xfs_trans_inode.c | 2 +- fs/xfs/xfs_inode.c | 4 ++-- fs/xfs/xfs_inode_item.c | 2 +- fs/xfs/xfs_inode_item.h | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index 518c6f0ec3a6..3e9a42f1e23b 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -592,7 +592,7 @@ void xfs_iflush_fork( xfs_inode_t *ip, xfs_dinode_t *dip, - xfs_inode_log_item_t *iip, + struct xfs_inode_log_item *iip, int whichfork) { char *cp; diff --git a/fs/xfs/libxfs/xfs_trans_inode.c b/fs/xfs/libxfs/xfs_trans_inode.c index 2b8ccb5b975d..b5dfb6654842 100644 --- a/fs/xfs/libxfs/xfs_trans_inode.c +++ b/fs/xfs/libxfs/xfs_trans_inode.c @@ -27,7 +27,7 @@ xfs_trans_ijoin( struct xfs_inode *ip, uint lock_flags) { - xfs_inode_log_item_t *iip; + struct xfs_inode_log_item *iip; ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL)); if (ip->i_itemp == NULL) diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index d1772786af29..0e2ef3f56be4 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -2602,7 +2602,7 @@ xfs_ifree_cluster( xfs_daddr_t blkno; xfs_buf_t *bp; xfs_inode_t *ip; - xfs_inode_log_item_t *iip; + struct xfs_inode_log_item *iip; struct xfs_log_item *lip; struct xfs_perag *pag; struct xfs_ino_geometry *igeo = M_IGEO(mp); @@ -2662,7 +2662,7 @@ xfs_ifree_cluster( */ list_for_each_entry(lip, &bp->b_li_list, li_bio_list) { if (lip->li_type == XFS_LI_INODE) { - iip = (xfs_inode_log_item_t *)lip; + iip = (struct xfs_inode_log_item *)lip; ASSERT(iip->ili_logged == 1); lip->li_cb = xfs_istale_done; xfs_trans_ail_copy_lsn(mp->m_ail, diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index f779cca2346f..75b74bbe38e4 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -780,7 +780,7 @@ xfs_iflush_abort( xfs_inode_t *ip, bool stale) { - xfs_inode_log_item_t *iip = ip->i_itemp; + struct xfs_inode_log_item *iip = ip->i_itemp; if (iip) { if (test_bit(XFS_LI_IN_AIL, &iip->ili_item.li_flags)) { diff --git a/fs/xfs/xfs_inode_item.h b/fs/xfs/xfs_inode_item.h index 07a60e74c39c..ad667fd4ae62 100644 --- a/fs/xfs/xfs_inode_item.h +++ b/fs/xfs/xfs_inode_item.h @@ -13,7 +13,7 @@ struct xfs_bmbt_rec; struct xfs_inode; struct xfs_mount; -typedef struct xfs_inode_log_item { +struct xfs_inode_log_item { struct xfs_log_item ili_item; /* common portion */ struct xfs_inode *ili_inode; /* inode ptr */ xfs_lsn_t ili_flush_lsn; /* lsn at last flush */ @@ -23,7 +23,7 @@ typedef struct xfs_inode_log_item { unsigned int ili_last_fields; /* fields when flushed */ unsigned int ili_fields; /* fields to be logged */ unsigned int ili_fsync_fields; /* logged since last fsync */ -} xfs_inode_log_item_t; +}; static inline int xfs_inode_clean(xfs_inode_t *ip) { From e046e949486ec92d83b2ccdf0e7e9144f74ef028 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 30 Apr 2020 12:52:20 -0700 Subject: [PATCH 0222/1043] xfs: factor out a xfs_defer_create_intent helper Create a helper that encapsulates the whole logic to create a defer intent. This reorders some of the work that was done, but none of that has an affect on the operation as only fields that don't directly interact are affected. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_defer.c | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index 22557527cfdb..8a38da602b7d 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -178,6 +178,23 @@ static const struct xfs_defer_op_type *defer_op_types[] = { [XFS_DEFER_OPS_TYPE_AGFL_FREE] = &xfs_agfl_free_defer_type, }; +static void +xfs_defer_create_intent( + struct xfs_trans *tp, + struct xfs_defer_pending *dfp, + bool sort) +{ + const struct xfs_defer_op_type *ops = defer_op_types[dfp->dfp_type]; + struct list_head *li; + + if (sort) + list_sort(tp->t_mountp, &dfp->dfp_work, ops->diff_items); + + dfp->dfp_intent = ops->create_intent(tp, dfp->dfp_count); + list_for_each(li, &dfp->dfp_work) + ops->log_item(tp, dfp->dfp_intent, li); +} + /* * For each pending item in the intake list, log its intent item and the * associated extents, then add the entire intake list to the end of @@ -187,17 +204,11 @@ STATIC void xfs_defer_create_intents( struct xfs_trans *tp) { - struct list_head *li; struct xfs_defer_pending *dfp; - const struct xfs_defer_op_type *ops; list_for_each_entry(dfp, &tp->t_dfops, dfp_list) { - ops = defer_op_types[dfp->dfp_type]; - dfp->dfp_intent = ops->create_intent(tp, dfp->dfp_count); trace_xfs_defer_create_intent(tp->t_mountp, dfp); - list_sort(tp->t_mountp, &dfp->dfp_work, ops->diff_items); - list_for_each(li, &dfp->dfp_work) - ops->log_item(tp, dfp->dfp_intent, li); + xfs_defer_create_intent(tp, dfp, true); } } @@ -419,17 +430,13 @@ xfs_defer_finish_noroll( } if (error == -EAGAIN) { /* - * Caller wants a fresh transaction, so log a - * new log intent item to replace the old one - * and roll the transaction. See "Requesting - * a Fresh Transaction while Finishing - * Deferred Work" above. + * Caller wants a fresh transaction, so log a new log + * intent item to replace the old one and roll the + * transaction. See "Requesting a Fresh Transaction + * while Finishing Deferred Work" above. */ - dfp->dfp_intent = ops->create_intent(*tp, - dfp->dfp_count); dfp->dfp_done = NULL; - list_for_each(li, &dfp->dfp_work) - ops->log_item(*tp, dfp->dfp_intent, li); + xfs_defer_create_intent(*tp, dfp, false); } else { /* Done with the dfp, free it. */ list_del(&dfp->dfp_list); From c1f09188e8de0ae65433cb9c8ace4feb66359bcc Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 30 Apr 2020 12:52:20 -0700 Subject: [PATCH 0223/1043] xfs: merge the ->log_item defer op into ->create_intent These are aways called together, and my merging them we reduce the amount of indirect calls, improve type safety and in general clean up the code a bit. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_defer.c | 6 ++--- fs/xfs/libxfs/xfs_defer.h | 4 ++-- fs/xfs/xfs_bmap_item.c | 47 +++++++++++++++--------------------- fs/xfs/xfs_extfree_item.c | 49 ++++++++++++++++---------------------- fs/xfs/xfs_refcount_item.c | 48 ++++++++++++++++--------------------- fs/xfs/xfs_rmap_item.c | 48 ++++++++++++++++--------------------- 6 files changed, 83 insertions(+), 119 deletions(-) diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index 8a38da602b7d..56d1357f9d13 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -185,14 +185,12 @@ xfs_defer_create_intent( bool sort) { const struct xfs_defer_op_type *ops = defer_op_types[dfp->dfp_type]; - struct list_head *li; if (sort) list_sort(tp->t_mountp, &dfp->dfp_work, ops->diff_items); - dfp->dfp_intent = ops->create_intent(tp, dfp->dfp_count); - list_for_each(li, &dfp->dfp_work) - ops->log_item(tp, dfp->dfp_intent, li); + dfp->dfp_intent = ops->create_intent(tp, &dfp->dfp_work, + dfp->dfp_count); } /* diff --git a/fs/xfs/libxfs/xfs_defer.h b/fs/xfs/libxfs/xfs_defer.h index 7c28d7608ac6..d6a4577c25b0 100644 --- a/fs/xfs/libxfs/xfs_defer.h +++ b/fs/xfs/libxfs/xfs_defer.h @@ -50,8 +50,8 @@ struct xfs_defer_op_type { void (*finish_cleanup)(struct xfs_trans *, void *, int); void (*cancel_item)(struct list_head *); int (*diff_items)(void *, struct list_head *, struct list_head *); - void *(*create_intent)(struct xfs_trans *, uint); - void (*log_item)(struct xfs_trans *, void *, struct list_head *); + void *(*create_intent)(struct xfs_trans *tp, struct list_head *items, + unsigned int count); unsigned int max_items; }; diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index ee6f4229cebc..dea97956d78d 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -278,27 +278,6 @@ xfs_bmap_update_diff_items( return ba->bi_owner->i_ino - bb->bi_owner->i_ino; } -/* Get an BUI. */ -STATIC void * -xfs_bmap_update_create_intent( - struct xfs_trans *tp, - unsigned int count) -{ - struct xfs_bui_log_item *buip; - - ASSERT(count == XFS_BUI_MAX_FAST_EXTENTS); - ASSERT(tp != NULL); - - buip = xfs_bui_init(tp->t_mountp); - ASSERT(buip != NULL); - - /* - * Get a log_item_desc to point at the new item. - */ - xfs_trans_add_item(tp, &buip->bui_item); - return buip; -} - /* Set the map extent flags for this mapping. */ static void xfs_trans_set_bmap_flags( @@ -326,16 +305,12 @@ xfs_trans_set_bmap_flags( STATIC void xfs_bmap_update_log_item( struct xfs_trans *tp, - void *intent, - struct list_head *item) + struct xfs_bui_log_item *buip, + struct xfs_bmap_intent *bmap) { - struct xfs_bui_log_item *buip = intent; - struct xfs_bmap_intent *bmap; uint next_extent; struct xfs_map_extent *map; - bmap = container_of(item, struct xfs_bmap_intent, bi_list); - tp->t_flags |= XFS_TRANS_DIRTY; set_bit(XFS_LI_DIRTY, &buip->bui_item.li_flags); @@ -355,6 +330,23 @@ xfs_bmap_update_log_item( bmap->bi_bmap.br_state); } +STATIC void * +xfs_bmap_update_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count) +{ + struct xfs_bui_log_item *buip = xfs_bui_init(tp->t_mountp); + struct xfs_bmap_intent *bmap; + + ASSERT(count == XFS_BUI_MAX_FAST_EXTENTS); + + xfs_trans_add_item(tp, &buip->bui_item); + list_for_each_entry(bmap, items, bi_list) + xfs_bmap_update_log_item(tp, buip, bmap); + return buip; +} + /* Get an BUD so we can process all the deferred rmap updates. */ STATIC void * xfs_bmap_update_create_done( @@ -419,7 +411,6 @@ const struct xfs_defer_op_type xfs_bmap_update_defer_type = { .diff_items = xfs_bmap_update_diff_items, .create_intent = xfs_bmap_update_create_intent, .abort_intent = xfs_bmap_update_abort_intent, - .log_item = xfs_bmap_update_log_item, .create_done = xfs_bmap_update_create_done, .finish_item = xfs_bmap_update_finish_item, .cancel_item = xfs_bmap_update_cancel_item, diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index 00309b81607c..cb22c7ad3181 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -412,41 +412,16 @@ xfs_extent_free_diff_items( XFS_FSB_TO_AGNO(mp, rb->xefi_startblock); } -/* Get an EFI. */ -STATIC void * -xfs_extent_free_create_intent( - struct xfs_trans *tp, - unsigned int count) -{ - struct xfs_efi_log_item *efip; - - ASSERT(tp != NULL); - ASSERT(count > 0); - - efip = xfs_efi_init(tp->t_mountp, count); - ASSERT(efip != NULL); - - /* - * Get a log_item_desc to point at the new item. - */ - xfs_trans_add_item(tp, &efip->efi_item); - return efip; -} - /* Log a free extent to the intent item. */ STATIC void xfs_extent_free_log_item( struct xfs_trans *tp, - void *intent, - struct list_head *item) + struct xfs_efi_log_item *efip, + struct xfs_extent_free_item *free) { - struct xfs_efi_log_item *efip = intent; - struct xfs_extent_free_item *free; uint next_extent; struct xfs_extent *extp; - free = container_of(item, struct xfs_extent_free_item, xefi_list); - tp->t_flags |= XFS_TRANS_DIRTY; set_bit(XFS_LI_DIRTY, &efip->efi_item.li_flags); @@ -462,6 +437,24 @@ xfs_extent_free_log_item( extp->ext_len = free->xefi_blockcount; } +STATIC void * +xfs_extent_free_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_efi_log_item *efip = xfs_efi_init(mp, count); + struct xfs_extent_free_item *free; + + ASSERT(count > 0); + + xfs_trans_add_item(tp, &efip->efi_item); + list_for_each_entry(free, items, xefi_list) + xfs_extent_free_log_item(tp, efip, free); + return efip; +} + /* Get an EFD so we can process all the free extents. */ STATIC void * xfs_extent_free_create_done( @@ -516,7 +509,6 @@ const struct xfs_defer_op_type xfs_extent_free_defer_type = { .diff_items = xfs_extent_free_diff_items, .create_intent = xfs_extent_free_create_intent, .abort_intent = xfs_extent_free_abort_intent, - .log_item = xfs_extent_free_log_item, .create_done = xfs_extent_free_create_done, .finish_item = xfs_extent_free_finish_item, .cancel_item = xfs_extent_free_cancel_item, @@ -582,7 +574,6 @@ const struct xfs_defer_op_type xfs_agfl_free_defer_type = { .diff_items = xfs_extent_free_diff_items, .create_intent = xfs_extent_free_create_intent, .abort_intent = xfs_extent_free_abort_intent, - .log_item = xfs_extent_free_log_item, .create_done = xfs_extent_free_create_done, .finish_item = xfs_agfl_free_finish_item, .cancel_item = xfs_extent_free_cancel_item, diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index 8eeed73928cd..325d49fc0406 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -284,27 +284,6 @@ xfs_refcount_update_diff_items( XFS_FSB_TO_AGNO(mp, rb->ri_startblock); } -/* Get an CUI. */ -STATIC void * -xfs_refcount_update_create_intent( - struct xfs_trans *tp, - unsigned int count) -{ - struct xfs_cui_log_item *cuip; - - ASSERT(tp != NULL); - ASSERT(count > 0); - - cuip = xfs_cui_init(tp->t_mountp, count); - ASSERT(cuip != NULL); - - /* - * Get a log_item_desc to point at the new item. - */ - xfs_trans_add_item(tp, &cuip->cui_item); - return cuip; -} - /* Set the phys extent flags for this reverse mapping. */ static void xfs_trans_set_refcount_flags( @@ -328,16 +307,12 @@ xfs_trans_set_refcount_flags( STATIC void xfs_refcount_update_log_item( struct xfs_trans *tp, - void *intent, - struct list_head *item) + struct xfs_cui_log_item *cuip, + struct xfs_refcount_intent *refc) { - struct xfs_cui_log_item *cuip = intent; - struct xfs_refcount_intent *refc; uint next_extent; struct xfs_phys_extent *ext; - refc = container_of(item, struct xfs_refcount_intent, ri_list); - tp->t_flags |= XFS_TRANS_DIRTY; set_bit(XFS_LI_DIRTY, &cuip->cui_item.li_flags); @@ -354,6 +329,24 @@ xfs_refcount_update_log_item( xfs_trans_set_refcount_flags(ext, refc->ri_type); } +STATIC void * +xfs_refcount_update_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_cui_log_item *cuip = xfs_cui_init(mp, count); + struct xfs_refcount_intent *refc; + + ASSERT(count > 0); + + xfs_trans_add_item(tp, &cuip->cui_item); + list_for_each_entry(refc, items, ri_list) + xfs_refcount_update_log_item(tp, cuip, refc); + return cuip; +} + /* Get an CUD so we can process all the deferred refcount updates. */ STATIC void * xfs_refcount_update_create_done( @@ -432,7 +425,6 @@ const struct xfs_defer_op_type xfs_refcount_update_defer_type = { .diff_items = xfs_refcount_update_diff_items, .create_intent = xfs_refcount_update_create_intent, .abort_intent = xfs_refcount_update_abort_intent, - .log_item = xfs_refcount_update_log_item, .create_done = xfs_refcount_update_create_done, .finish_item = xfs_refcount_update_finish_item, .finish_cleanup = xfs_refcount_update_finish_cleanup, diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index 4911b68f95dd..842d817f5168 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -352,41 +352,16 @@ xfs_rmap_update_diff_items( XFS_FSB_TO_AGNO(mp, rb->ri_bmap.br_startblock); } -/* Get an RUI. */ -STATIC void * -xfs_rmap_update_create_intent( - struct xfs_trans *tp, - unsigned int count) -{ - struct xfs_rui_log_item *ruip; - - ASSERT(tp != NULL); - ASSERT(count > 0); - - ruip = xfs_rui_init(tp->t_mountp, count); - ASSERT(ruip != NULL); - - /* - * Get a log_item_desc to point at the new item. - */ - xfs_trans_add_item(tp, &ruip->rui_item); - return ruip; -} - /* Log rmap updates in the intent item. */ STATIC void xfs_rmap_update_log_item( struct xfs_trans *tp, - void *intent, - struct list_head *item) + struct xfs_rui_log_item *ruip, + struct xfs_rmap_intent *rmap) { - struct xfs_rui_log_item *ruip = intent; - struct xfs_rmap_intent *rmap; uint next_extent; struct xfs_map_extent *map; - rmap = container_of(item, struct xfs_rmap_intent, ri_list); - tp->t_flags |= XFS_TRANS_DIRTY; set_bit(XFS_LI_DIRTY, &ruip->rui_item.li_flags); @@ -406,6 +381,24 @@ xfs_rmap_update_log_item( rmap->ri_bmap.br_state); } +STATIC void * +xfs_rmap_update_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_rui_log_item *ruip = xfs_rui_init(mp, count); + struct xfs_rmap_intent *rmap; + + ASSERT(count > 0); + + xfs_trans_add_item(tp, &ruip->rui_item); + list_for_each_entry(rmap, items, ri_list) + xfs_rmap_update_log_item(tp, ruip, rmap); + return ruip; +} + /* Get an RUD so we can process all the deferred rmap updates. */ STATIC void * xfs_rmap_update_create_done( @@ -476,7 +469,6 @@ const struct xfs_defer_op_type xfs_rmap_update_defer_type = { .diff_items = xfs_rmap_update_diff_items, .create_intent = xfs_rmap_update_create_intent, .abort_intent = xfs_rmap_update_abort_intent, - .log_item = xfs_rmap_update_log_item, .create_done = xfs_rmap_update_create_done, .finish_item = xfs_rmap_update_finish_item, .finish_cleanup = xfs_rmap_update_finish_cleanup, From d367a868e46b025a8ced8e00ef2b3a3c2f3bf732 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 30 Apr 2020 12:52:20 -0700 Subject: [PATCH 0224/1043] xfs: merge the ->diff_items defer op into ->create_intent This avoids a per-item indirect call, and also simplifies the interface a bit. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_defer.c | 5 +---- fs/xfs/libxfs/xfs_defer.h | 3 +-- fs/xfs/xfs_bmap_item.c | 9 ++++++--- fs/xfs/xfs_extfree_item.c | 7 ++++--- fs/xfs/xfs_refcount_item.c | 6 ++++-- fs/xfs/xfs_rmap_item.c | 6 ++++-- 6 files changed, 20 insertions(+), 16 deletions(-) diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index 56d1357f9d13..5402a7bf3110 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -186,11 +186,8 @@ xfs_defer_create_intent( { const struct xfs_defer_op_type *ops = defer_op_types[dfp->dfp_type]; - if (sort) - list_sort(tp->t_mountp, &dfp->dfp_work, ops->diff_items); - dfp->dfp_intent = ops->create_intent(tp, &dfp->dfp_work, - dfp->dfp_count); + dfp->dfp_count, sort); } /* diff --git a/fs/xfs/libxfs/xfs_defer.h b/fs/xfs/libxfs/xfs_defer.h index d6a4577c25b0..660f5c3821d6 100644 --- a/fs/xfs/libxfs/xfs_defer.h +++ b/fs/xfs/libxfs/xfs_defer.h @@ -49,9 +49,8 @@ struct xfs_defer_op_type { void **); void (*finish_cleanup)(struct xfs_trans *, void *, int); void (*cancel_item)(struct list_head *); - int (*diff_items)(void *, struct list_head *, struct list_head *); void *(*create_intent)(struct xfs_trans *tp, struct list_head *items, - unsigned int count); + unsigned int count, bool sort); unsigned int max_items; }; diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index dea97956d78d..f9505c5873bd 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -334,14 +334,18 @@ STATIC void * xfs_bmap_update_create_intent( struct xfs_trans *tp, struct list_head *items, - unsigned int count) + unsigned int count, + bool sort) { - struct xfs_bui_log_item *buip = xfs_bui_init(tp->t_mountp); + struct xfs_mount *mp = tp->t_mountp; + struct xfs_bui_log_item *buip = xfs_bui_init(mp); struct xfs_bmap_intent *bmap; ASSERT(count == XFS_BUI_MAX_FAST_EXTENTS); xfs_trans_add_item(tp, &buip->bui_item); + if (sort) + list_sort(mp, items, xfs_bmap_update_diff_items); list_for_each_entry(bmap, items, bi_list) xfs_bmap_update_log_item(tp, buip, bmap); return buip; @@ -408,7 +412,6 @@ xfs_bmap_update_cancel_item( const struct xfs_defer_op_type xfs_bmap_update_defer_type = { .max_items = XFS_BUI_MAX_FAST_EXTENTS, - .diff_items = xfs_bmap_update_diff_items, .create_intent = xfs_bmap_update_create_intent, .abort_intent = xfs_bmap_update_abort_intent, .create_done = xfs_bmap_update_create_done, diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index cb22c7ad3181..3e10eba9d22b 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -441,7 +441,8 @@ STATIC void * xfs_extent_free_create_intent( struct xfs_trans *tp, struct list_head *items, - unsigned int count) + unsigned int count, + bool sort) { struct xfs_mount *mp = tp->t_mountp; struct xfs_efi_log_item *efip = xfs_efi_init(mp, count); @@ -450,6 +451,8 @@ xfs_extent_free_create_intent( ASSERT(count > 0); xfs_trans_add_item(tp, &efip->efi_item); + if (sort) + list_sort(mp, items, xfs_extent_free_diff_items); list_for_each_entry(free, items, xefi_list) xfs_extent_free_log_item(tp, efip, free); return efip; @@ -506,7 +509,6 @@ xfs_extent_free_cancel_item( const struct xfs_defer_op_type xfs_extent_free_defer_type = { .max_items = XFS_EFI_MAX_FAST_EXTENTS, - .diff_items = xfs_extent_free_diff_items, .create_intent = xfs_extent_free_create_intent, .abort_intent = xfs_extent_free_abort_intent, .create_done = xfs_extent_free_create_done, @@ -571,7 +573,6 @@ xfs_agfl_free_finish_item( /* sub-type with special handling for AGFL deferred frees */ const struct xfs_defer_op_type xfs_agfl_free_defer_type = { .max_items = XFS_EFI_MAX_FAST_EXTENTS, - .diff_items = xfs_extent_free_diff_items, .create_intent = xfs_extent_free_create_intent, .abort_intent = xfs_extent_free_abort_intent, .create_done = xfs_extent_free_create_done, diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index 325d49fc0406..efc32ec55afd 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -333,7 +333,8 @@ STATIC void * xfs_refcount_update_create_intent( struct xfs_trans *tp, struct list_head *items, - unsigned int count) + unsigned int count, + bool sort) { struct xfs_mount *mp = tp->t_mountp; struct xfs_cui_log_item *cuip = xfs_cui_init(mp, count); @@ -342,6 +343,8 @@ xfs_refcount_update_create_intent( ASSERT(count > 0); xfs_trans_add_item(tp, &cuip->cui_item); + if (sort) + list_sort(mp, items, xfs_refcount_update_diff_items); list_for_each_entry(refc, items, ri_list) xfs_refcount_update_log_item(tp, cuip, refc); return cuip; @@ -422,7 +425,6 @@ xfs_refcount_update_cancel_item( const struct xfs_defer_op_type xfs_refcount_update_defer_type = { .max_items = XFS_CUI_MAX_FAST_EXTENTS, - .diff_items = xfs_refcount_update_diff_items, .create_intent = xfs_refcount_update_create_intent, .abort_intent = xfs_refcount_update_abort_intent, .create_done = xfs_refcount_update_create_done, diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index 842d817f5168..40567cf0c216 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -385,7 +385,8 @@ STATIC void * xfs_rmap_update_create_intent( struct xfs_trans *tp, struct list_head *items, - unsigned int count) + unsigned int count, + bool sort) { struct xfs_mount *mp = tp->t_mountp; struct xfs_rui_log_item *ruip = xfs_rui_init(mp, count); @@ -394,6 +395,8 @@ xfs_rmap_update_create_intent( ASSERT(count > 0); xfs_trans_add_item(tp, &ruip->rui_item); + if (sort) + list_sort(mp, items, xfs_rmap_update_diff_items); list_for_each_entry(rmap, items, ri_list) xfs_rmap_update_log_item(tp, ruip, rmap); return ruip; @@ -466,7 +469,6 @@ xfs_rmap_update_cancel_item( const struct xfs_defer_op_type xfs_rmap_update_defer_type = { .max_items = XFS_RUI_MAX_FAST_EXTENTS, - .diff_items = xfs_rmap_update_diff_items, .create_intent = xfs_rmap_update_create_intent, .abort_intent = xfs_rmap_update_abort_intent, .create_done = xfs_rmap_update_create_done, From 13a8333339072b8654c1d2c75550ee9f41ee15de Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 30 Apr 2020 12:52:21 -0700 Subject: [PATCH 0225/1043] xfs: turn dfp_intent into a xfs_log_item All defer op instance place their own extension of the log item into the dfp_intent field. Replace that with a xfs_log_item to improve type safety and make the code easier to follow. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_defer.h | 11 ++++++----- fs/xfs/xfs_bmap_item.c | 12 ++++++------ fs/xfs/xfs_extfree_item.c | 12 ++++++------ fs/xfs/xfs_refcount_item.c | 12 ++++++------ fs/xfs/xfs_rmap_item.c | 12 ++++++------ 5 files changed, 30 insertions(+), 29 deletions(-) diff --git a/fs/xfs/libxfs/xfs_defer.h b/fs/xfs/libxfs/xfs_defer.h index 660f5c3821d6..7b6cc3808a91 100644 --- a/fs/xfs/libxfs/xfs_defer.h +++ b/fs/xfs/libxfs/xfs_defer.h @@ -28,7 +28,7 @@ enum xfs_defer_ops_type { struct xfs_defer_pending { struct list_head dfp_list; /* pending items */ struct list_head dfp_work; /* work items */ - void *dfp_intent; /* log intent item */ + struct xfs_log_item *dfp_intent; /* log intent item */ void *dfp_done; /* log done item */ unsigned int dfp_count; /* # extent items */ enum xfs_defer_ops_type dfp_type; @@ -43,14 +43,15 @@ void xfs_defer_move(struct xfs_trans *dtp, struct xfs_trans *stp); /* Description of a deferred type. */ struct xfs_defer_op_type { - void (*abort_intent)(void *); - void *(*create_done)(struct xfs_trans *, void *, unsigned int); + struct xfs_log_item *(*create_intent)(struct xfs_trans *tp, + struct list_head *items, unsigned int count, bool sort); + void (*abort_intent)(struct xfs_log_item *intent); + void *(*create_done)(struct xfs_trans *tp, struct xfs_log_item *intent, + unsigned int count); int (*finish_item)(struct xfs_trans *, struct list_head *, void *, void **); void (*finish_cleanup)(struct xfs_trans *, void *, int); void (*cancel_item)(struct list_head *); - void *(*create_intent)(struct xfs_trans *tp, struct list_head *items, - unsigned int count, bool sort); unsigned int max_items; }; diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index f9505c5873bd..7b2153fca2d9 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -330,7 +330,7 @@ xfs_bmap_update_log_item( bmap->bi_bmap.br_state); } -STATIC void * +static struct xfs_log_item * xfs_bmap_update_create_intent( struct xfs_trans *tp, struct list_head *items, @@ -348,17 +348,17 @@ xfs_bmap_update_create_intent( list_sort(mp, items, xfs_bmap_update_diff_items); list_for_each_entry(bmap, items, bi_list) xfs_bmap_update_log_item(tp, buip, bmap); - return buip; + return &buip->bui_item; } /* Get an BUD so we can process all the deferred rmap updates. */ STATIC void * xfs_bmap_update_create_done( struct xfs_trans *tp, - void *intent, + struct xfs_log_item *intent, unsigned int count) { - return xfs_trans_get_bud(tp, intent); + return xfs_trans_get_bud(tp, BUI_ITEM(intent)); } /* Process a deferred rmap update. */ @@ -394,9 +394,9 @@ xfs_bmap_update_finish_item( /* Abort all pending BUIs. */ STATIC void xfs_bmap_update_abort_intent( - void *intent) + struct xfs_log_item *intent) { - xfs_bui_release(intent); + xfs_bui_release(BUI_ITEM(intent)); } /* Cancel a deferred rmap update. */ diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index 3e10eba9d22b..0453b6f2b1d6 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -437,7 +437,7 @@ xfs_extent_free_log_item( extp->ext_len = free->xefi_blockcount; } -STATIC void * +static struct xfs_log_item * xfs_extent_free_create_intent( struct xfs_trans *tp, struct list_head *items, @@ -455,17 +455,17 @@ xfs_extent_free_create_intent( list_sort(mp, items, xfs_extent_free_diff_items); list_for_each_entry(free, items, xefi_list) xfs_extent_free_log_item(tp, efip, free); - return efip; + return &efip->efi_item; } /* Get an EFD so we can process all the free extents. */ STATIC void * xfs_extent_free_create_done( struct xfs_trans *tp, - void *intent, + struct xfs_log_item *intent, unsigned int count) { - return xfs_trans_get_efd(tp, intent, count); + return xfs_trans_get_efd(tp, EFI_ITEM(intent), count); } /* Process a free extent. */ @@ -491,9 +491,9 @@ xfs_extent_free_finish_item( /* Abort all pending EFIs. */ STATIC void xfs_extent_free_abort_intent( - void *intent) + struct xfs_log_item *intent) { - xfs_efi_release(intent); + xfs_efi_release(EFI_ITEM(intent)); } /* Cancel a free extent. */ diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index efc32ec55afd..e8d3278e066e 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -329,7 +329,7 @@ xfs_refcount_update_log_item( xfs_trans_set_refcount_flags(ext, refc->ri_type); } -STATIC void * +static struct xfs_log_item * xfs_refcount_update_create_intent( struct xfs_trans *tp, struct list_head *items, @@ -347,17 +347,17 @@ xfs_refcount_update_create_intent( list_sort(mp, items, xfs_refcount_update_diff_items); list_for_each_entry(refc, items, ri_list) xfs_refcount_update_log_item(tp, cuip, refc); - return cuip; + return &cuip->cui_item; } /* Get an CUD so we can process all the deferred refcount updates. */ STATIC void * xfs_refcount_update_create_done( struct xfs_trans *tp, - void *intent, + struct xfs_log_item *intent, unsigned int count) { - return xfs_trans_get_cud(tp, intent); + return xfs_trans_get_cud(tp, CUI_ITEM(intent)); } /* Process a deferred refcount update. */ @@ -407,9 +407,9 @@ xfs_refcount_update_finish_cleanup( /* Abort all pending CUIs. */ STATIC void xfs_refcount_update_abort_intent( - void *intent) + struct xfs_log_item *intent) { - xfs_cui_release(intent); + xfs_cui_release(CUI_ITEM(intent)); } /* Cancel a deferred refcount update. */ diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index 40567cf0c216..a417e15fd0ce 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -381,7 +381,7 @@ xfs_rmap_update_log_item( rmap->ri_bmap.br_state); } -STATIC void * +static struct xfs_log_item * xfs_rmap_update_create_intent( struct xfs_trans *tp, struct list_head *items, @@ -399,17 +399,17 @@ xfs_rmap_update_create_intent( list_sort(mp, items, xfs_rmap_update_diff_items); list_for_each_entry(rmap, items, ri_list) xfs_rmap_update_log_item(tp, ruip, rmap); - return ruip; + return &ruip->rui_item; } /* Get an RUD so we can process all the deferred rmap updates. */ STATIC void * xfs_rmap_update_create_done( struct xfs_trans *tp, - void *intent, + struct xfs_log_item *intent, unsigned int count) { - return xfs_trans_get_rud(tp, intent); + return xfs_trans_get_rud(tp, RUI_ITEM(intent)); } /* Process a deferred rmap update. */ @@ -451,9 +451,9 @@ xfs_rmap_update_finish_cleanup( /* Abort all pending RUIs. */ STATIC void xfs_rmap_update_abort_intent( - void *intent) + struct xfs_log_item *intent) { - xfs_rui_release(intent); + xfs_rui_release(RUI_ITEM(intent)); } /* Cancel a deferred rmap update. */ From bb47d79750f1a68a75d4c7defc2da934ba31de14 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 30 Apr 2020 12:52:21 -0700 Subject: [PATCH 0226/1043] xfs: refactor xfs_defer_finish_noroll Split out a helper that operates on a single xfs_defer_pending structure to untangle the code. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_defer.c | 128 ++++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 69 deletions(-) diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index 5402a7bf3110..20950b56cdd0 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -351,6 +351,53 @@ xfs_defer_cancel_list( } } +/* + * Log an intent-done item for the first pending intent, and finish the work + * items. + */ +static int +xfs_defer_finish_one( + struct xfs_trans *tp, + struct xfs_defer_pending *dfp) +{ + const struct xfs_defer_op_type *ops = defer_op_types[dfp->dfp_type]; + void *state = NULL; + struct list_head *li, *n; + int error; + + trace_xfs_defer_pending_finish(tp->t_mountp, dfp); + + dfp->dfp_done = ops->create_done(tp, dfp->dfp_intent, dfp->dfp_count); + list_for_each_safe(li, n, &dfp->dfp_work) { + list_del(li); + dfp->dfp_count--; + error = ops->finish_item(tp, li, dfp->dfp_done, &state); + if (error == -EAGAIN) { + /* + * Caller wants a fresh transaction; put the work item + * back on the list and log a new log intent item to + * replace the old one. See "Requesting a Fresh + * Transaction while Finishing Deferred Work" above. + */ + list_add(li, &dfp->dfp_work); + dfp->dfp_count++; + dfp->dfp_done = NULL; + xfs_defer_create_intent(tp, dfp, false); + } + + if (error) + goto out; + } + + /* Done with the dfp, free it. */ + list_del(&dfp->dfp_list); + kmem_free(dfp); +out: + if (ops->finish_cleanup) + ops->finish_cleanup(tp, state, error); + return error; +} + /* * Finish all the pending work. This involves logging intent items for * any work items that wandered in since the last transaction roll (if @@ -364,11 +411,7 @@ xfs_defer_finish_noroll( struct xfs_trans **tp) { struct xfs_defer_pending *dfp; - struct list_head *li; - struct list_head *n; - void *state; int error = 0; - const struct xfs_defer_op_type *ops; LIST_HEAD(dop_pending); ASSERT((*tp)->t_flags & XFS_TRANS_PERM_LOG_RES); @@ -377,83 +420,30 @@ xfs_defer_finish_noroll( /* Until we run out of pending work to finish... */ while (!list_empty(&dop_pending) || !list_empty(&(*tp)->t_dfops)) { - /* log intents and pull in intake items */ xfs_defer_create_intents(*tp); list_splice_tail_init(&(*tp)->t_dfops, &dop_pending); - /* - * Roll the transaction. - */ error = xfs_defer_trans_roll(tp); if (error) - goto out; + goto out_shutdown; - /* Log an intent-done item for the first pending item. */ dfp = list_first_entry(&dop_pending, struct xfs_defer_pending, dfp_list); - ops = defer_op_types[dfp->dfp_type]; - trace_xfs_defer_pending_finish((*tp)->t_mountp, dfp); - dfp->dfp_done = ops->create_done(*tp, dfp->dfp_intent, - dfp->dfp_count); - - /* Finish the work items. */ - state = NULL; - list_for_each_safe(li, n, &dfp->dfp_work) { - list_del(li); - dfp->dfp_count--; - error = ops->finish_item(*tp, li, dfp->dfp_done, - &state); - if (error == -EAGAIN) { - /* - * Caller wants a fresh transaction; - * put the work item back on the list - * and jump out. - */ - list_add(li, &dfp->dfp_work); - dfp->dfp_count++; - break; - } else if (error) { - /* - * Clean up after ourselves and jump out. - * xfs_defer_cancel will take care of freeing - * all these lists and stuff. - */ - if (ops->finish_cleanup) - ops->finish_cleanup(*tp, state, error); - goto out; - } - } - if (error == -EAGAIN) { - /* - * Caller wants a fresh transaction, so log a new log - * intent item to replace the old one and roll the - * transaction. See "Requesting a Fresh Transaction - * while Finishing Deferred Work" above. - */ - dfp->dfp_done = NULL; - xfs_defer_create_intent(*tp, dfp, false); - } else { - /* Done with the dfp, free it. */ - list_del(&dfp->dfp_list); - kmem_free(dfp); - } - - if (ops->finish_cleanup) - ops->finish_cleanup(*tp, state, error); - } - -out: - if (error) { - xfs_defer_trans_abort(*tp, &dop_pending); - xfs_force_shutdown((*tp)->t_mountp, SHUTDOWN_CORRUPT_INCORE); - trace_xfs_defer_finish_error(*tp, error); - xfs_defer_cancel_list((*tp)->t_mountp, &dop_pending); - xfs_defer_cancel(*tp); - return error; + error = xfs_defer_finish_one(*tp, dfp); + if (error && error != -EAGAIN) + goto out_shutdown; } trace_xfs_defer_finish_done(*tp, _RET_IP_); return 0; + +out_shutdown: + xfs_defer_trans_abort(*tp, &dop_pending); + xfs_force_shutdown((*tp)->t_mountp, SHUTDOWN_CORRUPT_INCORE); + trace_xfs_defer_finish_error(*tp, error); + xfs_defer_cancel_list((*tp)->t_mountp, &dop_pending); + xfs_defer_cancel(*tp); + return error; } int From f09d167c20332ad1298ff82a6f538b4c7ea3fe1b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 30 Apr 2020 12:52:22 -0700 Subject: [PATCH 0227/1043] xfs: turn dfp_done into a xfs_log_item All defer op instance place their own extension of the log item into the dfp_done field. Replace that with a xfs_log_item to improve type safety and make the code easier to follow. Also use the opportunity to improve the ->finish_item calling conventions to place the done log item as the higher level structure before the list_entry used for the individual items. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_defer.c | 2 +- fs/xfs/libxfs/xfs_defer.h | 10 +++++----- fs/xfs/xfs_bmap_item.c | 8 ++++---- fs/xfs/xfs_extfree_item.c | 12 ++++++------ fs/xfs/xfs_refcount_item.c | 8 ++++---- fs/xfs/xfs_rmap_item.c | 8 ++++---- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index 20950b56cdd0..5f37f42cda67 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -371,7 +371,7 @@ xfs_defer_finish_one( list_for_each_safe(li, n, &dfp->dfp_work) { list_del(li); dfp->dfp_count--; - error = ops->finish_item(tp, li, dfp->dfp_done, &state); + error = ops->finish_item(tp, dfp->dfp_done, li, &state); if (error == -EAGAIN) { /* * Caller wants a fresh transaction; put the work item diff --git a/fs/xfs/libxfs/xfs_defer.h b/fs/xfs/libxfs/xfs_defer.h index 7b6cc3808a91..a86c890e63d2 100644 --- a/fs/xfs/libxfs/xfs_defer.h +++ b/fs/xfs/libxfs/xfs_defer.h @@ -29,7 +29,7 @@ struct xfs_defer_pending { struct list_head dfp_list; /* pending items */ struct list_head dfp_work; /* work items */ struct xfs_log_item *dfp_intent; /* log intent item */ - void *dfp_done; /* log done item */ + struct xfs_log_item *dfp_done; /* log done item */ unsigned int dfp_count; /* # extent items */ enum xfs_defer_ops_type dfp_type; }; @@ -46,10 +46,10 @@ struct xfs_defer_op_type { struct xfs_log_item *(*create_intent)(struct xfs_trans *tp, struct list_head *items, unsigned int count, bool sort); void (*abort_intent)(struct xfs_log_item *intent); - void *(*create_done)(struct xfs_trans *tp, struct xfs_log_item *intent, - unsigned int count); - int (*finish_item)(struct xfs_trans *, struct list_head *, void *, - void **); + struct xfs_log_item *(*create_done)(struct xfs_trans *tp, + struct xfs_log_item *intent, unsigned int count); + int (*finish_item)(struct xfs_trans *tp, struct xfs_log_item *done, + struct list_head *item, void **state); void (*finish_cleanup)(struct xfs_trans *, void *, int); void (*cancel_item)(struct list_head *); unsigned int max_items; diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index 7b2153fca2d9..feadd44a67e4 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -352,21 +352,21 @@ xfs_bmap_update_create_intent( } /* Get an BUD so we can process all the deferred rmap updates. */ -STATIC void * +static struct xfs_log_item * xfs_bmap_update_create_done( struct xfs_trans *tp, struct xfs_log_item *intent, unsigned int count) { - return xfs_trans_get_bud(tp, BUI_ITEM(intent)); + return &xfs_trans_get_bud(tp, BUI_ITEM(intent))->bud_item; } /* Process a deferred rmap update. */ STATIC int xfs_bmap_update_finish_item( struct xfs_trans *tp, + struct xfs_log_item *done, struct list_head *item, - void *done_item, void **state) { struct xfs_bmap_intent *bmap; @@ -375,7 +375,7 @@ xfs_bmap_update_finish_item( bmap = container_of(item, struct xfs_bmap_intent, bi_list); count = bmap->bi_bmap.br_blockcount; - error = xfs_trans_log_finish_bmap_update(tp, done_item, + error = xfs_trans_log_finish_bmap_update(tp, BUD_ITEM(done), bmap->bi_type, bmap->bi_owner, bmap->bi_whichfork, bmap->bi_bmap.br_startoff, diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index 0453b6f2b1d6..633628f70e12 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -459,28 +459,28 @@ xfs_extent_free_create_intent( } /* Get an EFD so we can process all the free extents. */ -STATIC void * +static struct xfs_log_item * xfs_extent_free_create_done( struct xfs_trans *tp, struct xfs_log_item *intent, unsigned int count) { - return xfs_trans_get_efd(tp, EFI_ITEM(intent), count); + return &xfs_trans_get_efd(tp, EFI_ITEM(intent), count)->efd_item; } /* Process a free extent. */ STATIC int xfs_extent_free_finish_item( struct xfs_trans *tp, + struct xfs_log_item *done, struct list_head *item, - void *done_item, void **state) { struct xfs_extent_free_item *free; int error; free = container_of(item, struct xfs_extent_free_item, xefi_list); - error = xfs_trans_free_extent(tp, done_item, + error = xfs_trans_free_extent(tp, EFD_ITEM(done), free->xefi_startblock, free->xefi_blockcount, &free->xefi_oinfo, free->xefi_skip_discard); @@ -523,12 +523,12 @@ const struct xfs_defer_op_type xfs_extent_free_defer_type = { STATIC int xfs_agfl_free_finish_item( struct xfs_trans *tp, + struct xfs_log_item *done, struct list_head *item, - void *done_item, void **state) { struct xfs_mount *mp = tp->t_mountp; - struct xfs_efd_log_item *efdp = done_item; + struct xfs_efd_log_item *efdp = EFD_ITEM(done); struct xfs_extent_free_item *free; struct xfs_extent *extp; struct xfs_buf *agbp; diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index e8d3278e066e..f1c2e559a7ae 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -351,21 +351,21 @@ xfs_refcount_update_create_intent( } /* Get an CUD so we can process all the deferred refcount updates. */ -STATIC void * +static struct xfs_log_item * xfs_refcount_update_create_done( struct xfs_trans *tp, struct xfs_log_item *intent, unsigned int count) { - return xfs_trans_get_cud(tp, CUI_ITEM(intent)); + return &xfs_trans_get_cud(tp, CUI_ITEM(intent))->cud_item; } /* Process a deferred refcount update. */ STATIC int xfs_refcount_update_finish_item( struct xfs_trans *tp, + struct xfs_log_item *done, struct list_head *item, - void *done_item, void **state) { struct xfs_refcount_intent *refc; @@ -374,7 +374,7 @@ xfs_refcount_update_finish_item( int error; refc = container_of(item, struct xfs_refcount_intent, ri_list); - error = xfs_trans_log_finish_refcount_update(tp, done_item, + error = xfs_trans_log_finish_refcount_update(tp, CUD_ITEM(done), refc->ri_type, refc->ri_startblock, refc->ri_blockcount, diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index a417e15fd0ce..f6a2a388e5ac 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -403,28 +403,28 @@ xfs_rmap_update_create_intent( } /* Get an RUD so we can process all the deferred rmap updates. */ -STATIC void * +static struct xfs_log_item * xfs_rmap_update_create_done( struct xfs_trans *tp, struct xfs_log_item *intent, unsigned int count) { - return xfs_trans_get_rud(tp, RUI_ITEM(intent)); + return &xfs_trans_get_rud(tp, RUI_ITEM(intent))->rud_item; } /* Process a deferred rmap update. */ STATIC int xfs_rmap_update_finish_item( struct xfs_trans *tp, + struct xfs_log_item *done, struct list_head *item, - void *done_item, void **state) { struct xfs_rmap_intent *rmap; int error; rmap = container_of(item, struct xfs_rmap_intent, ri_list); - error = xfs_trans_log_finish_rmap_update(tp, done_item, + error = xfs_trans_log_finish_rmap_update(tp, RUD_ITEM(done), rmap->ri_type, rmap->ri_owner, rmap->ri_whichfork, rmap->ri_bmap.br_startoff, From 3ec1b26c04d4910f37cdaad26d14b403c0240e30 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 30 Apr 2020 12:52:22 -0700 Subject: [PATCH 0228/1043] xfs: use a xfs_btree_cur for the ->finish_cleanup state Given how XFS is all based around btrees it doesn't make much sense to offer a totally generic state when we can just use the btree cursor. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_defer.c | 2 +- fs/xfs/libxfs/xfs_defer.h | 6 ++++-- fs/xfs/xfs_bmap_item.c | 2 +- fs/xfs/xfs_extfree_item.c | 4 ++-- fs/xfs/xfs_refcount_item.c | 24 +++++------------------- fs/xfs/xfs_rmap_item.c | 27 ++++++--------------------- 6 files changed, 19 insertions(+), 46 deletions(-) diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index 5f37f42cda67..1172fbf072d8 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -361,7 +361,7 @@ xfs_defer_finish_one( struct xfs_defer_pending *dfp) { const struct xfs_defer_op_type *ops = defer_op_types[dfp->dfp_type]; - void *state = NULL; + struct xfs_btree_cur *state = NULL; struct list_head *li, *n; int error; diff --git a/fs/xfs/libxfs/xfs_defer.h b/fs/xfs/libxfs/xfs_defer.h index a86c890e63d2..f2b65981bace 100644 --- a/fs/xfs/libxfs/xfs_defer.h +++ b/fs/xfs/libxfs/xfs_defer.h @@ -6,6 +6,7 @@ #ifndef __XFS_DEFER_H__ #define __XFS_DEFER_H__ +struct xfs_btree_cur; struct xfs_defer_op_type; /* @@ -49,8 +50,9 @@ struct xfs_defer_op_type { struct xfs_log_item *(*create_done)(struct xfs_trans *tp, struct xfs_log_item *intent, unsigned int count); int (*finish_item)(struct xfs_trans *tp, struct xfs_log_item *done, - struct list_head *item, void **state); - void (*finish_cleanup)(struct xfs_trans *, void *, int); + struct list_head *item, struct xfs_btree_cur **state); + void (*finish_cleanup)(struct xfs_trans *tp, + struct xfs_btree_cur *state, int error); void (*cancel_item)(struct list_head *); unsigned int max_items; }; diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index feadd44a67e4..7768fb2b7135 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -367,7 +367,7 @@ xfs_bmap_update_finish_item( struct xfs_trans *tp, struct xfs_log_item *done, struct list_head *item, - void **state) + struct xfs_btree_cur **state) { struct xfs_bmap_intent *bmap; xfs_filblks_t count; diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index 633628f70e12..c8cde4122a0f 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -474,7 +474,7 @@ xfs_extent_free_finish_item( struct xfs_trans *tp, struct xfs_log_item *done, struct list_head *item, - void **state) + struct xfs_btree_cur **state) { struct xfs_extent_free_item *free; int error; @@ -525,7 +525,7 @@ xfs_agfl_free_finish_item( struct xfs_trans *tp, struct xfs_log_item *done, struct list_head *item, - void **state) + struct xfs_btree_cur **state) { struct xfs_mount *mp = tp->t_mountp; struct xfs_efd_log_item *efdp = EFD_ITEM(done); diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index f1c2e559a7ae..0316eab2fc35 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -366,7 +366,7 @@ xfs_refcount_update_finish_item( struct xfs_trans *tp, struct xfs_log_item *done, struct list_head *item, - void **state) + struct xfs_btree_cur **state) { struct xfs_refcount_intent *refc; xfs_fsblock_t new_fsb; @@ -375,11 +375,9 @@ xfs_refcount_update_finish_item( refc = container_of(item, struct xfs_refcount_intent, ri_list); error = xfs_trans_log_finish_refcount_update(tp, CUD_ITEM(done), - refc->ri_type, - refc->ri_startblock, - refc->ri_blockcount, - &new_fsb, &new_aglen, - (struct xfs_btree_cur **)state); + refc->ri_type, refc->ri_startblock, refc->ri_blockcount, + &new_fsb, &new_aglen, state); + /* Did we run out of reservation? Requeue what we didn't finish. */ if (!error && new_aglen > 0) { ASSERT(refc->ri_type == XFS_REFCOUNT_INCREASE || @@ -392,18 +390,6 @@ xfs_refcount_update_finish_item( return error; } -/* Clean up after processing deferred refcounts. */ -STATIC void -xfs_refcount_update_finish_cleanup( - struct xfs_trans *tp, - void *state, - int error) -{ - struct xfs_btree_cur *rcur = state; - - xfs_refcount_finish_one_cleanup(tp, rcur, error); -} - /* Abort all pending CUIs. */ STATIC void xfs_refcount_update_abort_intent( @@ -429,7 +415,7 @@ const struct xfs_defer_op_type xfs_refcount_update_defer_type = { .abort_intent = xfs_refcount_update_abort_intent, .create_done = xfs_refcount_update_create_done, .finish_item = xfs_refcount_update_finish_item, - .finish_cleanup = xfs_refcount_update_finish_cleanup, + .finish_cleanup = xfs_refcount_finish_one_cleanup, .cancel_item = xfs_refcount_update_cancel_item, }; diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index f6a2a388e5ac..e3bba2aec868 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -418,36 +418,21 @@ xfs_rmap_update_finish_item( struct xfs_trans *tp, struct xfs_log_item *done, struct list_head *item, - void **state) + struct xfs_btree_cur **state) { struct xfs_rmap_intent *rmap; int error; rmap = container_of(item, struct xfs_rmap_intent, ri_list); error = xfs_trans_log_finish_rmap_update(tp, RUD_ITEM(done), - rmap->ri_type, - rmap->ri_owner, rmap->ri_whichfork, - rmap->ri_bmap.br_startoff, - rmap->ri_bmap.br_startblock, - rmap->ri_bmap.br_blockcount, - rmap->ri_bmap.br_state, - (struct xfs_btree_cur **)state); + rmap->ri_type, rmap->ri_owner, rmap->ri_whichfork, + rmap->ri_bmap.br_startoff, rmap->ri_bmap.br_startblock, + rmap->ri_bmap.br_blockcount, rmap->ri_bmap.br_state, + state); kmem_free(rmap); return error; } -/* Clean up after processing deferred rmaps. */ -STATIC void -xfs_rmap_update_finish_cleanup( - struct xfs_trans *tp, - void *state, - int error) -{ - struct xfs_btree_cur *rcur = state; - - xfs_rmap_finish_one_cleanup(tp, rcur, error); -} - /* Abort all pending RUIs. */ STATIC void xfs_rmap_update_abort_intent( @@ -473,7 +458,7 @@ const struct xfs_defer_op_type xfs_rmap_update_defer_type = { .abort_intent = xfs_rmap_update_abort_intent, .create_done = xfs_rmap_update_create_done, .finish_item = xfs_rmap_update_finish_item, - .finish_cleanup = xfs_rmap_update_finish_cleanup, + .finish_cleanup = xfs_rmap_finish_one_cleanup, .cancel_item = xfs_rmap_update_cancel_item, }; From 2f88f1efd02ddf76cb5973abc42474c4dac2b03a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 30 Apr 2020 12:52:23 -0700 Subject: [PATCH 0229/1043] xfs: spell out the parameter name for ->cancel_item Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_defer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_defer.h b/fs/xfs/libxfs/xfs_defer.h index f2b65981bace..3bf7c2c4d851 100644 --- a/fs/xfs/libxfs/xfs_defer.h +++ b/fs/xfs/libxfs/xfs_defer.h @@ -53,7 +53,7 @@ struct xfs_defer_op_type { struct list_head *item, struct xfs_btree_cur **state); void (*finish_cleanup)(struct xfs_trans *tp, struct xfs_btree_cur *state, int error); - void (*cancel_item)(struct list_head *); + void (*cancel_item)(struct list_head *item); unsigned int max_items; }; From d45344d6c49cf9863e1d526fc1d8cb27ab147148 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Wed, 22 Apr 2020 21:50:57 -0700 Subject: [PATCH 0230/1043] fs/xfs: Remove unnecessary initialization of i_rwsem An earlier call of xfs_reinit_inode() from xfs_iget_cache_hit() already handles initialization of i_rwsem. Doing so again is unneeded. Reviewed-by: Darrick J. Wong Reviewed-by: Dave Chinner Signed-off-by: Ira Weiny Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_icache.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 8bf1d15be3f6..17a0b86fe701 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -423,6 +423,7 @@ xfs_iget_cache_hit( spin_unlock(&ip->i_flags_lock); rcu_read_unlock(); + ASSERT(!rwsem_is_locked(&inode->i_rwsem)); error = xfs_reinit_inode(mp, inode); if (error) { bool wake; @@ -456,9 +457,6 @@ xfs_iget_cache_hit( ip->i_sick = 0; ip->i_checked = 0; - ASSERT(!rwsem_is_locked(&inode->i_rwsem)); - init_rwsem(&inode->i_rwsem); - spin_unlock(&ip->i_flags_lock); spin_unlock(&pag->pag_ici_lock); } else { From 606723d982939ab138a05a8070b1ec48ed532234 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Mon, 4 May 2020 09:02:41 -0700 Subject: [PATCH 0231/1043] fs/xfs: Change XFS_MOUNT_DAX to XFS_MOUNT_DAX_ALWAYS In prep for the new tri-state mount option which then introduces XFS_MOUNT_DAX_NEVER. Reviewed-by: Dave Chinner Reviewed-by: Darrick J. Wong Signed-off-by: Ira Weiny Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_iops.c | 2 +- fs/xfs/xfs_mount.h | 3 +-- fs/xfs/xfs_super.c | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index e34814590453..97c0e31241b7 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -1243,7 +1243,7 @@ xfs_inode_supports_dax( return false; /* DAX mount option or DAX iflag must be set. */ - if (!(mp->m_flags & XFS_MOUNT_DAX) && + if (!(mp->m_flags & XFS_MOUNT_DAX_ALWAYS) && !(ip->i_d.di_flags2 & XFS_DIFLAG2_DAX)) return false; diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index b2e4598fdf7d..f6123fb0113c 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -237,8 +237,7 @@ typedef struct xfs_mount { #define XFS_MOUNT_FILESTREAMS (1ULL << 24) /* enable the filestreams allocator */ #define XFS_MOUNT_NOATTR2 (1ULL << 25) /* disable use of attr2 format */ - -#define XFS_MOUNT_DAX (1ULL << 62) /* TEST ONLY! */ +#define XFS_MOUNT_DAX_ALWAYS (1ULL << 26) /* * Max and min values for mount-option defined I/O diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 424bb9a2d532..ce169d1c7474 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -129,7 +129,7 @@ xfs_fs_show_options( { XFS_MOUNT_GRPID, ",grpid" }, { XFS_MOUNT_DISCARD, ",discard" }, { XFS_MOUNT_LARGEIO, ",largeio" }, - { XFS_MOUNT_DAX, ",dax" }, + { XFS_MOUNT_DAX_ALWAYS, ",dax" }, { 0, NULL } }; struct xfs_mount *mp = XFS_M(root->d_sb); @@ -1261,7 +1261,7 @@ xfs_fc_parse_param( return 0; #ifdef CONFIG_FS_DAX case Opt_dax: - mp->m_flags |= XFS_MOUNT_DAX; + mp->m_flags |= XFS_MOUNT_DAX_ALWAYS; return 0; #endif default: @@ -1454,7 +1454,7 @@ xfs_fc_fill_super( if (XFS_SB_VERSION_NUM(&mp->m_sb) == XFS_SB_VERSION_5) sb->s_flags |= SB_I_VERSION; - if (mp->m_flags & XFS_MOUNT_DAX) { + if (mp->m_flags & XFS_MOUNT_DAX_ALWAYS) { bool rtdev_is_dax = false, datadev_is_dax; xfs_warn(mp, @@ -1468,7 +1468,7 @@ xfs_fc_fill_super( if (!rtdev_is_dax && !datadev_is_dax) { xfs_alert(mp, "DAX unsupported by block device. Turning off DAX."); - mp->m_flags &= ~XFS_MOUNT_DAX; + mp->m_flags &= ~XFS_MOUNT_DAX_ALWAYS; } if (xfs_sb_version_hasreflink(&mp->m_sb)) { xfs_alert(mp, From 8d6c3446ec23ecd97bc089ed224342baf9426c30 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Mon, 4 May 2020 09:02:42 -0700 Subject: [PATCH 0232/1043] fs/xfs: Make DAX mount option a tri-state As agreed upon[1]. We make the dax mount option a tri-state. '-o dax' continues to operate the same. We add 'always', 'never', and 'inode' (default). [1] https://lore.kernel.org/lkml/20200405061945.GA94792@iweiny-DESK2.sc.intel.com/ Signed-off-by: Ira Weiny Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_mount.h | 1 + fs/xfs/xfs_super.c | 46 ++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index f6123fb0113c..37bfb50db809 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -238,6 +238,7 @@ typedef struct xfs_mount { allocator */ #define XFS_MOUNT_NOATTR2 (1ULL << 25) /* disable use of attr2 format */ #define XFS_MOUNT_DAX_ALWAYS (1ULL << 26) +#define XFS_MOUNT_DAX_NEVER (1ULL << 27) /* * Max and min values for mount-option defined I/O diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index ce169d1c7474..e80bd2c4c279 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -47,6 +47,39 @@ static struct kset *xfs_kset; /* top-level xfs sysfs dir */ static struct xfs_kobj xfs_dbg_kobj; /* global debug sysfs attrs */ #endif +enum xfs_dax_mode { + XFS_DAX_INODE = 0, + XFS_DAX_ALWAYS = 1, + XFS_DAX_NEVER = 2, +}; + +static void +xfs_mount_set_dax_mode( + struct xfs_mount *mp, + enum xfs_dax_mode mode) +{ + switch (mode) { + case XFS_DAX_INODE: + mp->m_flags &= ~(XFS_MOUNT_DAX_ALWAYS | XFS_MOUNT_DAX_NEVER); + break; + case XFS_DAX_ALWAYS: + mp->m_flags |= XFS_MOUNT_DAX_ALWAYS; + mp->m_flags &= ~XFS_MOUNT_DAX_NEVER; + break; + case XFS_DAX_NEVER: + mp->m_flags |= XFS_MOUNT_DAX_NEVER; + mp->m_flags &= ~XFS_MOUNT_DAX_ALWAYS; + break; + } +} + +static const struct constant_table dax_param_enums[] = { + {"inode", XFS_DAX_INODE }, + {"always", XFS_DAX_ALWAYS }, + {"never", XFS_DAX_NEVER }, + {} +}; + /* * Table driven mount option parser. */ @@ -59,7 +92,7 @@ enum { Opt_filestreams, Opt_quota, Opt_noquota, Opt_usrquota, Opt_grpquota, Opt_prjquota, Opt_uquota, Opt_gquota, Opt_pquota, Opt_uqnoenforce, Opt_gqnoenforce, Opt_pqnoenforce, Opt_qnoenforce, - Opt_discard, Opt_nodiscard, Opt_dax, + Opt_discard, Opt_nodiscard, Opt_dax, Opt_dax_enum, }; static const struct fs_parameter_spec xfs_fs_parameters[] = { @@ -103,6 +136,7 @@ static const struct fs_parameter_spec xfs_fs_parameters[] = { fsparam_flag("discard", Opt_discard), fsparam_flag("nodiscard", Opt_nodiscard), fsparam_flag("dax", Opt_dax), + fsparam_enum("dax", Opt_dax_enum, dax_param_enums), {} }; @@ -129,7 +163,8 @@ xfs_fs_show_options( { XFS_MOUNT_GRPID, ",grpid" }, { XFS_MOUNT_DISCARD, ",discard" }, { XFS_MOUNT_LARGEIO, ",largeio" }, - { XFS_MOUNT_DAX_ALWAYS, ",dax" }, + { XFS_MOUNT_DAX_ALWAYS, ",dax=always" }, + { XFS_MOUNT_DAX_NEVER, ",dax=never" }, { 0, NULL } }; struct xfs_mount *mp = XFS_M(root->d_sb); @@ -1261,7 +1296,10 @@ xfs_fc_parse_param( return 0; #ifdef CONFIG_FS_DAX case Opt_dax: - mp->m_flags |= XFS_MOUNT_DAX_ALWAYS; + xfs_mount_set_dax_mode(mp, XFS_DAX_ALWAYS); + return 0; + case Opt_dax_enum: + xfs_mount_set_dax_mode(mp, result.uint_32); return 0; #endif default: @@ -1468,7 +1506,7 @@ xfs_fc_fill_super( if (!rtdev_is_dax && !datadev_is_dax) { xfs_alert(mp, "DAX unsupported by block device. Turning off DAX."); - mp->m_flags &= ~XFS_MOUNT_DAX_ALWAYS; + xfs_mount_set_dax_mode(mp, XFS_DAX_NEVER); } if (xfs_sb_version_hasreflink(&mp->m_sb)) { xfs_alert(mp, From 32dbc5655f1ccb3bffa5785523146f60fa4bf905 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Mon, 4 May 2020 09:02:42 -0700 Subject: [PATCH 0233/1043] fs/xfs: Create function xfs_inode_should_enable_dax() xfs_inode_supports_dax() should reflect if the inode can support DAX not that it is enabled for DAX. Change the use of xfs_inode_supports_dax() to reflect only if the inode and underlying storage support dax. Add a new function xfs_inode_should_enable_dax() which reflects if the inode should be enabled for DAX. Reviewed-by: Dave Chinner Reviewed-by: Darrick J. Wong Signed-off-by: Ira Weiny Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_iops.c | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 97c0e31241b7..6b9a29a05217 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -1238,13 +1238,12 @@ xfs_inode_supports_dax( { struct xfs_mount *mp = ip->i_mount; - /* Only supported on non-reflinked files. */ - if (!S_ISREG(VFS_I(ip)->i_mode) || xfs_is_reflink_inode(ip)) + /* Only supported on regular files. */ + if (!S_ISREG(VFS_I(ip)->i_mode)) return false; - /* DAX mount option or DAX iflag must be set. */ - if (!(mp->m_flags & XFS_MOUNT_DAX_ALWAYS) && - !(ip->i_d.di_flags2 & XFS_DIFLAG2_DAX)) + /* Only supported on non-reflinked files. */ + if (xfs_is_reflink_inode(ip)) return false; /* Block size must match page size */ @@ -1255,6 +1254,23 @@ xfs_inode_supports_dax( return xfs_inode_buftarg(ip)->bt_daxdev != NULL; } +static bool +xfs_inode_should_enable_dax( + struct xfs_inode *ip) +{ + if (!IS_ENABLED(CONFIG_FS_DAX)) + return false; + if (ip->i_mount->m_flags & XFS_MOUNT_DAX_NEVER) + return false; + if (!xfs_inode_supports_dax(ip)) + return false; + if (ip->i_mount->m_flags & XFS_MOUNT_DAX_ALWAYS) + return true; + if (ip->i_d.di_flags2 & XFS_DIFLAG2_DAX) + return true; + return false; +} + STATIC void xfs_diflags_to_iflags( struct inode *inode, @@ -1273,7 +1289,7 @@ xfs_diflags_to_iflags( inode->i_flags |= S_SYNC; if (flags & XFS_DIFLAG_NOATIME) inode->i_flags |= S_NOATIME; - if (xfs_inode_supports_dax(ip)) + if (xfs_inode_should_enable_dax(ip)) inode->i_flags |= S_DAX; } From 840d493dff1abb0cb0d73417148a3eeecd5f30d9 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Mon, 4 May 2020 09:02:43 -0700 Subject: [PATCH 0234/1043] fs/xfs: Combine xfs_diflags_to_linux() and xfs_diflags_to_iflags() The functionality in xfs_diflags_to_linux() and xfs_diflags_to_iflags() are nearly identical. The only difference is that *_to_linux() is called after inode setup and disallows changing the DAX flag. Combining them can be done with a flag which indicates if this is the initial setup to allow the DAX flag to be properly set only at init time. So remove xfs_diflags_to_linux() and call the modified xfs_diflags_to_iflags() directly. While we are here simplify xfs_diflags_to_iflags() to take struct xfs_inode and use xfs_ip2xflags() to ensure future diflags are included correctly. Reviewed-by: Dave Chinner Reviewed-by: Darrick J. Wong Signed-off-by: Ira Weiny Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_inode.h | 1 + fs/xfs/xfs_ioctl.c | 33 +-------------------------------- fs/xfs/xfs_iops.c | 42 +++++++++++++++++++++++++----------------- 3 files changed, 27 insertions(+), 49 deletions(-) diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index c6a63f6764a6..83073c883fbf 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -467,6 +467,7 @@ int xfs_break_layouts(struct inode *inode, uint *iolock, /* from xfs_iops.c */ extern void xfs_setup_inode(struct xfs_inode *ip); extern void xfs_setup_iops(struct xfs_inode *ip); +extern void xfs_diflags_to_iflags(struct xfs_inode *ip, bool init); /* * When setting up a newly allocated inode, we need to call diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 91936ed5e334..4ee0d13232f3 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -1201,37 +1201,6 @@ xfs_flags2diflags2( return di_flags2; } -STATIC void -xfs_diflags_to_linux( - struct xfs_inode *ip) -{ - struct inode *inode = VFS_I(ip); - unsigned int xflags = xfs_ip2xflags(ip); - - if (xflags & FS_XFLAG_IMMUTABLE) - inode->i_flags |= S_IMMUTABLE; - else - inode->i_flags &= ~S_IMMUTABLE; - if (xflags & FS_XFLAG_APPEND) - inode->i_flags |= S_APPEND; - else - inode->i_flags &= ~S_APPEND; - if (xflags & FS_XFLAG_SYNC) - inode->i_flags |= S_SYNC; - else - inode->i_flags &= ~S_SYNC; - if (xflags & FS_XFLAG_NOATIME) - inode->i_flags |= S_NOATIME; - else - inode->i_flags &= ~S_NOATIME; -#if 0 /* disabled until the flag switching races are sorted out */ - if (xflags & FS_XFLAG_DAX) - inode->i_flags |= S_DAX; - else - inode->i_flags &= ~S_DAX; -#endif -} - static int xfs_ioctl_setattr_xflags( struct xfs_trans *tp, @@ -1269,7 +1238,7 @@ xfs_ioctl_setattr_xflags( ip->i_d.di_flags = xfs_flags2diflags(ip, fa->fsx_xflags); ip->i_d.di_flags2 = di_flags2; - xfs_diflags_to_linux(ip); + xfs_diflags_to_iflags(ip, false); xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_CHG); xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); XFS_STATS_INC(mp, xs_ig_attrchg); diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 6b9a29a05217..26a71237d70f 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -1271,26 +1271,34 @@ xfs_inode_should_enable_dax( return false; } -STATIC void +void xfs_diflags_to_iflags( - struct inode *inode, - struct xfs_inode *ip) + struct xfs_inode *ip, + bool init) { - uint16_t flags = ip->i_d.di_flags; + struct inode *inode = VFS_I(ip); + unsigned int xflags = xfs_ip2xflags(ip); + unsigned int flags = 0; - inode->i_flags &= ~(S_IMMUTABLE | S_APPEND | S_SYNC | - S_NOATIME | S_DAX); + ASSERT(!(IS_DAX(inode) && init)); - if (flags & XFS_DIFLAG_IMMUTABLE) - inode->i_flags |= S_IMMUTABLE; - if (flags & XFS_DIFLAG_APPEND) - inode->i_flags |= S_APPEND; - if (flags & XFS_DIFLAG_SYNC) - inode->i_flags |= S_SYNC; - if (flags & XFS_DIFLAG_NOATIME) - inode->i_flags |= S_NOATIME; - if (xfs_inode_should_enable_dax(ip)) - inode->i_flags |= S_DAX; + if (xflags & FS_XFLAG_IMMUTABLE) + flags |= S_IMMUTABLE; + if (xflags & FS_XFLAG_APPEND) + flags |= S_APPEND; + if (xflags & FS_XFLAG_SYNC) + flags |= S_SYNC; + if (xflags & FS_XFLAG_NOATIME) + flags |= S_NOATIME; + if (init && xfs_inode_should_enable_dax(ip)) + flags |= S_DAX; + + /* + * S_DAX can only be set during inode initialization and is never set by + * the VFS, so we cannot mask off S_DAX in i_flags. + */ + inode->i_flags &= ~(S_IMMUTABLE | S_APPEND | S_SYNC | S_NOATIME); + inode->i_flags |= flags; } /* @@ -1316,7 +1324,7 @@ xfs_setup_inode( inode_fake_hash(inode); i_size_write(inode, ip->i_d.di_size); - xfs_diflags_to_iflags(inode, ip); + xfs_diflags_to_iflags(ip, true); if (S_ISDIR(inode->i_mode)) { /* From ac819eda7cc96656df50897848ffe5dfe9a3cb7c Mon Sep 17 00:00:00 2001 From: Lokesh Vutla Date: Mon, 27 Apr 2020 22:56:04 +0530 Subject: [PATCH 0235/1043] ARM: dts: Add 32KHz clock as default clock source Clocksource to timer configured in pwm mode can be selected using the DT property ti,clock-source. There are few pwm timers which are not selecting the clock source and relying on default value in hardware or selected by driver. Instead of relying on default value, always select the clock source from DT. Signed-off-by: Lokesh Vutla Reviewed-by: Suman Anna Signed-off-by: Tony Lindgren --- arch/arm/boot/dts/am335x-guardian.dts | 1 + arch/arm/boot/dts/am3517-evm.dts | 1 + arch/arm/boot/dts/logicpd-torpedo-baseboard.dtsi | 1 + arch/arm/boot/dts/omap3-gta04.dtsi | 1 + 4 files changed, 4 insertions(+) diff --git a/arch/arm/boot/dts/am335x-guardian.dts b/arch/arm/boot/dts/am335x-guardian.dts index 81e0f63e94d3..0ebe9e2c150e 100644 --- a/arch/arm/boot/dts/am335x-guardian.dts +++ b/arch/arm/boot/dts/am335x-guardian.dts @@ -105,6 +105,7 @@ ti,timers = <&timer7>; pinctrl-names = "default"; pinctrl-0 = <&dmtimer7_pins>; + ti,clock-source = <0x01>; }; vmmcsd_fixed: regulator-3v3 { diff --git a/arch/arm/boot/dts/am3517-evm.dts b/arch/arm/boot/dts/am3517-evm.dts index a1fd3e63e86e..92466b9eb6ba 100644 --- a/arch/arm/boot/dts/am3517-evm.dts +++ b/arch/arm/boot/dts/am3517-evm.dts @@ -156,6 +156,7 @@ pinctrl-0 = <&pwm_pins>; ti,timers = <&timer11>; #pwm-cells = <3>; + ti,clock-source = <0x01>; }; /* HS USB Host PHY on PORT 1 */ diff --git a/arch/arm/boot/dts/logicpd-torpedo-baseboard.dtsi b/arch/arm/boot/dts/logicpd-torpedo-baseboard.dtsi index f7b82ced4080..381f0e82bb70 100644 --- a/arch/arm/boot/dts/logicpd-torpedo-baseboard.dtsi +++ b/arch/arm/boot/dts/logicpd-torpedo-baseboard.dtsi @@ -65,6 +65,7 @@ pinctrl-0 = <&pwm_pins>; ti,timers = <&timer10>; #pwm-cells = <3>; + ti,clock-source = <0x01>; }; }; diff --git a/arch/arm/boot/dts/omap3-gta04.dtsi b/arch/arm/boot/dts/omap3-gta04.dtsi index 409a758c99f1..ecc45862b4f3 100644 --- a/arch/arm/boot/dts/omap3-gta04.dtsi +++ b/arch/arm/boot/dts/omap3-gta04.dtsi @@ -150,6 +150,7 @@ compatible = "ti,omap-dmtimer-pwm"; ti,timers = <&timer11>; #pwm-cells = <3>; + ti,clock-source = <0x01>; }; hsusb2_phy: hsusb2_phy { From 8bc3b5e4b70d28f8edcafc3c9e4de515998eea9e Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 4 May 2020 14:06:27 -0700 Subject: [PATCH 0236/1043] xfs: clean up the error handling in xfs_swap_extents Make sure we release resources properly if we cannot clean out the COW extents in preparation for an extent swap. Fixes: 96987eea537d6c ("xfs: cancel COW blocks before swapext") Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_bmap_util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index 4f800f7fe888..cc23a3e23e2d 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -1606,7 +1606,7 @@ xfs_swap_extents( if (xfs_inode_has_cow_data(tip)) { error = xfs_reflink_cancel_cow_range(tip, 0, NULLFILEOFF, true); if (error) - return error; + goto out_unlock; } /* From 80ab8eb6c5b6d00135d958844c6c03ad03a2eb1a Mon Sep 17 00:00:00 2001 From: Jason Yan Date: Wed, 6 May 2020 21:43:15 +0800 Subject: [PATCH 0237/1043] MIPS: Octeon: Remove dead code in __cvmx_helper_npi_probe() This code has been marked dead for more than 10 years. Seems no need to keep it now. Signed-off-by: Jason Yan Signed-off-by: Thomas Bogendoerfer --- arch/mips/cavium-octeon/executive/cvmx-helper-npi.c | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-npi.c b/arch/mips/cavium-octeon/executive/cvmx-helper-npi.c index cc94cfa545b4..cb210d2ef0c4 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-helper-npi.c +++ b/arch/mips/cavium-octeon/executive/cvmx-helper-npi.c @@ -59,18 +59,6 @@ int __cvmx_helper_npi_probe(int interface) && !OCTEON_IS_MODEL(OCTEON_CN52XX_PASS1_X)) /* The packet engines didn't exist before pass 2 */ return 4; -#if 0 - /* - * Technically CN30XX, CN31XX, and CN50XX contain packet - * engines, but nobody ever uses them. Since this is the case, - * we disable them here. - */ - else if (OCTEON_IS_MODEL(OCTEON_CN31XX) - || OCTEON_IS_MODEL(OCTEON_CN50XX)) - return 2; - else if (OCTEON_IS_MODEL(OCTEON_CN30XX)) - return 1; -#endif #endif return 0; } From fc9ab75e894b376b8f2960e59d32e8740d2c64e2 Mon Sep 17 00:00:00 2001 From: Jason Yan Date: Wed, 6 May 2020 21:43:07 +0800 Subject: [PATCH 0238/1043] MIPS: CFE: Remove dead code in cfe_getfwinfo() This code has been marked dead since the beginning of the git history. Seems no need to keep it now. Signed-off-by: Jason Yan Signed-off-by: Thomas Bogendoerfer --- arch/mips/fw/cfe/cfe_api.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/arch/mips/fw/cfe/cfe_api.c b/arch/mips/fw/cfe/cfe_api.c index c020b29f561c..0c9c97ab291e 100644 --- a/arch/mips/fw/cfe/cfe_api.c +++ b/arch/mips/fw/cfe/cfe_api.c @@ -243,11 +243,6 @@ int cfe_getfwinfo(cfe_fwinfo_t * info) info->fwi_bootarea_pa = xiocb.plist.xiocb_fwinfo.fwi_bootarea_pa; info->fwi_bootarea_size = xiocb.plist.xiocb_fwinfo.fwi_bootarea_size; -#if 0 - info->fwi_reserved1 = xiocb.plist.xiocb_fwinfo.fwi_reserved1; - info->fwi_reserved2 = xiocb.plist.xiocb_fwinfo.fwi_reserved2; - info->fwi_reserved3 = xiocb.plist.xiocb_fwinfo.fwi_reserved3; -#endif return 0; } From 53635eb2e24b96b7f2d308b7ac74641c8e2e112c Mon Sep 17 00:00:00 2001 From: Jason Yan Date: Wed, 6 May 2020 21:42:50 +0800 Subject: [PATCH 0239/1043] MIPS: Remove dead code in pci.h This code has been marked dead for more than 10 years. Seems no need to keep it now. Signed-off-by: Jason Yan Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mach-rc32434/pci.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/mips/include/asm/mach-rc32434/pci.h b/arch/mips/include/asm/mach-rc32434/pci.h index 6f40d1515580..9a6eefd12757 100644 --- a/arch/mips/include/asm/mach-rc32434/pci.h +++ b/arch/mips/include/asm/mach-rc32434/pci.h @@ -319,9 +319,6 @@ struct pci_msu { #define PCIM_H_EA 0x3 #define PCIM_H_IA_FIX 0x4 #define PCIM_H_IA_RR 0x5 -#if 0 -#define PCI_ADDR_START 0x13000000 -#endif #define PCI_ADDR_START 0x50000000 From ff487d41036035376e47972c7c522490b839ab37 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Wed, 6 May 2020 13:52:45 +0800 Subject: [PATCH 0240/1043] MIPS: Truncate link address into 32bit for 32bit kernel LLD failed to link vmlinux with 64bit load address for 32bit ELF while bfd will strip 64bit address into 32bit silently. To fix LLD build, we should truncate load address provided by platform into 32bit for 32bit kernel. Signed-off-by: Jiaxun Yang Link: https://github.com/ClangBuiltLinux/linux/issues/786 Link: https://sourceware.org/bugzilla/show_bug.cgi?id=25784 Reviewed-by: Fangrui Song Reviewed-by: Kees Cook Tested-by: Nathan Chancellor Cc: Maciej W. Rozycki Tested-by: Nick Desaulniers Signed-off-by: Thomas Bogendoerfer --- arch/mips/Makefile | 13 ++++++++++++- arch/mips/boot/compressed/Makefile | 2 +- arch/mips/kernel/vmlinux.lds.S | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/arch/mips/Makefile b/arch/mips/Makefile index e1c44aed8156..b6ee29e4565a 100644 --- a/arch/mips/Makefile +++ b/arch/mips/Makefile @@ -288,12 +288,23 @@ ifdef CONFIG_64BIT endif endif +# When linking a 32-bit executable the LLVM linker cannot cope with a +# 32-bit load address that has been sign-extended to 64 bits. Simply +# remove the upper 32 bits then, as it is safe to do so with other +# linkers. +ifdef CONFIG_64BIT + load-ld = $(load-y) +else + load-ld = $(subst 0xffffffff,0x,$(load-y)) +endif + KBUILD_AFLAGS += $(cflags-y) KBUILD_CFLAGS += $(cflags-y) -KBUILD_CPPFLAGS += -DVMLINUX_LOAD_ADDRESS=$(load-y) +KBUILD_CPPFLAGS += -DVMLINUX_LOAD_ADDRESS=$(load-y) -DLINKER_LOAD_ADDRESS=$(load-ld) KBUILD_CPPFLAGS += -DDATAOFFSET=$(if $(dataoffset-y),$(dataoffset-y),0) bootvars-y = VMLINUX_LOAD_ADDRESS=$(load-y) \ + LINKER_LOAD_ADDRESS=$(load-ld) \ VMLINUX_ENTRY_ADDRESS=$(entry-y) \ PLATFORM="$(platform-y)" \ ITS_INPUTS="$(its-y)" diff --git a/arch/mips/boot/compressed/Makefile b/arch/mips/boot/compressed/Makefile index 0df0ee8a298d..6e56caef69f0 100644 --- a/arch/mips/boot/compressed/Makefile +++ b/arch/mips/boot/compressed/Makefile @@ -90,7 +90,7 @@ ifneq ($(zload-y),) VMLINUZ_LOAD_ADDRESS := $(zload-y) else VMLINUZ_LOAD_ADDRESS = $(shell $(obj)/calc_vmlinuz_load_addr \ - $(obj)/vmlinux.bin $(VMLINUX_LOAD_ADDRESS)) + $(obj)/vmlinux.bin $(LINKER_LOAD_ADDRESS)) endif UIMAGE_LOADADDR = $(VMLINUZ_LOAD_ADDRESS) diff --git a/arch/mips/kernel/vmlinux.lds.S b/arch/mips/kernel/vmlinux.lds.S index a5f00ec73ea6..f185a85a27c1 100644 --- a/arch/mips/kernel/vmlinux.lds.S +++ b/arch/mips/kernel/vmlinux.lds.S @@ -55,7 +55,7 @@ SECTIONS /* . = 0xa800000000300000; */ . = 0xffffffff80300000; #endif - . = VMLINUX_LOAD_ADDRESS; + . = LINKER_LOAD_ADDRESS; /* read-only */ _text = .; /* Text and read-only data */ .text : { From c9b0299034665d594e56ee343f28033d1b24de6d Mon Sep 17 00:00:00 2001 From: Liangliang Huang Date: Mon, 4 May 2020 16:51:29 +0800 Subject: [PATCH 0241/1043] MIPS: Use fallthrough for arch/mips Convert the various /* fallthrough */ comments to the pseudo-keyword fallthrough; Done via script: https://lore.kernel.org/lkml/b56602fcf79f849e733e7b521bb0e17895d390fa.1582230379.git.joe@perches.com/ Signed-off-by: Liangliang Huang Reviewed-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/alchemy/devboards/db1550.c | 2 +- arch/mips/ar7/setup.c | 2 +- arch/mips/ath79/setup.c | 3 +-- arch/mips/bcm63xx/cpu.c | 2 +- arch/mips/bcm63xx/dev-flash.c | 2 +- arch/mips/cavium-octeon/executive/cvmx-pko.c | 2 +- arch/mips/cavium-octeon/octeon-platform.c | 4 +-- arch/mips/cavium-octeon/octeon-usb.c | 2 +- arch/mips/dec/tc.c | 2 +- arch/mips/include/asm/fpu.h | 2 +- arch/mips/include/asm/octeon/cvmx-sli-defs.h | 2 +- arch/mips/include/asm/page.h | 2 +- arch/mips/kernel/branch.c | 26 +++++++++--------- arch/mips/kernel/cpu-probe.c | 22 +++++++-------- arch/mips/kernel/idle.c | 2 +- arch/mips/kernel/mips-r2-to-r6-emul.c | 2 +- arch/mips/kernel/signal.c | 2 +- arch/mips/kernel/traps.c | 3 +-- arch/mips/kernel/watch.c | 26 +++++++++--------- arch/mips/kvm/emulate.c | 8 +++--- arch/mips/math-emu/cp1emu.c | 28 ++++++++++---------- arch/mips/math-emu/dp_add.c | 3 +-- arch/mips/math-emu/dp_div.c | 3 +-- arch/mips/math-emu/dp_fmax.c | 6 ++--- arch/mips/math-emu/dp_fmin.c | 6 ++--- arch/mips/math-emu/dp_maddf.c | 3 +-- arch/mips/math-emu/dp_mul.c | 3 +-- arch/mips/math-emu/dp_sqrt.c | 5 ++-- arch/mips/math-emu/dp_sub.c | 3 +-- arch/mips/math-emu/sp_add.c | 3 +-- arch/mips/math-emu/sp_div.c | 3 +-- arch/mips/math-emu/sp_fdp.c | 3 +-- arch/mips/math-emu/sp_fmax.c | 6 ++--- arch/mips/math-emu/sp_fmin.c | 6 ++--- arch/mips/math-emu/sp_maddf.c | 3 +-- arch/mips/math-emu/sp_mul.c | 3 +-- arch/mips/math-emu/sp_sub.c | 3 +-- arch/mips/mm/c-r4k.c | 6 ++--- arch/mips/mm/tlbex.c | 2 +- arch/mips/oprofile/op_model_mipsxx.c | 26 +++++++++--------- arch/mips/pci/fixup-sni.c | 3 +-- arch/mips/pci/ops-bcm63xx.c | 2 +- 42 files changed, 112 insertions(+), 135 deletions(-) diff --git a/arch/mips/alchemy/devboards/db1550.c b/arch/mips/alchemy/devboards/db1550.c index 3e0c75c0ece0..752b93d91ac9 100644 --- a/arch/mips/alchemy/devboards/db1550.c +++ b/arch/mips/alchemy/devboards/db1550.c @@ -225,7 +225,7 @@ static void __init pb1550_nand_setup(void) case 0: case 2: case 8: case 0xC: case 0xD: /* x16 NAND Flash */ pb1550_nand_pd.devwidth = 1; - /* fallthrough */ + fallthrough; case 1: case 3: case 9: case 0xE: case 0xF: /* x8 NAND, already set up */ platform_device_register(&pb1550_nand_dev); diff --git a/arch/mips/ar7/setup.c b/arch/mips/ar7/setup.c index b3ffe7c898eb..352d5dbc777c 100644 --- a/arch/mips/ar7/setup.c +++ b/arch/mips/ar7/setup.c @@ -57,7 +57,7 @@ const char *get_system_type(void) case TITAN_CHIP_1060: return "TI AR7 (TNETV1060)"; } - /* fall through */ + fallthrough; default: return "TI AR7 (unknown)"; } diff --git a/arch/mips/ath79/setup.c b/arch/mips/ath79/setup.c index acb4fd647a30..4b7c066ac88e 100644 --- a/arch/mips/ath79/setup.c +++ b/arch/mips/ath79/setup.c @@ -153,8 +153,7 @@ static void __init ath79_detect_sys_type(void) case REV_ID_MAJOR_QCA9533_V2: ver = 2; ath79_soc_rev = 2; - /* fall through */ - + fallthrough; case REV_ID_MAJOR_QCA9533: ath79_soc = ATH79_SOC_QCA9533; chip = "9533"; diff --git a/arch/mips/bcm63xx/cpu.c b/arch/mips/bcm63xx/cpu.c index f61c16f57a97..8e3e199dd35d 100644 --- a/arch/mips/bcm63xx/cpu.c +++ b/arch/mips/bcm63xx/cpu.c @@ -304,7 +304,7 @@ void __init bcm63xx_cpu_init(void) case CPU_BMIPS3300: if ((read_c0_prid() & PRID_IMP_MASK) != PRID_IMP_BMIPS3300_ALT) __cpu_name[cpu] = "Broadcom BCM6338"; - /* fall-through */ + fallthrough; case CPU_BMIPS32: chipid_reg = BCM_6345_PERF_BASE; break; diff --git a/arch/mips/bcm63xx/dev-flash.c b/arch/mips/bcm63xx/dev-flash.c index a1093934c616..f9cc015d3dc9 100644 --- a/arch/mips/bcm63xx/dev-flash.c +++ b/arch/mips/bcm63xx/dev-flash.c @@ -94,7 +94,7 @@ static int __init bcm63xx_detect_flash_type(void) case STRAPBUS_6368_BOOT_SEL_PARALLEL: return BCM63XX_FLASH_TYPE_PARALLEL; } - /* fall through */ + fallthrough; default: return -EINVAL; } diff --git a/arch/mips/cavium-octeon/executive/cvmx-pko.c b/arch/mips/cavium-octeon/executive/cvmx-pko.c index b077597c668a..b0efc35e95c4 100644 --- a/arch/mips/cavium-octeon/executive/cvmx-pko.c +++ b/arch/mips/cavium-octeon/executive/cvmx-pko.c @@ -489,7 +489,7 @@ cvmx_pko_status_t cvmx_pko_config_port(uint64_t port, uint64_t base_queue, config.s.qos_mask = 0xff; break; } - /* fall through - to the error case, when Pass 1 */ + fallthrough; /* to the error case, when Pass 1 */ default: cvmx_dprintf("ERROR: cvmx_pko_config_port: Invalid " "priority %llu\n", diff --git a/arch/mips/cavium-octeon/octeon-platform.c b/arch/mips/cavium-octeon/octeon-platform.c index 51685f893eab..d56e9b9d2e43 100644 --- a/arch/mips/cavium-octeon/octeon-platform.c +++ b/arch/mips/cavium-octeon/octeon-platform.c @@ -141,7 +141,7 @@ static void octeon2_usb_clocks_start(struct device *dev) default: pr_err("Invalid UCTL clock rate of %u, using 12000000 instead\n", clock_rate); - /* Fall through */ + fallthrough; case 12000000: clk_rst_ctl.s.p_refclk_div = 0; break; @@ -1116,7 +1116,7 @@ end_led: new_f[0] = cpu_to_be32(48000000); fdt_setprop_inplace(initial_boot_params, usbn, "refclk-frequency", new_f, sizeof(new_f)); - /* Fall through ...*/ + fallthrough; case USB_CLOCK_TYPE_REF_12: /* Missing "refclk-type" defaults to external. */ fdt_nop_property(initial_boot_params, usbn, "refclk-type"); diff --git a/arch/mips/cavium-octeon/octeon-usb.c b/arch/mips/cavium-octeon/octeon-usb.c index cc88a08bc1f7..1fd85c559700 100644 --- a/arch/mips/cavium-octeon/octeon-usb.c +++ b/arch/mips/cavium-octeon/octeon-usb.c @@ -398,7 +398,7 @@ static int dwc3_octeon_clocks_start(struct device *dev, u64 base) default: dev_err(dev, "Invalid ref_clk %u, using 100000000 instead\n", clock_rate); - /* fall through */ + fallthrough; case 100000000: mpll_mul = 0x19; if (ref_clk_sel < 2) diff --git a/arch/mips/dec/tc.c b/arch/mips/dec/tc.c index 732027c79834..dba58397668e 100644 --- a/arch/mips/dec/tc.c +++ b/arch/mips/dec/tc.c @@ -52,7 +52,7 @@ int __init tc_bus_get_info(struct tc_bus *tbus) case MACH_DS5900: tbus->ext_slot_base = 0x20000000; tbus->ext_slot_size = 0x20000000; - /* fall through */ + fallthrough; case MACH_DS5000_1XX: tbus->num_tcslots = 3; break; diff --git a/arch/mips/include/asm/fpu.h b/arch/mips/include/asm/fpu.h index 9476e0498d59..a9d5123e2a2a 100644 --- a/arch/mips/include/asm/fpu.h +++ b/arch/mips/include/asm/fpu.h @@ -76,7 +76,7 @@ static inline int __enable_fpu(enum fpu_mode mode) /* we only have a 32-bit FPU */ return SIGFPE; #endif - /* fall through */ + fallthrough; case FPU_32BIT: if (cpu_has_fre) { /* clear FRE */ diff --git a/arch/mips/include/asm/octeon/cvmx-sli-defs.h b/arch/mips/include/asm/octeon/cvmx-sli-defs.h index cbc7cdae1c6a..5ef6c38150f5 100644 --- a/arch/mips/include/asm/octeon/cvmx-sli-defs.h +++ b/arch/mips/include/asm/octeon/cvmx-sli-defs.h @@ -46,7 +46,7 @@ static inline uint64_t CVMX_SLI_PCIE_MSI_RCV_FUNC(void) case OCTEON_CN78XX & OCTEON_FAMILY_MASK: if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X)) return 0x0000000000003CB0ull; - /* Else, fall through */ + fallthrough; default: return 0x0000000000023CB0ull; } diff --git a/arch/mips/include/asm/page.h b/arch/mips/include/asm/page.h index e2f503fc7a84..6a77bc4a6eec 100644 --- a/arch/mips/include/asm/page.h +++ b/arch/mips/include/asm/page.h @@ -49,7 +49,7 @@ static inline unsigned int page_size_ftlb(unsigned int mmuextdef) return 6; if (PAGE_SIZE > (256 << 10)) return 7; /* reserved */ - /* fall through */ + fallthrough; case MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT: return (PAGE_SHIFT - 10) / 2; default: diff --git a/arch/mips/kernel/branch.c b/arch/mips/kernel/branch.c index 2c38f75d87ff..fb3e203698ea 100644 --- a/arch/mips/kernel/branch.c +++ b/arch/mips/kernel/branch.c @@ -90,7 +90,7 @@ int __mm_isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, regs->regs[31] = regs->cp0_epc + dec_insn.pc_inc + dec_insn.next_pc_inc; - /* Fall through */ + fallthrough; case mm_bltz_op: if ((long)regs->regs[insn.mm_i_format.rs] < 0) *contpc = regs->cp0_epc + @@ -106,7 +106,7 @@ int __mm_isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, regs->regs[31] = regs->cp0_epc + dec_insn.pc_inc + dec_insn.next_pc_inc; - /* Fall through */ + fallthrough; case mm_bgez_op: if ((long)regs->regs[insn.mm_i_format.rs] >= 0) *contpc = regs->cp0_epc + @@ -144,7 +144,7 @@ int __mm_isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, unsigned int bit; bc_false = 1; - /* Fall through */ + fallthrough; case mm_bc2t_op: case mm_bc1t_op: preempt_disable(); @@ -178,7 +178,7 @@ int __mm_isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, case mm_jalrs16_op: regs->regs[31] = regs->cp0_epc + dec_insn.pc_inc + dec_insn.next_pc_inc; - /* Fall through */ + fallthrough; case mm_jr16_op: *contpc = regs->regs[insn.mm_i_format.rs]; return 1; @@ -239,7 +239,7 @@ int __mm_isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, case mm_jal32_op: regs->regs[31] = regs->cp0_epc + dec_insn.pc_inc + dec_insn.next_pc_inc; - /* Fall through */ + fallthrough; case mm_j32_op: *contpc = regs->cp0_epc + dec_insn.pc_inc; *contpc >>= 27; @@ -432,7 +432,7 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, switch (insn.r_format.func) { case jalr_op: regs->regs[insn.r_format.rd] = epc + 8; - /* Fall through */ + fallthrough; case jr_op: if (NO_R6EMU && insn.r_format.func == jr_op) goto sigill_r2r6; @@ -451,7 +451,7 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, case bltzl_op: if (NO_R6EMU) goto sigill_r2r6; - /* fall through */ + fallthrough; case bltz_op: if ((long)regs->regs[insn.i_format.rs] < 0) { epc = epc + 4 + (insn.i_format.simmediate << 2); @@ -465,7 +465,7 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, case bgezl_op: if (NO_R6EMU) goto sigill_r2r6; - /* fall through */ + fallthrough; case bgez_op: if ((long)regs->regs[insn.i_format.rs] >= 0) { epc = epc + 4 + (insn.i_format.simmediate << 2); @@ -561,7 +561,7 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, case jalx_op: case jal_op: regs->regs[31] = regs->cp0_epc + 8; - /* fall through */ + fallthrough; case j_op: epc += 4; epc >>= 28; @@ -578,7 +578,7 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, case beql_op: if (NO_R6EMU) goto sigill_r2r6; - /* fall through */ + fallthrough; case beq_op: if (regs->regs[insn.i_format.rs] == regs->regs[insn.i_format.rt]) { @@ -593,7 +593,7 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, case bnel_op: if (NO_R6EMU) goto sigill_r2r6; - /* fall through */ + fallthrough; case bne_op: if (regs->regs[insn.i_format.rs] != regs->regs[insn.i_format.rt]) { @@ -608,7 +608,7 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, case blezl_op: /* not really i_format */ if (!insn.i_format.rt && NO_R6EMU) goto sigill_r2r6; - /* fall through */ + fallthrough; case blez_op: /* * Compact branches for R6 for the @@ -644,7 +644,7 @@ int __compute_return_epc_for_insn(struct pt_regs *regs, case bgtzl_op: if (!insn.i_format.rt && NO_R6EMU) goto sigill_r2r6; - /* fall through */ + fallthrough; case bgtz_op: /* * Compact branches for R6 for the diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index ca2e6f1af4fe..a0ef21b2d8b3 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -535,19 +535,19 @@ static void set_isa(struct cpuinfo_mips *c, unsigned int isa) case MIPS_CPU_ISA_M64R2: c->isa_level |= MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2; set_elf_base_platform("mips64r2"); - /* fall through */ + fallthrough; case MIPS_CPU_ISA_M64R1: c->isa_level |= MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1; set_elf_base_platform("mips64"); - /* fall through */ + fallthrough; case MIPS_CPU_ISA_V: c->isa_level |= MIPS_CPU_ISA_V; set_elf_base_platform("mips5"); - /* fall through */ + fallthrough; case MIPS_CPU_ISA_IV: c->isa_level |= MIPS_CPU_ISA_IV; set_elf_base_platform("mips4"); - /* fall through */ + fallthrough; case MIPS_CPU_ISA_III: c->isa_level |= MIPS_CPU_ISA_II | MIPS_CPU_ISA_III; set_elf_base_platform("mips3"); @@ -557,7 +557,7 @@ static void set_isa(struct cpuinfo_mips *c, unsigned int isa) case MIPS_CPU_ISA_M64R6: c->isa_level |= MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6; set_elf_base_platform("mips64r6"); - /* fall through */ + fallthrough; case MIPS_CPU_ISA_M32R6: c->isa_level |= MIPS_CPU_ISA_M32R6; set_elf_base_platform("mips32r6"); @@ -566,11 +566,11 @@ static void set_isa(struct cpuinfo_mips *c, unsigned int isa) case MIPS_CPU_ISA_M32R2: c->isa_level |= MIPS_CPU_ISA_M32R2; set_elf_base_platform("mips32r2"); - /* fall through */ + fallthrough; case MIPS_CPU_ISA_M32R1: c->isa_level |= MIPS_CPU_ISA_M32R1; set_elf_base_platform("mips32"); - /* fall through */ + fallthrough; case MIPS_CPU_ISA_II: c->isa_level |= MIPS_CPU_ISA_II; set_elf_base_platform("mips2"); @@ -850,7 +850,7 @@ static inline unsigned int decode_config4(struct cpuinfo_mips *c) MIPS_CONF4_VTLBSIZEEXT_SHIFT) * 0x40; c->tlbsize = c->tlbsizevtlb; ftlb_page = MIPS_CONF4_VFTLBPAGESIZE; - /* fall through */ + fallthrough; case MIPS_CONF4_MMUEXTDEF_FTLBSIZEEXT: if (mips_ftlb_disabled) break; @@ -1753,10 +1753,10 @@ static inline void cpu_probe_mips(struct cpuinfo_mips *c, unsigned int cpu) switch (__get_cpu_type(c->cputype)) { case CPU_I6500: c->options |= MIPS_CPU_SHARED_FTLB_ENTRIES; - /* fall-through */ + fallthrough; case CPU_I6400: c->options |= MIPS_CPU_SHARED_FTLB_RAM; - /* fall-through */ + fallthrough; default: break; } @@ -2077,7 +2077,7 @@ static inline void cpu_probe_ingenic(struct cpuinfo_mips *c, unsigned int cpu) default: break; } - /* fall-through */ + fallthrough; case PRID_IMP_XBURST_REV2: c->cputype = CPU_XBURST; c->writecombine = _CACHE_UNCACHED_ACCELERATED; diff --git a/arch/mips/kernel/idle.c b/arch/mips/kernel/idle.c index 60d8c2a380fe..5bc3b04693c7 100644 --- a/arch/mips/kernel/idle.c +++ b/arch/mips/kernel/idle.c @@ -202,7 +202,7 @@ void __init check_wait(void) */ if (IS_ENABLED(CONFIG_MIPS_EJTAG_FDC_TTY)) break; - /* fall through */ + fallthrough; case CPU_M14KC: case CPU_M14KEC: case CPU_24K: diff --git a/arch/mips/kernel/mips-r2-to-r6-emul.c b/arch/mips/kernel/mips-r2-to-r6-emul.c index b4d210bfcdae..a39ec755e4c2 100644 --- a/arch/mips/kernel/mips-r2-to-r6-emul.c +++ b/arch/mips/kernel/mips-r2-to-r6-emul.c @@ -1109,7 +1109,7 @@ repeat: err = SIGILL; break; } - /* fall through */ + fallthrough; case beql_op: case bnel_op: if (delay_slot(regs)) { diff --git a/arch/mips/kernel/signal.c b/arch/mips/kernel/signal.c index f6efabcb4e92..f926bf338dec 100644 --- a/arch/mips/kernel/signal.c +++ b/arch/mips/kernel/signal.c @@ -824,7 +824,7 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) regs->regs[2] = EINTR; break; } - /* fallthrough */ + fallthrough; case ERESTARTNOINTR: regs->regs[7] = regs->regs[26]; regs->regs[2] = regs->regs[0]; diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 31968cbd6464..89eb82f6c837 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -1401,8 +1401,7 @@ asmlinkage void do_cpu(struct pt_regs *regs) force_sig(SIGILL); break; } - /* Fall through. */ - + fallthrough; case 1: { void __user *fault_addr; unsigned long fcr31; diff --git a/arch/mips/kernel/watch.c b/arch/mips/kernel/watch.c index ba73b4077668..c9263b95cb2e 100644 --- a/arch/mips/kernel/watch.c +++ b/arch/mips/kernel/watch.c @@ -27,15 +27,15 @@ void mips_install_watch_registers(struct task_struct *t) case 4: write_c0_watchlo3(watches->watchlo[3]); write_c0_watchhi3(watchhi | watches->watchhi[3]); - /* fall through */ + fallthrough; case 3: write_c0_watchlo2(watches->watchlo[2]); write_c0_watchhi2(watchhi | watches->watchhi[2]); - /* fall through */ + fallthrough; case 2: write_c0_watchlo1(watches->watchlo[1]); write_c0_watchhi1(watchhi | watches->watchhi[1]); - /* fall through */ + fallthrough; case 1: write_c0_watchlo0(watches->watchlo[0]); write_c0_watchhi0(watchhi | watches->watchhi[0]); @@ -58,13 +58,13 @@ void mips_read_watch_registers(void) BUG(); case 4: watches->watchhi[3] = (read_c0_watchhi3() & watchhi_mask); - /* fall through */ + fallthrough; case 3: watches->watchhi[2] = (read_c0_watchhi2() & watchhi_mask); - /* fall through */ + fallthrough; case 2: watches->watchhi[1] = (read_c0_watchhi1() & watchhi_mask); - /* fall through */ + fallthrough; case 1: watches->watchhi[0] = (read_c0_watchhi0() & watchhi_mask); } @@ -91,25 +91,25 @@ void mips_clear_watch_registers(void) BUG(); case 8: write_c0_watchlo7(0); - /* fall through */ + fallthrough; case 7: write_c0_watchlo6(0); - /* fall through */ + fallthrough; case 6: write_c0_watchlo5(0); - /* fall through */ + fallthrough; case 5: write_c0_watchlo4(0); - /* fall through */ + fallthrough; case 4: write_c0_watchlo3(0); - /* fall through */ + fallthrough; case 3: write_c0_watchlo2(0); - /* fall through */ + fallthrough; case 2: write_c0_watchlo1(0); - /* fall through */ + fallthrough; case 1: write_c0_watchlo0(0); } diff --git a/arch/mips/kvm/emulate.c b/arch/mips/kvm/emulate.c index 754094b40a75..8c80333816fe 100644 --- a/arch/mips/kvm/emulate.c +++ b/arch/mips/kvm/emulate.c @@ -64,7 +64,7 @@ static int kvm_compute_return_epc(struct kvm_vcpu *vcpu, unsigned long instpc, switch (insn.r_format.func) { case jalr_op: arch->gprs[insn.r_format.rd] = epc + 8; - /* Fall through */ + fallthrough; case jr_op: nextpc = arch->gprs[insn.r_format.rs]; break; @@ -140,7 +140,7 @@ static int kvm_compute_return_epc(struct kvm_vcpu *vcpu, unsigned long instpc, /* These are unconditional and in j_format. */ case jal_op: arch->gprs[31] = instpc + 8; - /* fall through */ + fallthrough; case j_op: epc += 4; epc >>= 28; @@ -1724,14 +1724,14 @@ enum emulation_result kvm_mips_emulate_load(union mips_instruction inst, case lhu_op: vcpu->mmio_needed = 1; /* unsigned */ - /* fall through */ + fallthrough; case lh_op: run->mmio.len = 2; break; case lbu_op: vcpu->mmio_needed = 1; /* unsigned */ - /* fall through */ + fallthrough; case lb_op: run->mmio.len = 1; break; diff --git a/arch/mips/math-emu/cp1emu.c b/arch/mips/math-emu/cp1emu.c index 9701c89e7e14..587cf1d115e8 100644 --- a/arch/mips/math-emu/cp1emu.c +++ b/arch/mips/math-emu/cp1emu.c @@ -439,7 +439,7 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, regs->cp0_epc + dec_insn.pc_inc + dec_insn.next_pc_inc; } - /* fall through */ + fallthrough; case jr_op: /* For R6, JR already emulated in jalr_op */ if (NO_R6EMU && insn.r_format.func == jr_op) @@ -459,11 +459,11 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, regs->regs[31] = regs->cp0_epc + dec_insn.pc_inc + dec_insn.next_pc_inc; - /* fall through */ + fallthrough; case bltzl_op: if (NO_R6EMU) break; - /* fall through */ + fallthrough; case bltz_op: if ((long)regs->regs[insn.i_format.rs] < 0) *contpc = regs->cp0_epc + @@ -483,11 +483,11 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, regs->regs[31] = regs->cp0_epc + dec_insn.pc_inc + dec_insn.next_pc_inc; - /* fall through */ + fallthrough; case bgezl_op: if (NO_R6EMU) break; - /* fall through */ + fallthrough; case bgez_op: if ((long)regs->regs[insn.i_format.rs] >= 0) *contpc = regs->cp0_epc + @@ -502,12 +502,12 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, break; case jalx_op: set_isa16_mode(bit); - /* fall through */ + fallthrough; case jal_op: regs->regs[31] = regs->cp0_epc + dec_insn.pc_inc + dec_insn.next_pc_inc; - /* fall through */ + fallthrough; case j_op: *contpc = regs->cp0_epc + dec_insn.pc_inc; *contpc >>= 28; @@ -519,7 +519,7 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, case beql_op: if (NO_R6EMU) break; - /* fall through */ + fallthrough; case beq_op: if (regs->regs[insn.i_format.rs] == regs->regs[insn.i_format.rt]) @@ -534,7 +534,7 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, case bnel_op: if (NO_R6EMU) break; - /* fall through */ + fallthrough; case bne_op: if (regs->regs[insn.i_format.rs] != regs->regs[insn.i_format.rt]) @@ -549,7 +549,7 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, case blezl_op: if (!insn.i_format.rt && NO_R6EMU) break; - /* fall through */ + fallthrough; case blez_op: /* @@ -587,7 +587,7 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, case bgtzl_op: if (!insn.i_format.rt && NO_R6EMU) break; - /* fall through */ + fallthrough; case bgtz_op: /* * Compact branches for R6 for the @@ -725,7 +725,7 @@ int isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, return 1; } /* R2/R6 compatible cop1 instruction */ - /* fall through */ + fallthrough; case cop2_op: case cop1x_op: if (insn.i_format.rs == bc_op) { @@ -1217,14 +1217,14 @@ emul: case bcfl_op: if (cpu_has_mips_2_3_4_5_r) likely = 1; - /* fall through */ + fallthrough; case bcf_op: cond = !cond; break; case bctl_op: if (cpu_has_mips_2_3_4_5_r) likely = 1; - /* fall through */ + fallthrough; case bct_op: break; } diff --git a/arch/mips/math-emu/dp_add.c b/arch/mips/math-emu/dp_add.c index a8f98b8157f5..78504736be9e 100644 --- a/arch/mips/math-emu/dp_add.c +++ b/arch/mips/math-emu/dp_add.c @@ -92,8 +92,7 @@ union ieee754dp ieee754dp_add(union ieee754dp x, union ieee754dp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): DPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): DPDNORMY; break; diff --git a/arch/mips/math-emu/dp_div.c b/arch/mips/math-emu/dp_div.c index 2b682e930e39..ac1ecc46248d 100644 --- a/arch/mips/math-emu/dp_div.c +++ b/arch/mips/math-emu/dp_div.c @@ -91,8 +91,7 @@ union ieee754dp ieee754dp_div(union ieee754dp x, union ieee754dp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): DPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): DPDNORMY; break; diff --git a/arch/mips/math-emu/dp_fmax.c b/arch/mips/math-emu/dp_fmax.c index 3eda9ff7b491..126ec90bb4c7 100644 --- a/arch/mips/math-emu/dp_fmax.c +++ b/arch/mips/math-emu/dp_fmax.c @@ -93,8 +93,7 @@ union ieee754dp ieee754dp_fmax(union ieee754dp x, union ieee754dp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): DPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): DPDNORMY; break; @@ -222,8 +221,7 @@ union ieee754dp ieee754dp_fmaxa(union ieee754dp x, union ieee754dp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): DPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): DPDNORMY; break; diff --git a/arch/mips/math-emu/dp_fmin.c b/arch/mips/math-emu/dp_fmin.c index b3594a1704a7..35ded4c45989 100644 --- a/arch/mips/math-emu/dp_fmin.c +++ b/arch/mips/math-emu/dp_fmin.c @@ -93,8 +93,7 @@ union ieee754dp ieee754dp_fmin(union ieee754dp x, union ieee754dp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): DPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): DPDNORMY; break; @@ -222,8 +221,7 @@ union ieee754dp ieee754dp_fmina(union ieee754dp x, union ieee754dp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): DPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): DPDNORMY; break; diff --git a/arch/mips/math-emu/dp_maddf.c b/arch/mips/math-emu/dp_maddf.c index e24ef374d828..931e66f683ca 100644 --- a/arch/mips/math-emu/dp_maddf.c +++ b/arch/mips/math-emu/dp_maddf.c @@ -150,8 +150,7 @@ static union ieee754dp _dp_maddf(union ieee754dp z, union ieee754dp x, case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): DPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): if (zc == IEEE754_CLASS_INF) return ieee754dp_inf(zs); diff --git a/arch/mips/math-emu/dp_mul.c b/arch/mips/math-emu/dp_mul.c index e8a97d26472a..8a671bb7af12 100644 --- a/arch/mips/math-emu/dp_mul.c +++ b/arch/mips/math-emu/dp_mul.c @@ -89,8 +89,7 @@ union ieee754dp ieee754dp_mul(union ieee754dp x, union ieee754dp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): DPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): DPDNORMY; break; diff --git a/arch/mips/math-emu/dp_sqrt.c b/arch/mips/math-emu/dp_sqrt.c index 06be390ba79a..1ee38f8242fd 100644 --- a/arch/mips/math-emu/dp_sqrt.c +++ b/arch/mips/math-emu/dp_sqrt.c @@ -52,8 +52,7 @@ union ieee754dp ieee754dp_sqrt(union ieee754dp x) case IEEE754_CLASS_DNORM: DPDNORMX; - /* fall through */ - + fallthrough; case IEEE754_CLASS_NORM: if (xs) { /* sqrt(-x) = Nan */ @@ -130,7 +129,7 @@ union ieee754dp ieee754dp_sqrt(union ieee754dp x) switch (oldcsr.rm) { case FPU_CSR_RU: y.bits += 1; - /* fall through */ + fallthrough; case FPU_CSR_RN: t.bits += 1; break; diff --git a/arch/mips/math-emu/dp_sub.c b/arch/mips/math-emu/dp_sub.c index f08aecefceff..08474ad2a64e 100644 --- a/arch/mips/math-emu/dp_sub.c +++ b/arch/mips/math-emu/dp_sub.c @@ -94,8 +94,7 @@ union ieee754dp ieee754dp_sub(union ieee754dp x, union ieee754dp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): DPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): /* normalize ym,ye */ DPDNORMY; diff --git a/arch/mips/math-emu/sp_add.c b/arch/mips/math-emu/sp_add.c index 9af3ec7302fb..715cd0534301 100644 --- a/arch/mips/math-emu/sp_add.c +++ b/arch/mips/math-emu/sp_add.c @@ -92,8 +92,7 @@ union ieee754sp ieee754sp_add(union ieee754sp x, union ieee754sp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): SPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): SPDNORMY; break; diff --git a/arch/mips/math-emu/sp_div.c b/arch/mips/math-emu/sp_div.c index fcc285f3b48d..2bfa266fdc76 100644 --- a/arch/mips/math-emu/sp_div.c +++ b/arch/mips/math-emu/sp_div.c @@ -91,8 +91,7 @@ union ieee754sp ieee754sp_div(union ieee754sp x, union ieee754sp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): SPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): SPDNORMY; break; diff --git a/arch/mips/math-emu/sp_fdp.c b/arch/mips/math-emu/sp_fdp.c index 9f1456109aa8..56417497c88e 100644 --- a/arch/mips/math-emu/sp_fdp.c +++ b/arch/mips/math-emu/sp_fdp.c @@ -34,8 +34,7 @@ union ieee754sp ieee754sp_fdp(union ieee754dp x) case IEEE754_CLASS_SNAN: x = ieee754dp_nanxcpt(x); EXPLODEXDP; - /* fall through */ - + fallthrough; case IEEE754_CLASS_QNAN: y = ieee754sp_nan_fdp(xs, xm); if (!ieee754_csr.nan2008) { diff --git a/arch/mips/math-emu/sp_fmax.c b/arch/mips/math-emu/sp_fmax.c index 4ce1d1f8b499..3fb16a1df3b8 100644 --- a/arch/mips/math-emu/sp_fmax.c +++ b/arch/mips/math-emu/sp_fmax.c @@ -93,8 +93,7 @@ union ieee754sp ieee754sp_fmax(union ieee754sp x, union ieee754sp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): SPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): SPDNORMY; break; @@ -222,8 +221,7 @@ union ieee754sp ieee754sp_fmaxa(union ieee754sp x, union ieee754sp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): SPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): SPDNORMY; break; diff --git a/arch/mips/math-emu/sp_fmin.c b/arch/mips/math-emu/sp_fmin.c index 7ad867fd7de2..ad2599d4a892 100644 --- a/arch/mips/math-emu/sp_fmin.c +++ b/arch/mips/math-emu/sp_fmin.c @@ -93,8 +93,7 @@ union ieee754sp ieee754sp_fmin(union ieee754sp x, union ieee754sp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): SPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): SPDNORMY; break; @@ -222,8 +221,7 @@ union ieee754sp ieee754sp_fmina(union ieee754sp x, union ieee754sp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): SPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): SPDNORMY; break; diff --git a/arch/mips/math-emu/sp_maddf.c b/arch/mips/math-emu/sp_maddf.c index 1b85b1a527ac..473ee222d90c 100644 --- a/arch/mips/math-emu/sp_maddf.c +++ b/arch/mips/math-emu/sp_maddf.c @@ -119,8 +119,7 @@ static union ieee754sp _sp_maddf(union ieee754sp z, union ieee754sp x, case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): SPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): if (zc == IEEE754_CLASS_INF) return ieee754sp_inf(zs); diff --git a/arch/mips/math-emu/sp_mul.c b/arch/mips/math-emu/sp_mul.c index ded17e28e8bc..26cfd63025e9 100644 --- a/arch/mips/math-emu/sp_mul.c +++ b/arch/mips/math-emu/sp_mul.c @@ -89,8 +89,7 @@ union ieee754sp ieee754sp_mul(union ieee754sp x, union ieee754sp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): SPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): SPDNORMY; break; diff --git a/arch/mips/math-emu/sp_sub.c b/arch/mips/math-emu/sp_sub.c index f3d26a1f162c..16c8e9ae63ed 100644 --- a/arch/mips/math-emu/sp_sub.c +++ b/arch/mips/math-emu/sp_sub.c @@ -94,8 +94,7 @@ union ieee754sp ieee754sp_sub(union ieee754sp x, union ieee754sp y) case CLPAIR(IEEE754_CLASS_DNORM, IEEE754_CLASS_DNORM): SPDNORMX; - /* fall through */ - + fallthrough; case CLPAIR(IEEE754_CLASS_NORM, IEEE754_CLASS_DNORM): SPDNORMY; break; diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index 85eb62e40e2b..54c18b8a2406 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -1200,7 +1200,7 @@ static void probe_pcache(void) case CPU_VR4133: write_c0_config(config & ~VR41_CONF_P4K); - /* fall through */ + fallthrough; case CPU_VR4131: /* Workaround for cache instruction bug of VR4131 */ if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U || @@ -1426,7 +1426,7 @@ static void probe_pcache(void) case CPU_74K: case CPU_1074K: has_74k_erratum = alias_74k_erratum(c); - /* Fall through. */ + fallthrough; case CPU_M14KC: case CPU_M14KEC: case CPU_24K: @@ -1450,7 +1450,7 @@ static void probe_pcache(void) c->dcache.flags |= MIPS_CACHE_PINDEX; break; } - /* fall through */ + fallthrough; default: if (has_74k_erratum || c->dcache.waysize > PAGE_SIZE) c->dcache.flags |= MIPS_CACHE_ALIASES; diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index da407cdc2135..38c204204529 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -576,7 +576,7 @@ void build_tlb_write_entry(u32 **p, struct uasm_label **l, case CPU_R5500: if (m4kc_tlbp_war()) uasm_i_nop(p); - /* fall through */ + fallthrough; case CPU_ALCHEMY: tlbw(p); break; diff --git a/arch/mips/oprofile/op_model_mipsxx.c b/arch/mips/oprofile/op_model_mipsxx.c index a537bf98912c..1493c49ca47a 100644 --- a/arch/mips/oprofile/op_model_mipsxx.c +++ b/arch/mips/oprofile/op_model_mipsxx.c @@ -172,15 +172,15 @@ static void mipsxx_cpu_setup(void *args) case 4: w_c0_perfctrl3(0); w_c0_perfcntr3(reg.counter[3]); - /* fall through */ + fallthrough; case 3: w_c0_perfctrl2(0); w_c0_perfcntr2(reg.counter[2]); - /* fall through */ + fallthrough; case 2: w_c0_perfctrl1(0); w_c0_perfcntr1(reg.counter[1]); - /* fall through */ + fallthrough; case 1: w_c0_perfctrl0(0); w_c0_perfcntr0(reg.counter[0]); @@ -198,13 +198,13 @@ static void mipsxx_cpu_start(void *args) switch (counters) { case 4: w_c0_perfctrl3(WHAT | reg.control[3]); - /* fall through */ + fallthrough; case 3: w_c0_perfctrl2(WHAT | reg.control[2]); - /* fall through */ + fallthrough; case 2: w_c0_perfctrl1(WHAT | reg.control[1]); - /* fall through */ + fallthrough; case 1: w_c0_perfctrl0(WHAT | reg.control[0]); } @@ -221,13 +221,13 @@ static void mipsxx_cpu_stop(void *args) switch (counters) { case 4: w_c0_perfctrl3(0); - /* fall through */ + fallthrough; case 3: w_c0_perfctrl2(0); - /* fall through */ + fallthrough; case 2: w_c0_perfctrl1(0); - /* fall through */ + fallthrough; case 1: w_c0_perfctrl0(0); } @@ -245,7 +245,7 @@ static int mipsxx_perfcount_handler(void) switch (counters) { #define HANDLE_COUNTER(n) \ - /* fall through */ \ + fallthrough; \ case n + 1: \ control = r_c0_perfctrl ## n(); \ counter = r_c0_perfcntr ## n(); \ @@ -307,15 +307,15 @@ static void reset_counters(void *arg) case 4: w_c0_perfctrl3(0); w_c0_perfcntr3(0); - /* fall through */ + fallthrough; case 3: w_c0_perfctrl2(0); w_c0_perfcntr2(0); - /* fall through */ + fallthrough; case 2: w_c0_perfctrl1(0); w_c0_perfcntr1(0); - /* fall through */ + fallthrough; case 1: w_c0_perfctrl0(0); w_c0_perfcntr0(0); diff --git a/arch/mips/pci/fixup-sni.c b/arch/mips/pci/fixup-sni.c index adb9a58641e8..de012f8bd8c3 100644 --- a/arch/mips/pci/fixup-sni.c +++ b/arch/mips/pci/fixup-sni.c @@ -151,8 +151,7 @@ int pcibios_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) case SNI_BRD_PCI_MTOWER: if (is_rm300_revd()) return irq_tab_rm300d[slot][pin]; - /* fall through */ - + fallthrough; case SNI_BRD_PCI_DESKTOP: return irq_tab_rm200[slot][pin]; diff --git a/arch/mips/pci/ops-bcm63xx.c b/arch/mips/pci/ops-bcm63xx.c index 925c72348fb6..dc6dc2741272 100644 --- a/arch/mips/pci/ops-bcm63xx.c +++ b/arch/mips/pci/ops-bcm63xx.c @@ -474,7 +474,7 @@ static int bcm63xx_pcie_can_access(struct pci_bus *bus, int devfn) if (PCI_SLOT(devfn) == 0) return bcm_pcie_readl(PCIE_DLSTATUS_REG) & DLSTATUS_PHYLINKUP; - /* else, fall through */ + fallthrough; default: return false; } From ce61b490965e6d4068641cf33b8ecf6864c3e105 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 6 May 2020 11:04:52 +0200 Subject: [PATCH 0242/1043] MIPS: ingenic: Replace by The JZ4740 setup code is not a clock provider, and just needs to call of_clk_init(). Hence it can include instead of . Fixes: f932449c11dabb4b ("MIPS: ingenic: Drop obsolete code, merge the rest in setup.c") Signed-off-by: Geert Uytterhoeven Reviewed-by: Paul Cercueil Signed-off-by: Thomas Bogendoerfer --- arch/mips/jz4740/setup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/jz4740/setup.c b/arch/mips/jz4740/setup.c index 81428ddcaa97..142cf127bf9e 100644 --- a/arch/mips/jz4740/setup.c +++ b/arch/mips/jz4740/setup.c @@ -5,13 +5,13 @@ * JZ4740 setup code */ -#include #include #include #include #include #include #include +#include #include #include #include From 8a0efb8b101665a843205eab3d67ab09cb2d9a8d Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Wed, 6 May 2020 20:42:22 +0300 Subject: [PATCH 0243/1043] mips: cm: Fix an invalid error code of INTVN_*_ERR Commit 3885c2b463f6 ("MIPS: CM: Add support for reporting CM cache errors") adds cm2_causes[] array with map of error type ID and pointers to the short description string. There is a mistake in the table, since according to MIPS32 manual CM2_ERROR_TYPE = {17,18} correspond to INTVN_WR_ERR and INTVN_RD_ERR, while the table claims they have {0x17,0x18} codes. This is obviously hex-dec copy-paste bug. Moreover codes {0x18 - 0x1a} indicate L2 ECC errors. Fixes: 3885c2b463f6 ("MIPS: CM: Add support for reporting CM cache errors") Signed-off-by: Serge Semin Cc: Alexey Malahov Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Arnd Bergmann Cc: Rob Herring Cc: linux-pm@vger.kernel.org Cc: devicetree@vger.kernel.org Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/mips-cm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/mips/kernel/mips-cm.c b/arch/mips/kernel/mips-cm.c index cdb93ed91cde..361bfc91a0e6 100644 --- a/arch/mips/kernel/mips-cm.c +++ b/arch/mips/kernel/mips-cm.c @@ -119,9 +119,9 @@ static char *cm2_causes[32] = { "COH_RD_ERR", "MMIO_WR_ERR", "MMIO_RD_ERR", "0x07", "0x08", "0x09", "0x0a", "0x0b", "0x0c", "0x0d", "0x0e", "0x0f", - "0x10", "0x11", "0x12", "0x13", - "0x14", "0x15", "0x16", "INTVN_WR_ERR", - "INTVN_RD_ERR", "0x19", "0x1a", "0x1b", + "0x10", "INTVN_WR_ERR", "INTVN_RD_ERR", "0x13", + "0x14", "0x15", "0x16", "0x17", + "0x18", "0x19", "0x1a", "0x1b", "0x1c", "0x1d", "0x1e", "0x1f" }; From 109111b33202e19c956c32cb65960b3b31730650 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Wed, 6 May 2020 20:42:23 +0300 Subject: [PATCH 0244/1043] mips: cm: Add L2 ECC/parity errors reporting According to the MIPS32 InterAptiv software manual error codes 24 - 26 of CM2 indicate L2 ECC/parity error with switching to a corresponding errors info fields. This patch provides these errors parsing code, which handles the read/write uncorrectable and correctable ECC/parity errors, and prints instruction causing the fault, RAM array type, cache way/dword and syndrome associated with the faulty data. Co-developed-by: Alexey Malahov Signed-off-by: Alexey Malahov Signed-off-by: Serge Semin Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Arnd Bergmann Cc: Rob Herring Cc: linux-pm@vger.kernel.org Cc: devicetree@vger.kernel.org Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/mips-cm.c | 62 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/arch/mips/kernel/mips-cm.c b/arch/mips/kernel/mips-cm.c index 361bfc91a0e6..f60af512c877 100644 --- a/arch/mips/kernel/mips-cm.c +++ b/arch/mips/kernel/mips-cm.c @@ -114,6 +114,48 @@ static char *cm2_core[8] = { "Exclusive/OK", "Exclusive/Data" }; +static char *cm2_l2_type[4] = { + [0x0] = "None", + [0x1] = "Tag RAM single/double ECC error", + [0x2] = "Data RAM single/double ECC error", + [0x3] = "WS RAM uncorrectable dirty parity" +}; + +static char *cm2_l2_instr[32] = { + [0x00] = "L2_NOP", + [0x01] = "L2_ERR_CORR", + [0x02] = "L2_TAG_INV", + [0x03] = "L2_WS_CLEAN", + [0x04] = "L2_RD_MDYFY_WR", + [0x05] = "L2_WS_MRU", + [0x06] = "L2_EVICT_LN2", + [0x07] = "0x07", + [0x08] = "L2_EVICT", + [0x09] = "L2_REFL", + [0x0a] = "L2_RD", + [0x0b] = "L2_WR", + [0x0c] = "L2_EVICT_MRU", + [0x0d] = "L2_SYNC", + [0x0e] = "L2_REFL_ERR", + [0x0f] = "0x0f", + [0x10] = "L2_INDX_WB_INV", + [0x11] = "L2_INDX_LD_TAG", + [0x12] = "L2_INDX_ST_TAG", + [0x13] = "L2_INDX_ST_DATA", + [0x14] = "L2_INDX_ST_ECC", + [0x15] = "0x15", + [0x16] = "0x16", + [0x17] = "0x17", + [0x18] = "L2_FTCH_AND_LCK", + [0x19] = "L2_HIT_INV", + [0x1a] = "L2_HIT_WB_INV", + [0x1b] = "L2_HIT_WB", + [0x1c] = "0x1c", + [0x1d] = "0x1d", + [0x1e] = "0x1e", + [0x1f] = "0x1f" +}; + static char *cm2_causes[32] = { "None", "GC_WR_ERR", "GC_RD_ERR", "COH_WR_ERR", "COH_RD_ERR", "MMIO_WR_ERR", "MMIO_RD_ERR", "0x07", @@ -121,7 +163,7 @@ static char *cm2_causes[32] = { "0x0c", "0x0d", "0x0e", "0x0f", "0x10", "INTVN_WR_ERR", "INTVN_RD_ERR", "0x13", "0x14", "0x15", "0x16", "0x17", - "0x18", "0x19", "0x1a", "0x1b", + "L2_RD_UNCORR", "L2_WR_UNCORR", "L2_CORR", "0x1b", "0x1c", "0x1d", "0x1e", "0x1f" }; @@ -360,7 +402,7 @@ void mips_cm_error_report(void) "CCA=%lu TR=%s MCmd=%s STag=%lu " "SPort=%lu\n", cca_bits, cm2_tr[tr_bits], cm2_cmd[cmd_bits], stag_bits, sport_bits); - } else { + } else if (cause < 24) { /* glob state & sresp together */ unsigned long c3_bits = (cm_error >> 18) & 7; unsigned long c2_bits = (cm_error >> 15) & 7; @@ -377,6 +419,22 @@ void mips_cm_error_report(void) cm2_core[c1_bits], cm2_core[c0_bits], sc_bit ? "True" : "False", cm2_cmd[cmd_bits], sport_bits); + } else { + unsigned long muc_bit = (cm_error >> 23) & 1; + unsigned long ins_bits = (cm_error >> 18) & 0x1f; + unsigned long arr_bits = (cm_error >> 16) & 3; + unsigned long dw_bits = (cm_error >> 12) & 15; + unsigned long way_bits = (cm_error >> 9) & 7; + unsigned long mway_bit = (cm_error >> 8) & 1; + unsigned long syn_bits = (cm_error >> 0) & 0xFF; + + snprintf(buf, sizeof(buf), + "Type=%s%s Instr=%s DW=%lu Way=%lu " + "MWay=%s Syndrome=0x%02lx", + muc_bit ? "Multi-UC " : "", + cm2_l2_type[arr_bits], + cm2_l2_instr[ins_bits], dw_bits, way_bits, + mway_bit ? "True" : "False", syn_bits); } pr_err("CM_ERROR=%08llx %s <%s>\n", cm_error, cm2_causes[cause], buf); From da706e5034c34602e0fb199e5be712ab7bf2c028 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Thu, 7 May 2020 17:49:18 +0800 Subject: [PATCH 0245/1043] MIPS: asm: Rename some macros to avoid build errors Use ASM_ prefix to rename some macros (PANIC and PRINT), in order to avoid build errors (all users are updated as well): 1, PANIC conflicts with drivers/scsi/smartpqi/smartpqi_init.c 2, PRINT conflicts with net/netfilter/nf_conntrack_h323_asn1.c and net/ mac80211/debugfs_sta.c Fixes: d339cd02b888eb8 ("MIPS: Move unaligned load/store helpers to inst.h") Reported-by: kbuild test robot Signed-off-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/dec/int-handler.S | 4 ++-- arch/mips/include/asm/asm.h | 20 ++++++++++---------- arch/mips/kernel/genex.S | 6 +++--- arch/mips/kernel/scall64-o32.S | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/arch/mips/dec/int-handler.S b/arch/mips/dec/int-handler.S index a25ef822e725..ea5b5a83f1e1 100644 --- a/arch/mips/dec/int-handler.S +++ b/arch/mips/dec/int-handler.S @@ -304,8 +304,8 @@ spurious: */ FEXPORT(dec_intr_unimplemented) move a1,t0 # cheats way of printing an arg! - PANIC("Unimplemented cpu interrupt! CP0_CAUSE: 0x%08x"); + ASM_PANIC("Unimplemented cpu interrupt! CP0_CAUSE: 0x%08x"); FEXPORT(asic_intr_unimplemented) move a1,t0 # cheats way of printing an arg! - PANIC("Unimplemented asic interrupt! ASIC ISR: 0x%08x"); + ASM_PANIC("Unimplemented asic interrupt! ASIC ISR: 0x%08x"); diff --git a/arch/mips/include/asm/asm.h b/arch/mips/include/asm/asm.h index 934465de470e..3682d1a0bb80 100644 --- a/arch/mips/include/asm/asm.h +++ b/arch/mips/include/asm/asm.h @@ -74,10 +74,15 @@ symbol: .insn .globl symbol; \ symbol = value -#define PANIC(msg) \ +#define TEXT(msg) \ + .pushsection .data; \ +8: .asciiz msg; \ + .popsection; + +#define ASM_PANIC(msg) \ .set push; \ .set reorder; \ - PTR_LA a0, 8f; \ + PTR_LA a0, 8f; \ jal panic; \ 9: b 9b; \ .set pop; \ @@ -87,22 +92,17 @@ symbol = value * Print formatted string */ #ifdef CONFIG_PRINTK -#define PRINT(string) \ +#define ASM_PRINT(string) \ .set push; \ .set reorder; \ - PTR_LA a0, 8f; \ + PTR_LA a0, 8f; \ jal printk; \ .set pop; \ TEXT(string) #else -#define PRINT(string) +#define ASM_PRINT(string) #endif -#define TEXT(msg) \ - .pushsection .data; \ -8: .asciiz msg; \ - .popsection; - /* * Stack alignment */ diff --git a/arch/mips/kernel/genex.S b/arch/mips/kernel/genex.S index 0a43c9125267..8236fb291e3f 100644 --- a/arch/mips/kernel/genex.S +++ b/arch/mips/kernel/genex.S @@ -501,17 +501,17 @@ NESTED(nmi_handler, PT_SIZE, sp) .macro __BUILD_silent exception .endm - /* Gas tries to parse the PRINT argument as a string containing + /* Gas tries to parse the ASM_PRINT argument as a string containing string escapes and emits bogus warnings if it believes to recognize an unknown escape code. So make the arguments start with an n and gas will believe \n is ok ... */ .macro __BUILD_verbose nexception LONG_L a1, PT_EPC(sp) #ifdef CONFIG_32BIT - PRINT("Got \nexception at %08lx\012") + ASM_PRINT("Got \nexception at %08lx\012") #endif #ifdef CONFIG_64BIT - PRINT("Got \nexception at %016lx\012") + ASM_PRINT("Got \nexception at %016lx\012") #endif .endm diff --git a/arch/mips/kernel/scall64-o32.S b/arch/mips/kernel/scall64-o32.S index 41df8221bb8f..50c9a57e0d3a 100644 --- a/arch/mips/kernel/scall64-o32.S +++ b/arch/mips/kernel/scall64-o32.S @@ -41,7 +41,7 @@ NESTED(handle_sys, PT_SIZE, sp) #if 0 SAVE_ALL move a1, v0 - PRINT("Scall %ld\n") + ASM_PRINT("Scall %ld\n") RESTORE_ALL #endif From e8824890249355656968d8846908a313fe231f11 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 24 Apr 2020 12:37:54 -0700 Subject: [PATCH 0246/1043] x86/delay: Preparatory code cleanup The naming conventions in the delay code are confusing at best. All delay variants use a loops argument and or variable which originates from the original delay_loop() implementation. But all variants except delay_loop() are based on TSC cycles. Rename the argument to cycles and make it type u64 to avoid these weird expansions to u64 in the functions. Rename MWAITX_MAX_LOOPS to MWAITX_MAX_WAIT_CYCLES for the same reason and fixup the comment of delay_mwaitx() as well. Mark the delay_fn function pointer __ro_after_init and fixup the comment for it. No functional change and preparation for the upcoming TPAUSE based delay variant. [ Kyung Min Park: Added __init to use_tsc_delay() ] Signed-off-by: Thomas Gleixner Signed-off-by: Kyung Min Park Signed-off-by: Thomas Gleixner Link: https://lkml.kernel.org/r/1587757076-30337-2-git-send-email-kyung.min.park@intel.com --- arch/x86/include/asm/delay.h | 3 ++- arch/x86/include/asm/mwait.h | 2 +- arch/x86/lib/delay.c | 45 +++++++++++++++++++----------------- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/arch/x86/include/asm/delay.h b/arch/x86/include/asm/delay.h index de9e7841f953..9aa38de7bd72 100644 --- a/arch/x86/include/asm/delay.h +++ b/arch/x86/include/asm/delay.h @@ -3,8 +3,9 @@ #define _ASM_X86_DELAY_H #include +#include -void use_tsc_delay(void); +void __init use_tsc_delay(void); void use_mwaitx_delay(void); #endif /* _ASM_X86_DELAY_H */ diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h index b809f117f3f4..a43b35b35049 100644 --- a/arch/x86/include/asm/mwait.h +++ b/arch/x86/include/asm/mwait.h @@ -20,7 +20,7 @@ #define MWAIT_ECX_INTERRUPT_BREAK 0x1 #define MWAITX_ECX_TIMER_ENABLE BIT(1) -#define MWAITX_MAX_LOOPS ((u32)-1) +#define MWAITX_MAX_WAIT_CYCLES UINT_MAX #define MWAITX_DISABLE_CSTATES 0xf0 u32 get_umwait_control_msr(void); diff --git a/arch/x86/lib/delay.c b/arch/x86/lib/delay.c index c126571e5e2e..887d52d5a7cc 100644 --- a/arch/x86/lib/delay.c +++ b/arch/x86/lib/delay.c @@ -27,9 +27,19 @@ # include #endif +static void delay_loop(u64 __loops); + +/* + * Calibration and selection of the delay mechanism happens only once + * during boot. + */ +static void (*delay_fn)(u64) __ro_after_init = delay_loop; + /* simple loop based delay: */ -static void delay_loop(unsigned long loops) +static void delay_loop(u64 __loops) { + unsigned long loops = (unsigned long)__loops; + asm volatile( " test %0,%0 \n" " jz 3f \n" @@ -49,9 +59,9 @@ static void delay_loop(unsigned long loops) } /* TSC based delay: */ -static void delay_tsc(unsigned long __loops) +static void delay_tsc(u64 cycles) { - u64 bclock, now, loops = __loops; + u64 bclock, now; int cpu; preempt_disable(); @@ -59,7 +69,7 @@ static void delay_tsc(unsigned long __loops) bclock = rdtsc_ordered(); for (;;) { now = rdtsc_ordered(); - if ((now - bclock) >= loops) + if ((now - bclock) >= cycles) break; /* Allow RT tasks to run */ @@ -77,7 +87,7 @@ static void delay_tsc(unsigned long __loops) * counter for this CPU. */ if (unlikely(cpu != smp_processor_id())) { - loops -= (now - bclock); + cycles -= (now - bclock); cpu = smp_processor_id(); bclock = rdtsc_ordered(); } @@ -87,24 +97,24 @@ static void delay_tsc(unsigned long __loops) /* * On some AMD platforms, MWAITX has a configurable 32-bit timer, that - * counts with TSC frequency. The input value is the loop of the - * counter, it will exit when the timer expires. + * counts with TSC frequency. The input value is the number of TSC cycles + * to wait. MWAITX will also exit when the timer expires. */ -static void delay_mwaitx(unsigned long __loops) +static void delay_mwaitx(u64 cycles) { - u64 start, end, delay, loops = __loops; + u64 start, end, delay; /* * Timer value of 0 causes MWAITX to wait indefinitely, unless there * is a store on the memory monitored by MONITORX. */ - if (loops == 0) + if (!cycles) return; start = rdtsc_ordered(); for (;;) { - delay = min_t(u64, MWAITX_MAX_LOOPS, loops); + delay = min_t(u64, MWAITX_MAX_WAIT_CYCLES, cycles); /* * Use cpu_tss_rw as a cacheline-aligned, seldomly @@ -121,22 +131,15 @@ static void delay_mwaitx(unsigned long __loops) end = rdtsc_ordered(); - if (loops <= end - start) + if (cycles <= end - start) break; - loops -= end - start; - + cycles -= end - start; start = end; } } -/* - * Since we calibrate only once at boot, this - * function should be set once at boot and not changed - */ -static void (*delay_fn)(unsigned long) = delay_loop; - -void use_tsc_delay(void) +void __init use_tsc_delay(void) { if (delay_fn == delay_loop) delay_fn = delay_tsc; From 46f90c7aad62be1af76588108c730d826308a801 Mon Sep 17 00:00:00 2001 From: Kyung Min Park Date: Fri, 24 Apr 2020 12:37:55 -0700 Subject: [PATCH 0247/1043] x86/delay: Refactor delay_mwaitx() for TPAUSE support Refactor code to make it easier to add a new model specific function to delay for a number of cycles. No functional change. Co-developed-by: Fenghua Yu Signed-off-by: Fenghua Yu Signed-off-by: Kyung Min Park Signed-off-by: Thomas Gleixner Reviewed-by: Tony Luck Link: https://lkml.kernel.org/r/1587757076-30337-3-git-send-email-kyung.min.park@intel.com --- arch/x86/lib/delay.c | 48 +++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/arch/x86/lib/delay.c b/arch/x86/lib/delay.c index 887d52d5a7cc..fe91dc171cf8 100644 --- a/arch/x86/lib/delay.c +++ b/arch/x86/lib/delay.c @@ -34,6 +34,7 @@ static void delay_loop(u64 __loops); * during boot. */ static void (*delay_fn)(u64) __ro_after_init = delay_loop; +static void (*delay_halt_fn)(u64 start, u64 cycles) __ro_after_init; /* simple loop based delay: */ static void delay_loop(u64 __loops) @@ -100,9 +101,33 @@ static void delay_tsc(u64 cycles) * counts with TSC frequency. The input value is the number of TSC cycles * to wait. MWAITX will also exit when the timer expires. */ -static void delay_mwaitx(u64 cycles) +static void delay_halt_mwaitx(u64 unused, u64 cycles) { - u64 start, end, delay; + u64 delay; + + delay = min_t(u64, MWAITX_MAX_WAIT_CYCLES, cycles); + /* + * Use cpu_tss_rw as a cacheline-aligned, seldomly accessed per-cpu + * variable as the monitor target. + */ + __monitorx(raw_cpu_ptr(&cpu_tss_rw), 0, 0); + + /* + * AMD, like Intel, supports the EAX hint and EAX=0xf means, do not + * enter any deep C-state and we use it here in delay() to minimize + * wakeup latency. + */ + __mwaitx(MWAITX_DISABLE_CSTATES, delay, MWAITX_ECX_TIMER_ENABLE); +} + +/* + * Call a vendor specific function to delay for a given amount of time. Because + * these functions may return earlier than requested, check for actual elapsed + * time and call again until done. + */ +static void delay_halt(u64 __cycles) +{ + u64 start, end, cycles = __cycles; /* * Timer value of 0 causes MWAITX to wait indefinitely, unless there @@ -114,21 +139,7 @@ static void delay_mwaitx(u64 cycles) start = rdtsc_ordered(); for (;;) { - delay = min_t(u64, MWAITX_MAX_WAIT_CYCLES, cycles); - - /* - * Use cpu_tss_rw as a cacheline-aligned, seldomly - * accessed per-cpu variable as the monitor target. - */ - __monitorx(raw_cpu_ptr(&cpu_tss_rw), 0, 0); - - /* - * AMD, like Intel's MWAIT version, supports the EAX hint and - * EAX=0xf0 means, do not enter any deep C-state and we use it - * here in delay() to minimize wakeup latency. - */ - __mwaitx(MWAITX_DISABLE_CSTATES, delay, MWAITX_ECX_TIMER_ENABLE); - + delay_halt_fn(start, cycles); end = rdtsc_ordered(); if (cycles <= end - start) @@ -147,7 +158,8 @@ void __init use_tsc_delay(void) void use_mwaitx_delay(void) { - delay_fn = delay_mwaitx; + delay_halt_fn = delay_halt_mwaitx; + delay_fn = delay_halt; } int read_current_timer(unsigned long *timer_val) From cec5f268cd02d25d2d74807843d8ae0292fe0fb7 Mon Sep 17 00:00:00 2001 From: Kyung Min Park Date: Fri, 24 Apr 2020 12:37:56 -0700 Subject: [PATCH 0248/1043] x86/delay: Introduce TPAUSE delay TPAUSE instructs the processor to enter an implementation-dependent optimized state. The instruction execution wakes up when the time-stamp counter reaches or exceeds the implicit EDX:EAX 64-bit input value. The instruction execution also wakes up due to the expiration of the operating system time-limit or by an external interrupt or exceptions such as a debug exception or a machine check exception. TPAUSE offers a choice of two lower power states: 1. Light-weight power/performance optimized state C0.1 2. Improved power/performance optimized state C0.2 This way, it can save power with low wake-up latency in comparison to spinloop based delay. The selection between the two is governed by the input register. TPAUSE is available on processors with X86_FEATURE_WAITPKG. Co-developed-by: Fenghua Yu Signed-off-by: Fenghua Yu Signed-off-by: Kyung Min Park Signed-off-by: Thomas Gleixner Reviewed-by: Tony Luck Link: https://lkml.kernel.org/r/1587757076-30337-4-git-send-email-kyung.min.park@intel.com --- arch/x86/Kconfig.assembler | 4 ++++ arch/x86/include/asm/delay.h | 1 + arch/x86/include/asm/mwait.h | 22 ++++++++++++++++++++++ arch/x86/kernel/time.c | 3 +++ arch/x86/lib/delay.c | 27 +++++++++++++++++++++++++++ 5 files changed, 57 insertions(+) diff --git a/arch/x86/Kconfig.assembler b/arch/x86/Kconfig.assembler index 13de0db38d4e..26b8c08e2fc4 100644 --- a/arch/x86/Kconfig.assembler +++ b/arch/x86/Kconfig.assembler @@ -15,3 +15,7 @@ config AS_SHA256_NI def_bool $(as-instr,sha256msg1 %xmm0$(comma)%xmm1) help Supported by binutils >= 2.24 and LLVM integrated assembler +config AS_TPAUSE + def_bool $(as-instr,tpause %ecx) + help + Supported by binutils >= 2.31.1 and LLVM integrated assembler >= V7 diff --git a/arch/x86/include/asm/delay.h b/arch/x86/include/asm/delay.h index 9aa38de7bd72..630891d25819 100644 --- a/arch/x86/include/asm/delay.h +++ b/arch/x86/include/asm/delay.h @@ -6,6 +6,7 @@ #include void __init use_tsc_delay(void); +void __init use_tpause_delay(void); void use_mwaitx_delay(void); #endif /* _ASM_X86_DELAY_H */ diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h index a43b35b35049..73d997aa2966 100644 --- a/arch/x86/include/asm/mwait.h +++ b/arch/x86/include/asm/mwait.h @@ -22,6 +22,8 @@ #define MWAITX_ECX_TIMER_ENABLE BIT(1) #define MWAITX_MAX_WAIT_CYCLES UINT_MAX #define MWAITX_DISABLE_CSTATES 0xf0 +#define TPAUSE_C01_STATE 1 +#define TPAUSE_C02_STATE 0 u32 get_umwait_control_msr(void); @@ -122,4 +124,24 @@ static inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx) current_clr_polling(); } +/* + * Caller can specify whether to enter C0.1 (low latency, less + * power saving) or C0.2 state (saves more power, but longer wakeup + * latency). This may be overridden by the IA32_UMWAIT_CONTROL MSR + * which can force requests for C0.2 to be downgraded to C0.1. + */ +static inline void __tpause(u32 ecx, u32 edx, u32 eax) +{ + /* "tpause %ecx, %edx, %eax;" */ + #ifdef CONFIG_AS_TPAUSE + asm volatile("tpause %%ecx\n" + : + : "c"(ecx), "d"(edx), "a"(eax)); + #else + asm volatile(".byte 0x66, 0x0f, 0xae, 0xf1\t\n" + : + : "c"(ecx), "d"(edx), "a"(eax)); + #endif +} + #endif /* _ASM_X86_MWAIT_H */ diff --git a/arch/x86/kernel/time.c b/arch/x86/kernel/time.c index 106e7f87f534..371a6b348e44 100644 --- a/arch/x86/kernel/time.c +++ b/arch/x86/kernel/time.c @@ -103,6 +103,9 @@ static __init void x86_late_time_init(void) */ x86_init.irqs.intr_mode_init(); tsc_init(); + + if (static_cpu_has(X86_FEATURE_WAITPKG)) + use_tpause_delay(); } /* diff --git a/arch/x86/lib/delay.c b/arch/x86/lib/delay.c index fe91dc171cf8..65d15df6212d 100644 --- a/arch/x86/lib/delay.c +++ b/arch/x86/lib/delay.c @@ -96,6 +96,27 @@ static void delay_tsc(u64 cycles) preempt_enable(); } +/* + * On Intel the TPAUSE instruction waits until any of: + * 1) the TSC counter exceeds the value provided in EDX:EAX + * 2) global timeout in IA32_UMWAIT_CONTROL is exceeded + * 3) an external interrupt occurs + */ +static void delay_halt_tpause(u64 start, u64 cycles) +{ + u64 until = start + cycles; + u32 eax, edx; + + eax = lower_32_bits(until); + edx = upper_32_bits(until); + + /* + * Hard code the deeper (C0.2) sleep state because exit latency is + * small compared to the "microseconds" that usleep() will delay. + */ + __tpause(TPAUSE_C02_STATE, edx, eax); +} + /* * On some AMD platforms, MWAITX has a configurable 32-bit timer, that * counts with TSC frequency. The input value is the number of TSC cycles @@ -156,6 +177,12 @@ void __init use_tsc_delay(void) delay_fn = delay_tsc; } +void __init use_tpause_delay(void) +{ + delay_halt_fn = delay_halt_tpause; + delay_fn = delay_halt; +} + void use_mwaitx_delay(void) { delay_halt_fn = delay_halt_mwaitx; From 3fec4aecb311995189217e64d725cfe84a568de3 Mon Sep 17 00:00:00 2001 From: Daniel Thompson Date: Wed, 6 May 2020 17:42:23 +0100 Subject: [PATCH 0249/1043] kgdb: Fix spurious true from in_dbg_master() Currently there is a small window where a badly timed migration could cause in_dbg_master() to spuriously return true. Specifically if we migrate to a new core after reading the processor id and the previous core takes a breakpoint then we will evaluate true if we read kgdb_active before we get the IPI to bring us to halt. Fix this by checking irqs_disabled() first. Interrupts are always disabled when we are executing the kgdb trap so this is an acceptable prerequisite. This also allows us to replace raw_smp_processor_id() with smp_processor_id() since the short circuit logic will prevent warnings from PREEMPT_DEBUG. Fixes: dcc7871128e9 ("kgdb: core changes to support kdb") Suggested-by: Will Deacon Link: https://lore.kernel.org/r/20200506164223.2875760-1-daniel.thompson@linaro.org Reviewed-by: Douglas Anderson Signed-off-by: Daniel Thompson --- include/linux/kgdb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/kgdb.h b/include/linux/kgdb.h index b072aeb1fd78..4d6fe87fd38f 100644 --- a/include/linux/kgdb.h +++ b/include/linux/kgdb.h @@ -323,7 +323,7 @@ extern void gdbstub_exit(int status); extern int kgdb_single_step; extern atomic_t kgdb_active; #define in_dbg_master() \ - (raw_smp_processor_id() == atomic_read(&kgdb_active)) + (irqs_disabled() && (smp_processor_id() == atomic_read(&kgdb_active))) extern bool dbg_is_early; extern void __init dbg_late_init(void); extern void kgdb_panic(const char *msg); From a13502073638a0b28d84099fa985971fe3287aee Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 6 May 2020 18:17:27 +0300 Subject: [PATCH 0250/1043] kgdb: Drop malformed kernel doc comment Kernel doc does not understand POD variables to be referred to. .../debug_core.c:73: warning: cannot understand function prototype: 'int kgdb_connected; ' Convert kernel doc to pure comment. Fixes: dc7d55270521 ("kgdb: core") Cc: Jason Wessel Signed-off-by: Andy Shevchenko Reviewed-by: Douglas Anderson Signed-off-by: Daniel Thompson --- kernel/debug/debug_core.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c index 2b7c9b67931d..2266ba27f27d 100644 --- a/kernel/debug/debug_core.c +++ b/kernel/debug/debug_core.c @@ -67,9 +67,7 @@ static int kgdb_break_asap; struct debuggerinfo_struct kgdb_info[NR_CPUS]; -/** - * kgdb_connected - Is a host GDB connected to us? - */ +/* kgdb_connected - Is a host GDB connected to us? */ int kgdb_connected; EXPORT_SYMBOL_GPL(kgdb_connected); From 1137a96f9b5a8615296bd319151896c859ead292 Mon Sep 17 00:00:00 2001 From: Jason Yan Date: Thu, 7 May 2020 19:06:49 +0800 Subject: [PATCH 0251/1043] kgdb: Return true in kgdb_nmi_poll_knock() Fix the following coccicheck warning: include/linux/kgdb.h:301:54-55: WARNING: return of 0/1 in function 'kgdb_nmi_poll_knock' with return type bool Signed-off-by: Jason Yan Link: https://lore.kernel.org/r/20200507110649.37426-1-yanaijie@huawei.com Signed-off-by: Daniel Thompson --- include/linux/kgdb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/kgdb.h b/include/linux/kgdb.h index 4d6fe87fd38f..c2caee08e418 100644 --- a/include/linux/kgdb.h +++ b/include/linux/kgdb.h @@ -298,7 +298,7 @@ extern bool kgdb_nmi_poll_knock(void); #else static inline int kgdb_register_nmi_console(void) { return 0; } static inline int kgdb_unregister_nmi_console(void) { return 0; } -static inline bool kgdb_nmi_poll_knock(void) { return 1; } +static inline bool kgdb_nmi_poll_knock(void) { return true; } #endif extern int kgdb_register_io_module(struct kgdb_io *local_kgdb_io_ops); From cb6ad0993eb8973533235de8a35aa4b9c872d733 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:25:19 -0700 Subject: [PATCH 0252/1043] xfs: refactor failed buffer resubmission into xfsaild Flush locked log items whose underlying buffers fail metadata writeback are tagged with a special flag to indicate that the flush lock is already held. This is currently implemented in the type specific ->iop_push() callback, but the processing required for such items is not type specific because we're only doing basic state management on the underlying buffer. Factor the failed log item handling out of the inode and dquot ->iop_push() callbacks and open code the buffer resubmit helper into a single helper called from xfsaild_push_item(). This provides a generic mechanism for handling failed metadata buffer writeback with a bit less code. Signed-off-by: Brian Foster Reviewed-by: Allison Collins Reviewed-by: Dave Chinner Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_buf_item.c | 39 --------------------------------------- fs/xfs/xfs_buf_item.h | 2 -- fs/xfs/xfs_dquot_item.c | 15 --------------- fs/xfs/xfs_inode_item.c | 15 --------------- fs/xfs/xfs_trans_ail.c | 41 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 41 insertions(+), 71 deletions(-) diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c index 1545657c3ca0..8796adde2d12 100644 --- a/fs/xfs/xfs_buf_item.c +++ b/fs/xfs/xfs_buf_item.c @@ -1248,42 +1248,3 @@ xfs_buf_iodone( xfs_trans_ail_delete(ailp, lip, SHUTDOWN_CORRUPT_INCORE); xfs_buf_item_free(BUF_ITEM(lip)); } - -/* - * Requeue a failed buffer for writeback. - * - * We clear the log item failed state here as well, but we have to be careful - * about reference counts because the only active reference counts on the buffer - * may be the failed log items. Hence if we clear the log item failed state - * before queuing the buffer for IO we can release all active references to - * the buffer and free it, leading to use after free problems in - * xfs_buf_delwri_queue. It makes no difference to the buffer or log items which - * order we process them in - the buffer is locked, and we own the buffer list - * so nothing on them is going to change while we are performing this action. - * - * Hence we can safely queue the buffer for IO before we clear the failed log - * item state, therefore always having an active reference to the buffer and - * avoiding the transient zero-reference state that leads to use-after-free. - * - * Return true if the buffer was added to the buffer list, false if it was - * already on the buffer list. - */ -bool -xfs_buf_resubmit_failed_buffers( - struct xfs_buf *bp, - struct list_head *buffer_list) -{ - struct xfs_log_item *lip; - bool ret; - - ret = xfs_buf_delwri_queue(bp, buffer_list); - - /* - * XFS_LI_FAILED set/clear is protected by ail_lock, caller of this - * function already have it acquired - */ - list_for_each_entry(lip, &bp->b_li_list, li_bio_list) - xfs_clear_li_failed(lip); - - return ret; -} diff --git a/fs/xfs/xfs_buf_item.h b/fs/xfs/xfs_buf_item.h index 30114b510332..c9c57e2da932 100644 --- a/fs/xfs/xfs_buf_item.h +++ b/fs/xfs/xfs_buf_item.h @@ -59,8 +59,6 @@ void xfs_buf_attach_iodone(struct xfs_buf *, struct xfs_log_item *); void xfs_buf_iodone_callbacks(struct xfs_buf *); void xfs_buf_iodone(struct xfs_buf *, struct xfs_log_item *); -bool xfs_buf_resubmit_failed_buffers(struct xfs_buf *, - struct list_head *); bool xfs_buf_log_check_iovec(struct xfs_log_iovec *iovec); extern kmem_zone_t *xfs_buf_item_zone; diff --git a/fs/xfs/xfs_dquot_item.c b/fs/xfs/xfs_dquot_item.c index baad1748d0d1..5a7808299a32 100644 --- a/fs/xfs/xfs_dquot_item.c +++ b/fs/xfs/xfs_dquot_item.c @@ -145,21 +145,6 @@ xfs_qm_dquot_logitem_push( if (atomic_read(&dqp->q_pincount) > 0) return XFS_ITEM_PINNED; - /* - * The buffer containing this item failed to be written back - * previously. Resubmit the buffer for IO - */ - if (test_bit(XFS_LI_FAILED, &lip->li_flags)) { - if (!xfs_buf_trylock(bp)) - return XFS_ITEM_LOCKED; - - if (!xfs_buf_resubmit_failed_buffers(bp, buffer_list)) - rval = XFS_ITEM_FLUSHING; - - xfs_buf_unlock(bp); - return rval; - } - if (!xfs_dqlock_nowait(dqp)) return XFS_ITEM_LOCKED; diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index 75b74bbe38e4..a4027f4ca6c4 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -497,21 +497,6 @@ xfs_inode_item_push( if (xfs_ipincount(ip) > 0) return XFS_ITEM_PINNED; - /* - * The buffer containing this item failed to be written back - * previously. Resubmit the buffer for IO. - */ - if (test_bit(XFS_LI_FAILED, &lip->li_flags)) { - if (!xfs_buf_trylock(bp)) - return XFS_ITEM_LOCKED; - - if (!xfs_buf_resubmit_failed_buffers(bp, buffer_list)) - rval = XFS_ITEM_FLUSHING; - - xfs_buf_unlock(bp); - return rval; - } - if (!xfs_ilock_nowait(ip, XFS_ILOCK_SHARED)) return XFS_ITEM_LOCKED; diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c index 564253550b75..2574d01e4a83 100644 --- a/fs/xfs/xfs_trans_ail.c +++ b/fs/xfs/xfs_trans_ail.c @@ -345,6 +345,45 @@ xfs_ail_delete( xfs_trans_ail_cursor_clear(ailp, lip); } +/* + * Requeue a failed buffer for writeback. + * + * We clear the log item failed state here as well, but we have to be careful + * about reference counts because the only active reference counts on the buffer + * may be the failed log items. Hence if we clear the log item failed state + * before queuing the buffer for IO we can release all active references to + * the buffer and free it, leading to use after free problems in + * xfs_buf_delwri_queue. It makes no difference to the buffer or log items which + * order we process them in - the buffer is locked, and we own the buffer list + * so nothing on them is going to change while we are performing this action. + * + * Hence we can safely queue the buffer for IO before we clear the failed log + * item state, therefore always having an active reference to the buffer and + * avoiding the transient zero-reference state that leads to use-after-free. + */ +static inline int +xfsaild_resubmit_item( + struct xfs_log_item *lip, + struct list_head *buffer_list) +{ + struct xfs_buf *bp = lip->li_buf; + + if (!xfs_buf_trylock(bp)) + return XFS_ITEM_LOCKED; + + if (!xfs_buf_delwri_queue(bp, buffer_list)) { + xfs_buf_unlock(bp); + return XFS_ITEM_FLUSHING; + } + + /* protected by ail_lock */ + list_for_each_entry(lip, &bp->b_li_list, li_bio_list) + xfs_clear_li_failed(lip); + + xfs_buf_unlock(bp); + return XFS_ITEM_SUCCESS; +} + static inline uint xfsaild_push_item( struct xfs_ail *ailp, @@ -365,6 +404,8 @@ xfsaild_push_item( */ if (!lip->li_ops->iop_push) return XFS_ITEM_PINNED; + if (test_bit(XFS_LI_FAILED, &lip->li_flags)) + return xfsaild_resubmit_item(lip, &ailp->ail_buf_list); return lip->li_ops->iop_push(lip, &ailp->ail_buf_list); } From 54b3b1f619efbc838870735db2ddde9450f93a8f Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:25:19 -0700 Subject: [PATCH 0253/1043] xfs: factor out buffer I/O failure code We use the same buffer I/O failure code in a few different places. It's not much code, but it's not necessarily self-explanatory. Factor it into a helper and document it in one place. Signed-off-by: Brian Foster Reviewed-by: Allison Collins Reviewed-by: Dave Chinner Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_buf.c | 21 +++++++++++++++++---- fs/xfs/xfs_buf.h | 1 + fs/xfs/xfs_buf_item.c | 21 +++------------------ fs/xfs/xfs_inode.c | 6 +----- 4 files changed, 22 insertions(+), 27 deletions(-) diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 9ec3eaf1c618..d5d6a68bb1e6 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -1248,6 +1248,22 @@ xfs_buf_ioerror_alert( -bp->b_error); } +/* + * To simulate an I/O failure, the buffer must be locked and held with at least + * three references. The LRU reference is dropped by the stale call. The buf + * item reference is dropped via ioend processing. The third reference is owned + * by the caller and is dropped on I/O completion if the buffer is XBF_ASYNC. + */ +void +xfs_buf_ioend_fail( + struct xfs_buf *bp) +{ + bp->b_flags &= ~XBF_DONE; + xfs_buf_stale(bp); + xfs_buf_ioerror(bp, -EIO); + xfs_buf_ioend(bp); +} + int xfs_bwrite( struct xfs_buf *bp) @@ -1480,10 +1496,7 @@ __xfs_buf_submit( /* on shutdown we stale and complete the buffer immediately */ if (XFS_FORCED_SHUTDOWN(bp->b_mount)) { - xfs_buf_ioerror(bp, -EIO); - bp->b_flags &= ~XBF_DONE; - xfs_buf_stale(bp); - xfs_buf_ioend(bp); + xfs_buf_ioend_fail(bp); return -EIO; } diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h index 9a04c53c2488..06ea3eef866e 100644 --- a/fs/xfs/xfs_buf.h +++ b/fs/xfs/xfs_buf.h @@ -263,6 +263,7 @@ extern void __xfs_buf_ioerror(struct xfs_buf *bp, int error, xfs_failaddr_t failaddr); #define xfs_buf_ioerror(bp, err) __xfs_buf_ioerror((bp), (err), __this_address) extern void xfs_buf_ioerror_alert(struct xfs_buf *bp, xfs_failaddr_t fa); +void xfs_buf_ioend_fail(struct xfs_buf *); extern int __xfs_buf_submit(struct xfs_buf *bp, bool); static inline int xfs_buf_submit(struct xfs_buf *bp) diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c index 8796adde2d12..b452a399a441 100644 --- a/fs/xfs/xfs_buf_item.c +++ b/fs/xfs/xfs_buf_item.c @@ -471,28 +471,13 @@ xfs_buf_item_unpin( xfs_buf_relse(bp); } else if (freed && remove) { /* - * There are currently two references to the buffer - the active - * LRU reference and the buf log item. What we are about to do - * here - simulate a failed IO completion - requires 3 - * references. - * - * The LRU reference is removed by the xfs_buf_stale() call. The - * buf item reference is removed by the xfs_buf_iodone() - * callback that is run by xfs_buf_do_callbacks() during ioend - * processing (via the bp->b_iodone callback), and then finally - * the ioend processing will drop the IO reference if the buffer - * is marked XBF_ASYNC. - * - * Hence we need to take an additional reference here so that IO - * completion processing doesn't free the buffer prematurely. + * The buffer must be locked and held by the caller to simulate + * an async I/O failure. */ xfs_buf_lock(bp); xfs_buf_hold(bp); bp->b_flags |= XBF_ASYNC; - xfs_buf_ioerror(bp, -EIO); - bp->b_flags &= ~XBF_DONE; - xfs_buf_stale(bp); - xfs_buf_ioend(bp); + xfs_buf_ioend_fail(bp); } } diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 0e2ef3f56be4..8ce8b6bd992c 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -3630,11 +3630,7 @@ cluster_corrupt_out: */ ASSERT(bp->b_iodone); bp->b_flags |= XBF_ASYNC; - bp->b_flags &= ~XBF_DONE; - xfs_buf_stale(bp); - xfs_buf_ioerror(bp, -EIO); - xfs_buf_ioend(bp); - + xfs_buf_ioend_fail(bp); xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); /* abort the corrupt inode, as it was not attached to the buffer */ From f20192991d791293ccb7f6a073bb28de572f799a Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:25:20 -0700 Subject: [PATCH 0254/1043] xfs: simplify inode flush error handling The inode flush code has several layers of error handling between the inode and cluster flushing code. If the inode flush fails before acquiring the backing buffer, the inode flush is aborted. If the cluster flush fails, the current inode flush is aborted and the cluster buffer is failed to handle the initial inode and any others that might have been attached before the error. Since xfs_iflush() is the only caller of xfs_iflush_cluster(), the error handling between the two can be condensed in the top-level function. If we update xfs_iflush_int() to always fall through to the log item update and attach the item completion handler to the buffer, any errors that occur after the first call to xfs_iflush_int() can be handled with a buffer I/O failure. Lift the error handling from xfs_iflush_cluster() into xfs_iflush() and consolidate with the existing error handling. This also replaces the need to release the buffer because failing the buffer with XBF_ASYNC drops the current reference. Signed-off-by: Brian Foster Reviewed-by: Christoph Hellwig Reviewed-by: Allison Collins Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_inode.c | 117 +++++++++++++++++---------------------------- 1 file changed, 45 insertions(+), 72 deletions(-) diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 8ce8b6bd992c..788a68ef8d10 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -3496,6 +3496,7 @@ xfs_iflush_cluster( struct xfs_inode **cilist; struct xfs_inode *cip; struct xfs_ino_geometry *igeo = M_IGEO(mp); + int error = 0; int nr_found; int clcount = 0; int i; @@ -3588,11 +3589,10 @@ xfs_iflush_cluster( * re-check that it's dirty before flushing. */ if (!xfs_inode_clean(cip)) { - int error; error = xfs_iflush_int(cip, bp); if (error) { xfs_iunlock(cip, XFS_ILOCK_SHARED); - goto cluster_corrupt_out; + goto out_free; } clcount++; } else { @@ -3611,33 +3611,7 @@ out_free: kmem_free(cilist); out_put: xfs_perag_put(pag); - return 0; - - -cluster_corrupt_out: - /* - * Corruption detected in the clustering loop. Invalidate the - * inode buffer and shut down the filesystem. - */ - rcu_read_unlock(); - - /* - * We'll always have an inode attached to the buffer for completion - * process by the time we are called from xfs_iflush(). Hence we have - * always need to do IO completion processing to abort the inodes - * attached to the buffer. handle them just like the shutdown case in - * xfs_buf_submit(). - */ - ASSERT(bp->b_iodone); - bp->b_flags |= XBF_ASYNC; - xfs_buf_ioend_fail(bp); - xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); - - /* abort the corrupt inode, as it was not attached to the buffer */ - xfs_iflush_abort(cip, false); - kmem_free(cilist); - xfs_perag_put(pag); - return -EFSCORRUPTED; + return error; } /* @@ -3693,17 +3667,16 @@ xfs_iflush( */ if (XFS_FORCED_SHUTDOWN(mp)) { error = -EIO; - goto abort_out; + goto abort; } /* * Get the buffer containing the on-disk inode. We are doing a try-lock - * operation here, so we may get an EAGAIN error. In that case, we - * simply want to return with the inode still dirty. + * operation here, so we may get an EAGAIN error. In that case, return + * leaving the inode dirty. * * If we get any other error, we effectively have a corruption situation - * and we cannot flush the inode, so we treat it the same as failing - * xfs_iflush_int(). + * and we cannot flush the inode. Abort the flush and shut down. */ error = xfs_imap_to_bp(mp, NULL, &ip->i_imap, &dip, &bp, XBF_TRYLOCK, 0); @@ -3712,14 +3685,7 @@ xfs_iflush( return error; } if (error) - goto corrupt_out; - - /* - * First flush out the inode that xfs_iflush was called with. - */ - error = xfs_iflush_int(ip, bp); - if (error) - goto corrupt_out; + goto abort; /* * If the buffer is pinned then push on the log now so we won't @@ -3729,28 +3695,29 @@ xfs_iflush( xfs_log_force(mp, 0); /* - * inode clustering: try to gather other inodes into this write + * Flush the provided inode then attempt to gather others from the + * cluster into the write. * - * Note: Any error during clustering will result in the filesystem - * being shut down and completion callbacks run on the cluster buffer. - * As we have already flushed and attached this inode to the buffer, - * it has already been aborted and released by xfs_iflush_cluster() and - * so we have no further error handling to do here. + * Note: Once we attempt to flush an inode, we must run buffer + * completion callbacks on any failure. If this fails, simulate an I/O + * failure on the buffer and shut down. */ - error = xfs_iflush_cluster(ip, bp); - if (error) - return error; + error = xfs_iflush_int(ip, bp); + if (!error) + error = xfs_iflush_cluster(ip, bp); + if (error) { + bp->b_flags |= XBF_ASYNC; + xfs_buf_ioend_fail(bp); + goto shutdown; + } *bpp = bp; return 0; -corrupt_out: - if (bp) - xfs_buf_relse(bp); - xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); -abort_out: - /* abort the corrupt inode, as it was not attached to the buffer */ +abort: xfs_iflush_abort(ip, false); +shutdown: + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); return error; } @@ -3792,6 +3759,7 @@ xfs_iflush_int( struct xfs_inode_log_item *iip = ip->i_itemp; struct xfs_dinode *dip; struct xfs_mount *mp = ip->i_mount; + int error; ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL|XFS_ILOCK_SHARED)); ASSERT(xfs_isiflocked(ip)); @@ -3799,15 +3767,21 @@ xfs_iflush_int( ip->i_d.di_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)); ASSERT(iip != NULL && iip->ili_fields != 0); - /* set *dip = inode's place in the buffer */ dip = xfs_buf_offset(bp, ip->i_imap.im_boffset); + /* + * We don't flush the inode if any of the following checks fail, but we + * do still update the log item and attach to the backing buffer as if + * the flush happened. This is a formality to facilitate predictable + * error handling as the caller will shutdown and fail the buffer. + */ + error = -EFSCORRUPTED; if (XFS_TEST_ERROR(dip->di_magic != cpu_to_be16(XFS_DINODE_MAGIC), mp, XFS_ERRTAG_IFLUSH_1)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: Bad inode %Lu magic number 0x%x, ptr "PTR_FMT, __func__, ip->i_ino, be16_to_cpu(dip->di_magic), dip); - goto corrupt_out; + goto flush_out; } if (S_ISREG(VFS_I(ip)->i_mode)) { if (XFS_TEST_ERROR( @@ -3817,7 +3791,7 @@ xfs_iflush_int( xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: Bad regular inode %Lu, ptr "PTR_FMT, __func__, ip->i_ino, ip); - goto corrupt_out; + goto flush_out; } } else if (S_ISDIR(VFS_I(ip)->i_mode)) { if (XFS_TEST_ERROR( @@ -3828,7 +3802,7 @@ xfs_iflush_int( xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: Bad directory inode %Lu, ptr "PTR_FMT, __func__, ip->i_ino, ip); - goto corrupt_out; + goto flush_out; } } if (XFS_TEST_ERROR(ip->i_d.di_nextents + ip->i_d.di_anextents > @@ -3839,14 +3813,14 @@ xfs_iflush_int( __func__, ip->i_ino, ip->i_d.di_nextents + ip->i_d.di_anextents, ip->i_d.di_nblocks, ip); - goto corrupt_out; + goto flush_out; } if (XFS_TEST_ERROR(ip->i_d.di_forkoff > mp->m_sb.sb_inodesize, mp, XFS_ERRTAG_IFLUSH_6)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: bad inode %Lu, forkoff 0x%x, ptr "PTR_FMT, __func__, ip->i_ino, ip->i_d.di_forkoff, ip); - goto corrupt_out; + goto flush_out; } /* @@ -3863,7 +3837,7 @@ xfs_iflush_int( /* Check the inline fork data before we write out. */ if (!xfs_inode_verify_forks(ip)) - goto corrupt_out; + goto flush_out; /* * Copy the dirty parts of the inode into the on-disk inode. We always @@ -3906,6 +3880,8 @@ xfs_iflush_int( * need the AIL lock, because it is a 64 bit value that cannot be read * atomically. */ + error = 0; +flush_out: iip->ili_last_fields = iip->ili_fields; iip->ili_fields = 0; iip->ili_fsync_fields = 0; @@ -3915,10 +3891,10 @@ xfs_iflush_int( &iip->ili_item.li_lsn); /* - * Attach the function xfs_iflush_done to the inode's - * buffer. This will remove the inode from the AIL - * and unlock the inode's flush lock when the inode is - * completely written to disk. + * Attach the inode item callback to the buffer whether the flush + * succeeded or not. If not, the caller will shut down and fail I/O + * completion on the buffer to remove the inode from the AIL and release + * the flush lock. */ xfs_buf_attach_iodone(bp, xfs_iflush_done, &iip->ili_item); @@ -3927,10 +3903,7 @@ xfs_iflush_int( ASSERT(!list_empty(&bp->b_li_list)); ASSERT(bp->b_iodone != NULL); - return 0; - -corrupt_out: - return -EFSCORRUPTED; + return error; } /* Release an inode. */ From 15fab3b9be2255be70ba1c598a11622fa03c9d5e Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:25:20 -0700 Subject: [PATCH 0255/1043] xfs: remove unnecessary shutdown check from xfs_iflush() The shutdown check in xfs_iflush() duplicates checks down in the buffer code. If the fs is shut down, xfs_trans_read_buf_map() always returns an error and falls into the same error path. Remove the unnecessary check along with the warning in xfs_imap_to_bp() that generates excessive noise in the log if the fs is shut down. Signed-off-by: Brian Foster Reviewed-by: Allison Collins Reviewed-by: Dave Chinner Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_buf.c | 7 +------ fs/xfs/xfs_inode.c | 13 ------------- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index 39c5a6e24915..b102e611bf54 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -172,12 +172,7 @@ xfs_imap_to_bp( (int)imap->im_len, buf_flags, &bp, &xfs_inode_buf_ops); if (error) { - if (error == -EAGAIN) { - ASSERT(buf_flags & XBF_TRYLOCK); - return error; - } - xfs_warn(mp, "%s: xfs_trans_read_buf() returned error %d.", - __func__, error); + ASSERT(error != -EAGAIN || (buf_flags & XBF_TRYLOCK)); return error; } diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 788a68ef8d10..d1f7da2fc2ec 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -3657,19 +3657,6 @@ xfs_iflush( return 0; } - /* - * This may have been unpinned because the filesystem is shutting - * down forcibly. If that's the case we must not write this inode - * to disk, because the log record didn't make it to disk. - * - * We also have to remove the log item from the AIL in this case, - * as we wait for an empty AIL as part of the unmount process. - */ - if (XFS_FORCED_SHUTDOWN(mp)) { - error = -EIO; - goto abort; - } - /* * Get the buffer containing the on-disk inode. We are doing a try-lock * operation here, so we may get an EAGAIN error. In that case, return From b6983e80b03bd4fd42de71993b3ac7403edac758 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:25:20 -0700 Subject: [PATCH 0256/1043] xfs: reset buffer write failure state on successful completion The buffer write failure flag is intended to control the internal write retry that XFS has historically implemented to help mitigate the severity of transient I/O errors. The flag is set when a buffer is resubmitted from the I/O completion path due to a previous failure. It is checked on subsequent I/O completions to skip the internal retry and fall through to the higher level configurable error handling mechanism. The flag is cleared in the synchronous and delwri submission paths and also checked in various places to log write failure messages. There are a couple minor problems with the current usage of this flag. One is that we issue an internal retry after every submission from xfsaild due to how delwri submission clears the flag. This results in double the expected or configured number of write attempts when under sustained failures. Another more subtle issue is that the flag is never cleared on successful I/O completion. This can cause xfs_wait_buftarg() to suggest that dirty buffers are being thrown away due to the existence of the flag, when the reality is that the flag might still be set because the write succeeded on the retry. Clear the write failure flag on successful I/O completion to address both of these problems. This means that the internal retry attempt occurs once since the last time a buffer write failed and that various other contexts only see the flag set when the immediately previous write attempt has failed. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Allison Collins Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_buf.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index d5d6a68bb1e6..fd76a84cefdd 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -1197,8 +1197,10 @@ xfs_buf_ioend( bp->b_ops->verify_read(bp); } - if (!bp->b_error) + if (!bp->b_error) { + bp->b_flags &= ~XBF_WRITE_FAIL; bp->b_flags |= XBF_DONE; + } if (bp->b_iodone) (*(bp->b_iodone))(bp); @@ -1274,7 +1276,7 @@ xfs_bwrite( bp->b_flags |= XBF_WRITE; bp->b_flags &= ~(XBF_ASYNC | XBF_READ | _XBF_DELWRI_Q | - XBF_WRITE_FAIL | XBF_DONE); + XBF_DONE); error = xfs_buf_submit(bp); if (error) @@ -1996,7 +1998,7 @@ xfs_buf_delwri_submit_buffers( * synchronously. Otherwise, drop the buffer from the delwri * queue and submit async. */ - bp->b_flags &= ~(_XBF_DELWRI_Q | XBF_WRITE_FAIL); + bp->b_flags &= ~_XBF_DELWRI_Q; bp->b_flags |= XBF_WRITE; if (wait_list) { bp->b_flags &= ~XBF_ASYNC; From f9bccfcc3b59b9aba64791ab3a2bfefe681ab75b Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:25:21 -0700 Subject: [PATCH 0257/1043] xfs: refactor ratelimited buffer error messages into helper XFS has some inconsistent log message rate limiting with respect to buffer alerts. The metadata I/O error notification uses the generic ratelimited alert, the buffer push code uses a custom rate limit and the similar quiesce time failure checks are not rate limited at all (when they should be). The custom rate limit defined in the buf item code is specifically crafted for buffer alerts. It is more aggressive than generic rate limiting code because it must accommodate a high frequency of I/O error events in a relative short timeframe. Factor out the custom rate limit state from the buf item code into a per-buftarg rate limit so various alerts are limited based on the target. Define a buffer alert helper function and use it for the buffer alerts that are already ratelimited. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Allison Collins Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_buf.c | 15 +++++++++++---- fs/xfs/xfs_buf.h | 1 + fs/xfs/xfs_buf_item.c | 17 ++++------------- fs/xfs/xfs_message.c | 22 ++++++++++++++++++++++ fs/xfs/xfs_message.h | 3 +++ 5 files changed, 41 insertions(+), 17 deletions(-) diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index fd76a84cefdd..594d5e1df6f8 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -1244,10 +1244,10 @@ xfs_buf_ioerror_alert( struct xfs_buf *bp, xfs_failaddr_t func) { - xfs_alert_ratelimited(bp->b_mount, -"metadata I/O error in \"%pS\" at daddr 0x%llx len %d error %d", - func, (uint64_t)XFS_BUF_ADDR(bp), bp->b_length, - -bp->b_error); + xfs_buf_alert_ratelimited(bp, "XFS: metadata IO error", + "metadata I/O error in \"%pS\" at daddr 0x%llx len %d error %d", + func, (uint64_t)XFS_BUF_ADDR(bp), + bp->b_length, -bp->b_error); } /* @@ -1828,6 +1828,13 @@ xfs_alloc_buftarg( btp->bt_bdev = bdev; btp->bt_daxdev = dax_dev; + /* + * Buffer IO error rate limiting. Limit it to no more than 10 messages + * per 30 seconds so as to not spam logs too much on repeated errors. + */ + ratelimit_state_init(&btp->bt_ioerror_rl, 30 * HZ, + DEFAULT_RATELIMIT_BURST); + if (xfs_setsize_buftarg_early(btp, bdev)) goto error_free; diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h index 06ea3eef866e..050c53b739e2 100644 --- a/fs/xfs/xfs_buf.h +++ b/fs/xfs/xfs_buf.h @@ -91,6 +91,7 @@ typedef struct xfs_buftarg { struct list_lru bt_lru; struct percpu_counter bt_io_count; + struct ratelimit_state bt_ioerror_rl; } xfs_buftarg_t; struct xfs_buf; diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c index b452a399a441..1f7acffc99ba 100644 --- a/fs/xfs/xfs_buf_item.c +++ b/fs/xfs/xfs_buf_item.c @@ -481,14 +481,6 @@ xfs_buf_item_unpin( } } -/* - * Buffer IO error rate limiting. Limit it to no more than 10 messages per 30 - * seconds so as to not spam logs too much on repeated detection of the same - * buffer being bad.. - */ - -static DEFINE_RATELIMIT_STATE(xfs_buf_write_fail_rl_state, 30 * HZ, 10); - STATIC uint xfs_buf_item_push( struct xfs_log_item *lip, @@ -518,11 +510,10 @@ xfs_buf_item_push( trace_xfs_buf_item_push(bip); /* has a previous flush failed due to IO errors? */ - if ((bp->b_flags & XBF_WRITE_FAIL) && - ___ratelimit(&xfs_buf_write_fail_rl_state, "XFS: Failing async write")) { - xfs_warn(bp->b_mount, -"Failing async write on buffer block 0x%llx. Retrying async write.", - (long long)bp->b_bn); + if (bp->b_flags & XBF_WRITE_FAIL) { + xfs_buf_alert_ratelimited(bp, "XFS: Failing async write", + "Failing async write on buffer block 0x%llx. Retrying async write.", + (long long)bp->b_bn); } if (!xfs_buf_delwri_queue(bp, buffer_list)) diff --git a/fs/xfs/xfs_message.c b/fs/xfs/xfs_message.c index e0f9d3b6abe9..bc66d95c8d4c 100644 --- a/fs/xfs/xfs_message.c +++ b/fs/xfs/xfs_message.c @@ -117,3 +117,25 @@ xfs_hex_dump(const void *p, int length) { print_hex_dump(KERN_ALERT, "", DUMP_PREFIX_OFFSET, 16, 1, p, length, 1); } + +void +xfs_buf_alert_ratelimited( + struct xfs_buf *bp, + const char *rlmsg, + const char *fmt, + ...) +{ + struct xfs_mount *mp = bp->b_mount; + struct va_format vaf; + va_list args; + + /* use the more aggressive per-target rate limit for buffers */ + if (!___ratelimit(&bp->b_target->bt_ioerror_rl, rlmsg)) + return; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + __xfs_printk(KERN_ALERT, mp, &vaf); + va_end(args); +} diff --git a/fs/xfs/xfs_message.h b/fs/xfs/xfs_message.h index 802a96190d22..4d9bd6bb63ca 100644 --- a/fs/xfs/xfs_message.h +++ b/fs/xfs/xfs_message.h @@ -79,4 +79,7 @@ void asswarn(struct xfs_mount *mp, char *expr, char *f, int l); extern void xfs_hex_dump(const void *p, int length); +void xfs_buf_alert_ratelimited(struct xfs_buf *bp, const char *rlmsg, + const char *fmt, ...); + #endif /* __XFS_MESSAGE_H */ From 61948b6fb276ad95ad63f82863e607719a31a628 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:25:21 -0700 Subject: [PATCH 0258/1043] xfs: ratelimit unmount time per-buffer I/O error alert At unmount time, XFS emits an alert for every in-core buffer that might have undergone a write error. In practice this behavior is probably reasonable given that the filesystem is likely short lived once I/O errors begin to occur consistently. Under certain test or otherwise expected error conditions, this can spam the logs and slow down the unmount. Now that we have a ratelimit mechanism specifically for buffer alerts, reuse it for the per-buffer alerts in xfs_wait_buftarg(). Also lift the final repair message out of the loop so it always prints and assert that the metadata error handling code has shut down the fs. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Allison Collins Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_buf.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 594d5e1df6f8..3918270f4eab 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -1657,7 +1657,8 @@ xfs_wait_buftarg( struct xfs_buftarg *btp) { LIST_HEAD(dispose); - int loop = 0; + int loop = 0; + bool write_fail = false; /* * First wait on the buftarg I/O count for all in-flight buffers to be @@ -1685,17 +1686,29 @@ xfs_wait_buftarg( bp = list_first_entry(&dispose, struct xfs_buf, b_lru); list_del_init(&bp->b_lru); if (bp->b_flags & XBF_WRITE_FAIL) { - xfs_alert(btp->bt_mount, + write_fail = true; + xfs_buf_alert_ratelimited(bp, + "XFS: Corruption Alert", "Corruption Alert: Buffer at daddr 0x%llx had permanent write failures!", (long long)bp->b_bn); - xfs_alert(btp->bt_mount, -"Please run xfs_repair to determine the extent of the problem."); } xfs_buf_rele(bp); } if (loop++ != 0) delay(100); } + + /* + * If one or more failed buffers were freed, that means dirty metadata + * was thrown away. This should only ever happen after I/O completion + * handling has elevated I/O error(s) to permanent failures and shuts + * down the fs. + */ + if (write_fail) { + ASSERT(XFS_FORCED_SHUTDOWN(btp->bt_mount)); + xfs_alert(btp->bt_mount, + "Please run xfs_repair to determine the extent of the problem."); + } } static enum lru_status From 629dcb38dc351947ed6a26a997d4b587f3bd5c7e Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:25:22 -0700 Subject: [PATCH 0259/1043] xfs: fix duplicate verification from xfs_qm_dqflush() The pre-flush dquot verification in xfs_qm_dqflush() duplicates the read verifier by checking the dquot in the on-disk buffer. Instead, verify the in-core variant before it is flushed to the buffer. Fixes: 7224fa482a6d ("xfs: add full xfs_dqblk verifier") Signed-off-by: Brian Foster Reviewed-by: Dave Chinner Reviewed-by: Christoph Hellwig Reviewed-by: Allison Collins Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_dquot.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index af2c8e5ceea0..265feb62290d 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -1116,13 +1116,12 @@ xfs_qm_dqflush( dqb = bp->b_addr + dqp->q_bufoffset; ddqp = &dqb->dd_diskdq; - /* - * A simple sanity check in case we got a corrupted dquot. - */ - fa = xfs_dqblk_verify(mp, dqb, be32_to_cpu(ddqp->d_id), 0); + /* sanity check the in-core structure before we flush */ + fa = xfs_dquot_verify(mp, &dqp->q_core, be32_to_cpu(dqp->q_core.d_id), + 0); if (fa) { xfs_alert(mp, "corrupt dquot ID 0x%x in memory at %pS", - be32_to_cpu(ddqp->d_id), fa); + be32_to_cpu(dqp->q_core.d_id), fa); xfs_buf_relse(bp); xfs_dqfunlock(dqp); xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); From b707fffda6a3e1d0a46fceaa4c3f6c06b90f448a Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:25:22 -0700 Subject: [PATCH 0260/1043] xfs: abort consistently on dquot flush failure The dquot flush handler effectively aborts the dquot flush if the filesystem is already shut down, but doesn't actually shut down if the flush fails. Update xfs_qm_dqflush() to consistently abort the dquot flush and shutdown the fs if the flush fails with an unexpected error. Signed-off-by: Brian Foster Reviewed-by: Dave Chinner Reviewed-by: Allison Collins Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_dquot.c | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 265feb62290d..ffe607733c50 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -1068,6 +1068,7 @@ xfs_qm_dqflush( struct xfs_buf **bpp) { struct xfs_mount *mp = dqp->q_mount; + struct xfs_log_item *lip = &dqp->q_logitem.qli_item; struct xfs_buf *bp; struct xfs_dqblk *dqb; struct xfs_disk_dquot *ddqp; @@ -1083,32 +1084,16 @@ xfs_qm_dqflush( xfs_qm_dqunpin_wait(dqp); - /* - * This may have been unpinned because the filesystem is shutting - * down forcibly. If that's the case we must not write this dquot - * to disk, because the log record didn't make it to disk. - * - * We also have to remove the log item from the AIL in this case, - * as we wait for an emptry AIL as part of the unmount process. - */ - if (XFS_FORCED_SHUTDOWN(mp)) { - struct xfs_log_item *lip = &dqp->q_logitem.qli_item; - dqp->dq_flags &= ~XFS_DQ_DIRTY; - - xfs_trans_ail_remove(lip, SHUTDOWN_CORRUPT_INCORE); - - error = -EIO; - goto out_unlock; - } - /* * Get the buffer containing the on-disk dquot */ error = xfs_trans_read_buf(mp, NULL, mp->m_ddev_targp, dqp->q_blkno, mp->m_quotainfo->qi_dqchunklen, XBF_TRYLOCK, &bp, &xfs_dquot_buf_ops); - if (error) + if (error == -EAGAIN) goto out_unlock; + if (error) + goto out_abort; /* * Calculate the location of the dquot inside the buffer. @@ -1123,9 +1108,8 @@ xfs_qm_dqflush( xfs_alert(mp, "corrupt dquot ID 0x%x in memory at %pS", be32_to_cpu(dqp->q_core.d_id), fa); xfs_buf_relse(bp); - xfs_dqfunlock(dqp); - xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); - return -EFSCORRUPTED; + error = -EFSCORRUPTED; + goto out_abort; } /* This is the only portion of data that needs to persist */ @@ -1174,6 +1158,10 @@ xfs_qm_dqflush( *bpp = bp; return 0; +out_abort: + dqp->dq_flags &= ~XFS_DQ_DIRTY; + xfs_trans_ail_remove(lip, SHUTDOWN_CORRUPT_INCORE); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); out_unlock: xfs_dqfunlock(dqp); return error; From 849274c103aeb149f23a81ba4a6bab42a5d77a78 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:25:23 -0700 Subject: [PATCH 0261/1043] xfs: acquire ->ail_lock from xfs_trans_ail_delete() Several callers acquire the lock just prior to the call. Callers that require ->ail_lock for other purposes already check IN_AIL state and thus don't require the additional shutdown check in the helper. Push the lock down into xfs_trans_ail_delete(), open code the instances that still acquire it, and remove the unnecessary ailp parameter. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Allison Collins Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_buf_item.c | 27 +++++++++++---------------- fs/xfs/xfs_dquot.c | 6 ++++-- fs/xfs/xfs_trans_ail.c | 3 ++- fs/xfs/xfs_trans_priv.h | 14 ++++++++------ 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c index 1f7acffc99ba..06e306b49283 100644 --- a/fs/xfs/xfs_buf_item.c +++ b/fs/xfs/xfs_buf_item.c @@ -410,7 +410,6 @@ xfs_buf_item_unpin( { struct xfs_buf_log_item *bip = BUF_ITEM(lip); xfs_buf_t *bp = bip->bli_buf; - struct xfs_ail *ailp = lip->li_ailp; int stale = bip->bli_flags & XFS_BLI_STALE; int freed; @@ -452,10 +451,10 @@ xfs_buf_item_unpin( } /* - * If we get called here because of an IO error, we may - * or may not have the item on the AIL. xfs_trans_ail_delete() - * will take care of that situation. - * xfs_trans_ail_delete() drops the AIL lock. + * If we get called here because of an IO error, we may or may + * not have the item on the AIL. xfs_trans_ail_delete() will + * take care of that situation. xfs_trans_ail_delete() drops + * the AIL lock. */ if (bip->bli_flags & XFS_BLI_STALE_INODE) { xfs_buf_do_callbacks(bp); @@ -463,8 +462,7 @@ xfs_buf_item_unpin( list_del_init(&bp->b_li_list); bp->b_iodone = NULL; } else { - spin_lock(&ailp->ail_lock); - xfs_trans_ail_delete(ailp, lip, SHUTDOWN_LOG_IO_ERROR); + xfs_trans_ail_delete(lip, SHUTDOWN_LOG_IO_ERROR); xfs_buf_item_relse(bp); ASSERT(bp->b_log_item == NULL); } @@ -1205,22 +1203,19 @@ xfs_buf_iodone( struct xfs_buf *bp, struct xfs_log_item *lip) { - struct xfs_ail *ailp = lip->li_ailp; - ASSERT(BUF_ITEM(lip)->bli_buf == bp); xfs_buf_rele(bp); /* - * If we are forcibly shutting down, this may well be - * off the AIL already. That's because we simulate the - * log-committed callbacks to unpin these buffers. Or we may never - * have put this item on AIL because of the transaction was - * aborted forcibly. xfs_trans_ail_delete() takes care of these. + * If we are forcibly shutting down, this may well be off the AIL + * already. That's because we simulate the log-committed callbacks to + * unpin these buffers. Or we may never have put this item on AIL + * because of the transaction was aborted forcibly. + * xfs_trans_ail_delete() takes care of these. * * Either way, AIL is useless if we're forcing a shutdown. */ - spin_lock(&ailp->ail_lock); - xfs_trans_ail_delete(ailp, lip, SHUTDOWN_CORRUPT_INCORE); + xfs_trans_ail_delete(lip, SHUTDOWN_CORRUPT_INCORE); xfs_buf_item_free(BUF_ITEM(lip)); } diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index ffe607733c50..5fb65f43b980 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -1021,6 +1021,7 @@ xfs_qm_dqflush_done( struct xfs_dq_logitem *qip = (struct xfs_dq_logitem *)lip; struct xfs_dquot *dqp = qip->qli_dquot; struct xfs_ail *ailp = lip->li_ailp; + xfs_lsn_t tail_lsn; /* * We only want to pull the item from the AIL if its @@ -1034,10 +1035,11 @@ xfs_qm_dqflush_done( ((lip->li_lsn == qip->qli_flush_lsn) || test_bit(XFS_LI_FAILED, &lip->li_flags))) { - /* xfs_trans_ail_delete() drops the AIL lock. */ spin_lock(&ailp->ail_lock); if (lip->li_lsn == qip->qli_flush_lsn) { - xfs_trans_ail_delete(ailp, lip, SHUTDOWN_CORRUPT_INCORE); + /* xfs_ail_update_finish() drops the AIL lock */ + tail_lsn = xfs_ail_delete_one(ailp, lip); + xfs_ail_update_finish(ailp, tail_lsn); } else { /* * Clear the failed state since we are about to drop the diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c index 2574d01e4a83..cfba691664c7 100644 --- a/fs/xfs/xfs_trans_ail.c +++ b/fs/xfs/xfs_trans_ail.c @@ -864,13 +864,14 @@ xfs_ail_delete_one( */ void xfs_trans_ail_delete( - struct xfs_ail *ailp, struct xfs_log_item *lip, int shutdown_type) { + struct xfs_ail *ailp = lip->li_ailp; struct xfs_mount *mp = ailp->ail_mount; xfs_lsn_t tail_lsn; + spin_lock(&ailp->ail_lock); if (!test_bit(XFS_LI_IN_AIL, &lip->li_flags)) { spin_unlock(&ailp->ail_lock); if (!XFS_FORCED_SHUTDOWN(mp)) { diff --git a/fs/xfs/xfs_trans_priv.h b/fs/xfs/xfs_trans_priv.h index 35655eac01a6..e4362fb8d483 100644 --- a/fs/xfs/xfs_trans_priv.h +++ b/fs/xfs/xfs_trans_priv.h @@ -94,8 +94,7 @@ xfs_trans_ail_update( xfs_lsn_t xfs_ail_delete_one(struct xfs_ail *ailp, struct xfs_log_item *lip); void xfs_ail_update_finish(struct xfs_ail *ailp, xfs_lsn_t old_lsn) __releases(ailp->ail_lock); -void xfs_trans_ail_delete(struct xfs_ail *ailp, struct xfs_log_item *lip, - int shutdown_type); +void xfs_trans_ail_delete(struct xfs_log_item *lip, int shutdown_type); static inline void xfs_trans_ail_remove( @@ -103,13 +102,16 @@ xfs_trans_ail_remove( int shutdown_type) { struct xfs_ail *ailp = lip->li_ailp; + xfs_lsn_t tail_lsn; spin_lock(&ailp->ail_lock); - /* xfs_trans_ail_delete() drops the AIL lock */ - if (test_bit(XFS_LI_IN_AIL, &lip->li_flags)) - xfs_trans_ail_delete(ailp, lip, shutdown_type); - else + /* xfs_ail_update_finish() drops the AIL lock */ + if (test_bit(XFS_LI_IN_AIL, &lip->li_flags)) { + tail_lsn = xfs_ail_delete_one(ailp, lip); + xfs_ail_update_finish(ailp, tail_lsn); + } else { spin_unlock(&ailp->ail_lock); + } } void xfs_ail_push(struct xfs_ail *, xfs_lsn_t); From 655879290c28bc5678a30a7b41c9b61f4f7f90c2 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:25:23 -0700 Subject: [PATCH 0262/1043] xfs: use delete helper for items expected to be in AIL Various intent log items call xfs_trans_ail_remove() with a log I/O error shutdown type, but this helper historically checks whether an item is in the AIL before calling xfs_trans_ail_delete(). This means the shutdown check is essentially a no-op for users of xfs_trans_ail_remove(). It is possible that some items might not be AIL resident when the AIL remove attempt occurs, but this should be isolated to cases where the filesystem has already shutdown. For example, this includes abort of the transaction committing the intent and I/O error of the iclog buffer committing the intent to the log. Therefore, update these callsites to use xfs_trans_ail_delete() to provide AIL state validation for the common path of items being released and removed when associated done items commit to the physical log. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Allison Collins Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_bmap_item.c | 2 +- fs/xfs/xfs_extfree_item.c | 2 +- fs/xfs/xfs_refcount_item.c | 2 +- fs/xfs/xfs_rmap_item.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index 7768fb2b7135..17eb7cfad5d9 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -51,7 +51,7 @@ xfs_bui_release( { ASSERT(atomic_read(&buip->bui_refcount) > 0); if (atomic_dec_and_test(&buip->bui_refcount)) { - xfs_trans_ail_remove(&buip->bui_item, SHUTDOWN_LOG_IO_ERROR); + xfs_trans_ail_delete(&buip->bui_item, SHUTDOWN_LOG_IO_ERROR); xfs_bui_item_free(buip); } } diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index c8cde4122a0f..9809637fb84d 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -55,7 +55,7 @@ xfs_efi_release( { ASSERT(atomic_read(&efip->efi_refcount) > 0); if (atomic_dec_and_test(&efip->efi_refcount)) { - xfs_trans_ail_remove(&efip->efi_item, SHUTDOWN_LOG_IO_ERROR); + xfs_trans_ail_delete(&efip->efi_item, SHUTDOWN_LOG_IO_ERROR); xfs_efi_item_free(efip); } } diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index 0316eab2fc35..01bb77daeaee 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -50,7 +50,7 @@ xfs_cui_release( { ASSERT(atomic_read(&cuip->cui_refcount) > 0); if (atomic_dec_and_test(&cuip->cui_refcount)) { - xfs_trans_ail_remove(&cuip->cui_item, SHUTDOWN_LOG_IO_ERROR); + xfs_trans_ail_delete(&cuip->cui_item, SHUTDOWN_LOG_IO_ERROR); xfs_cui_item_free(cuip); } } diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index e3bba2aec868..fdb12b01b178 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -50,7 +50,7 @@ xfs_rui_release( { ASSERT(atomic_read(&ruip->rui_refcount) > 0); if (atomic_dec_and_test(&ruip->rui_refcount)) { - xfs_trans_ail_remove(&ruip->rui_item, SHUTDOWN_LOG_IO_ERROR); + xfs_trans_ail_delete(&ruip->rui_item, SHUTDOWN_LOG_IO_ERROR); xfs_rui_item_free(ruip); } } From 6af0479d8b6b162b7f006581553705521af9984b Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:25:50 -0700 Subject: [PATCH 0263/1043] xfs: drop unused shutdown parameter from xfs_trans_ail_remove() The shutdown parameter of xfs_trans_ail_remove() is no longer used. The remaining callers use it for items that legitimately might not be in the AIL or from contexts where AIL state has already been checked. Remove the unnecessary parameter and fix up the callers. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Allison Collins Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_buf_item.c | 2 +- fs/xfs/xfs_dquot.c | 2 +- fs/xfs/xfs_dquot_item.c | 2 +- fs/xfs/xfs_inode_item.c | 6 +----- fs/xfs/xfs_trans_priv.h | 3 +-- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c index 06e306b49283..47c547aca1f1 100644 --- a/fs/xfs/xfs_buf_item.c +++ b/fs/xfs/xfs_buf_item.c @@ -558,7 +558,7 @@ xfs_buf_item_put( * state. */ if (aborted) - xfs_trans_ail_remove(lip, SHUTDOWN_LOG_IO_ERROR); + xfs_trans_ail_remove(lip); xfs_buf_item_relse(bip->bli_buf); return true; } diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 5fb65f43b980..497a9dbef1c9 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -1162,7 +1162,7 @@ xfs_qm_dqflush( out_abort: dqp->dq_flags &= ~XFS_DQ_DIRTY; - xfs_trans_ail_remove(lip, SHUTDOWN_CORRUPT_INCORE); + xfs_trans_ail_remove(lip); xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); out_unlock: xfs_dqfunlock(dqp); diff --git a/fs/xfs/xfs_dquot_item.c b/fs/xfs/xfs_dquot_item.c index 5a7808299a32..8bd46810d5db 100644 --- a/fs/xfs/xfs_dquot_item.c +++ b/fs/xfs/xfs_dquot_item.c @@ -343,7 +343,7 @@ xfs_qm_qoff_logitem_relse( ASSERT(test_bit(XFS_LI_IN_AIL, &lip->li_flags) || test_bit(XFS_LI_ABORTED, &lip->li_flags) || XFS_FORCED_SHUTDOWN(lip->li_mountp)); - xfs_trans_ail_remove(lip, SHUTDOWN_LOG_IO_ERROR); + xfs_trans_ail_remove(lip); kmem_free(lip->li_lv_shadow); kmem_free(qoff); } diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index a4027f4ca6c4..73a37b18ab3e 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -768,11 +768,7 @@ xfs_iflush_abort( struct xfs_inode_log_item *iip = ip->i_itemp; if (iip) { - if (test_bit(XFS_LI_IN_AIL, &iip->ili_item.li_flags)) { - xfs_trans_ail_remove(&iip->ili_item, - stale ? SHUTDOWN_LOG_IO_ERROR : - SHUTDOWN_CORRUPT_INCORE); - } + xfs_trans_ail_remove(&iip->ili_item); iip->ili_logged = 0; /* * Clear the ili_last_fields bits now that we know that the diff --git a/fs/xfs/xfs_trans_priv.h b/fs/xfs/xfs_trans_priv.h index e4362fb8d483..ab0a82e90825 100644 --- a/fs/xfs/xfs_trans_priv.h +++ b/fs/xfs/xfs_trans_priv.h @@ -98,8 +98,7 @@ void xfs_trans_ail_delete(struct xfs_log_item *lip, int shutdown_type); static inline void xfs_trans_ail_remove( - struct xfs_log_item *lip, - int shutdown_type) + struct xfs_log_item *lip) { struct xfs_ail *ailp = lip->li_ailp; xfs_lsn_t tail_lsn; From 2b3cf09356d54711b6afdc7694b382c379ea42c4 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:27:04 -0700 Subject: [PATCH 0264/1043] xfs: combine xfs_trans_ail_[remove|delete]() Now that the functions and callers of xfs_trans_ail_[remove|delete]() have been fixed up appropriately, the only difference between the two is the shutdown behavior. There are only a few callers of the _remove() variant, so make the shutdown conditional on the parameter and combine the two functions. Suggested-by: Dave Chinner Signed-off-by: Brian Foster Reviewed-by: Dave Chinner Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Allison Collins Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_buf_item.c | 2 +- fs/xfs/xfs_dquot.c | 2 +- fs/xfs/xfs_dquot_item.c | 2 +- fs/xfs/xfs_inode_item.c | 2 +- fs/xfs/xfs_trans_ail.c | 24 ++---------------------- fs/xfs/xfs_trans_priv.h | 17 ----------------- 6 files changed, 6 insertions(+), 43 deletions(-) diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c index 47c547aca1f1..9e75e8d6042e 100644 --- a/fs/xfs/xfs_buf_item.c +++ b/fs/xfs/xfs_buf_item.c @@ -558,7 +558,7 @@ xfs_buf_item_put( * state. */ if (aborted) - xfs_trans_ail_remove(lip); + xfs_trans_ail_delete(lip, 0); xfs_buf_item_relse(bip->bli_buf); return true; } diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 497a9dbef1c9..52e0f7245afc 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -1162,7 +1162,7 @@ xfs_qm_dqflush( out_abort: dqp->dq_flags &= ~XFS_DQ_DIRTY; - xfs_trans_ail_remove(lip); + xfs_trans_ail_delete(lip, 0); xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); out_unlock: xfs_dqfunlock(dqp); diff --git a/fs/xfs/xfs_dquot_item.c b/fs/xfs/xfs_dquot_item.c index 8bd46810d5db..349c92d26570 100644 --- a/fs/xfs/xfs_dquot_item.c +++ b/fs/xfs/xfs_dquot_item.c @@ -343,7 +343,7 @@ xfs_qm_qoff_logitem_relse( ASSERT(test_bit(XFS_LI_IN_AIL, &lip->li_flags) || test_bit(XFS_LI_ABORTED, &lip->li_flags) || XFS_FORCED_SHUTDOWN(lip->li_mountp)); - xfs_trans_ail_remove(lip); + xfs_trans_ail_delete(lip, 0); kmem_free(lip->li_lv_shadow); kmem_free(qoff); } diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index 73a37b18ab3e..664163be5778 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -768,7 +768,7 @@ xfs_iflush_abort( struct xfs_inode_log_item *iip = ip->i_itemp; if (iip) { - xfs_trans_ail_remove(&iip->ili_item); + xfs_trans_ail_delete(&iip->ili_item, 0); iip->ili_logged = 0; /* * Clear the ili_last_fields bits now that we know that the diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c index cfba691664c7..bf09d4b4df58 100644 --- a/fs/xfs/xfs_trans_ail.c +++ b/fs/xfs/xfs_trans_ail.c @@ -841,27 +841,6 @@ xfs_ail_delete_one( return 0; } -/** - * Remove a log items from the AIL - * - * @xfs_trans_ail_delete_bulk takes an array of log items that all need to - * removed from the AIL. The caller is already holding the AIL lock, and done - * all the checks necessary to ensure the items passed in via @log_items are - * ready for deletion. This includes checking that the items are in the AIL. - * - * For each log item to be removed, unlink it from the AIL, clear the IN_AIL - * flag from the item and reset the item's lsn to 0. If we remove the first - * item in the AIL, update the log tail to match the new minimum LSN in the - * AIL. - * - * This function will not drop the AIL lock until all items are removed from - * the AIL to minimise the amount of lock traffic on the AIL. This does not - * greatly increase the AIL hold time, but does significantly reduce the amount - * of traffic on the lock, especially during IO completion. - * - * This function must be called with the AIL lock held. The lock is dropped - * before returning. - */ void xfs_trans_ail_delete( struct xfs_log_item *lip, @@ -874,7 +853,7 @@ xfs_trans_ail_delete( spin_lock(&ailp->ail_lock); if (!test_bit(XFS_LI_IN_AIL, &lip->li_flags)) { spin_unlock(&ailp->ail_lock); - if (!XFS_FORCED_SHUTDOWN(mp)) { + if (shutdown_type && !XFS_FORCED_SHUTDOWN(mp)) { xfs_alert_tag(mp, XFS_PTAG_AILDELETE, "%s: attempting to delete a log item that is not in the AIL", __func__); @@ -883,6 +862,7 @@ xfs_trans_ail_delete( return; } + /* xfs_ail_update_finish() drops the AIL lock */ tail_lsn = xfs_ail_delete_one(ailp, lip); xfs_ail_update_finish(ailp, tail_lsn); } diff --git a/fs/xfs/xfs_trans_priv.h b/fs/xfs/xfs_trans_priv.h index ab0a82e90825..cc046d9557ae 100644 --- a/fs/xfs/xfs_trans_priv.h +++ b/fs/xfs/xfs_trans_priv.h @@ -96,23 +96,6 @@ void xfs_ail_update_finish(struct xfs_ail *ailp, xfs_lsn_t old_lsn) __releases(ailp->ail_lock); void xfs_trans_ail_delete(struct xfs_log_item *lip, int shutdown_type); -static inline void -xfs_trans_ail_remove( - struct xfs_log_item *lip) -{ - struct xfs_ail *ailp = lip->li_ailp; - xfs_lsn_t tail_lsn; - - spin_lock(&ailp->ail_lock); - /* xfs_ail_update_finish() drops the AIL lock */ - if (test_bit(XFS_LI_IN_AIL, &lip->li_flags)) { - tail_lsn = xfs_ail_delete_one(ailp, lip); - xfs_ail_update_finish(ailp, tail_lsn); - } else { - spin_unlock(&ailp->ail_lock); - } -} - void xfs_ail_push(struct xfs_ail *, xfs_lsn_t); void xfs_ail_push_all(struct xfs_ail *); void xfs_ail_push_all_sync(struct xfs_ail *); From 88fc187984c968c02b3b41b27049e52a70b2d941 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:27:40 -0700 Subject: [PATCH 0265/1043] xfs: remove unused iflush stale parameter The stale parameter was used to control the now unused shutdown parameter of xfs_trans_ail_remove(). Signed-off-by: Brian Foster Reviewed-by: Dave Chinner Reviewed-by: Allison Collins Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_icache.c | 2 +- fs/xfs/xfs_inode.c | 2 +- fs/xfs/xfs_inode_item.c | 7 +++---- fs/xfs/xfs_inode_item.h | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 17a0b86fe701..922a29032e37 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1126,7 +1126,7 @@ restart: if (XFS_FORCED_SHUTDOWN(ip->i_mount)) { xfs_iunpin_wait(ip); /* xfs_iflush_abort() drops the flush lock */ - xfs_iflush_abort(ip, false); + xfs_iflush_abort(ip); goto reclaim; } if (xfs_ipincount(ip)) { diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index d1f7da2fc2ec..e094afe28439 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -3702,7 +3702,7 @@ xfs_iflush( return 0; abort: - xfs_iflush_abort(ip, false); + xfs_iflush_abort(ip); shutdown: xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); return error; diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index 664163be5778..cefa2484f0db 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -762,10 +762,9 @@ xfs_iflush_done( */ void xfs_iflush_abort( - xfs_inode_t *ip, - bool stale) + struct xfs_inode *ip) { - struct xfs_inode_log_item *iip = ip->i_itemp; + struct xfs_inode_log_item *iip = ip->i_itemp; if (iip) { xfs_trans_ail_delete(&iip->ili_item, 0); @@ -793,7 +792,7 @@ xfs_istale_done( struct xfs_buf *bp, struct xfs_log_item *lip) { - xfs_iflush_abort(INODE_ITEM(lip)->ili_inode, true); + xfs_iflush_abort(INODE_ITEM(lip)->ili_inode); } /* diff --git a/fs/xfs/xfs_inode_item.h b/fs/xfs/xfs_inode_item.h index ad667fd4ae62..60b34bb66e8e 100644 --- a/fs/xfs/xfs_inode_item.h +++ b/fs/xfs/xfs_inode_item.h @@ -34,7 +34,7 @@ extern void xfs_inode_item_init(struct xfs_inode *, struct xfs_mount *); extern void xfs_inode_item_destroy(struct xfs_inode *); extern void xfs_iflush_done(struct xfs_buf *, struct xfs_log_item *); extern void xfs_istale_done(struct xfs_buf *, struct xfs_log_item *); -extern void xfs_iflush_abort(struct xfs_inode *, bool); +extern void xfs_iflush_abort(struct xfs_inode *); extern int xfs_inode_item_format_convert(xfs_log_iovec_t *, struct xfs_inode_log_format *); From 7376d74547344598008d00419eae0caa5f50f4f0 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:29:19 -0700 Subject: [PATCH 0266/1043] xfs: random buffer write failure errortag Introduce an error tag to randomly fail async buffer writes. This is primarily to facilitate testing of the XFS error configuration mechanism. Signed-off-by: Brian Foster Reviewed-by: Allison Collins Reviewed-by: Dave Chinner Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_errortag.h | 4 +++- fs/xfs/xfs_buf.c | 6 ++++++ fs/xfs/xfs_error.c | 3 +++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_errortag.h b/fs/xfs/libxfs/xfs_errortag.h index 79e6c4fb1d8a..2486dab19023 100644 --- a/fs/xfs/libxfs/xfs_errortag.h +++ b/fs/xfs/libxfs/xfs_errortag.h @@ -55,7 +55,8 @@ #define XFS_ERRTAG_FORCE_SCRUB_REPAIR 32 #define XFS_ERRTAG_FORCE_SUMMARY_RECALC 33 #define XFS_ERRTAG_IUNLINK_FALLBACK 34 -#define XFS_ERRTAG_MAX 35 +#define XFS_ERRTAG_BUF_IOERROR 35 +#define XFS_ERRTAG_MAX 36 /* * Random factors for above tags, 1 means always, 2 means 1/2 time, etc. @@ -95,5 +96,6 @@ #define XFS_RANDOM_FORCE_SCRUB_REPAIR 1 #define XFS_RANDOM_FORCE_SUMMARY_RECALC 1 #define XFS_RANDOM_IUNLINK_FALLBACK (XFS_RANDOM_DEFAULT/10) +#define XFS_RANDOM_BUF_IOERROR XFS_RANDOM_DEFAULT #endif /* __XFS_ERRORTAG_H_ */ diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 3918270f4eab..9d8841ac7375 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -1289,6 +1289,12 @@ xfs_buf_bio_end_io( struct bio *bio) { struct xfs_buf *bp = (struct xfs_buf *)bio->bi_private; + struct xfs_mount *mp = bp->b_mount; + + if (!bio->bi_status && + (bp->b_flags & XBF_WRITE) && (bp->b_flags & XBF_ASYNC) && + XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BUF_IOERROR)) + bio->bi_status = BLK_STS_IOERR; /* * don't overwrite existing errors - otherwise we can lose errors on diff --git a/fs/xfs/xfs_error.c b/fs/xfs/xfs_error.c index a21e9cc6516a..7f6e20899473 100644 --- a/fs/xfs/xfs_error.c +++ b/fs/xfs/xfs_error.c @@ -53,6 +53,7 @@ static unsigned int xfs_errortag_random_default[] = { XFS_RANDOM_FORCE_SCRUB_REPAIR, XFS_RANDOM_FORCE_SUMMARY_RECALC, XFS_RANDOM_IUNLINK_FALLBACK, + XFS_RANDOM_BUF_IOERROR, }; struct xfs_errortag_attr { @@ -162,6 +163,7 @@ XFS_ERRORTAG_ATTR_RW(buf_lru_ref, XFS_ERRTAG_BUF_LRU_REF); XFS_ERRORTAG_ATTR_RW(force_repair, XFS_ERRTAG_FORCE_SCRUB_REPAIR); XFS_ERRORTAG_ATTR_RW(bad_summary, XFS_ERRTAG_FORCE_SUMMARY_RECALC); XFS_ERRORTAG_ATTR_RW(iunlink_fallback, XFS_ERRTAG_IUNLINK_FALLBACK); +XFS_ERRORTAG_ATTR_RW(buf_ioerror, XFS_ERRTAG_BUF_IOERROR); static struct attribute *xfs_errortag_attrs[] = { XFS_ERRORTAG_ATTR_LIST(noerror), @@ -199,6 +201,7 @@ static struct attribute *xfs_errortag_attrs[] = { XFS_ERRORTAG_ATTR_LIST(force_repair), XFS_ERRORTAG_ATTR_LIST(bad_summary), XFS_ERRORTAG_ATTR_LIST(iunlink_fallback), + XFS_ERRORTAG_ATTR_LIST(buf_ioerror), NULL, }; From 28d84620797e04e983bcd515ac0a6146396b0e31 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:29:19 -0700 Subject: [PATCH 0267/1043] xfs: remove unused shutdown types Both types control shutdown messaging and neither is used in the current codebase. Signed-off-by: Brian Foster Reviewed-by: Dave Chinner Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Reviewed-by: Allison Collins Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_fsops.c | 5 +---- fs/xfs/xfs_mount.h | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/fs/xfs/xfs_fsops.c b/fs/xfs/xfs_fsops.c index 3e61d0cc23f8..ef1d5bb88b93 100644 --- a/fs/xfs/xfs_fsops.c +++ b/fs/xfs/xfs_fsops.c @@ -504,10 +504,7 @@ xfs_do_force_shutdown( } else if (logerror) { xfs_alert_tag(mp, XFS_PTAG_SHUTDOWN_LOGERROR, "Log I/O Error Detected. Shutting down filesystem"); - } else if (flags & SHUTDOWN_DEVICE_REQ) { - xfs_alert_tag(mp, XFS_PTAG_SHUTDOWN_IOERROR, - "All device paths lost. Shutting down filesystem"); - } else if (!(flags & SHUTDOWN_REMOTE_REQ)) { + } else { xfs_alert_tag(mp, XFS_PTAG_SHUTDOWN_IOERROR, "I/O Error Detected. Shutting down filesystem"); } diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index 37bfb50db809..aba5a1579279 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -259,8 +259,6 @@ void xfs_do_force_shutdown(struct xfs_mount *mp, int flags, char *fname, #define SHUTDOWN_LOG_IO_ERROR 0x0002 /* write attempt to the log failed */ #define SHUTDOWN_FORCE_UMOUNT 0x0004 /* shutdown from a forced unmount */ #define SHUTDOWN_CORRUPT_INCORE 0x0008 /* corrupt in-memory data structures */ -#define SHUTDOWN_REMOTE_REQ 0x0010 /* shutdown came from remote cell */ -#define SHUTDOWN_DEVICE_REQ 0x0020 /* failed all paths to the device */ /* * Flags for xfs_mountfs From c199507993ede3f63d0deae7e2cbc2f5462c6452 Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Wed, 6 May 2020 13:29:20 -0700 Subject: [PATCH 0268/1043] xfs: remove unused iget_flags param from xfs_imap_to_bp() iget_flags is unused in xfs_imap_to_bp(). Remove the parameter and fix up the callers. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Allison Collins Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_buf.c | 5 ++--- fs/xfs/libxfs/xfs_inode_buf.h | 2 +- fs/xfs/scrub/ialloc.c | 3 +-- fs/xfs/xfs_inode.c | 7 +++---- fs/xfs/xfs_log_recover.c | 2 +- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index b102e611bf54..81a010422bea 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -161,8 +161,7 @@ xfs_imap_to_bp( struct xfs_imap *imap, struct xfs_dinode **dipp, struct xfs_buf **bpp, - uint buf_flags, - uint iget_flags) + uint buf_flags) { struct xfs_buf *bp; int error; @@ -621,7 +620,7 @@ xfs_iread( /* * Get pointers to the on-disk inode and the buffer containing it. */ - error = xfs_imap_to_bp(mp, tp, &ip->i_imap, &dip, &bp, 0, iget_flags); + error = xfs_imap_to_bp(mp, tp, &ip->i_imap, &dip, &bp, 0); if (error) return error; diff --git a/fs/xfs/libxfs/xfs_inode_buf.h b/fs/xfs/libxfs/xfs_inode_buf.h index 9b373dcf9e34..d9b4781ac9fd 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.h +++ b/fs/xfs/libxfs/xfs_inode_buf.h @@ -48,7 +48,7 @@ struct xfs_imap { int xfs_imap_to_bp(struct xfs_mount *, struct xfs_trans *, struct xfs_imap *, struct xfs_dinode **, - struct xfs_buf **, uint, uint); + struct xfs_buf **, uint); int xfs_iread(struct xfs_mount *, struct xfs_trans *, struct xfs_inode *, uint); void xfs_dinode_calc_crc(struct xfs_mount *, struct xfs_dinode *); diff --git a/fs/xfs/scrub/ialloc.c b/fs/xfs/scrub/ialloc.c index 64c217eb06a7..6517d67e8d51 100644 --- a/fs/xfs/scrub/ialloc.c +++ b/fs/xfs/scrub/ialloc.c @@ -278,8 +278,7 @@ xchk_iallocbt_check_cluster( &XFS_RMAP_OINFO_INODES); /* Grab the inode cluster buffer. */ - error = xfs_imap_to_bp(mp, bs->cur->bc_tp, &imap, &dip, &cluster_bp, - 0, 0); + error = xfs_imap_to_bp(mp, bs->cur->bc_tp, &imap, &dip, &cluster_bp, 0); if (!xchk_btree_xref_process_error(bs->sc, bs->cur, 0, &error)) return error; diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index e094afe28439..ab31a5dec7aa 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -2172,7 +2172,7 @@ xfs_iunlink_update_inode( ASSERT(xfs_verify_agino_or_null(mp, agno, next_agino)); - error = xfs_imap_to_bp(mp, tp, &ip->i_imap, &dip, &ibp, 0, 0); + error = xfs_imap_to_bp(mp, tp, &ip->i_imap, &dip, &ibp, 0); if (error) return error; @@ -2302,7 +2302,7 @@ xfs_iunlink_map_ino( return error; } - error = xfs_imap_to_bp(mp, tp, imap, dipp, bpp, 0, 0); + error = xfs_imap_to_bp(mp, tp, imap, dipp, bpp, 0); if (error) { xfs_warn(mp, "%s: xfs_imap_to_bp returned error %d.", __func__, error); @@ -3665,8 +3665,7 @@ xfs_iflush( * If we get any other error, we effectively have a corruption situation * and we cannot flush the inode. Abort the flush and shut down. */ - error = xfs_imap_to_bp(mp, NULL, &ip->i_imap, &dip, &bp, XBF_TRYLOCK, - 0); + error = xfs_imap_to_bp(mp, NULL, &ip->i_imap, &dip, &bp, XBF_TRYLOCK); if (error == -EAGAIN) { xfs_ifunlock(ip); return error; diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index d0e2dd81de53..320785115833 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -4969,7 +4969,7 @@ xlog_recover_process_one_iunlink( /* * Get the on disk inode to find the next inode in the bucket. */ - error = xfs_imap_to_bp(mp, NULL, &ip->i_imap, &dip, &ibp, 0, 0); + error = xfs_imap_to_bp(mp, NULL, &ip->i_imap, &dip, &ibp, 0); if (error) goto fail_iput; From 0090c1edebf464f34629e14ae03d764cca7e0a3b Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 7 May 2020 13:50:41 -0500 Subject: [PATCH 0269/1043] audit: Replace zero-length array with flexible-array The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] sizeof(flexible-array-member) triggers a warning because flexible array members have incomplete type[1]. There are some instances of code in which the sizeof operator is being incorrectly/erroneously applied to zero-length arrays and the result is zero. Such instances may be hiding some bugs. So, this work (flexible-array member conversions) will also help to get completely rid of those sorts of issues. This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Reviewed-by: Richard Guy Briggs Signed-off-by: Paul Moore --- include/linux/audit.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/audit.h b/include/linux/audit.h index 1a2351508211..3fcd9ee49734 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -19,7 +19,7 @@ struct audit_sig_info { uid_t uid; pid_t pid; - char ctx[0]; + char ctx[]; }; struct audit_buffer; From e701656ec4db60f560f7b790235fe94819837db3 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Fri, 8 May 2020 17:30:12 +0800 Subject: [PATCH 0270/1043] MIPS: inst.h: Stop including asm.h to avoid various build failures Commit d339cd02b888eb8 ("MIPS: Move unaligned load/store helpers to inst.h") causes a lot of build failures because macros in asm.h conflict with various subsystems. Some of these conflictions has been fixed (such as LONG, PANIC and PRINT) by adjusting asm.h, but some of them is nearly impossible to fix (such as PTR and END). The only reason of including asm.h in inst.h is that we need the PTR macro which is used by unaligned load/store helpers. So in this patch we define a new PTR_STR macro and use it to replace STR(PTR), then we can stop including asm.h to avoid various build failures. Fixes: d339cd02b888eb8 ("MIPS: Move unaligned load/store helpers to inst.h") Reported-by: kbuild test robot Signed-off-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/inst.h | 184 ++++++++++++++++++----------------- 1 file changed, 95 insertions(+), 89 deletions(-) diff --git a/arch/mips/include/asm/inst.h b/arch/mips/include/asm/inst.h index 82545454ad3e..6d74ba33b923 100644 --- a/arch/mips/include/asm/inst.h +++ b/arch/mips/include/asm/inst.h @@ -11,9 +11,15 @@ #ifndef _ASM_INST_H #define _ASM_INST_H -#include #include +#if (_MIPS_SZPTR == 32) +#define PTR_STR ".word" +#endif +#if (_MIPS_SZPTR == 64) +#define PTR_STR ".dword" +#endif + /* HACHACHAHCAHC ... */ /* In case some other massaging is needed, keep MIPSInst as wrapper */ @@ -102,8 +108,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -123,8 +129,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -156,10 +162,10 @@ do { \ "j\t10b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ + PTR_STR"\t1b, 11b\n\t" \ + PTR_STR"\t2b, 11b\n\t" \ + PTR_STR"\t3b, 11b\n\t" \ + PTR_STR"\t4b, 11b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -184,8 +190,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -207,8 +213,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -227,8 +233,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -260,10 +266,10 @@ do { \ "j\t10b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ + PTR_STR"\t1b, 11b\n\t" \ + PTR_STR"\t2b, 11b\n\t" \ + PTR_STR"\t3b, 11b\n\t" \ + PTR_STR"\t4b, 11b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -305,14 +311,14 @@ do { \ "j\t10b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ - STR(PTR)"\t5b, 11b\n\t" \ - STR(PTR)"\t6b, 11b\n\t" \ - STR(PTR)"\t7b, 11b\n\t" \ - STR(PTR)"\t8b, 11b\n\t" \ + PTR_STR"\t1b, 11b\n\t" \ + PTR_STR"\t2b, 11b\n\t" \ + PTR_STR"\t3b, 11b\n\t" \ + PTR_STR"\t4b, 11b\n\t" \ + PTR_STR"\t5b, 11b\n\t" \ + PTR_STR"\t6b, 11b\n\t" \ + PTR_STR"\t7b, 11b\n\t" \ + PTR_STR"\t8b, 11b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -337,8 +343,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=r" (res) \ : "r" (value), "r" (addr), "i" (-EFAULT));\ @@ -358,8 +364,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=r" (res) \ : "r" (value), "r" (addr), "i" (-EFAULT)); \ @@ -378,8 +384,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=r" (res) \ : "r" (value), "r" (addr), "i" (-EFAULT)); \ @@ -407,10 +413,10 @@ do { \ "j\t10b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ + PTR_STR"\t1b, 11b\n\t" \ + PTR_STR"\t2b, 11b\n\t" \ + PTR_STR"\t3b, 11b\n\t" \ + PTR_STR"\t4b, 11b\n\t" \ ".previous" \ : "=&r" (res) \ : "r" (value), "r" (addr), "i" (-EFAULT) \ @@ -447,14 +453,14 @@ do { \ "j\t10b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ - STR(PTR)"\t5b, 11b\n\t" \ - STR(PTR)"\t6b, 11b\n\t" \ - STR(PTR)"\t7b, 11b\n\t" \ - STR(PTR)"\t8b, 11b\n\t" \ + PTR_STR"\t1b, 11b\n\t" \ + PTR_STR"\t2b, 11b\n\t" \ + PTR_STR"\t3b, 11b\n\t" \ + PTR_STR"\t4b, 11b\n\t" \ + PTR_STR"\t5b, 11b\n\t" \ + PTR_STR"\t6b, 11b\n\t" \ + PTR_STR"\t7b, 11b\n\t" \ + PTR_STR"\t8b, 11b\n\t" \ ".previous" \ : "=&r" (res) \ : "r" (value), "r" (addr), "i" (-EFAULT) \ @@ -480,8 +486,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -501,8 +507,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -534,10 +540,10 @@ do { \ "j\t10b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ + PTR_STR"\t1b, 11b\n\t" \ + PTR_STR"\t2b, 11b\n\t" \ + PTR_STR"\t3b, 11b\n\t" \ + PTR_STR"\t4b, 11b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -563,8 +569,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -586,8 +592,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -606,8 +612,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -639,10 +645,10 @@ do { \ "j\t10b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ + PTR_STR"\t1b, 11b\n\t" \ + PTR_STR"\t2b, 11b\n\t" \ + PTR_STR"\t3b, 11b\n\t" \ + PTR_STR"\t4b, 11b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -684,14 +690,14 @@ do { \ "j\t10b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ - STR(PTR)"\t5b, 11b\n\t" \ - STR(PTR)"\t6b, 11b\n\t" \ - STR(PTR)"\t7b, 11b\n\t" \ - STR(PTR)"\t8b, 11b\n\t" \ + PTR_STR"\t1b, 11b\n\t" \ + PTR_STR"\t2b, 11b\n\t" \ + PTR_STR"\t3b, 11b\n\t" \ + PTR_STR"\t4b, 11b\n\t" \ + PTR_STR"\t5b, 11b\n\t" \ + PTR_STR"\t6b, 11b\n\t" \ + PTR_STR"\t7b, 11b\n\t" \ + PTR_STR"\t8b, 11b\n\t" \ ".previous" \ : "=&r" (value), "=r" (res) \ : "r" (addr), "i" (-EFAULT)); \ @@ -714,8 +720,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=r" (res) \ : "r" (value), "r" (addr), "i" (-EFAULT));\ @@ -735,8 +741,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=r" (res) \ : "r" (value), "r" (addr), "i" (-EFAULT)); \ @@ -755,8 +761,8 @@ do { \ "j\t3b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 4b\n\t" \ - STR(PTR)"\t2b, 4b\n\t" \ + PTR_STR"\t1b, 4b\n\t" \ + PTR_STR"\t2b, 4b\n\t" \ ".previous" \ : "=r" (res) \ : "r" (value), "r" (addr), "i" (-EFAULT)); \ @@ -785,10 +791,10 @@ do { \ "j\t10b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ + PTR_STR"\t1b, 11b\n\t" \ + PTR_STR"\t2b, 11b\n\t" \ + PTR_STR"\t3b, 11b\n\t" \ + PTR_STR"\t4b, 11b\n\t" \ ".previous" \ : "=&r" (res) \ : "r" (value), "r" (addr), "i" (-EFAULT) \ @@ -825,14 +831,14 @@ do { \ "j\t10b\n\t" \ ".previous\n\t" \ ".section\t__ex_table,\"a\"\n\t" \ - STR(PTR)"\t1b, 11b\n\t" \ - STR(PTR)"\t2b, 11b\n\t" \ - STR(PTR)"\t3b, 11b\n\t" \ - STR(PTR)"\t4b, 11b\n\t" \ - STR(PTR)"\t5b, 11b\n\t" \ - STR(PTR)"\t6b, 11b\n\t" \ - STR(PTR)"\t7b, 11b\n\t" \ - STR(PTR)"\t8b, 11b\n\t" \ + PTR_STR"\t1b, 11b\n\t" \ + PTR_STR"\t2b, 11b\n\t" \ + PTR_STR"\t3b, 11b\n\t" \ + PTR_STR"\t4b, 11b\n\t" \ + PTR_STR"\t5b, 11b\n\t" \ + PTR_STR"\t6b, 11b\n\t" \ + PTR_STR"\t7b, 11b\n\t" \ + PTR_STR"\t8b, 11b\n\t" \ ".previous" \ : "=&r" (res) \ : "r" (value), "r" (addr), "i" (-EFAULT) \ From 68fbb9721ea76b2a3088b749e004456f2de05b9f Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Fri, 8 May 2020 16:36:05 +0800 Subject: [PATCH 0271/1043] MIPS: Loongson: Add DMA support for LS7A In the current market, the most used bridge chip on the Loongson platform are RS780E and LS7A, the RS780E bridge chip is already supported by the mainline kernel. If use the default implementation of __phys_to_dma() and __dma_to_phys() in dma-direct.h when CONFIG_ARCH_HAS_PHYS_TO_DMA is not set, it works well used with LS7A on the Loongson single-way and multi-way platform, and also works well used with RS780E on the Loongson single-way platform, but the DMA address will be wrong on the non-node0 used with RS780E on the Loongson multi-way platform. Just as the description in the code comment, the devices get node id from 40 bit of HyperTransport bus, so we extract 2 bit node id (bit 44~45) from 48 bit address space of Loongson CPU and embed it into HyperTransport bus (bit 37-38), this operation can be done only at the software level used with RS780E on the Loongson multi-way platform, because it has no hardware function to translate address of node id, this is a hardware compatibility problem. Device | | DMA address | Host Bridge | | HT bus address (40 bit) | CPU | | physical address (48 bit) | RAM The LS7A has dma_node_id_offset field in the DMA route config register, the hardware can use the dma_node_id_offset to translate address of node id automatically, so we can get correct address when just use the dma_pfn_offset field in struct device. For the above reasons, in order to maintain downward compatibility to support the RS780E bridge chip, it is better to use the platform dependent implementation of __phys_to_dma() and __dma_to_phys(). Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- .../include/asm/mach-loongson64/boot_param.h | 5 +++++ arch/mips/loongson64/dma.c | 9 ++++++--- arch/mips/loongson64/env.c | 2 ++ arch/mips/loongson64/init.c | 17 +++++++++++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/arch/mips/include/asm/mach-loongson64/boot_param.h b/arch/mips/include/asm/mach-loongson64/boot_param.h index f082d87665bc..b35be709f9da 100644 --- a/arch/mips/include/asm/mach-loongson64/boot_param.h +++ b/arch/mips/include/asm/mach-loongson64/boot_param.h @@ -221,9 +221,14 @@ struct loongson_system_configuration { u32 nr_sensors; struct sensor_device sensors[MAX_SENSORS]; u64 workarounds; + void (*early_config)(void); }; extern struct efi_memory_map_loongson *loongson_memmap; extern struct loongson_system_configuration loongson_sysconf; +extern u32 node_id_offset; +extern void ls7a_early_config(void); +extern void rs780e_early_config(void); + #endif diff --git a/arch/mips/loongson64/dma.c b/arch/mips/loongson64/dma.c index 5e86635f71db..dbfe6e82fddd 100644 --- a/arch/mips/loongson64/dma.c +++ b/arch/mips/loongson64/dma.c @@ -2,21 +2,24 @@ #include #include #include +#include dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr) { /* We extract 2bit node id (bit 44~47, only bit 44~45 used now) from * Loongson-3's 48bit address space and embed it into 40bit */ long nid = (paddr >> 44) & 0x3; - return ((nid << 44) ^ paddr) | (nid << 37); + + return ((nid << 44) ^ paddr) | (nid << node_id_offset); } phys_addr_t __dma_to_phys(struct device *dev, dma_addr_t daddr) { /* We extract 2bit node id (bit 44~47, only bit 44~45 used now) from * Loongson-3's 48bit address space and embed it into 40bit */ - long nid = (daddr >> 37) & 0x3; - return ((nid << 37) ^ daddr) | (nid << 44); + long nid = (daddr >> node_id_offset) & 0x3; + + return ((nid << node_id_offset) ^ daddr) | (nid << 44); } void __init plat_swiotlb_setup(void) diff --git a/arch/mips/loongson64/env.c b/arch/mips/loongson64/env.c index 71f4aaf58791..d11bc346bbca 100644 --- a/arch/mips/loongson64/env.c +++ b/arch/mips/loongson64/env.c @@ -192,8 +192,10 @@ void __init prom_init_env(void) if (vendor == PCI_VENDOR_ID_LOONGSON && device == 0x7a00) { pr_info("The bridge chip is LS7A\n"); loongson_sysconf.bridgetype = LS7A; + loongson_sysconf.early_config = ls7a_early_config; } else { pr_info("The bridge chip is RS780E or SR5690\n"); loongson_sysconf.bridgetype = RS780E; + loongson_sysconf.early_config = rs780e_early_config; } } diff --git a/arch/mips/loongson64/init.c b/arch/mips/loongson64/init.c index da38944471f4..2b45ca6ca98d 100644 --- a/arch/mips/loongson64/init.c +++ b/arch/mips/loongson64/init.c @@ -13,6 +13,11 @@ #include #include +#include + +#define NODE_ID_OFFSET_ADDR ((void __iomem *)TO_UNCAC(0x1001041c)) + +u32 node_id_offset; static void __init mips_nmi_setup(void) { @@ -24,6 +29,16 @@ static void __init mips_nmi_setup(void) flush_icache_range((unsigned long)base, (unsigned long)base + 0x80); } +void ls7a_early_config(void) +{ + node_id_offset = ((readl(NODE_ID_OFFSET_ADDR) >> 8) & 0x1f) + 36; +} + +void rs780e_early_config(void) +{ + node_id_offset = 37; +} + void __init prom_init(void) { fw_init_cmdline(); @@ -33,6 +48,8 @@ void __init prom_init(void) set_io_port_base((unsigned long) ioremap(LOONGSON_PCIIO_BASE, LOONGSON_PCIIO_SIZE)); + loongson_sysconf.early_config(); + prom_init_numa_memory(); /* Hardcode to CPU UART 0 */ From c4ad6ea9574bfa29ca00ddf6b6b1508ec344ff45 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 7 May 2020 14:00:52 -0500 Subject: [PATCH 0272/1043] MIPS: Replace zero-length array with flexible-array The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] sizeof(flexible-array-member) triggers a warning because flexible array members have incomplete type[1]. There are some instances of code in which the sizeof operator is being incorrectly/erroneously applied to zero-length arrays and the result is zero. Such instances may be hiding some bugs. So, this work (flexible-array member conversions) will also help to get completely rid of those sorts of issues. This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/signal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/kernel/signal.c b/arch/mips/kernel/signal.c index f926bf338dec..a0262729cd4c 100644 --- a/arch/mips/kernel/signal.c +++ b/arch/mips/kernel/signal.c @@ -52,7 +52,7 @@ struct sigframe { /* Matches struct ucontext from its uc_mcontext field onwards */ struct sigcontext sf_sc; sigset_t sf_mask; - unsigned long long sf_extcontext[0]; + unsigned long long sf_extcontext[]; }; struct rt_sigframe { From 35f4521fd3a001fb290a1780f8beeffb06d99a04 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 30 Apr 2020 10:45:41 -0700 Subject: [PATCH 0273/1043] xfs: convert xfs_log_recover_item_t to struct xfs_log_recover_item Remove the old typedefs. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_log_recover.h | 4 ++-- fs/xfs/xfs_log_recover.c | 26 ++++++++++++++------------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/fs/xfs/libxfs/xfs_log_recover.h b/fs/xfs/libxfs/xfs_log_recover.h index 3bf671637a91..148e0cb5d379 100644 --- a/fs/xfs/libxfs/xfs_log_recover.h +++ b/fs/xfs/libxfs/xfs_log_recover.h @@ -22,13 +22,13 @@ /* * item headers are in ri_buf[0]. Additional buffers follow. */ -typedef struct xlog_recover_item { +struct xlog_recover_item { struct list_head ri_list; int ri_type; int ri_cnt; /* count of regions found */ int ri_total; /* total regions */ xfs_log_iovec_t *ri_buf; /* ptr to regions buffer */ -} xlog_recover_item_t; +}; struct xlog_recover { struct hlist_node r_list; diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 320785115833..ed4ed76f8e9c 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -1841,7 +1841,7 @@ xlog_recover_reorder_trans( struct xlog_recover *trans, int pass) { - xlog_recover_item_t *item, *n; + struct xlog_recover_item *item, *n; int error = 0; LIST_HEAD(sort_list); LIST_HEAD(cancel_list); @@ -2056,7 +2056,7 @@ xlog_recover_buffer_pass1( STATIC int xlog_recover_do_inode_buffer( struct xfs_mount *mp, - xlog_recover_item_t *item, + struct xlog_recover_item *item, struct xfs_buf *bp, xfs_buf_log_format_t *buf_f) { @@ -2561,7 +2561,7 @@ xlog_recover_validate_buf_type( STATIC void xlog_recover_do_reg_buffer( struct xfs_mount *mp, - xlog_recover_item_t *item, + struct xlog_recover_item *item, struct xfs_buf *bp, xfs_buf_log_format_t *buf_f, xfs_lsn_t current_lsn) @@ -3759,7 +3759,7 @@ STATIC int xlog_recover_do_icreate_pass2( struct xlog *log, struct list_head *buffer_list, - xlog_recover_item_t *item) + struct xlog_recover_item *item) { struct xfs_mount *mp = log->l_mp; struct xfs_icreate_log *icl; @@ -4134,9 +4134,9 @@ STATIC void xlog_recover_add_item( struct list_head *head) { - xlog_recover_item_t *item; + struct xlog_recover_item *item; - item = kmem_zalloc(sizeof(xlog_recover_item_t), 0); + item = kmem_zalloc(sizeof(struct xlog_recover_item), 0); INIT_LIST_HEAD(&item->ri_list); list_add_tail(&item->ri_list, head); } @@ -4148,7 +4148,7 @@ xlog_recover_add_to_cont_trans( char *dp, int len) { - xlog_recover_item_t *item; + struct xlog_recover_item *item; char *ptr, *old_ptr; int old_len; @@ -4171,7 +4171,8 @@ xlog_recover_add_to_cont_trans( } /* take the tail entry */ - item = list_entry(trans->r_itemq.prev, xlog_recover_item_t, ri_list); + item = list_entry(trans->r_itemq.prev, struct xlog_recover_item, + ri_list); old_ptr = item->ri_buf[item->ri_cnt-1].i_addr; old_len = item->ri_buf[item->ri_cnt-1].i_len; @@ -4205,7 +4206,7 @@ xlog_recover_add_to_trans( int len) { struct xfs_inode_log_format *in_f; /* any will do */ - xlog_recover_item_t *item; + struct xlog_recover_item *item; char *ptr; if (!len) @@ -4241,13 +4242,14 @@ xlog_recover_add_to_trans( in_f = (struct xfs_inode_log_format *)ptr; /* take the tail entry */ - item = list_entry(trans->r_itemq.prev, xlog_recover_item_t, ri_list); + item = list_entry(trans->r_itemq.prev, struct xlog_recover_item, + ri_list); if (item->ri_total != 0 && item->ri_total == item->ri_cnt) { /* tail item is in use, get a new one */ xlog_recover_add_item(&trans->r_itemq); item = list_entry(trans->r_itemq.prev, - xlog_recover_item_t, ri_list); + struct xlog_recover_item, ri_list); } if (item->ri_total == 0) { /* first region to be added */ @@ -4293,7 +4295,7 @@ STATIC void xlog_recover_free_trans( struct xlog_recover *trans) { - xlog_recover_item_t *item, *n; + struct xlog_recover_item *item, *n; int i; hlist_del_init(&trans->r_list); From 86ffa471d9ce6ac3fda66f704c3143c3d55181f5 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:45 -0700 Subject: [PATCH 0274/1043] xfs: refactor log recovery item sorting into a generic dispatch structure Create a generic dispatch structure to delegate recovery of different log item types into various code modules. This will enable us to move code specific to a particular log item type out of xfs_log_recover.c and into the log item source. The first operation we virtualize is the log item sorting. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/Makefile | 3 ++ fs/xfs/libxfs/xfs_log_recover.h | 45 +++++++++++++++- fs/xfs/xfs_bmap_item.c | 9 ++++ fs/xfs/xfs_buf_item_recover.c | 52 ++++++++++++++++++ fs/xfs/xfs_dquot_item_recover.c | 29 ++++++++++ fs/xfs/xfs_extfree_item.c | 9 ++++ fs/xfs/xfs_icreate_item.c | 20 +++++++ fs/xfs/xfs_inode_item_recover.c | 26 +++++++++ fs/xfs/xfs_log_recover.c | 93 ++++++++++++++++++++------------- fs/xfs/xfs_refcount_item.c | 9 ++++ fs/xfs/xfs_rmap_item.c | 9 ++++ 11 files changed, 265 insertions(+), 39 deletions(-) create mode 100644 fs/xfs/xfs_buf_item_recover.c create mode 100644 fs/xfs/xfs_dquot_item_recover.c create mode 100644 fs/xfs/xfs_inode_item_recover.c diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile index ff94fb90a2ee..04611a1068b4 100644 --- a/fs/xfs/Makefile +++ b/fs/xfs/Makefile @@ -99,9 +99,12 @@ xfs-y += xfs_log.o \ xfs_log_cil.o \ xfs_bmap_item.o \ xfs_buf_item.o \ + xfs_buf_item_recover.o \ + xfs_dquot_item_recover.o \ xfs_extfree_item.o \ xfs_icreate_item.o \ xfs_inode_item.o \ + xfs_inode_item_recover.o \ xfs_refcount_item.o \ xfs_rmap_item.o \ xfs_log_recover.o \ diff --git a/fs/xfs/libxfs/xfs_log_recover.h b/fs/xfs/libxfs/xfs_log_recover.h index 148e0cb5d379..c9c27e6367bb 100644 --- a/fs/xfs/libxfs/xfs_log_recover.h +++ b/fs/xfs/libxfs/xfs_log_recover.h @@ -6,6 +6,47 @@ #ifndef __XFS_LOG_RECOVER_H__ #define __XFS_LOG_RECOVER_H__ +/* + * Each log item type (XFS_LI_*) gets its own xlog_recover_item_ops to + * define how recovery should work for that type of log item. + */ +struct xlog_recover_item; + +/* Sorting hat for log items as they're read in. */ +enum xlog_recover_reorder { + XLOG_REORDER_BUFFER_LIST, + XLOG_REORDER_ITEM_LIST, + XLOG_REORDER_INODE_BUFFER_LIST, + XLOG_REORDER_CANCEL_LIST, +}; + +struct xlog_recover_item_ops { + uint16_t item_type; /* XFS_LI_* type code. */ + + /* + * Help sort recovered log items into the order required to replay them + * correctly. Log item types that always use XLOG_REORDER_ITEM_LIST do + * not have to supply a function here. See the comment preceding + * xlog_recover_reorder_trans for more details about what the return + * values mean. + */ + enum xlog_recover_reorder (*reorder)(struct xlog_recover_item *item); +}; + +extern const struct xlog_recover_item_ops xlog_icreate_item_ops; +extern const struct xlog_recover_item_ops xlog_buf_item_ops; +extern const struct xlog_recover_item_ops xlog_inode_item_ops; +extern const struct xlog_recover_item_ops xlog_dquot_item_ops; +extern const struct xlog_recover_item_ops xlog_quotaoff_item_ops; +extern const struct xlog_recover_item_ops xlog_bui_item_ops; +extern const struct xlog_recover_item_ops xlog_bud_item_ops; +extern const struct xlog_recover_item_ops xlog_efi_item_ops; +extern const struct xlog_recover_item_ops xlog_efd_item_ops; +extern const struct xlog_recover_item_ops xlog_rui_item_ops; +extern const struct xlog_recover_item_ops xlog_rud_item_ops; +extern const struct xlog_recover_item_ops xlog_cui_item_ops; +extern const struct xlog_recover_item_ops xlog_cud_item_ops; + /* * Macros, structures, prototypes for internal log manager use. */ @@ -24,10 +65,10 @@ */ struct xlog_recover_item { struct list_head ri_list; - int ri_type; int ri_cnt; /* count of regions found */ int ri_total; /* total regions */ - xfs_log_iovec_t *ri_buf; /* ptr to regions buffer */ + struct xfs_log_iovec *ri_buf; /* ptr to regions buffer */ + const struct xlog_recover_item_ops *ri_ops; }; struct xlog_recover { diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index 17eb7cfad5d9..508b48ca5ced 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -22,6 +22,7 @@ #include "xfs_bmap_btree.h" #include "xfs_trans_space.h" #include "xfs_error.h" +#include "xfs_log_recover.h" kmem_zone_t *xfs_bui_zone; kmem_zone_t *xfs_bud_zone; @@ -557,3 +558,11 @@ err_inode: } return error; } + +const struct xlog_recover_item_ops xlog_bui_item_ops = { + .item_type = XFS_LI_BUI, +}; + +const struct xlog_recover_item_ops xlog_bud_item_ops = { + .item_type = XFS_LI_BUD, +}; diff --git a/fs/xfs/xfs_buf_item_recover.c b/fs/xfs/xfs_buf_item_recover.c new file mode 100644 index 000000000000..5dea6323ff1f --- /dev/null +++ b/fs/xfs/xfs_buf_item_recover.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_trans_priv.h" +#include "xfs_trace.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" + +/* + * Sort buffer items for log recovery. Most buffer items should end up on the + * buffer list and are recovered first, with the following exceptions: + * + * 1. XFS_BLF_CANCEL buffers must be processed last because some log items + * might depend on the incor ecancellation record, and replaying a cancelled + * buffer item can remove the incore record. + * + * 2. XFS_BLF_INODE_BUF buffers are handled after most regular items so that + * we replay di_next_unlinked only after flushing the inode 'free' state + * to the inode buffer. + * + * See xlog_recover_reorder_trans for more details. + */ +STATIC enum xlog_recover_reorder +xlog_recover_buf_reorder( + struct xlog_recover_item *item) +{ + struct xfs_buf_log_format *buf_f = item->ri_buf[0].i_addr; + + if (buf_f->blf_flags & XFS_BLF_CANCEL) + return XLOG_REORDER_CANCEL_LIST; + if (buf_f->blf_flags & XFS_BLF_INODE_BUF) + return XLOG_REORDER_INODE_BUFFER_LIST; + return XLOG_REORDER_BUFFER_LIST; +} + +const struct xlog_recover_item_ops xlog_buf_item_ops = { + .item_type = XFS_LI_BUF, + .reorder = xlog_recover_buf_reorder, +}; diff --git a/fs/xfs/xfs_dquot_item_recover.c b/fs/xfs/xfs_dquot_item_recover.c new file mode 100644 index 000000000000..78fe644e9907 --- /dev/null +++ b/fs/xfs/xfs_dquot_item_recover.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_quota.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_trans_priv.h" +#include "xfs_qm.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" + +const struct xlog_recover_item_ops xlog_dquot_item_ops = { + .item_type = XFS_LI_DQUOT, +}; + +const struct xlog_recover_item_ops xlog_quotaoff_item_ops = { + .item_type = XFS_LI_QUOTAOFF, +}; diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index 9809637fb84d..163d01cb9f9f 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -22,6 +22,7 @@ #include "xfs_bmap.h" #include "xfs_trace.h" #include "xfs_error.h" +#include "xfs_log_recover.h" kmem_zone_t *xfs_efi_zone; kmem_zone_t *xfs_efd_zone; @@ -644,3 +645,11 @@ abort_error: xfs_trans_cancel(tp); return error; } + +const struct xlog_recover_item_ops xlog_efi_item_ops = { + .item_type = XFS_LI_EFI, +}; + +const struct xlog_recover_item_ops xlog_efd_item_ops = { + .item_type = XFS_LI_EFD, +}; diff --git a/fs/xfs/xfs_icreate_item.c b/fs/xfs/xfs_icreate_item.c index 490fee22b878..366c1e722a29 100644 --- a/fs/xfs/xfs_icreate_item.c +++ b/fs/xfs/xfs_icreate_item.c @@ -11,6 +11,8 @@ #include "xfs_trans_priv.h" #include "xfs_icreate_item.h" #include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" kmem_zone_t *xfs_icreate_zone; /* inode create item zone */ @@ -107,3 +109,21 @@ xfs_icreate_log( tp->t_flags |= XFS_TRANS_DIRTY; set_bit(XFS_LI_DIRTY, &icp->ic_item.li_flags); } + +static enum xlog_recover_reorder +xlog_recover_icreate_reorder( + struct xlog_recover_item *item) +{ + /* + * Inode allocation buffers must be replayed before subsequent inode + * items try to modify those buffers. ICREATE items are the logical + * equivalent of logging a newly initialized inode buffer, so recover + * these at the same time that we recover logged buffers. + */ + return XLOG_REORDER_BUFFER_LIST; +} + +const struct xlog_recover_item_ops xlog_icreate_item_ops = { + .item_type = XFS_LI_ICREATE, + .reorder = xlog_recover_icreate_reorder, +}; diff --git a/fs/xfs/xfs_inode_item_recover.c b/fs/xfs/xfs_inode_item_recover.c new file mode 100644 index 000000000000..b19a151efb10 --- /dev/null +++ b/fs/xfs/xfs_inode_item_recover.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_inode_item.h" +#include "xfs_trace.h" +#include "xfs_trans_priv.h" +#include "xfs_buf_item.h" +#include "xfs_log.h" +#include "xfs_error.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" + +const struct xlog_recover_item_ops xlog_inode_item_ops = { + .item_type = XFS_LI_INODE, +}; diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index ed4ed76f8e9c..e44c64fca65f 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -1785,6 +1785,34 @@ xlog_clear_stale_blocks( * ****************************************************************************** */ +static const struct xlog_recover_item_ops *xlog_recover_item_ops[] = { + &xlog_buf_item_ops, + &xlog_inode_item_ops, + &xlog_dquot_item_ops, + &xlog_quotaoff_item_ops, + &xlog_icreate_item_ops, + &xlog_efi_item_ops, + &xlog_efd_item_ops, + &xlog_rui_item_ops, + &xlog_rud_item_ops, + &xlog_cui_item_ops, + &xlog_cud_item_ops, + &xlog_bui_item_ops, + &xlog_bud_item_ops, +}; + +static const struct xlog_recover_item_ops * +xlog_find_item_ops( + struct xlog_recover_item *item) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(xlog_recover_item_ops); i++) + if (ITEM_TYPE(item) == xlog_recover_item_ops[i]->item_type) + return xlog_recover_item_ops[i]; + + return NULL; +} /* * Sort the log items in the transaction. @@ -1851,41 +1879,10 @@ xlog_recover_reorder_trans( list_splice_init(&trans->r_itemq, &sort_list); list_for_each_entry_safe(item, n, &sort_list, ri_list) { - xfs_buf_log_format_t *buf_f = item->ri_buf[0].i_addr; + enum xlog_recover_reorder fate = XLOG_REORDER_ITEM_LIST; - switch (ITEM_TYPE(item)) { - case XFS_LI_ICREATE: - list_move_tail(&item->ri_list, &buffer_list); - break; - case XFS_LI_BUF: - if (buf_f->blf_flags & XFS_BLF_CANCEL) { - trace_xfs_log_recover_item_reorder_head(log, - trans, item, pass); - list_move(&item->ri_list, &cancel_list); - break; - } - if (buf_f->blf_flags & XFS_BLF_INODE_BUF) { - list_move(&item->ri_list, &inode_buffer_list); - break; - } - list_move_tail(&item->ri_list, &buffer_list); - break; - case XFS_LI_INODE: - case XFS_LI_DQUOT: - case XFS_LI_QUOTAOFF: - case XFS_LI_EFD: - case XFS_LI_EFI: - case XFS_LI_RUI: - case XFS_LI_RUD: - case XFS_LI_CUI: - case XFS_LI_CUD: - case XFS_LI_BUI: - case XFS_LI_BUD: - trace_xfs_log_recover_item_reorder_tail(log, - trans, item, pass); - list_move_tail(&item->ri_list, &item_list); - break; - default: + item->ri_ops = xlog_find_item_ops(item); + if (!item->ri_ops) { xfs_warn(log->l_mp, "%s: unrecognized type of log operation (%d)", __func__, ITEM_TYPE(item)); @@ -1896,11 +1893,33 @@ xlog_recover_reorder_trans( */ if (!list_empty(&sort_list)) list_splice_init(&sort_list, &trans->r_itemq); - error = -EIO; - goto out; + error = -EFSCORRUPTED; + break; + } + + if (item->ri_ops->reorder) + fate = item->ri_ops->reorder(item); + + switch (fate) { + case XLOG_REORDER_BUFFER_LIST: + list_move_tail(&item->ri_list, &buffer_list); + break; + case XLOG_REORDER_CANCEL_LIST: + trace_xfs_log_recover_item_reorder_head(log, + trans, item, pass); + list_move(&item->ri_list, &cancel_list); + break; + case XLOG_REORDER_INODE_BUFFER_LIST: + list_move(&item->ri_list, &inode_buffer_list); + break; + case XLOG_REORDER_ITEM_LIST: + trace_xfs_log_recover_item_reorder_tail(log, + trans, item, pass); + list_move_tail(&item->ri_list, &item_list); + break; } } -out: + ASSERT(list_empty(&sort_list)); if (!list_empty(&buffer_list)) list_splice(&buffer_list, &trans->r_itemq); diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index 01bb77daeaee..2a9465d9a77f 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -18,6 +18,7 @@ #include "xfs_log.h" #include "xfs_refcount.h" #include "xfs_error.h" +#include "xfs_log_recover.h" kmem_zone_t *xfs_cui_zone; kmem_zone_t *xfs_cud_zone; @@ -570,3 +571,11 @@ abort_error: xfs_trans_cancel(tp); return error; } + +const struct xlog_recover_item_ops xlog_cui_item_ops = { + .item_type = XFS_LI_CUI, +}; + +const struct xlog_recover_item_ops xlog_cud_item_ops = { + .item_type = XFS_LI_CUD, +}; diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index fdb12b01b178..0f3af9f05764 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -18,6 +18,7 @@ #include "xfs_log.h" #include "xfs_rmap.h" #include "xfs_error.h" +#include "xfs_log_recover.h" kmem_zone_t *xfs_rui_zone; kmem_zone_t *xfs_rud_zone; @@ -585,3 +586,11 @@ abort_error: xfs_trans_cancel(tp); return error; } + +const struct xlog_recover_item_ops xlog_rui_item_ops = { + .item_type = XFS_LI_RUI, +}; + +const struct xlog_recover_item_ops xlog_rud_item_ops = { + .item_type = XFS_LI_RUD, +}; From 8ea5682d07115b422e923bb4f55fe081964f484a Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:46 -0700 Subject: [PATCH 0275/1043] xfs: refactor log recovery item dispatch for pass2 readhead functions Move the pass2 readhead code into the per-item source code files and use the dispatch function to call them. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_log_recover.h | 6 +++ fs/xfs/xfs_buf_item_recover.c | 11 ++++ fs/xfs/xfs_dquot_item_recover.c | 34 ++++++++++++ fs/xfs/xfs_inode_item_recover.c | 19 +++++++ fs/xfs/xfs_log_recover.c | 95 ++------------------------------- 5 files changed, 73 insertions(+), 92 deletions(-) diff --git a/fs/xfs/libxfs/xfs_log_recover.h b/fs/xfs/libxfs/xfs_log_recover.h index c9c27e6367bb..ceb1e1e9d1d1 100644 --- a/fs/xfs/libxfs/xfs_log_recover.h +++ b/fs/xfs/libxfs/xfs_log_recover.h @@ -31,6 +31,9 @@ struct xlog_recover_item_ops { * values mean. */ enum xlog_recover_reorder (*reorder)(struct xlog_recover_item *item); + + /* Start readahead for pass2, if provided. */ + void (*ra_pass2)(struct xlog *log, struct xlog_recover_item *item); }; extern const struct xlog_recover_item_ops xlog_icreate_item_ops; @@ -92,4 +95,7 @@ struct xlog_recover { #define XLOG_RECOVER_PASS1 1 #define XLOG_RECOVER_PASS2 2 +void xlog_buf_readahead(struct xlog *log, xfs_daddr_t blkno, uint len, + const struct xfs_buf_ops *ops); + #endif /* __XFS_LOG_RECOVER_H__ */ diff --git a/fs/xfs/xfs_buf_item_recover.c b/fs/xfs/xfs_buf_item_recover.c index 5dea6323ff1f..e35892534aaa 100644 --- a/fs/xfs/xfs_buf_item_recover.c +++ b/fs/xfs/xfs_buf_item_recover.c @@ -46,7 +46,18 @@ xlog_recover_buf_reorder( return XLOG_REORDER_BUFFER_LIST; } +STATIC void +xlog_recover_buf_ra_pass2( + struct xlog *log, + struct xlog_recover_item *item) +{ + struct xfs_buf_log_format *buf_f = item->ri_buf[0].i_addr; + + xlog_buf_readahead(log, buf_f->blf_blkno, buf_f->blf_len, NULL); +} + const struct xlog_recover_item_ops xlog_buf_item_ops = { .item_type = XFS_LI_BUF, .reorder = xlog_recover_buf_reorder, + .ra_pass2 = xlog_recover_buf_ra_pass2, }; diff --git a/fs/xfs/xfs_dquot_item_recover.c b/fs/xfs/xfs_dquot_item_recover.c index 78fe644e9907..215274173b70 100644 --- a/fs/xfs/xfs_dquot_item_recover.c +++ b/fs/xfs/xfs_dquot_item_recover.c @@ -20,8 +20,42 @@ #include "xfs_log_priv.h" #include "xfs_log_recover.h" +STATIC void +xlog_recover_dquot_ra_pass2( + struct xlog *log, + struct xlog_recover_item *item) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_disk_dquot *recddq; + struct xfs_dq_logformat *dq_f; + uint type; + + if (mp->m_qflags == 0) + return; + + recddq = item->ri_buf[1].i_addr; + if (recddq == NULL) + return; + if (item->ri_buf[1].i_len < sizeof(struct xfs_disk_dquot)) + return; + + type = recddq->d_flags & (XFS_DQ_USER | XFS_DQ_PROJ | XFS_DQ_GROUP); + ASSERT(type); + if (log->l_quotaoffs_flag & type) + return; + + dq_f = item->ri_buf[0].i_addr; + ASSERT(dq_f); + ASSERT(dq_f->qlf_len == 1); + + xlog_buf_readahead(log, dq_f->qlf_blkno, + XFS_FSB_TO_BB(mp, dq_f->qlf_len), + &xfs_dquot_buf_ra_ops); +} + const struct xlog_recover_item_ops xlog_dquot_item_ops = { .item_type = XFS_LI_DQUOT, + .ra_pass2 = xlog_recover_dquot_ra_pass2, }; const struct xlog_recover_item_ops xlog_quotaoff_item_ops = { diff --git a/fs/xfs/xfs_inode_item_recover.c b/fs/xfs/xfs_inode_item_recover.c index b19a151efb10..a132cacd8d48 100644 --- a/fs/xfs/xfs_inode_item_recover.c +++ b/fs/xfs/xfs_inode_item_recover.c @@ -21,6 +21,25 @@ #include "xfs_log_priv.h" #include "xfs_log_recover.h" +STATIC void +xlog_recover_inode_ra_pass2( + struct xlog *log, + struct xlog_recover_item *item) +{ + if (item->ri_buf[0].i_len == sizeof(struct xfs_inode_log_format)) { + struct xfs_inode_log_format *ilfp = item->ri_buf[0].i_addr; + + xlog_buf_readahead(log, ilfp->ilf_blkno, ilfp->ilf_len, + &xfs_inode_buf_ra_ops); + } else { + struct xfs_inode_log_format_32 *ilfp = item->ri_buf[0].i_addr; + + xlog_buf_readahead(log, ilfp->ilf_blkno, ilfp->ilf_len, + &xfs_inode_buf_ra_ops); + } +} + const struct xlog_recover_item_ops xlog_inode_item_ops = { .item_type = XFS_LI_INODE, + .ra_pass2 = xlog_recover_inode_ra_pass2, }; diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index e44c64fca65f..7fd0126a80bf 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2023,7 +2023,7 @@ xlog_put_buffer_cancelled( return true; } -static void +void xlog_buf_readahead( struct xlog *log, xfs_daddr_t blkno, @@ -3890,96 +3890,6 @@ xlog_recover_do_icreate_pass2( length, be32_to_cpu(icl->icl_gen)); } -STATIC void -xlog_recover_buffer_ra_pass2( - struct xlog *log, - struct xlog_recover_item *item) -{ - struct xfs_buf_log_format *buf_f = item->ri_buf[0].i_addr; - - xlog_buf_readahead(log, buf_f->blf_blkno, buf_f->blf_len, NULL); -} - -STATIC void -xlog_recover_inode_ra_pass2( - struct xlog *log, - struct xlog_recover_item *item) -{ - if (item->ri_buf[0].i_len == sizeof(struct xfs_inode_log_format)) { - struct xfs_inode_log_format *ilfp = item->ri_buf[0].i_addr; - - xlog_buf_readahead(log, ilfp->ilf_blkno, ilfp->ilf_len, - &xfs_inode_buf_ra_ops); - } else { - struct xfs_inode_log_format_32 *ilfp = item->ri_buf[0].i_addr; - - xlog_buf_readahead(log, ilfp->ilf_blkno, ilfp->ilf_len, - &xfs_inode_buf_ra_ops); - } -} - -STATIC void -xlog_recover_dquot_ra_pass2( - struct xlog *log, - struct xlog_recover_item *item) -{ - struct xfs_mount *mp = log->l_mp; - struct xfs_disk_dquot *recddq; - struct xfs_dq_logformat *dq_f; - uint type; - - if (mp->m_qflags == 0) - return; - - recddq = item->ri_buf[1].i_addr; - if (recddq == NULL) - return; - if (item->ri_buf[1].i_len < sizeof(struct xfs_disk_dquot)) - return; - - type = recddq->d_flags & (XFS_DQ_USER | XFS_DQ_PROJ | XFS_DQ_GROUP); - ASSERT(type); - if (log->l_quotaoffs_flag & type) - return; - - dq_f = item->ri_buf[0].i_addr; - ASSERT(dq_f); - ASSERT(dq_f->qlf_len == 1); - - xlog_buf_readahead(log, dq_f->qlf_blkno, - XFS_FSB_TO_BB(mp, dq_f->qlf_len), - &xfs_dquot_buf_ra_ops); -} - -STATIC void -xlog_recover_ra_pass2( - struct xlog *log, - struct xlog_recover_item *item) -{ - switch (ITEM_TYPE(item)) { - case XFS_LI_BUF: - xlog_recover_buffer_ra_pass2(log, item); - break; - case XFS_LI_INODE: - xlog_recover_inode_ra_pass2(log, item); - break; - case XFS_LI_DQUOT: - xlog_recover_dquot_ra_pass2(log, item); - break; - case XFS_LI_EFI: - case XFS_LI_EFD: - case XFS_LI_QUOTAOFF: - case XFS_LI_RUI: - case XFS_LI_RUD: - case XFS_LI_CUI: - case XFS_LI_CUD: - case XFS_LI_BUI: - case XFS_LI_BUD: - default: - break; - } -} - STATIC int xlog_recover_commit_pass1( struct xlog *log, @@ -4116,7 +4026,8 @@ xlog_recover_commit_trans( error = xlog_recover_commit_pass1(log, trans, item); break; case XLOG_RECOVER_PASS2: - xlog_recover_ra_pass2(log, item); + if (item->ri_ops->ra_pass2) + item->ri_ops->ra_pass2(log, item); list_move_tail(&item->ri_list, &ra_list); items_queued++; if (items_queued >= XLOG_RECOVER_COMMIT_QUEUE_MAX) { From 3304a4fabd099820df99de1acac345dd6fe16d1d Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:46 -0700 Subject: [PATCH 0276/1043] xfs: refactor log recovery item dispatch for pass1 commit functions Move the pass1 commit code into the per-item source code files and use the dispatch function to call them. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_log_recover.h | 4 ++ fs/xfs/xfs_buf_item_recover.c | 27 ++++++++++ fs/xfs/xfs_dquot_item_recover.c | 28 ++++++++++ fs/xfs/xfs_log_recover.c | 94 ++------------------------------- 4 files changed, 64 insertions(+), 89 deletions(-) diff --git a/fs/xfs/libxfs/xfs_log_recover.h b/fs/xfs/libxfs/xfs_log_recover.h index ceb1e1e9d1d1..19e24b8877c9 100644 --- a/fs/xfs/libxfs/xfs_log_recover.h +++ b/fs/xfs/libxfs/xfs_log_recover.h @@ -34,6 +34,9 @@ struct xlog_recover_item_ops { /* Start readahead for pass2, if provided. */ void (*ra_pass2)(struct xlog *log, struct xlog_recover_item *item); + + /* Do whatever work we need to do for pass1, if provided. */ + int (*commit_pass1)(struct xlog *log, struct xlog_recover_item *item); }; extern const struct xlog_recover_item_ops xlog_icreate_item_ops; @@ -97,5 +100,6 @@ struct xlog_recover { void xlog_buf_readahead(struct xlog *log, xfs_daddr_t blkno, uint len, const struct xfs_buf_ops *ops); +bool xlog_add_buffer_cancelled(struct xlog *log, xfs_daddr_t blkno, uint len); #endif /* __XFS_LOG_RECOVER_H__ */ diff --git a/fs/xfs/xfs_buf_item_recover.c b/fs/xfs/xfs_buf_item_recover.c index e35892534aaa..e2d9599f67df 100644 --- a/fs/xfs/xfs_buf_item_recover.c +++ b/fs/xfs/xfs_buf_item_recover.c @@ -56,8 +56,35 @@ xlog_recover_buf_ra_pass2( xlog_buf_readahead(log, buf_f->blf_blkno, buf_f->blf_len, NULL); } +/* + * Build up the table of buf cancel records so that we don't replay cancelled + * data in the second pass. + */ +static int +xlog_recover_buf_commit_pass1( + struct xlog *log, + struct xlog_recover_item *item) +{ + struct xfs_buf_log_format *bf = item->ri_buf[0].i_addr; + + if (!xfs_buf_log_check_iovec(&item->ri_buf[0])) { + xfs_err(log->l_mp, "bad buffer log item size (%d)", + item->ri_buf[0].i_len); + return -EFSCORRUPTED; + } + + if (!(bf->blf_flags & XFS_BLF_CANCEL)) + trace_xfs_log_recover_buf_not_cancel(log, bf); + else if (xlog_add_buffer_cancelled(log, bf->blf_blkno, bf->blf_len)) + trace_xfs_log_recover_buf_cancel_add(log, bf); + else + trace_xfs_log_recover_buf_cancel_ref_inc(log, bf); + return 0; +} + const struct xlog_recover_item_ops xlog_buf_item_ops = { .item_type = XFS_LI_BUF, .reorder = xlog_recover_buf_reorder, .ra_pass2 = xlog_recover_buf_ra_pass2, + .commit_pass1 = xlog_recover_buf_commit_pass1, }; diff --git a/fs/xfs/xfs_dquot_item_recover.c b/fs/xfs/xfs_dquot_item_recover.c index 215274173b70..ebc44c1bc2b1 100644 --- a/fs/xfs/xfs_dquot_item_recover.c +++ b/fs/xfs/xfs_dquot_item_recover.c @@ -58,6 +58,34 @@ const struct xlog_recover_item_ops xlog_dquot_item_ops = { .ra_pass2 = xlog_recover_dquot_ra_pass2, }; +/* + * Recover QUOTAOFF records. We simply make a note of it in the xlog + * structure, so that we know not to do any dquot item or dquot buffer recovery, + * of that type. + */ +STATIC int +xlog_recover_quotaoff_commit_pass1( + struct xlog *log, + struct xlog_recover_item *item) +{ + struct xfs_qoff_logformat *qoff_f = item->ri_buf[0].i_addr; + ASSERT(qoff_f); + + /* + * The logitem format's flag tells us if this was user quotaoff, + * group/project quotaoff or both. + */ + if (qoff_f->qf_flags & XFS_UQUOTA_ACCT) + log->l_quotaoffs_flag |= XFS_DQ_USER; + if (qoff_f->qf_flags & XFS_PQUOTA_ACCT) + log->l_quotaoffs_flag |= XFS_DQ_PROJ; + if (qoff_f->qf_flags & XFS_GQUOTA_ACCT) + log->l_quotaoffs_flag |= XFS_DQ_GROUP; + + return 0; +} + const struct xlog_recover_item_ops xlog_quotaoff_item_ops = { .item_type = XFS_LI_QUOTAOFF, + .commit_pass1 = xlog_recover_quotaoff_commit_pass1, }; diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 7fd0126a80bf..2511f2874464 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -1953,7 +1953,7 @@ xlog_find_buffer_cancelled( return NULL; } -static bool +bool xlog_add_buffer_cancelled( struct xlog *log, xfs_daddr_t blkno, @@ -2034,32 +2034,6 @@ xlog_buf_readahead( xfs_buf_readahead(log->l_mp->m_ddev_targp, blkno, len, ops); } -/* - * Build up the table of buf cancel records so that we don't replay cancelled - * data in the second pass. - */ -static int -xlog_recover_buffer_pass1( - struct xlog *log, - struct xlog_recover_item *item) -{ - struct xfs_buf_log_format *bf = item->ri_buf[0].i_addr; - - if (!xfs_buf_log_check_iovec(&item->ri_buf[0])) { - xfs_err(log->l_mp, "bad buffer log item size (%d)", - item->ri_buf[0].i_len); - return -EFSCORRUPTED; - } - - if (!(bf->blf_flags & XFS_BLF_CANCEL)) - trace_xfs_log_recover_buf_not_cancel(log, bf); - else if (xlog_add_buffer_cancelled(log, bf->blf_blkno, bf->blf_len)) - trace_xfs_log_recover_buf_cancel_add(log, bf); - else - trace_xfs_log_recover_buf_cancel_ref_inc(log, bf); - return 0; -} - /* * Perform recovery for a buffer full of inodes. In these buffers, the only * data which should be recovered is that which corresponds to the @@ -3197,33 +3171,6 @@ error: return error; } -/* - * Recover QUOTAOFF records. We simply make a note of it in the xlog - * structure, so that we know not to do any dquot item or dquot buffer recovery, - * of that type. - */ -STATIC int -xlog_recover_quotaoff_pass1( - struct xlog *log, - struct xlog_recover_item *item) -{ - xfs_qoff_logformat_t *qoff_f = item->ri_buf[0].i_addr; - ASSERT(qoff_f); - - /* - * The logitem format's flag tells us if this was user quotaoff, - * group/project quotaoff or both. - */ - if (qoff_f->qf_flags & XFS_UQUOTA_ACCT) - log->l_quotaoffs_flag |= XFS_DQ_USER; - if (qoff_f->qf_flags & XFS_PQUOTA_ACCT) - log->l_quotaoffs_flag |= XFS_DQ_PROJ; - if (qoff_f->qf_flags & XFS_GQUOTA_ACCT) - log->l_quotaoffs_flag |= XFS_DQ_GROUP; - - return 0; -} - /* * Recover a dquot record */ @@ -3890,40 +3837,6 @@ xlog_recover_do_icreate_pass2( length, be32_to_cpu(icl->icl_gen)); } -STATIC int -xlog_recover_commit_pass1( - struct xlog *log, - struct xlog_recover *trans, - struct xlog_recover_item *item) -{ - trace_xfs_log_recover_item_recover(log, trans, item, XLOG_RECOVER_PASS1); - - switch (ITEM_TYPE(item)) { - case XFS_LI_BUF: - return xlog_recover_buffer_pass1(log, item); - case XFS_LI_QUOTAOFF: - return xlog_recover_quotaoff_pass1(log, item); - case XFS_LI_INODE: - case XFS_LI_EFI: - case XFS_LI_EFD: - case XFS_LI_DQUOT: - case XFS_LI_ICREATE: - case XFS_LI_RUI: - case XFS_LI_RUD: - case XFS_LI_CUI: - case XFS_LI_CUD: - case XFS_LI_BUI: - case XFS_LI_BUD: - /* nothing to do in pass 1 */ - return 0; - default: - xfs_warn(log->l_mp, "%s: invalid item type (%d)", - __func__, ITEM_TYPE(item)); - ASSERT(0); - return -EFSCORRUPTED; - } -} - STATIC int xlog_recover_commit_pass2( struct xlog *log, @@ -4021,9 +3934,12 @@ xlog_recover_commit_trans( return error; list_for_each_entry_safe(item, next, &trans->r_itemq, ri_list) { + trace_xfs_log_recover_item_recover(log, trans, item, pass); + switch (pass) { case XLOG_RECOVER_PASS1: - error = xlog_recover_commit_pass1(log, trans, item); + if (item->ri_ops->commit_pass1) + error = item->ri_ops->commit_pass1(log, item); break; case XLOG_RECOVER_PASS2: if (item->ri_ops->ra_pass2) From 1094d3f12363474b2a3d1a6c06124bec25dd1555 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:47 -0700 Subject: [PATCH 0277/1043] xfs: refactor log recovery buffer item dispatch for pass2 commit functions Move the log buffer item pass2 commit code into the per-item source code files and use the dispatch function to call it. We do these one at a time because there's a lot of code to move. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_log_recover.h | 23 + fs/xfs/xfs_buf_item_recover.c | 790 +++++++++++++++++++++++++++++++ fs/xfs/xfs_log_recover.c | 798 +------------------------------- 3 files changed, 820 insertions(+), 791 deletions(-) diff --git a/fs/xfs/libxfs/xfs_log_recover.h b/fs/xfs/libxfs/xfs_log_recover.h index 19e24b8877c9..91fe954a796c 100644 --- a/fs/xfs/libxfs/xfs_log_recover.h +++ b/fs/xfs/libxfs/xfs_log_recover.h @@ -37,6 +37,26 @@ struct xlog_recover_item_ops { /* Do whatever work we need to do for pass1, if provided. */ int (*commit_pass1)(struct xlog *log, struct xlog_recover_item *item); + + /* + * This function should do whatever work is needed for pass2 of log + * recovery, if provided. + * + * If the recovered item is an intent item, this function should parse + * the recovered item to construct an in-core log intent item and + * insert it into the AIL. The in-core log intent item should have 1 + * refcount so that the item is freed either (a) when we commit the + * recovered log item for the intent-done item; (b) replay the work and + * log a new intent-done item; or (c) recovery fails and we have to + * abort. + * + * If the recovered item is an intent-done item, this function should + * parse the recovered item to find the id of the corresponding intent + * log item. Next, it should find the in-core log intent item in the + * AIL and release it. + */ + int (*commit_pass2)(struct xlog *log, struct list_head *buffer_list, + struct xlog_recover_item *item, xfs_lsn_t lsn); }; extern const struct xlog_recover_item_ops xlog_icreate_item_ops; @@ -101,5 +121,8 @@ struct xlog_recover { void xlog_buf_readahead(struct xlog *log, xfs_daddr_t blkno, uint len, const struct xfs_buf_ops *ops); bool xlog_add_buffer_cancelled(struct xlog *log, xfs_daddr_t blkno, uint len); +bool xlog_is_buffer_cancelled(struct xlog *log, xfs_daddr_t blkno, uint len); +bool xlog_put_buffer_cancelled(struct xlog *log, xfs_daddr_t blkno, uint len); +void xlog_recover_iodone(struct xfs_buf *bp); #endif /* __XFS_LOG_RECOVER_H__ */ diff --git a/fs/xfs/xfs_buf_item_recover.c b/fs/xfs/xfs_buf_item_recover.c index e2d9599f67df..4ba2e27a15ca 100644 --- a/fs/xfs/xfs_buf_item_recover.c +++ b/fs/xfs/xfs_buf_item_recover.c @@ -18,6 +18,10 @@ #include "xfs_log.h" #include "xfs_log_priv.h" #include "xfs_log_recover.h" +#include "xfs_error.h" +#include "xfs_inode.h" +#include "xfs_dir2.h" +#include "xfs_quota.h" /* * Sort buffer items for log recovery. Most buffer items should end up on the @@ -82,9 +86,795 @@ xlog_recover_buf_commit_pass1( return 0; } +/* + * Validate the recovered buffer is of the correct type and attach the + * appropriate buffer operations to them for writeback. Magic numbers are in a + * few places: + * the first 16 bits of the buffer (inode buffer, dquot buffer), + * the first 32 bits of the buffer (most blocks), + * inside a struct xfs_da_blkinfo at the start of the buffer. + */ +static void +xlog_recover_validate_buf_type( + struct xfs_mount *mp, + struct xfs_buf *bp, + struct xfs_buf_log_format *buf_f, + xfs_lsn_t current_lsn) +{ + struct xfs_da_blkinfo *info = bp->b_addr; + uint32_t magic32; + uint16_t magic16; + uint16_t magicda; + char *warnmsg = NULL; + + /* + * We can only do post recovery validation on items on CRC enabled + * fielsystems as we need to know when the buffer was written to be able + * to determine if we should have replayed the item. If we replay old + * metadata over a newer buffer, then it will enter a temporarily + * inconsistent state resulting in verification failures. Hence for now + * just avoid the verification stage for non-crc filesystems + */ + if (!xfs_sb_version_hascrc(&mp->m_sb)) + return; + + magic32 = be32_to_cpu(*(__be32 *)bp->b_addr); + magic16 = be16_to_cpu(*(__be16*)bp->b_addr); + magicda = be16_to_cpu(info->magic); + switch (xfs_blft_from_flags(buf_f)) { + case XFS_BLFT_BTREE_BUF: + switch (magic32) { + case XFS_ABTB_CRC_MAGIC: + case XFS_ABTB_MAGIC: + bp->b_ops = &xfs_bnobt_buf_ops; + break; + case XFS_ABTC_CRC_MAGIC: + case XFS_ABTC_MAGIC: + bp->b_ops = &xfs_cntbt_buf_ops; + break; + case XFS_IBT_CRC_MAGIC: + case XFS_IBT_MAGIC: + bp->b_ops = &xfs_inobt_buf_ops; + break; + case XFS_FIBT_CRC_MAGIC: + case XFS_FIBT_MAGIC: + bp->b_ops = &xfs_finobt_buf_ops; + break; + case XFS_BMAP_CRC_MAGIC: + case XFS_BMAP_MAGIC: + bp->b_ops = &xfs_bmbt_buf_ops; + break; + case XFS_RMAP_CRC_MAGIC: + bp->b_ops = &xfs_rmapbt_buf_ops; + break; + case XFS_REFC_CRC_MAGIC: + bp->b_ops = &xfs_refcountbt_buf_ops; + break; + default: + warnmsg = "Bad btree block magic!"; + break; + } + break; + case XFS_BLFT_AGF_BUF: + if (magic32 != XFS_AGF_MAGIC) { + warnmsg = "Bad AGF block magic!"; + break; + } + bp->b_ops = &xfs_agf_buf_ops; + break; + case XFS_BLFT_AGFL_BUF: + if (magic32 != XFS_AGFL_MAGIC) { + warnmsg = "Bad AGFL block magic!"; + break; + } + bp->b_ops = &xfs_agfl_buf_ops; + break; + case XFS_BLFT_AGI_BUF: + if (magic32 != XFS_AGI_MAGIC) { + warnmsg = "Bad AGI block magic!"; + break; + } + bp->b_ops = &xfs_agi_buf_ops; + break; + case XFS_BLFT_UDQUOT_BUF: + case XFS_BLFT_PDQUOT_BUF: + case XFS_BLFT_GDQUOT_BUF: +#ifdef CONFIG_XFS_QUOTA + if (magic16 != XFS_DQUOT_MAGIC) { + warnmsg = "Bad DQUOT block magic!"; + break; + } + bp->b_ops = &xfs_dquot_buf_ops; +#else + xfs_alert(mp, + "Trying to recover dquots without QUOTA support built in!"); + ASSERT(0); +#endif + break; + case XFS_BLFT_DINO_BUF: + if (magic16 != XFS_DINODE_MAGIC) { + warnmsg = "Bad INODE block magic!"; + break; + } + bp->b_ops = &xfs_inode_buf_ops; + break; + case XFS_BLFT_SYMLINK_BUF: + if (magic32 != XFS_SYMLINK_MAGIC) { + warnmsg = "Bad symlink block magic!"; + break; + } + bp->b_ops = &xfs_symlink_buf_ops; + break; + case XFS_BLFT_DIR_BLOCK_BUF: + if (magic32 != XFS_DIR2_BLOCK_MAGIC && + magic32 != XFS_DIR3_BLOCK_MAGIC) { + warnmsg = "Bad dir block magic!"; + break; + } + bp->b_ops = &xfs_dir3_block_buf_ops; + break; + case XFS_BLFT_DIR_DATA_BUF: + if (magic32 != XFS_DIR2_DATA_MAGIC && + magic32 != XFS_DIR3_DATA_MAGIC) { + warnmsg = "Bad dir data magic!"; + break; + } + bp->b_ops = &xfs_dir3_data_buf_ops; + break; + case XFS_BLFT_DIR_FREE_BUF: + if (magic32 != XFS_DIR2_FREE_MAGIC && + magic32 != XFS_DIR3_FREE_MAGIC) { + warnmsg = "Bad dir3 free magic!"; + break; + } + bp->b_ops = &xfs_dir3_free_buf_ops; + break; + case XFS_BLFT_DIR_LEAF1_BUF: + if (magicda != XFS_DIR2_LEAF1_MAGIC && + magicda != XFS_DIR3_LEAF1_MAGIC) { + warnmsg = "Bad dir leaf1 magic!"; + break; + } + bp->b_ops = &xfs_dir3_leaf1_buf_ops; + break; + case XFS_BLFT_DIR_LEAFN_BUF: + if (magicda != XFS_DIR2_LEAFN_MAGIC && + magicda != XFS_DIR3_LEAFN_MAGIC) { + warnmsg = "Bad dir leafn magic!"; + break; + } + bp->b_ops = &xfs_dir3_leafn_buf_ops; + break; + case XFS_BLFT_DA_NODE_BUF: + if (magicda != XFS_DA_NODE_MAGIC && + magicda != XFS_DA3_NODE_MAGIC) { + warnmsg = "Bad da node magic!"; + break; + } + bp->b_ops = &xfs_da3_node_buf_ops; + break; + case XFS_BLFT_ATTR_LEAF_BUF: + if (magicda != XFS_ATTR_LEAF_MAGIC && + magicda != XFS_ATTR3_LEAF_MAGIC) { + warnmsg = "Bad attr leaf magic!"; + break; + } + bp->b_ops = &xfs_attr3_leaf_buf_ops; + break; + case XFS_BLFT_ATTR_RMT_BUF: + if (magic32 != XFS_ATTR3_RMT_MAGIC) { + warnmsg = "Bad attr remote magic!"; + break; + } + bp->b_ops = &xfs_attr3_rmt_buf_ops; + break; + case XFS_BLFT_SB_BUF: + if (magic32 != XFS_SB_MAGIC) { + warnmsg = "Bad SB block magic!"; + break; + } + bp->b_ops = &xfs_sb_buf_ops; + break; +#ifdef CONFIG_XFS_RT + case XFS_BLFT_RTBITMAP_BUF: + case XFS_BLFT_RTSUMMARY_BUF: + /* no magic numbers for verification of RT buffers */ + bp->b_ops = &xfs_rtbuf_ops; + break; +#endif /* CONFIG_XFS_RT */ + default: + xfs_warn(mp, "Unknown buffer type %d!", + xfs_blft_from_flags(buf_f)); + break; + } + + /* + * Nothing else to do in the case of a NULL current LSN as this means + * the buffer is more recent than the change in the log and will be + * skipped. + */ + if (current_lsn == NULLCOMMITLSN) + return; + + if (warnmsg) { + xfs_warn(mp, warnmsg); + ASSERT(0); + } + + /* + * We must update the metadata LSN of the buffer as it is written out to + * ensure that older transactions never replay over this one and corrupt + * the buffer. This can occur if log recovery is interrupted at some + * point after the current transaction completes, at which point a + * subsequent mount starts recovery from the beginning. + * + * Write verifiers update the metadata LSN from log items attached to + * the buffer. Therefore, initialize a bli purely to carry the LSN to + * the verifier. We'll clean it up in our ->iodone() callback. + */ + if (bp->b_ops) { + struct xfs_buf_log_item *bip; + + ASSERT(!bp->b_iodone || bp->b_iodone == xlog_recover_iodone); + bp->b_iodone = xlog_recover_iodone; + xfs_buf_item_init(bp, mp); + bip = bp->b_log_item; + bip->bli_item.li_lsn = current_lsn; + } +} + +/* + * Perform a 'normal' buffer recovery. Each logged region of the + * buffer should be copied over the corresponding region in the + * given buffer. The bitmap in the buf log format structure indicates + * where to place the logged data. + */ +STATIC void +xlog_recover_do_reg_buffer( + struct xfs_mount *mp, + struct xlog_recover_item *item, + struct xfs_buf *bp, + struct xfs_buf_log_format *buf_f, + xfs_lsn_t current_lsn) +{ + int i; + int bit; + int nbits; + xfs_failaddr_t fa; + const size_t size_disk_dquot = sizeof(struct xfs_disk_dquot); + + trace_xfs_log_recover_buf_reg_buf(mp->m_log, buf_f); + + bit = 0; + i = 1; /* 0 is the buf format structure */ + while (1) { + bit = xfs_next_bit(buf_f->blf_data_map, + buf_f->blf_map_size, bit); + if (bit == -1) + break; + nbits = xfs_contig_bits(buf_f->blf_data_map, + buf_f->blf_map_size, bit); + ASSERT(nbits > 0); + ASSERT(item->ri_buf[i].i_addr != NULL); + ASSERT(item->ri_buf[i].i_len % XFS_BLF_CHUNK == 0); + ASSERT(BBTOB(bp->b_length) >= + ((uint)bit << XFS_BLF_SHIFT) + (nbits << XFS_BLF_SHIFT)); + + /* + * The dirty regions logged in the buffer, even though + * contiguous, may span multiple chunks. This is because the + * dirty region may span a physical page boundary in a buffer + * and hence be split into two separate vectors for writing into + * the log. Hence we need to trim nbits back to the length of + * the current region being copied out of the log. + */ + if (item->ri_buf[i].i_len < (nbits << XFS_BLF_SHIFT)) + nbits = item->ri_buf[i].i_len >> XFS_BLF_SHIFT; + + /* + * Do a sanity check if this is a dquot buffer. Just checking + * the first dquot in the buffer should do. XXXThis is + * probably a good thing to do for other buf types also. + */ + fa = NULL; + if (buf_f->blf_flags & + (XFS_BLF_UDQUOT_BUF|XFS_BLF_PDQUOT_BUF|XFS_BLF_GDQUOT_BUF)) { + if (item->ri_buf[i].i_addr == NULL) { + xfs_alert(mp, + "XFS: NULL dquot in %s.", __func__); + goto next; + } + if (item->ri_buf[i].i_len < size_disk_dquot) { + xfs_alert(mp, + "XFS: dquot too small (%d) in %s.", + item->ri_buf[i].i_len, __func__); + goto next; + } + fa = xfs_dquot_verify(mp, item->ri_buf[i].i_addr, + -1, 0); + if (fa) { + xfs_alert(mp, + "dquot corrupt at %pS trying to replay into block 0x%llx", + fa, bp->b_bn); + goto next; + } + } + + memcpy(xfs_buf_offset(bp, + (uint)bit << XFS_BLF_SHIFT), /* dest */ + item->ri_buf[i].i_addr, /* source */ + nbits<ri_total); + + xlog_recover_validate_buf_type(mp, bp, buf_f, current_lsn); +} + +/* + * Perform a dquot buffer recovery. + * Simple algorithm: if we have found a QUOTAOFF log item of the same type + * (ie. USR or GRP), then just toss this buffer away; don't recover it. + * Else, treat it as a regular buffer and do recovery. + * + * Return false if the buffer was tossed and true if we recovered the buffer to + * indicate to the caller if the buffer needs writing. + */ +STATIC bool +xlog_recover_do_dquot_buffer( + struct xfs_mount *mp, + struct xlog *log, + struct xlog_recover_item *item, + struct xfs_buf *bp, + struct xfs_buf_log_format *buf_f) +{ + uint type; + + trace_xfs_log_recover_buf_dquot_buf(log, buf_f); + + /* + * Filesystems are required to send in quota flags at mount time. + */ + if (!mp->m_qflags) + return false; + + type = 0; + if (buf_f->blf_flags & XFS_BLF_UDQUOT_BUF) + type |= XFS_DQ_USER; + if (buf_f->blf_flags & XFS_BLF_PDQUOT_BUF) + type |= XFS_DQ_PROJ; + if (buf_f->blf_flags & XFS_BLF_GDQUOT_BUF) + type |= XFS_DQ_GROUP; + /* + * This type of quotas was turned off, so ignore this buffer + */ + if (log->l_quotaoffs_flag & type) + return false; + + xlog_recover_do_reg_buffer(mp, item, bp, buf_f, NULLCOMMITLSN); + return true; +} + +/* + * Perform recovery for a buffer full of inodes. In these buffers, the only + * data which should be recovered is that which corresponds to the + * di_next_unlinked pointers in the on disk inode structures. The rest of the + * data for the inodes is always logged through the inodes themselves rather + * than the inode buffer and is recovered in xlog_recover_inode_pass2(). + * + * The only time when buffers full of inodes are fully recovered is when the + * buffer is full of newly allocated inodes. In this case the buffer will + * not be marked as an inode buffer and so will be sent to + * xlog_recover_do_reg_buffer() below during recovery. + */ +STATIC int +xlog_recover_do_inode_buffer( + struct xfs_mount *mp, + struct xlog_recover_item *item, + struct xfs_buf *bp, + struct xfs_buf_log_format *buf_f) +{ + int i; + int item_index = 0; + int bit = 0; + int nbits = 0; + int reg_buf_offset = 0; + int reg_buf_bytes = 0; + int next_unlinked_offset; + int inodes_per_buf; + xfs_agino_t *logged_nextp; + xfs_agino_t *buffer_nextp; + + trace_xfs_log_recover_buf_inode_buf(mp->m_log, buf_f); + + /* + * Post recovery validation only works properly on CRC enabled + * filesystems. + */ + if (xfs_sb_version_hascrc(&mp->m_sb)) + bp->b_ops = &xfs_inode_buf_ops; + + inodes_per_buf = BBTOB(bp->b_length) >> mp->m_sb.sb_inodelog; + for (i = 0; i < inodes_per_buf; i++) { + next_unlinked_offset = (i * mp->m_sb.sb_inodesize) + + offsetof(xfs_dinode_t, di_next_unlinked); + + while (next_unlinked_offset >= + (reg_buf_offset + reg_buf_bytes)) { + /* + * The next di_next_unlinked field is beyond + * the current logged region. Find the next + * logged region that contains or is beyond + * the current di_next_unlinked field. + */ + bit += nbits; + bit = xfs_next_bit(buf_f->blf_data_map, + buf_f->blf_map_size, bit); + + /* + * If there are no more logged regions in the + * buffer, then we're done. + */ + if (bit == -1) + return 0; + + nbits = xfs_contig_bits(buf_f->blf_data_map, + buf_f->blf_map_size, bit); + ASSERT(nbits > 0); + reg_buf_offset = bit << XFS_BLF_SHIFT; + reg_buf_bytes = nbits << XFS_BLF_SHIFT; + item_index++; + } + + /* + * If the current logged region starts after the current + * di_next_unlinked field, then move on to the next + * di_next_unlinked field. + */ + if (next_unlinked_offset < reg_buf_offset) + continue; + + ASSERT(item->ri_buf[item_index].i_addr != NULL); + ASSERT((item->ri_buf[item_index].i_len % XFS_BLF_CHUNK) == 0); + ASSERT((reg_buf_offset + reg_buf_bytes) <= BBTOB(bp->b_length)); + + /* + * The current logged region contains a copy of the + * current di_next_unlinked field. Extract its value + * and copy it to the buffer copy. + */ + logged_nextp = item->ri_buf[item_index].i_addr + + next_unlinked_offset - reg_buf_offset; + if (XFS_IS_CORRUPT(mp, *logged_nextp == 0)) { + xfs_alert(mp, + "Bad inode buffer log record (ptr = "PTR_FMT", bp = "PTR_FMT"). " + "Trying to replay bad (0) inode di_next_unlinked field.", + item, bp); + return -EFSCORRUPTED; + } + + buffer_nextp = xfs_buf_offset(bp, next_unlinked_offset); + *buffer_nextp = *logged_nextp; + + /* + * If necessary, recalculate the CRC in the on-disk inode. We + * have to leave the inode in a consistent state for whoever + * reads it next.... + */ + xfs_dinode_calc_crc(mp, + xfs_buf_offset(bp, i * mp->m_sb.sb_inodesize)); + + } + + return 0; +} + +/* + * V5 filesystems know the age of the buffer on disk being recovered. We can + * have newer objects on disk than we are replaying, and so for these cases we + * don't want to replay the current change as that will make the buffer contents + * temporarily invalid on disk. + * + * The magic number might not match the buffer type we are going to recover + * (e.g. reallocated blocks), so we ignore the xfs_buf_log_format flags. Hence + * extract the LSN of the existing object in the buffer based on it's current + * magic number. If we don't recognise the magic number in the buffer, then + * return a LSN of -1 so that the caller knows it was an unrecognised block and + * so can recover the buffer. + * + * Note: we cannot rely solely on magic number matches to determine that the + * buffer has a valid LSN - we also need to verify that it belongs to this + * filesystem, so we need to extract the object's LSN and compare it to that + * which we read from the superblock. If the UUIDs don't match, then we've got a + * stale metadata block from an old filesystem instance that we need to recover + * over the top of. + */ +static xfs_lsn_t +xlog_recover_get_buf_lsn( + struct xfs_mount *mp, + struct xfs_buf *bp) +{ + uint32_t magic32; + uint16_t magic16; + uint16_t magicda; + void *blk = bp->b_addr; + uuid_t *uuid; + xfs_lsn_t lsn = -1; + + /* v4 filesystems always recover immediately */ + if (!xfs_sb_version_hascrc(&mp->m_sb)) + goto recover_immediately; + + magic32 = be32_to_cpu(*(__be32 *)blk); + switch (magic32) { + case XFS_ABTB_CRC_MAGIC: + case XFS_ABTC_CRC_MAGIC: + case XFS_ABTB_MAGIC: + case XFS_ABTC_MAGIC: + case XFS_RMAP_CRC_MAGIC: + case XFS_REFC_CRC_MAGIC: + case XFS_IBT_CRC_MAGIC: + case XFS_IBT_MAGIC: { + struct xfs_btree_block *btb = blk; + + lsn = be64_to_cpu(btb->bb_u.s.bb_lsn); + uuid = &btb->bb_u.s.bb_uuid; + break; + } + case XFS_BMAP_CRC_MAGIC: + case XFS_BMAP_MAGIC: { + struct xfs_btree_block *btb = blk; + + lsn = be64_to_cpu(btb->bb_u.l.bb_lsn); + uuid = &btb->bb_u.l.bb_uuid; + break; + } + case XFS_AGF_MAGIC: + lsn = be64_to_cpu(((struct xfs_agf *)blk)->agf_lsn); + uuid = &((struct xfs_agf *)blk)->agf_uuid; + break; + case XFS_AGFL_MAGIC: + lsn = be64_to_cpu(((struct xfs_agfl *)blk)->agfl_lsn); + uuid = &((struct xfs_agfl *)blk)->agfl_uuid; + break; + case XFS_AGI_MAGIC: + lsn = be64_to_cpu(((struct xfs_agi *)blk)->agi_lsn); + uuid = &((struct xfs_agi *)blk)->agi_uuid; + break; + case XFS_SYMLINK_MAGIC: + lsn = be64_to_cpu(((struct xfs_dsymlink_hdr *)blk)->sl_lsn); + uuid = &((struct xfs_dsymlink_hdr *)blk)->sl_uuid; + break; + case XFS_DIR3_BLOCK_MAGIC: + case XFS_DIR3_DATA_MAGIC: + case XFS_DIR3_FREE_MAGIC: + lsn = be64_to_cpu(((struct xfs_dir3_blk_hdr *)blk)->lsn); + uuid = &((struct xfs_dir3_blk_hdr *)blk)->uuid; + break; + case XFS_ATTR3_RMT_MAGIC: + /* + * Remote attr blocks are written synchronously, rather than + * being logged. That means they do not contain a valid LSN + * (i.e. transactionally ordered) in them, and hence any time we + * see a buffer to replay over the top of a remote attribute + * block we should simply do so. + */ + goto recover_immediately; + case XFS_SB_MAGIC: + /* + * superblock uuids are magic. We may or may not have a + * sb_meta_uuid on disk, but it will be set in the in-core + * superblock. We set the uuid pointer for verification + * according to the superblock feature mask to ensure we check + * the relevant UUID in the superblock. + */ + lsn = be64_to_cpu(((struct xfs_dsb *)blk)->sb_lsn); + if (xfs_sb_version_hasmetauuid(&mp->m_sb)) + uuid = &((struct xfs_dsb *)blk)->sb_meta_uuid; + else + uuid = &((struct xfs_dsb *)blk)->sb_uuid; + break; + default: + break; + } + + if (lsn != (xfs_lsn_t)-1) { + if (!uuid_equal(&mp->m_sb.sb_meta_uuid, uuid)) + goto recover_immediately; + return lsn; + } + + magicda = be16_to_cpu(((struct xfs_da_blkinfo *)blk)->magic); + switch (magicda) { + case XFS_DIR3_LEAF1_MAGIC: + case XFS_DIR3_LEAFN_MAGIC: + case XFS_DA3_NODE_MAGIC: + lsn = be64_to_cpu(((struct xfs_da3_blkinfo *)blk)->lsn); + uuid = &((struct xfs_da3_blkinfo *)blk)->uuid; + break; + default: + break; + } + + if (lsn != (xfs_lsn_t)-1) { + if (!uuid_equal(&mp->m_sb.sb_uuid, uuid)) + goto recover_immediately; + return lsn; + } + + /* + * We do individual object checks on dquot and inode buffers as they + * have their own individual LSN records. Also, we could have a stale + * buffer here, so we have to at least recognise these buffer types. + * + * A notd complexity here is inode unlinked list processing - it logs + * the inode directly in the buffer, but we don't know which inodes have + * been modified, and there is no global buffer LSN. Hence we need to + * recover all inode buffer types immediately. This problem will be + * fixed by logical logging of the unlinked list modifications. + */ + magic16 = be16_to_cpu(*(__be16 *)blk); + switch (magic16) { + case XFS_DQUOT_MAGIC: + case XFS_DINODE_MAGIC: + goto recover_immediately; + default: + break; + } + + /* unknown buffer contents, recover immediately */ + +recover_immediately: + return (xfs_lsn_t)-1; + +} + +/* + * This routine replays a modification made to a buffer at runtime. + * There are actually two types of buffer, regular and inode, which + * are handled differently. Inode buffers are handled differently + * in that we only recover a specific set of data from them, namely + * the inode di_next_unlinked fields. This is because all other inode + * data is actually logged via inode records and any data we replay + * here which overlaps that may be stale. + * + * When meta-data buffers are freed at run time we log a buffer item + * with the XFS_BLF_CANCEL bit set to indicate that previous copies + * of the buffer in the log should not be replayed at recovery time. + * This is so that if the blocks covered by the buffer are reused for + * file data before we crash we don't end up replaying old, freed + * meta-data into a user's file. + * + * To handle the cancellation of buffer log items, we make two passes + * over the log during recovery. During the first we build a table of + * those buffers which have been cancelled, and during the second we + * only replay those buffers which do not have corresponding cancel + * records in the table. See xlog_recover_buf_pass[1,2] above + * for more details on the implementation of the table of cancel records. + */ +STATIC int +xlog_recover_buf_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t current_lsn) +{ + struct xfs_buf_log_format *buf_f = item->ri_buf[0].i_addr; + struct xfs_mount *mp = log->l_mp; + struct xfs_buf *bp; + int error; + uint buf_flags; + xfs_lsn_t lsn; + + /* + * In this pass we only want to recover all the buffers which have + * not been cancelled and are not cancellation buffers themselves. + */ + if (buf_f->blf_flags & XFS_BLF_CANCEL) { + if (xlog_put_buffer_cancelled(log, buf_f->blf_blkno, + buf_f->blf_len)) + goto cancelled; + } else { + + if (xlog_is_buffer_cancelled(log, buf_f->blf_blkno, + buf_f->blf_len)) + goto cancelled; + } + + trace_xfs_log_recover_buf_recover(log, buf_f); + + buf_flags = 0; + if (buf_f->blf_flags & XFS_BLF_INODE_BUF) + buf_flags |= XBF_UNMAPPED; + + error = xfs_buf_read(mp->m_ddev_targp, buf_f->blf_blkno, buf_f->blf_len, + buf_flags, &bp, NULL); + if (error) + return error; + + /* + * Recover the buffer only if we get an LSN from it and it's less than + * the lsn of the transaction we are replaying. + * + * Note that we have to be extremely careful of readahead here. + * Readahead does not attach verfiers to the buffers so if we don't + * actually do any replay after readahead because of the LSN we found + * in the buffer if more recent than that current transaction then we + * need to attach the verifier directly. Failure to do so can lead to + * future recovery actions (e.g. EFI and unlinked list recovery) can + * operate on the buffers and they won't get the verifier attached. This + * can lead to blocks on disk having the correct content but a stale + * CRC. + * + * It is safe to assume these clean buffers are currently up to date. + * If the buffer is dirtied by a later transaction being replayed, then + * the verifier will be reset to match whatever recover turns that + * buffer into. + */ + lsn = xlog_recover_get_buf_lsn(mp, bp); + if (lsn && lsn != -1 && XFS_LSN_CMP(lsn, current_lsn) >= 0) { + trace_xfs_log_recover_buf_skip(log, buf_f); + xlog_recover_validate_buf_type(mp, bp, buf_f, NULLCOMMITLSN); + goto out_release; + } + + if (buf_f->blf_flags & XFS_BLF_INODE_BUF) { + error = xlog_recover_do_inode_buffer(mp, item, bp, buf_f); + if (error) + goto out_release; + } else if (buf_f->blf_flags & + (XFS_BLF_UDQUOT_BUF|XFS_BLF_PDQUOT_BUF|XFS_BLF_GDQUOT_BUF)) { + bool dirty; + + dirty = xlog_recover_do_dquot_buffer(mp, log, item, bp, buf_f); + if (!dirty) + goto out_release; + } else { + xlog_recover_do_reg_buffer(mp, item, bp, buf_f, current_lsn); + } + + /* + * Perform delayed write on the buffer. Asynchronous writes will be + * slower when taking into account all the buffers to be flushed. + * + * Also make sure that only inode buffers with good sizes stay in + * the buffer cache. The kernel moves inodes in buffers of 1 block + * or inode_cluster_size bytes, whichever is bigger. The inode + * buffers in the log can be a different size if the log was generated + * by an older kernel using unclustered inode buffers or a newer kernel + * running with a different inode cluster size. Regardless, if the + * the inode buffer size isn't max(blocksize, inode_cluster_size) + * for *our* value of inode_cluster_size, then we need to keep + * the buffer out of the buffer cache so that the buffer won't + * overlap with future reads of those inodes. + */ + if (XFS_DINODE_MAGIC == + be16_to_cpu(*((__be16 *)xfs_buf_offset(bp, 0))) && + (BBTOB(bp->b_length) != M_IGEO(log->l_mp)->inode_cluster_size)) { + xfs_buf_stale(bp); + error = xfs_bwrite(bp); + } else { + ASSERT(bp->b_mount == mp); + bp->b_iodone = xlog_recover_iodone; + xfs_buf_delwri_queue(bp, buffer_list); + } + +out_release: + xfs_buf_relse(bp); + return error; +cancelled: + trace_xfs_log_recover_buf_cancel(log, buf_f); + return 0; +} + const struct xlog_recover_item_ops xlog_buf_item_ops = { .item_type = XFS_LI_BUF, .reorder = xlog_recover_buf_reorder, .ra_pass2 = xlog_recover_buf_ra_pass2, .commit_pass1 = xlog_recover_buf_commit_pass1, + .commit_pass2 = xlog_recover_buf_commit_pass2, }; diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 2511f2874464..eaf2ea8da5d6 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -284,7 +284,7 @@ xlog_header_check_mount( return 0; } -STATIC void +void xlog_recover_iodone( struct xfs_buf *bp) { @@ -1985,7 +1985,7 @@ xlog_add_buffer_cancelled( /* * Check if there is and entry for blkno, len in the buffer cancel record table. */ -static bool +bool xlog_is_buffer_cancelled( struct xlog *log, xfs_daddr_t blkno, @@ -2002,7 +2002,7 @@ xlog_is_buffer_cancelled( * buffer is re-used again after its last cancellation we actually replay the * changes made at that point. */ -static bool +bool xlog_put_buffer_cancelled( struct xlog *log, xfs_daddr_t blkno, @@ -2034,791 +2034,6 @@ xlog_buf_readahead( xfs_buf_readahead(log->l_mp->m_ddev_targp, blkno, len, ops); } -/* - * Perform recovery for a buffer full of inodes. In these buffers, the only - * data which should be recovered is that which corresponds to the - * di_next_unlinked pointers in the on disk inode structures. The rest of the - * data for the inodes is always logged through the inodes themselves rather - * than the inode buffer and is recovered in xlog_recover_inode_pass2(). - * - * The only time when buffers full of inodes are fully recovered is when the - * buffer is full of newly allocated inodes. In this case the buffer will - * not be marked as an inode buffer and so will be sent to - * xlog_recover_do_reg_buffer() below during recovery. - */ -STATIC int -xlog_recover_do_inode_buffer( - struct xfs_mount *mp, - struct xlog_recover_item *item, - struct xfs_buf *bp, - xfs_buf_log_format_t *buf_f) -{ - int i; - int item_index = 0; - int bit = 0; - int nbits = 0; - int reg_buf_offset = 0; - int reg_buf_bytes = 0; - int next_unlinked_offset; - int inodes_per_buf; - xfs_agino_t *logged_nextp; - xfs_agino_t *buffer_nextp; - - trace_xfs_log_recover_buf_inode_buf(mp->m_log, buf_f); - - /* - * Post recovery validation only works properly on CRC enabled - * filesystems. - */ - if (xfs_sb_version_hascrc(&mp->m_sb)) - bp->b_ops = &xfs_inode_buf_ops; - - inodes_per_buf = BBTOB(bp->b_length) >> mp->m_sb.sb_inodelog; - for (i = 0; i < inodes_per_buf; i++) { - next_unlinked_offset = (i * mp->m_sb.sb_inodesize) + - offsetof(xfs_dinode_t, di_next_unlinked); - - while (next_unlinked_offset >= - (reg_buf_offset + reg_buf_bytes)) { - /* - * The next di_next_unlinked field is beyond - * the current logged region. Find the next - * logged region that contains or is beyond - * the current di_next_unlinked field. - */ - bit += nbits; - bit = xfs_next_bit(buf_f->blf_data_map, - buf_f->blf_map_size, bit); - - /* - * If there are no more logged regions in the - * buffer, then we're done. - */ - if (bit == -1) - return 0; - - nbits = xfs_contig_bits(buf_f->blf_data_map, - buf_f->blf_map_size, bit); - ASSERT(nbits > 0); - reg_buf_offset = bit << XFS_BLF_SHIFT; - reg_buf_bytes = nbits << XFS_BLF_SHIFT; - item_index++; - } - - /* - * If the current logged region starts after the current - * di_next_unlinked field, then move on to the next - * di_next_unlinked field. - */ - if (next_unlinked_offset < reg_buf_offset) - continue; - - ASSERT(item->ri_buf[item_index].i_addr != NULL); - ASSERT((item->ri_buf[item_index].i_len % XFS_BLF_CHUNK) == 0); - ASSERT((reg_buf_offset + reg_buf_bytes) <= BBTOB(bp->b_length)); - - /* - * The current logged region contains a copy of the - * current di_next_unlinked field. Extract its value - * and copy it to the buffer copy. - */ - logged_nextp = item->ri_buf[item_index].i_addr + - next_unlinked_offset - reg_buf_offset; - if (XFS_IS_CORRUPT(mp, *logged_nextp == 0)) { - xfs_alert(mp, - "Bad inode buffer log record (ptr = "PTR_FMT", bp = "PTR_FMT"). " - "Trying to replay bad (0) inode di_next_unlinked field.", - item, bp); - return -EFSCORRUPTED; - } - - buffer_nextp = xfs_buf_offset(bp, next_unlinked_offset); - *buffer_nextp = *logged_nextp; - - /* - * If necessary, recalculate the CRC in the on-disk inode. We - * have to leave the inode in a consistent state for whoever - * reads it next.... - */ - xfs_dinode_calc_crc(mp, - xfs_buf_offset(bp, i * mp->m_sb.sb_inodesize)); - - } - - return 0; -} - -/* - * V5 filesystems know the age of the buffer on disk being recovered. We can - * have newer objects on disk than we are replaying, and so for these cases we - * don't want to replay the current change as that will make the buffer contents - * temporarily invalid on disk. - * - * The magic number might not match the buffer type we are going to recover - * (e.g. reallocated blocks), so we ignore the xfs_buf_log_format flags. Hence - * extract the LSN of the existing object in the buffer based on it's current - * magic number. If we don't recognise the magic number in the buffer, then - * return a LSN of -1 so that the caller knows it was an unrecognised block and - * so can recover the buffer. - * - * Note: we cannot rely solely on magic number matches to determine that the - * buffer has a valid LSN - we also need to verify that it belongs to this - * filesystem, so we need to extract the object's LSN and compare it to that - * which we read from the superblock. If the UUIDs don't match, then we've got a - * stale metadata block from an old filesystem instance that we need to recover - * over the top of. - */ -static xfs_lsn_t -xlog_recover_get_buf_lsn( - struct xfs_mount *mp, - struct xfs_buf *bp) -{ - uint32_t magic32; - uint16_t magic16; - uint16_t magicda; - void *blk = bp->b_addr; - uuid_t *uuid; - xfs_lsn_t lsn = -1; - - /* v4 filesystems always recover immediately */ - if (!xfs_sb_version_hascrc(&mp->m_sb)) - goto recover_immediately; - - magic32 = be32_to_cpu(*(__be32 *)blk); - switch (magic32) { - case XFS_ABTB_CRC_MAGIC: - case XFS_ABTC_CRC_MAGIC: - case XFS_ABTB_MAGIC: - case XFS_ABTC_MAGIC: - case XFS_RMAP_CRC_MAGIC: - case XFS_REFC_CRC_MAGIC: - case XFS_IBT_CRC_MAGIC: - case XFS_IBT_MAGIC: { - struct xfs_btree_block *btb = blk; - - lsn = be64_to_cpu(btb->bb_u.s.bb_lsn); - uuid = &btb->bb_u.s.bb_uuid; - break; - } - case XFS_BMAP_CRC_MAGIC: - case XFS_BMAP_MAGIC: { - struct xfs_btree_block *btb = blk; - - lsn = be64_to_cpu(btb->bb_u.l.bb_lsn); - uuid = &btb->bb_u.l.bb_uuid; - break; - } - case XFS_AGF_MAGIC: - lsn = be64_to_cpu(((struct xfs_agf *)blk)->agf_lsn); - uuid = &((struct xfs_agf *)blk)->agf_uuid; - break; - case XFS_AGFL_MAGIC: - lsn = be64_to_cpu(((struct xfs_agfl *)blk)->agfl_lsn); - uuid = &((struct xfs_agfl *)blk)->agfl_uuid; - break; - case XFS_AGI_MAGIC: - lsn = be64_to_cpu(((struct xfs_agi *)blk)->agi_lsn); - uuid = &((struct xfs_agi *)blk)->agi_uuid; - break; - case XFS_SYMLINK_MAGIC: - lsn = be64_to_cpu(((struct xfs_dsymlink_hdr *)blk)->sl_lsn); - uuid = &((struct xfs_dsymlink_hdr *)blk)->sl_uuid; - break; - case XFS_DIR3_BLOCK_MAGIC: - case XFS_DIR3_DATA_MAGIC: - case XFS_DIR3_FREE_MAGIC: - lsn = be64_to_cpu(((struct xfs_dir3_blk_hdr *)blk)->lsn); - uuid = &((struct xfs_dir3_blk_hdr *)blk)->uuid; - break; - case XFS_ATTR3_RMT_MAGIC: - /* - * Remote attr blocks are written synchronously, rather than - * being logged. That means they do not contain a valid LSN - * (i.e. transactionally ordered) in them, and hence any time we - * see a buffer to replay over the top of a remote attribute - * block we should simply do so. - */ - goto recover_immediately; - case XFS_SB_MAGIC: - /* - * superblock uuids are magic. We may or may not have a - * sb_meta_uuid on disk, but it will be set in the in-core - * superblock. We set the uuid pointer for verification - * according to the superblock feature mask to ensure we check - * the relevant UUID in the superblock. - */ - lsn = be64_to_cpu(((struct xfs_dsb *)blk)->sb_lsn); - if (xfs_sb_version_hasmetauuid(&mp->m_sb)) - uuid = &((struct xfs_dsb *)blk)->sb_meta_uuid; - else - uuid = &((struct xfs_dsb *)blk)->sb_uuid; - break; - default: - break; - } - - if (lsn != (xfs_lsn_t)-1) { - if (!uuid_equal(&mp->m_sb.sb_meta_uuid, uuid)) - goto recover_immediately; - return lsn; - } - - magicda = be16_to_cpu(((struct xfs_da_blkinfo *)blk)->magic); - switch (magicda) { - case XFS_DIR3_LEAF1_MAGIC: - case XFS_DIR3_LEAFN_MAGIC: - case XFS_DA3_NODE_MAGIC: - lsn = be64_to_cpu(((struct xfs_da3_blkinfo *)blk)->lsn); - uuid = &((struct xfs_da3_blkinfo *)blk)->uuid; - break; - default: - break; - } - - if (lsn != (xfs_lsn_t)-1) { - if (!uuid_equal(&mp->m_sb.sb_uuid, uuid)) - goto recover_immediately; - return lsn; - } - - /* - * We do individual object checks on dquot and inode buffers as they - * have their own individual LSN records. Also, we could have a stale - * buffer here, so we have to at least recognise these buffer types. - * - * A notd complexity here is inode unlinked list processing - it logs - * the inode directly in the buffer, but we don't know which inodes have - * been modified, and there is no global buffer LSN. Hence we need to - * recover all inode buffer types immediately. This problem will be - * fixed by logical logging of the unlinked list modifications. - */ - magic16 = be16_to_cpu(*(__be16 *)blk); - switch (magic16) { - case XFS_DQUOT_MAGIC: - case XFS_DINODE_MAGIC: - goto recover_immediately; - default: - break; - } - - /* unknown buffer contents, recover immediately */ - -recover_immediately: - return (xfs_lsn_t)-1; - -} - -/* - * Validate the recovered buffer is of the correct type and attach the - * appropriate buffer operations to them for writeback. Magic numbers are in a - * few places: - * the first 16 bits of the buffer (inode buffer, dquot buffer), - * the first 32 bits of the buffer (most blocks), - * inside a struct xfs_da_blkinfo at the start of the buffer. - */ -static void -xlog_recover_validate_buf_type( - struct xfs_mount *mp, - struct xfs_buf *bp, - xfs_buf_log_format_t *buf_f, - xfs_lsn_t current_lsn) -{ - struct xfs_da_blkinfo *info = bp->b_addr; - uint32_t magic32; - uint16_t magic16; - uint16_t magicda; - char *warnmsg = NULL; - - /* - * We can only do post recovery validation on items on CRC enabled - * fielsystems as we need to know when the buffer was written to be able - * to determine if we should have replayed the item. If we replay old - * metadata over a newer buffer, then it will enter a temporarily - * inconsistent state resulting in verification failures. Hence for now - * just avoid the verification stage for non-crc filesystems - */ - if (!xfs_sb_version_hascrc(&mp->m_sb)) - return; - - magic32 = be32_to_cpu(*(__be32 *)bp->b_addr); - magic16 = be16_to_cpu(*(__be16*)bp->b_addr); - magicda = be16_to_cpu(info->magic); - switch (xfs_blft_from_flags(buf_f)) { - case XFS_BLFT_BTREE_BUF: - switch (magic32) { - case XFS_ABTB_CRC_MAGIC: - case XFS_ABTB_MAGIC: - bp->b_ops = &xfs_bnobt_buf_ops; - break; - case XFS_ABTC_CRC_MAGIC: - case XFS_ABTC_MAGIC: - bp->b_ops = &xfs_cntbt_buf_ops; - break; - case XFS_IBT_CRC_MAGIC: - case XFS_IBT_MAGIC: - bp->b_ops = &xfs_inobt_buf_ops; - break; - case XFS_FIBT_CRC_MAGIC: - case XFS_FIBT_MAGIC: - bp->b_ops = &xfs_finobt_buf_ops; - break; - case XFS_BMAP_CRC_MAGIC: - case XFS_BMAP_MAGIC: - bp->b_ops = &xfs_bmbt_buf_ops; - break; - case XFS_RMAP_CRC_MAGIC: - bp->b_ops = &xfs_rmapbt_buf_ops; - break; - case XFS_REFC_CRC_MAGIC: - bp->b_ops = &xfs_refcountbt_buf_ops; - break; - default: - warnmsg = "Bad btree block magic!"; - break; - } - break; - case XFS_BLFT_AGF_BUF: - if (magic32 != XFS_AGF_MAGIC) { - warnmsg = "Bad AGF block magic!"; - break; - } - bp->b_ops = &xfs_agf_buf_ops; - break; - case XFS_BLFT_AGFL_BUF: - if (magic32 != XFS_AGFL_MAGIC) { - warnmsg = "Bad AGFL block magic!"; - break; - } - bp->b_ops = &xfs_agfl_buf_ops; - break; - case XFS_BLFT_AGI_BUF: - if (magic32 != XFS_AGI_MAGIC) { - warnmsg = "Bad AGI block magic!"; - break; - } - bp->b_ops = &xfs_agi_buf_ops; - break; - case XFS_BLFT_UDQUOT_BUF: - case XFS_BLFT_PDQUOT_BUF: - case XFS_BLFT_GDQUOT_BUF: -#ifdef CONFIG_XFS_QUOTA - if (magic16 != XFS_DQUOT_MAGIC) { - warnmsg = "Bad DQUOT block magic!"; - break; - } - bp->b_ops = &xfs_dquot_buf_ops; -#else - xfs_alert(mp, - "Trying to recover dquots without QUOTA support built in!"); - ASSERT(0); -#endif - break; - case XFS_BLFT_DINO_BUF: - if (magic16 != XFS_DINODE_MAGIC) { - warnmsg = "Bad INODE block magic!"; - break; - } - bp->b_ops = &xfs_inode_buf_ops; - break; - case XFS_BLFT_SYMLINK_BUF: - if (magic32 != XFS_SYMLINK_MAGIC) { - warnmsg = "Bad symlink block magic!"; - break; - } - bp->b_ops = &xfs_symlink_buf_ops; - break; - case XFS_BLFT_DIR_BLOCK_BUF: - if (magic32 != XFS_DIR2_BLOCK_MAGIC && - magic32 != XFS_DIR3_BLOCK_MAGIC) { - warnmsg = "Bad dir block magic!"; - break; - } - bp->b_ops = &xfs_dir3_block_buf_ops; - break; - case XFS_BLFT_DIR_DATA_BUF: - if (magic32 != XFS_DIR2_DATA_MAGIC && - magic32 != XFS_DIR3_DATA_MAGIC) { - warnmsg = "Bad dir data magic!"; - break; - } - bp->b_ops = &xfs_dir3_data_buf_ops; - break; - case XFS_BLFT_DIR_FREE_BUF: - if (magic32 != XFS_DIR2_FREE_MAGIC && - magic32 != XFS_DIR3_FREE_MAGIC) { - warnmsg = "Bad dir3 free magic!"; - break; - } - bp->b_ops = &xfs_dir3_free_buf_ops; - break; - case XFS_BLFT_DIR_LEAF1_BUF: - if (magicda != XFS_DIR2_LEAF1_MAGIC && - magicda != XFS_DIR3_LEAF1_MAGIC) { - warnmsg = "Bad dir leaf1 magic!"; - break; - } - bp->b_ops = &xfs_dir3_leaf1_buf_ops; - break; - case XFS_BLFT_DIR_LEAFN_BUF: - if (magicda != XFS_DIR2_LEAFN_MAGIC && - magicda != XFS_DIR3_LEAFN_MAGIC) { - warnmsg = "Bad dir leafn magic!"; - break; - } - bp->b_ops = &xfs_dir3_leafn_buf_ops; - break; - case XFS_BLFT_DA_NODE_BUF: - if (magicda != XFS_DA_NODE_MAGIC && - magicda != XFS_DA3_NODE_MAGIC) { - warnmsg = "Bad da node magic!"; - break; - } - bp->b_ops = &xfs_da3_node_buf_ops; - break; - case XFS_BLFT_ATTR_LEAF_BUF: - if (magicda != XFS_ATTR_LEAF_MAGIC && - magicda != XFS_ATTR3_LEAF_MAGIC) { - warnmsg = "Bad attr leaf magic!"; - break; - } - bp->b_ops = &xfs_attr3_leaf_buf_ops; - break; - case XFS_BLFT_ATTR_RMT_BUF: - if (magic32 != XFS_ATTR3_RMT_MAGIC) { - warnmsg = "Bad attr remote magic!"; - break; - } - bp->b_ops = &xfs_attr3_rmt_buf_ops; - break; - case XFS_BLFT_SB_BUF: - if (magic32 != XFS_SB_MAGIC) { - warnmsg = "Bad SB block magic!"; - break; - } - bp->b_ops = &xfs_sb_buf_ops; - break; -#ifdef CONFIG_XFS_RT - case XFS_BLFT_RTBITMAP_BUF: - case XFS_BLFT_RTSUMMARY_BUF: - /* no magic numbers for verification of RT buffers */ - bp->b_ops = &xfs_rtbuf_ops; - break; -#endif /* CONFIG_XFS_RT */ - default: - xfs_warn(mp, "Unknown buffer type %d!", - xfs_blft_from_flags(buf_f)); - break; - } - - /* - * Nothing else to do in the case of a NULL current LSN as this means - * the buffer is more recent than the change in the log and will be - * skipped. - */ - if (current_lsn == NULLCOMMITLSN) - return; - - if (warnmsg) { - xfs_warn(mp, warnmsg); - ASSERT(0); - } - - /* - * We must update the metadata LSN of the buffer as it is written out to - * ensure that older transactions never replay over this one and corrupt - * the buffer. This can occur if log recovery is interrupted at some - * point after the current transaction completes, at which point a - * subsequent mount starts recovery from the beginning. - * - * Write verifiers update the metadata LSN from log items attached to - * the buffer. Therefore, initialize a bli purely to carry the LSN to - * the verifier. We'll clean it up in our ->iodone() callback. - */ - if (bp->b_ops) { - struct xfs_buf_log_item *bip; - - ASSERT(!bp->b_iodone || bp->b_iodone == xlog_recover_iodone); - bp->b_iodone = xlog_recover_iodone; - xfs_buf_item_init(bp, mp); - bip = bp->b_log_item; - bip->bli_item.li_lsn = current_lsn; - } -} - -/* - * Perform a 'normal' buffer recovery. Each logged region of the - * buffer should be copied over the corresponding region in the - * given buffer. The bitmap in the buf log format structure indicates - * where to place the logged data. - */ -STATIC void -xlog_recover_do_reg_buffer( - struct xfs_mount *mp, - struct xlog_recover_item *item, - struct xfs_buf *bp, - xfs_buf_log_format_t *buf_f, - xfs_lsn_t current_lsn) -{ - int i; - int bit; - int nbits; - xfs_failaddr_t fa; - const size_t size_disk_dquot = sizeof(struct xfs_disk_dquot); - - trace_xfs_log_recover_buf_reg_buf(mp->m_log, buf_f); - - bit = 0; - i = 1; /* 0 is the buf format structure */ - while (1) { - bit = xfs_next_bit(buf_f->blf_data_map, - buf_f->blf_map_size, bit); - if (bit == -1) - break; - nbits = xfs_contig_bits(buf_f->blf_data_map, - buf_f->blf_map_size, bit); - ASSERT(nbits > 0); - ASSERT(item->ri_buf[i].i_addr != NULL); - ASSERT(item->ri_buf[i].i_len % XFS_BLF_CHUNK == 0); - ASSERT(BBTOB(bp->b_length) >= - ((uint)bit << XFS_BLF_SHIFT) + (nbits << XFS_BLF_SHIFT)); - - /* - * The dirty regions logged in the buffer, even though - * contiguous, may span multiple chunks. This is because the - * dirty region may span a physical page boundary in a buffer - * and hence be split into two separate vectors for writing into - * the log. Hence we need to trim nbits back to the length of - * the current region being copied out of the log. - */ - if (item->ri_buf[i].i_len < (nbits << XFS_BLF_SHIFT)) - nbits = item->ri_buf[i].i_len >> XFS_BLF_SHIFT; - - /* - * Do a sanity check if this is a dquot buffer. Just checking - * the first dquot in the buffer should do. XXXThis is - * probably a good thing to do for other buf types also. - */ - fa = NULL; - if (buf_f->blf_flags & - (XFS_BLF_UDQUOT_BUF|XFS_BLF_PDQUOT_BUF|XFS_BLF_GDQUOT_BUF)) { - if (item->ri_buf[i].i_addr == NULL) { - xfs_alert(mp, - "XFS: NULL dquot in %s.", __func__); - goto next; - } - if (item->ri_buf[i].i_len < size_disk_dquot) { - xfs_alert(mp, - "XFS: dquot too small (%d) in %s.", - item->ri_buf[i].i_len, __func__); - goto next; - } - fa = xfs_dquot_verify(mp, item->ri_buf[i].i_addr, - -1, 0); - if (fa) { - xfs_alert(mp, - "dquot corrupt at %pS trying to replay into block 0x%llx", - fa, bp->b_bn); - goto next; - } - } - - memcpy(xfs_buf_offset(bp, - (uint)bit << XFS_BLF_SHIFT), /* dest */ - item->ri_buf[i].i_addr, /* source */ - nbits<ri_total); - - xlog_recover_validate_buf_type(mp, bp, buf_f, current_lsn); -} - -/* - * Perform a dquot buffer recovery. - * Simple algorithm: if we have found a QUOTAOFF log item of the same type - * (ie. USR or GRP), then just toss this buffer away; don't recover it. - * Else, treat it as a regular buffer and do recovery. - * - * Return false if the buffer was tossed and true if we recovered the buffer to - * indicate to the caller if the buffer needs writing. - */ -STATIC bool -xlog_recover_do_dquot_buffer( - struct xfs_mount *mp, - struct xlog *log, - struct xlog_recover_item *item, - struct xfs_buf *bp, - struct xfs_buf_log_format *buf_f) -{ - uint type; - - trace_xfs_log_recover_buf_dquot_buf(log, buf_f); - - /* - * Filesystems are required to send in quota flags at mount time. - */ - if (!mp->m_qflags) - return false; - - type = 0; - if (buf_f->blf_flags & XFS_BLF_UDQUOT_BUF) - type |= XFS_DQ_USER; - if (buf_f->blf_flags & XFS_BLF_PDQUOT_BUF) - type |= XFS_DQ_PROJ; - if (buf_f->blf_flags & XFS_BLF_GDQUOT_BUF) - type |= XFS_DQ_GROUP; - /* - * This type of quotas was turned off, so ignore this buffer - */ - if (log->l_quotaoffs_flag & type) - return false; - - xlog_recover_do_reg_buffer(mp, item, bp, buf_f, NULLCOMMITLSN); - return true; -} - -/* - * This routine replays a modification made to a buffer at runtime. - * There are actually two types of buffer, regular and inode, which - * are handled differently. Inode buffers are handled differently - * in that we only recover a specific set of data from them, namely - * the inode di_next_unlinked fields. This is because all other inode - * data is actually logged via inode records and any data we replay - * here which overlaps that may be stale. - * - * When meta-data buffers are freed at run time we log a buffer item - * with the XFS_BLF_CANCEL bit set to indicate that previous copies - * of the buffer in the log should not be replayed at recovery time. - * This is so that if the blocks covered by the buffer are reused for - * file data before we crash we don't end up replaying old, freed - * meta-data into a user's file. - * - * To handle the cancellation of buffer log items, we make two passes - * over the log during recovery. During the first we build a table of - * those buffers which have been cancelled, and during the second we - * only replay those buffers which do not have corresponding cancel - * records in the table. See xlog_recover_buffer_pass[1,2] above - * for more details on the implementation of the table of cancel records. - */ -STATIC int -xlog_recover_buffer_pass2( - struct xlog *log, - struct list_head *buffer_list, - struct xlog_recover_item *item, - xfs_lsn_t current_lsn) -{ - xfs_buf_log_format_t *buf_f = item->ri_buf[0].i_addr; - xfs_mount_t *mp = log->l_mp; - xfs_buf_t *bp; - int error; - uint buf_flags; - xfs_lsn_t lsn; - - /* - * In this pass we only want to recover all the buffers which have - * not been cancelled and are not cancellation buffers themselves. - */ - if (buf_f->blf_flags & XFS_BLF_CANCEL) { - if (xlog_put_buffer_cancelled(log, buf_f->blf_blkno, - buf_f->blf_len)) - goto cancelled; - } else { - - if (xlog_is_buffer_cancelled(log, buf_f->blf_blkno, - buf_f->blf_len)) - goto cancelled; - } - - trace_xfs_log_recover_buf_recover(log, buf_f); - - buf_flags = 0; - if (buf_f->blf_flags & XFS_BLF_INODE_BUF) - buf_flags |= XBF_UNMAPPED; - - error = xfs_buf_read(mp->m_ddev_targp, buf_f->blf_blkno, buf_f->blf_len, - buf_flags, &bp, NULL); - if (error) - return error; - - /* - * Recover the buffer only if we get an LSN from it and it's less than - * the lsn of the transaction we are replaying. - * - * Note that we have to be extremely careful of readahead here. - * Readahead does not attach verfiers to the buffers so if we don't - * actually do any replay after readahead because of the LSN we found - * in the buffer if more recent than that current transaction then we - * need to attach the verifier directly. Failure to do so can lead to - * future recovery actions (e.g. EFI and unlinked list recovery) can - * operate on the buffers and they won't get the verifier attached. This - * can lead to blocks on disk having the correct content but a stale - * CRC. - * - * It is safe to assume these clean buffers are currently up to date. - * If the buffer is dirtied by a later transaction being replayed, then - * the verifier will be reset to match whatever recover turns that - * buffer into. - */ - lsn = xlog_recover_get_buf_lsn(mp, bp); - if (lsn && lsn != -1 && XFS_LSN_CMP(lsn, current_lsn) >= 0) { - trace_xfs_log_recover_buf_skip(log, buf_f); - xlog_recover_validate_buf_type(mp, bp, buf_f, NULLCOMMITLSN); - goto out_release; - } - - if (buf_f->blf_flags & XFS_BLF_INODE_BUF) { - error = xlog_recover_do_inode_buffer(mp, item, bp, buf_f); - if (error) - goto out_release; - } else if (buf_f->blf_flags & - (XFS_BLF_UDQUOT_BUF|XFS_BLF_PDQUOT_BUF|XFS_BLF_GDQUOT_BUF)) { - bool dirty; - - dirty = xlog_recover_do_dquot_buffer(mp, log, item, bp, buf_f); - if (!dirty) - goto out_release; - } else { - xlog_recover_do_reg_buffer(mp, item, bp, buf_f, current_lsn); - } - - /* - * Perform delayed write on the buffer. Asynchronous writes will be - * slower when taking into account all the buffers to be flushed. - * - * Also make sure that only inode buffers with good sizes stay in - * the buffer cache. The kernel moves inodes in buffers of 1 block - * or inode_cluster_size bytes, whichever is bigger. The inode - * buffers in the log can be a different size if the log was generated - * by an older kernel using unclustered inode buffers or a newer kernel - * running with a different inode cluster size. Regardless, if the - * the inode buffer size isn't max(blocksize, inode_cluster_size) - * for *our* value of inode_cluster_size, then we need to keep - * the buffer out of the buffer cache so that the buffer won't - * overlap with future reads of those inodes. - */ - if (XFS_DINODE_MAGIC == - be16_to_cpu(*((__be16 *)xfs_buf_offset(bp, 0))) && - (BBTOB(bp->b_length) != M_IGEO(log->l_mp)->inode_cluster_size)) { - xfs_buf_stale(bp); - error = xfs_bwrite(bp); - } else { - ASSERT(bp->b_mount == mp); - bp->b_iodone = xlog_recover_iodone; - xfs_buf_delwri_queue(bp, buffer_list); - } - -out_release: - xfs_buf_relse(bp); - return error; -cancelled: - trace_xfs_log_recover_buf_cancel(log, buf_f); - return 0; -} - /* * Inode fork owner changes * @@ -3846,10 +3061,11 @@ xlog_recover_commit_pass2( { trace_xfs_log_recover_item_recover(log, trans, item, XLOG_RECOVER_PASS2); + if (item->ri_ops->commit_pass2) + return item->ri_ops->commit_pass2(log, buffer_list, item, + trans->r_lsn); + switch (ITEM_TYPE(item)) { - case XFS_LI_BUF: - return xlog_recover_buffer_pass2(log, buffer_list, item, - trans->r_lsn); case XFS_LI_INODE: return xlog_recover_inode_pass2(log, buffer_list, item, trans->r_lsn); From 658fa68b6f34f73c7e4023489accd34c1db91cb1 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:47 -0700 Subject: [PATCH 0278/1043] xfs: refactor log recovery inode item dispatch for pass2 commit functions Move the log inode item pass2 commit code into the per-item source code files and use the dispatch function to call it. We do these one at a time because there's a lot of code to move. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_inode_item_recover.c | 355 ++++++++++++++++++++++++++++++++ fs/xfs/xfs_log_recover.c | 355 -------------------------------- 2 files changed, 355 insertions(+), 355 deletions(-) diff --git a/fs/xfs/xfs_inode_item_recover.c b/fs/xfs/xfs_inode_item_recover.c index a132cacd8d48..2bdba612aa71 100644 --- a/fs/xfs/xfs_inode_item_recover.c +++ b/fs/xfs/xfs_inode_item_recover.c @@ -20,6 +20,8 @@ #include "xfs_error.h" #include "xfs_log_priv.h" #include "xfs_log_recover.h" +#include "xfs_icache.h" +#include "xfs_bmap_btree.h" STATIC void xlog_recover_inode_ra_pass2( @@ -39,7 +41,360 @@ xlog_recover_inode_ra_pass2( } } +/* + * Inode fork owner changes + * + * If we have been told that we have to reparent the inode fork, it's because an + * extent swap operation on a CRC enabled filesystem has been done and we are + * replaying it. We need to walk the BMBT of the appropriate fork and change the + * owners of it. + * + * The complexity here is that we don't have an inode context to work with, so + * after we've replayed the inode we need to instantiate one. This is where the + * fun begins. + * + * We are in the middle of log recovery, so we can't run transactions. That + * means we cannot use cache coherent inode instantiation via xfs_iget(), as + * that will result in the corresponding iput() running the inode through + * xfs_inactive(). If we've just replayed an inode core that changes the link + * count to zero (i.e. it's been unlinked), then xfs_inactive() will run + * transactions (bad!). + * + * So, to avoid this, we instantiate an inode directly from the inode core we've + * just recovered. We have the buffer still locked, and all we really need to + * instantiate is the inode core and the forks being modified. We can do this + * manually, then run the inode btree owner change, and then tear down the + * xfs_inode without having to run any transactions at all. + * + * Also, because we don't have a transaction context available here but need to + * gather all the buffers we modify for writeback so we pass the buffer_list + * instead for the operation to use. + */ + +STATIC int +xfs_recover_inode_owner_change( + struct xfs_mount *mp, + struct xfs_dinode *dip, + struct xfs_inode_log_format *in_f, + struct list_head *buffer_list) +{ + struct xfs_inode *ip; + int error; + + ASSERT(in_f->ilf_fields & (XFS_ILOG_DOWNER|XFS_ILOG_AOWNER)); + + ip = xfs_inode_alloc(mp, in_f->ilf_ino); + if (!ip) + return -ENOMEM; + + /* instantiate the inode */ + ASSERT(dip->di_version >= 3); + xfs_inode_from_disk(ip, dip); + + error = xfs_iformat_fork(ip, dip); + if (error) + goto out_free_ip; + + if (!xfs_inode_verify_forks(ip)) { + error = -EFSCORRUPTED; + goto out_free_ip; + } + + if (in_f->ilf_fields & XFS_ILOG_DOWNER) { + ASSERT(in_f->ilf_fields & XFS_ILOG_DBROOT); + error = xfs_bmbt_change_owner(NULL, ip, XFS_DATA_FORK, + ip->i_ino, buffer_list); + if (error) + goto out_free_ip; + } + + if (in_f->ilf_fields & XFS_ILOG_AOWNER) { + ASSERT(in_f->ilf_fields & XFS_ILOG_ABROOT); + error = xfs_bmbt_change_owner(NULL, ip, XFS_ATTR_FORK, + ip->i_ino, buffer_list); + if (error) + goto out_free_ip; + } + +out_free_ip: + xfs_inode_free(ip); + return error; +} + +STATIC int +xlog_recover_inode_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t current_lsn) +{ + struct xfs_inode_log_format *in_f; + struct xfs_mount *mp = log->l_mp; + struct xfs_buf *bp; + struct xfs_dinode *dip; + int len; + char *src; + char *dest; + int error; + int attr_index; + uint fields; + struct xfs_log_dinode *ldip; + uint isize; + int need_free = 0; + + if (item->ri_buf[0].i_len == sizeof(struct xfs_inode_log_format)) { + in_f = item->ri_buf[0].i_addr; + } else { + in_f = kmem_alloc(sizeof(struct xfs_inode_log_format), 0); + need_free = 1; + error = xfs_inode_item_format_convert(&item->ri_buf[0], in_f); + if (error) + goto error; + } + + /* + * Inode buffers can be freed, look out for it, + * and do not replay the inode. + */ + if (xlog_is_buffer_cancelled(log, in_f->ilf_blkno, in_f->ilf_len)) { + error = 0; + trace_xfs_log_recover_inode_cancel(log, in_f); + goto error; + } + trace_xfs_log_recover_inode_recover(log, in_f); + + error = xfs_buf_read(mp->m_ddev_targp, in_f->ilf_blkno, in_f->ilf_len, + 0, &bp, &xfs_inode_buf_ops); + if (error) + goto error; + ASSERT(in_f->ilf_fields & XFS_ILOG_CORE); + dip = xfs_buf_offset(bp, in_f->ilf_boffset); + + /* + * Make sure the place we're flushing out to really looks + * like an inode! + */ + if (XFS_IS_CORRUPT(mp, !xfs_verify_magic16(bp, dip->di_magic))) { + xfs_alert(mp, + "%s: Bad inode magic number, dip = "PTR_FMT", dino bp = "PTR_FMT", ino = %Ld", + __func__, dip, bp, in_f->ilf_ino); + error = -EFSCORRUPTED; + goto out_release; + } + ldip = item->ri_buf[1].i_addr; + if (XFS_IS_CORRUPT(mp, ldip->di_magic != XFS_DINODE_MAGIC)) { + xfs_alert(mp, + "%s: Bad inode log record, rec ptr "PTR_FMT", ino %Ld", + __func__, item, in_f->ilf_ino); + error = -EFSCORRUPTED; + goto out_release; + } + + /* + * If the inode has an LSN in it, recover the inode only if it's less + * than the lsn of the transaction we are replaying. Note: we still + * need to replay an owner change even though the inode is more recent + * than the transaction as there is no guarantee that all the btree + * blocks are more recent than this transaction, too. + */ + if (dip->di_version >= 3) { + xfs_lsn_t lsn = be64_to_cpu(dip->di_lsn); + + if (lsn && lsn != -1 && XFS_LSN_CMP(lsn, current_lsn) >= 0) { + trace_xfs_log_recover_inode_skip(log, in_f); + error = 0; + goto out_owner_change; + } + } + + /* + * di_flushiter is only valid for v1/2 inodes. All changes for v3 inodes + * are transactional and if ordering is necessary we can determine that + * more accurately by the LSN field in the V3 inode core. Don't trust + * the inode versions we might be changing them here - use the + * superblock flag to determine whether we need to look at di_flushiter + * to skip replay when the on disk inode is newer than the log one + */ + if (!xfs_sb_version_has_v3inode(&mp->m_sb) && + ldip->di_flushiter < be16_to_cpu(dip->di_flushiter)) { + /* + * Deal with the wrap case, DI_MAX_FLUSH is less + * than smaller numbers + */ + if (be16_to_cpu(dip->di_flushiter) == DI_MAX_FLUSH && + ldip->di_flushiter < (DI_MAX_FLUSH >> 1)) { + /* do nothing */ + } else { + trace_xfs_log_recover_inode_skip(log, in_f); + error = 0; + goto out_release; + } + } + + /* Take the opportunity to reset the flush iteration count */ + ldip->di_flushiter = 0; + + if (unlikely(S_ISREG(ldip->di_mode))) { + if ((ldip->di_format != XFS_DINODE_FMT_EXTENTS) && + (ldip->di_format != XFS_DINODE_FMT_BTREE)) { + XFS_CORRUPTION_ERROR("xlog_recover_inode_pass2(3)", + XFS_ERRLEVEL_LOW, mp, ldip, + sizeof(*ldip)); + xfs_alert(mp, + "%s: Bad regular inode log record, rec ptr "PTR_FMT", " + "ino ptr = "PTR_FMT", ino bp = "PTR_FMT", ino %Ld", + __func__, item, dip, bp, in_f->ilf_ino); + error = -EFSCORRUPTED; + goto out_release; + } + } else if (unlikely(S_ISDIR(ldip->di_mode))) { + if ((ldip->di_format != XFS_DINODE_FMT_EXTENTS) && + (ldip->di_format != XFS_DINODE_FMT_BTREE) && + (ldip->di_format != XFS_DINODE_FMT_LOCAL)) { + XFS_CORRUPTION_ERROR("xlog_recover_inode_pass2(4)", + XFS_ERRLEVEL_LOW, mp, ldip, + sizeof(*ldip)); + xfs_alert(mp, + "%s: Bad dir inode log record, rec ptr "PTR_FMT", " + "ino ptr = "PTR_FMT", ino bp = "PTR_FMT", ino %Ld", + __func__, item, dip, bp, in_f->ilf_ino); + error = -EFSCORRUPTED; + goto out_release; + } + } + if (unlikely(ldip->di_nextents + ldip->di_anextents > ldip->di_nblocks)){ + XFS_CORRUPTION_ERROR("xlog_recover_inode_pass2(5)", + XFS_ERRLEVEL_LOW, mp, ldip, + sizeof(*ldip)); + xfs_alert(mp, + "%s: Bad inode log record, rec ptr "PTR_FMT", dino ptr "PTR_FMT", " + "dino bp "PTR_FMT", ino %Ld, total extents = %d, nblocks = %Ld", + __func__, item, dip, bp, in_f->ilf_ino, + ldip->di_nextents + ldip->di_anextents, + ldip->di_nblocks); + error = -EFSCORRUPTED; + goto out_release; + } + if (unlikely(ldip->di_forkoff > mp->m_sb.sb_inodesize)) { + XFS_CORRUPTION_ERROR("xlog_recover_inode_pass2(6)", + XFS_ERRLEVEL_LOW, mp, ldip, + sizeof(*ldip)); + xfs_alert(mp, + "%s: Bad inode log record, rec ptr "PTR_FMT", dino ptr "PTR_FMT", " + "dino bp "PTR_FMT", ino %Ld, forkoff 0x%x", __func__, + item, dip, bp, in_f->ilf_ino, ldip->di_forkoff); + error = -EFSCORRUPTED; + goto out_release; + } + isize = xfs_log_dinode_size(mp); + if (unlikely(item->ri_buf[1].i_len > isize)) { + XFS_CORRUPTION_ERROR("xlog_recover_inode_pass2(7)", + XFS_ERRLEVEL_LOW, mp, ldip, + sizeof(*ldip)); + xfs_alert(mp, + "%s: Bad inode log record length %d, rec ptr "PTR_FMT, + __func__, item->ri_buf[1].i_len, item); + error = -EFSCORRUPTED; + goto out_release; + } + + /* recover the log dinode inode into the on disk inode */ + xfs_log_dinode_to_disk(ldip, dip); + + fields = in_f->ilf_fields; + if (fields & XFS_ILOG_DEV) + xfs_dinode_put_rdev(dip, in_f->ilf_u.ilfu_rdev); + + if (in_f->ilf_size == 2) + goto out_owner_change; + len = item->ri_buf[2].i_len; + src = item->ri_buf[2].i_addr; + ASSERT(in_f->ilf_size <= 4); + ASSERT((in_f->ilf_size == 3) || (fields & XFS_ILOG_AFORK)); + ASSERT(!(fields & XFS_ILOG_DFORK) || + (len == in_f->ilf_dsize)); + + switch (fields & XFS_ILOG_DFORK) { + case XFS_ILOG_DDATA: + case XFS_ILOG_DEXT: + memcpy(XFS_DFORK_DPTR(dip), src, len); + break; + + case XFS_ILOG_DBROOT: + xfs_bmbt_to_bmdr(mp, (struct xfs_btree_block *)src, len, + (struct xfs_bmdr_block *)XFS_DFORK_DPTR(dip), + XFS_DFORK_DSIZE(dip, mp)); + break; + + default: + /* + * There are no data fork flags set. + */ + ASSERT((fields & XFS_ILOG_DFORK) == 0); + break; + } + + /* + * If we logged any attribute data, recover it. There may or + * may not have been any other non-core data logged in this + * transaction. + */ + if (in_f->ilf_fields & XFS_ILOG_AFORK) { + if (in_f->ilf_fields & XFS_ILOG_DFORK) { + attr_index = 3; + } else { + attr_index = 2; + } + len = item->ri_buf[attr_index].i_len; + src = item->ri_buf[attr_index].i_addr; + ASSERT(len == in_f->ilf_asize); + + switch (in_f->ilf_fields & XFS_ILOG_AFORK) { + case XFS_ILOG_ADATA: + case XFS_ILOG_AEXT: + dest = XFS_DFORK_APTR(dip); + ASSERT(len <= XFS_DFORK_ASIZE(dip, mp)); + memcpy(dest, src, len); + break; + + case XFS_ILOG_ABROOT: + dest = XFS_DFORK_APTR(dip); + xfs_bmbt_to_bmdr(mp, (struct xfs_btree_block *)src, + len, (struct xfs_bmdr_block *)dest, + XFS_DFORK_ASIZE(dip, mp)); + break; + + default: + xfs_warn(log->l_mp, "%s: Invalid flag", __func__); + ASSERT(0); + error = -EFSCORRUPTED; + goto out_release; + } + } + +out_owner_change: + /* Recover the swapext owner change unless inode has been deleted */ + if ((in_f->ilf_fields & (XFS_ILOG_DOWNER|XFS_ILOG_AOWNER)) && + (dip->di_mode != 0)) + error = xfs_recover_inode_owner_change(mp, dip, in_f, + buffer_list); + /* re-generate the checksum. */ + xfs_dinode_calc_crc(log->l_mp, dip); + + ASSERT(bp->b_mount == mp); + bp->b_iodone = xlog_recover_iodone; + xfs_buf_delwri_queue(bp, buffer_list); + +out_release: + xfs_buf_relse(bp); +error: + if (need_free) + kmem_free(in_f); + return error; +} + const struct xlog_recover_item_ops xlog_inode_item_ops = { .item_type = XFS_LI_INODE, .ra_pass2 = xlog_recover_inode_ra_pass2, + .commit_pass2 = xlog_recover_inode_commit_pass2, }; diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index eaf2ea8da5d6..8bf8d4dec0d7 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2034,358 +2034,6 @@ xlog_buf_readahead( xfs_buf_readahead(log->l_mp->m_ddev_targp, blkno, len, ops); } -/* - * Inode fork owner changes - * - * If we have been told that we have to reparent the inode fork, it's because an - * extent swap operation on a CRC enabled filesystem has been done and we are - * replaying it. We need to walk the BMBT of the appropriate fork and change the - * owners of it. - * - * The complexity here is that we don't have an inode context to work with, so - * after we've replayed the inode we need to instantiate one. This is where the - * fun begins. - * - * We are in the middle of log recovery, so we can't run transactions. That - * means we cannot use cache coherent inode instantiation via xfs_iget(), as - * that will result in the corresponding iput() running the inode through - * xfs_inactive(). If we've just replayed an inode core that changes the link - * count to zero (i.e. it's been unlinked), then xfs_inactive() will run - * transactions (bad!). - * - * So, to avoid this, we instantiate an inode directly from the inode core we've - * just recovered. We have the buffer still locked, and all we really need to - * instantiate is the inode core and the forks being modified. We can do this - * manually, then run the inode btree owner change, and then tear down the - * xfs_inode without having to run any transactions at all. - * - * Also, because we don't have a transaction context available here but need to - * gather all the buffers we modify for writeback so we pass the buffer_list - * instead for the operation to use. - */ - -STATIC int -xfs_recover_inode_owner_change( - struct xfs_mount *mp, - struct xfs_dinode *dip, - struct xfs_inode_log_format *in_f, - struct list_head *buffer_list) -{ - struct xfs_inode *ip; - int error; - - ASSERT(in_f->ilf_fields & (XFS_ILOG_DOWNER|XFS_ILOG_AOWNER)); - - ip = xfs_inode_alloc(mp, in_f->ilf_ino); - if (!ip) - return -ENOMEM; - - /* instantiate the inode */ - ASSERT(dip->di_version >= 3); - xfs_inode_from_disk(ip, dip); - - error = xfs_iformat_fork(ip, dip); - if (error) - goto out_free_ip; - - if (!xfs_inode_verify_forks(ip)) { - error = -EFSCORRUPTED; - goto out_free_ip; - } - - if (in_f->ilf_fields & XFS_ILOG_DOWNER) { - ASSERT(in_f->ilf_fields & XFS_ILOG_DBROOT); - error = xfs_bmbt_change_owner(NULL, ip, XFS_DATA_FORK, - ip->i_ino, buffer_list); - if (error) - goto out_free_ip; - } - - if (in_f->ilf_fields & XFS_ILOG_AOWNER) { - ASSERT(in_f->ilf_fields & XFS_ILOG_ABROOT); - error = xfs_bmbt_change_owner(NULL, ip, XFS_ATTR_FORK, - ip->i_ino, buffer_list); - if (error) - goto out_free_ip; - } - -out_free_ip: - xfs_inode_free(ip); - return error; -} - -STATIC int -xlog_recover_inode_pass2( - struct xlog *log, - struct list_head *buffer_list, - struct xlog_recover_item *item, - xfs_lsn_t current_lsn) -{ - struct xfs_inode_log_format *in_f; - xfs_mount_t *mp = log->l_mp; - xfs_buf_t *bp; - xfs_dinode_t *dip; - int len; - char *src; - char *dest; - int error; - int attr_index; - uint fields; - struct xfs_log_dinode *ldip; - uint isize; - int need_free = 0; - - if (item->ri_buf[0].i_len == sizeof(struct xfs_inode_log_format)) { - in_f = item->ri_buf[0].i_addr; - } else { - in_f = kmem_alloc(sizeof(struct xfs_inode_log_format), 0); - need_free = 1; - error = xfs_inode_item_format_convert(&item->ri_buf[0], in_f); - if (error) - goto error; - } - - /* - * Inode buffers can be freed, look out for it, - * and do not replay the inode. - */ - if (xlog_is_buffer_cancelled(log, in_f->ilf_blkno, in_f->ilf_len)) { - error = 0; - trace_xfs_log_recover_inode_cancel(log, in_f); - goto error; - } - trace_xfs_log_recover_inode_recover(log, in_f); - - error = xfs_buf_read(mp->m_ddev_targp, in_f->ilf_blkno, in_f->ilf_len, - 0, &bp, &xfs_inode_buf_ops); - if (error) - goto error; - ASSERT(in_f->ilf_fields & XFS_ILOG_CORE); - dip = xfs_buf_offset(bp, in_f->ilf_boffset); - - /* - * Make sure the place we're flushing out to really looks - * like an inode! - */ - if (XFS_IS_CORRUPT(mp, !xfs_verify_magic16(bp, dip->di_magic))) { - xfs_alert(mp, - "%s: Bad inode magic number, dip = "PTR_FMT", dino bp = "PTR_FMT", ino = %Ld", - __func__, dip, bp, in_f->ilf_ino); - error = -EFSCORRUPTED; - goto out_release; - } - ldip = item->ri_buf[1].i_addr; - if (XFS_IS_CORRUPT(mp, ldip->di_magic != XFS_DINODE_MAGIC)) { - xfs_alert(mp, - "%s: Bad inode log record, rec ptr "PTR_FMT", ino %Ld", - __func__, item, in_f->ilf_ino); - error = -EFSCORRUPTED; - goto out_release; - } - - /* - * If the inode has an LSN in it, recover the inode only if it's less - * than the lsn of the transaction we are replaying. Note: we still - * need to replay an owner change even though the inode is more recent - * than the transaction as there is no guarantee that all the btree - * blocks are more recent than this transaction, too. - */ - if (dip->di_version >= 3) { - xfs_lsn_t lsn = be64_to_cpu(dip->di_lsn); - - if (lsn && lsn != -1 && XFS_LSN_CMP(lsn, current_lsn) >= 0) { - trace_xfs_log_recover_inode_skip(log, in_f); - error = 0; - goto out_owner_change; - } - } - - /* - * di_flushiter is only valid for v1/2 inodes. All changes for v3 inodes - * are transactional and if ordering is necessary we can determine that - * more accurately by the LSN field in the V3 inode core. Don't trust - * the inode versions we might be changing them here - use the - * superblock flag to determine whether we need to look at di_flushiter - * to skip replay when the on disk inode is newer than the log one - */ - if (!xfs_sb_version_has_v3inode(&mp->m_sb) && - ldip->di_flushiter < be16_to_cpu(dip->di_flushiter)) { - /* - * Deal with the wrap case, DI_MAX_FLUSH is less - * than smaller numbers - */ - if (be16_to_cpu(dip->di_flushiter) == DI_MAX_FLUSH && - ldip->di_flushiter < (DI_MAX_FLUSH >> 1)) { - /* do nothing */ - } else { - trace_xfs_log_recover_inode_skip(log, in_f); - error = 0; - goto out_release; - } - } - - /* Take the opportunity to reset the flush iteration count */ - ldip->di_flushiter = 0; - - if (unlikely(S_ISREG(ldip->di_mode))) { - if ((ldip->di_format != XFS_DINODE_FMT_EXTENTS) && - (ldip->di_format != XFS_DINODE_FMT_BTREE)) { - XFS_CORRUPTION_ERROR("xlog_recover_inode_pass2(3)", - XFS_ERRLEVEL_LOW, mp, ldip, - sizeof(*ldip)); - xfs_alert(mp, - "%s: Bad regular inode log record, rec ptr "PTR_FMT", " - "ino ptr = "PTR_FMT", ino bp = "PTR_FMT", ino %Ld", - __func__, item, dip, bp, in_f->ilf_ino); - error = -EFSCORRUPTED; - goto out_release; - } - } else if (unlikely(S_ISDIR(ldip->di_mode))) { - if ((ldip->di_format != XFS_DINODE_FMT_EXTENTS) && - (ldip->di_format != XFS_DINODE_FMT_BTREE) && - (ldip->di_format != XFS_DINODE_FMT_LOCAL)) { - XFS_CORRUPTION_ERROR("xlog_recover_inode_pass2(4)", - XFS_ERRLEVEL_LOW, mp, ldip, - sizeof(*ldip)); - xfs_alert(mp, - "%s: Bad dir inode log record, rec ptr "PTR_FMT", " - "ino ptr = "PTR_FMT", ino bp = "PTR_FMT", ino %Ld", - __func__, item, dip, bp, in_f->ilf_ino); - error = -EFSCORRUPTED; - goto out_release; - } - } - if (unlikely(ldip->di_nextents + ldip->di_anextents > ldip->di_nblocks)){ - XFS_CORRUPTION_ERROR("xlog_recover_inode_pass2(5)", - XFS_ERRLEVEL_LOW, mp, ldip, - sizeof(*ldip)); - xfs_alert(mp, - "%s: Bad inode log record, rec ptr "PTR_FMT", dino ptr "PTR_FMT", " - "dino bp "PTR_FMT", ino %Ld, total extents = %d, nblocks = %Ld", - __func__, item, dip, bp, in_f->ilf_ino, - ldip->di_nextents + ldip->di_anextents, - ldip->di_nblocks); - error = -EFSCORRUPTED; - goto out_release; - } - if (unlikely(ldip->di_forkoff > mp->m_sb.sb_inodesize)) { - XFS_CORRUPTION_ERROR("xlog_recover_inode_pass2(6)", - XFS_ERRLEVEL_LOW, mp, ldip, - sizeof(*ldip)); - xfs_alert(mp, - "%s: Bad inode log record, rec ptr "PTR_FMT", dino ptr "PTR_FMT", " - "dino bp "PTR_FMT", ino %Ld, forkoff 0x%x", __func__, - item, dip, bp, in_f->ilf_ino, ldip->di_forkoff); - error = -EFSCORRUPTED; - goto out_release; - } - isize = xfs_log_dinode_size(mp); - if (unlikely(item->ri_buf[1].i_len > isize)) { - XFS_CORRUPTION_ERROR("xlog_recover_inode_pass2(7)", - XFS_ERRLEVEL_LOW, mp, ldip, - sizeof(*ldip)); - xfs_alert(mp, - "%s: Bad inode log record length %d, rec ptr "PTR_FMT, - __func__, item->ri_buf[1].i_len, item); - error = -EFSCORRUPTED; - goto out_release; - } - - /* recover the log dinode inode into the on disk inode */ - xfs_log_dinode_to_disk(ldip, dip); - - fields = in_f->ilf_fields; - if (fields & XFS_ILOG_DEV) - xfs_dinode_put_rdev(dip, in_f->ilf_u.ilfu_rdev); - - if (in_f->ilf_size == 2) - goto out_owner_change; - len = item->ri_buf[2].i_len; - src = item->ri_buf[2].i_addr; - ASSERT(in_f->ilf_size <= 4); - ASSERT((in_f->ilf_size == 3) || (fields & XFS_ILOG_AFORK)); - ASSERT(!(fields & XFS_ILOG_DFORK) || - (len == in_f->ilf_dsize)); - - switch (fields & XFS_ILOG_DFORK) { - case XFS_ILOG_DDATA: - case XFS_ILOG_DEXT: - memcpy(XFS_DFORK_DPTR(dip), src, len); - break; - - case XFS_ILOG_DBROOT: - xfs_bmbt_to_bmdr(mp, (struct xfs_btree_block *)src, len, - (xfs_bmdr_block_t *)XFS_DFORK_DPTR(dip), - XFS_DFORK_DSIZE(dip, mp)); - break; - - default: - /* - * There are no data fork flags set. - */ - ASSERT((fields & XFS_ILOG_DFORK) == 0); - break; - } - - /* - * If we logged any attribute data, recover it. There may or - * may not have been any other non-core data logged in this - * transaction. - */ - if (in_f->ilf_fields & XFS_ILOG_AFORK) { - if (in_f->ilf_fields & XFS_ILOG_DFORK) { - attr_index = 3; - } else { - attr_index = 2; - } - len = item->ri_buf[attr_index].i_len; - src = item->ri_buf[attr_index].i_addr; - ASSERT(len == in_f->ilf_asize); - - switch (in_f->ilf_fields & XFS_ILOG_AFORK) { - case XFS_ILOG_ADATA: - case XFS_ILOG_AEXT: - dest = XFS_DFORK_APTR(dip); - ASSERT(len <= XFS_DFORK_ASIZE(dip, mp)); - memcpy(dest, src, len); - break; - - case XFS_ILOG_ABROOT: - dest = XFS_DFORK_APTR(dip); - xfs_bmbt_to_bmdr(mp, (struct xfs_btree_block *)src, - len, (xfs_bmdr_block_t*)dest, - XFS_DFORK_ASIZE(dip, mp)); - break; - - default: - xfs_warn(log->l_mp, "%s: Invalid flag", __func__); - ASSERT(0); - error = -EFSCORRUPTED; - goto out_release; - } - } - -out_owner_change: - /* Recover the swapext owner change unless inode has been deleted */ - if ((in_f->ilf_fields & (XFS_ILOG_DOWNER|XFS_ILOG_AOWNER)) && - (dip->di_mode != 0)) - error = xfs_recover_inode_owner_change(mp, dip, in_f, - buffer_list); - /* re-generate the checksum. */ - xfs_dinode_calc_crc(log->l_mp, dip); - - ASSERT(bp->b_mount == mp); - bp->b_iodone = xlog_recover_iodone; - xfs_buf_delwri_queue(bp, buffer_list); - -out_release: - xfs_buf_relse(bp); -error: - if (need_free) - kmem_free(in_f); - return error; -} - /* * Recover a dquot record */ @@ -3066,9 +2714,6 @@ xlog_recover_commit_pass2( trans->r_lsn); switch (ITEM_TYPE(item)) { - case XFS_LI_INODE: - return xlog_recover_inode_pass2(log, buffer_list, item, - trans->r_lsn); case XFS_LI_EFI: return xlog_recover_efi_pass2(log, item, trans->r_lsn); case XFS_LI_EFD: From fcbdf91e0c9ff099ca5b6ea4c69cb5223874cec7 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:47 -0700 Subject: [PATCH 0279/1043] xfs: refactor log recovery dquot item dispatch for pass2 commit functions Move the log dquot item pass2 commit code into the per-item source code files and use the dispatch function to call it. We do these one at a time because there's a lot of code to move. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_dquot_item_recover.c | 109 +++++++++++++++++++++++++++++++ fs/xfs/xfs_log_recover.c | 112 -------------------------------- 2 files changed, 109 insertions(+), 112 deletions(-) diff --git a/fs/xfs/xfs_dquot_item_recover.c b/fs/xfs/xfs_dquot_item_recover.c index ebc44c1bc2b1..07ff943972a3 100644 --- a/fs/xfs/xfs_dquot_item_recover.c +++ b/fs/xfs/xfs_dquot_item_recover.c @@ -53,9 +53,118 @@ xlog_recover_dquot_ra_pass2( &xfs_dquot_buf_ra_ops); } +/* + * Recover a dquot record + */ +STATIC int +xlog_recover_dquot_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t current_lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_buf *bp; + struct xfs_disk_dquot *ddq, *recddq; + struct xfs_dq_logformat *dq_f; + xfs_failaddr_t fa; + int error; + uint type; + + /* + * Filesystems are required to send in quota flags at mount time. + */ + if (mp->m_qflags == 0) + return 0; + + recddq = item->ri_buf[1].i_addr; + if (recddq == NULL) { + xfs_alert(log->l_mp, "NULL dquot in %s.", __func__); + return -EFSCORRUPTED; + } + if (item->ri_buf[1].i_len < sizeof(struct xfs_disk_dquot)) { + xfs_alert(log->l_mp, "dquot too small (%d) in %s.", + item->ri_buf[1].i_len, __func__); + return -EFSCORRUPTED; + } + + /* + * This type of quotas was turned off, so ignore this record. + */ + type = recddq->d_flags & (XFS_DQ_USER | XFS_DQ_PROJ | XFS_DQ_GROUP); + ASSERT(type); + if (log->l_quotaoffs_flag & type) + return 0; + + /* + * At this point we know that quota was _not_ turned off. + * Since the mount flags are not indicating to us otherwise, this + * must mean that quota is on, and the dquot needs to be replayed. + * Remember that we may not have fully recovered the superblock yet, + * so we can't do the usual trick of looking at the SB quota bits. + * + * The other possibility, of course, is that the quota subsystem was + * removed since the last mount - ENOSYS. + */ + dq_f = item->ri_buf[0].i_addr; + ASSERT(dq_f); + fa = xfs_dquot_verify(mp, recddq, dq_f->qlf_id, 0); + if (fa) { + xfs_alert(mp, "corrupt dquot ID 0x%x in log at %pS", + dq_f->qlf_id, fa); + return -EFSCORRUPTED; + } + ASSERT(dq_f->qlf_len == 1); + + /* + * At this point we are assuming that the dquots have been allocated + * and hence the buffer has valid dquots stamped in it. It should, + * therefore, pass verifier validation. If the dquot is bad, then the + * we'll return an error here, so we don't need to specifically check + * the dquot in the buffer after the verifier has run. + */ + error = xfs_trans_read_buf(mp, NULL, mp->m_ddev_targp, dq_f->qlf_blkno, + XFS_FSB_TO_BB(mp, dq_f->qlf_len), 0, &bp, + &xfs_dquot_buf_ops); + if (error) + return error; + + ASSERT(bp); + ddq = xfs_buf_offset(bp, dq_f->qlf_boffset); + + /* + * If the dquot has an LSN in it, recover the dquot only if it's less + * than the lsn of the transaction we are replaying. + */ + if (xfs_sb_version_hascrc(&mp->m_sb)) { + struct xfs_dqblk *dqb = (struct xfs_dqblk *)ddq; + xfs_lsn_t lsn = be64_to_cpu(dqb->dd_lsn); + + if (lsn && lsn != -1 && XFS_LSN_CMP(lsn, current_lsn) >= 0) { + goto out_release; + } + } + + memcpy(ddq, recddq, item->ri_buf[1].i_len); + if (xfs_sb_version_hascrc(&mp->m_sb)) { + xfs_update_cksum((char *)ddq, sizeof(struct xfs_dqblk), + XFS_DQUOT_CRC_OFF); + } + + ASSERT(dq_f->qlf_size == 2); + ASSERT(bp->b_mount == mp); + bp->b_iodone = xlog_recover_iodone; + xfs_buf_delwri_queue(bp, buffer_list); + +out_release: + xfs_buf_relse(bp); + return 0; +} + const struct xlog_recover_item_ops xlog_dquot_item_ops = { .item_type = XFS_LI_DQUOT, .ra_pass2 = xlog_recover_dquot_ra_pass2, + .commit_pass2 = xlog_recover_dquot_commit_pass2, }; /* diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 8bf8d4dec0d7..1b96df783756 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2034,115 +2034,6 @@ xlog_buf_readahead( xfs_buf_readahead(log->l_mp->m_ddev_targp, blkno, len, ops); } -/* - * Recover a dquot record - */ -STATIC int -xlog_recover_dquot_pass2( - struct xlog *log, - struct list_head *buffer_list, - struct xlog_recover_item *item, - xfs_lsn_t current_lsn) -{ - xfs_mount_t *mp = log->l_mp; - xfs_buf_t *bp; - struct xfs_disk_dquot *ddq, *recddq; - xfs_failaddr_t fa; - int error; - xfs_dq_logformat_t *dq_f; - uint type; - - - /* - * Filesystems are required to send in quota flags at mount time. - */ - if (mp->m_qflags == 0) - return 0; - - recddq = item->ri_buf[1].i_addr; - if (recddq == NULL) { - xfs_alert(log->l_mp, "NULL dquot in %s.", __func__); - return -EFSCORRUPTED; - } - if (item->ri_buf[1].i_len < sizeof(struct xfs_disk_dquot)) { - xfs_alert(log->l_mp, "dquot too small (%d) in %s.", - item->ri_buf[1].i_len, __func__); - return -EFSCORRUPTED; - } - - /* - * This type of quotas was turned off, so ignore this record. - */ - type = recddq->d_flags & (XFS_DQ_USER | XFS_DQ_PROJ | XFS_DQ_GROUP); - ASSERT(type); - if (log->l_quotaoffs_flag & type) - return 0; - - /* - * At this point we know that quota was _not_ turned off. - * Since the mount flags are not indicating to us otherwise, this - * must mean that quota is on, and the dquot needs to be replayed. - * Remember that we may not have fully recovered the superblock yet, - * so we can't do the usual trick of looking at the SB quota bits. - * - * The other possibility, of course, is that the quota subsystem was - * removed since the last mount - ENOSYS. - */ - dq_f = item->ri_buf[0].i_addr; - ASSERT(dq_f); - fa = xfs_dquot_verify(mp, recddq, dq_f->qlf_id, 0); - if (fa) { - xfs_alert(mp, "corrupt dquot ID 0x%x in log at %pS", - dq_f->qlf_id, fa); - return -EFSCORRUPTED; - } - ASSERT(dq_f->qlf_len == 1); - - /* - * At this point we are assuming that the dquots have been allocated - * and hence the buffer has valid dquots stamped in it. It should, - * therefore, pass verifier validation. If the dquot is bad, then the - * we'll return an error here, so we don't need to specifically check - * the dquot in the buffer after the verifier has run. - */ - error = xfs_trans_read_buf(mp, NULL, mp->m_ddev_targp, dq_f->qlf_blkno, - XFS_FSB_TO_BB(mp, dq_f->qlf_len), 0, &bp, - &xfs_dquot_buf_ops); - if (error) - return error; - - ASSERT(bp); - ddq = xfs_buf_offset(bp, dq_f->qlf_boffset); - - /* - * If the dquot has an LSN in it, recover the dquot only if it's less - * than the lsn of the transaction we are replaying. - */ - if (xfs_sb_version_hascrc(&mp->m_sb)) { - struct xfs_dqblk *dqb = (struct xfs_dqblk *)ddq; - xfs_lsn_t lsn = be64_to_cpu(dqb->dd_lsn); - - if (lsn && lsn != -1 && XFS_LSN_CMP(lsn, current_lsn) >= 0) { - goto out_release; - } - } - - memcpy(ddq, recddq, item->ri_buf[1].i_len); - if (xfs_sb_version_hascrc(&mp->m_sb)) { - xfs_update_cksum((char *)ddq, sizeof(struct xfs_dqblk), - XFS_DQUOT_CRC_OFF); - } - - ASSERT(dq_f->qlf_size == 2); - ASSERT(bp->b_mount == mp); - bp->b_iodone = xlog_recover_iodone; - xfs_buf_delwri_queue(bp, buffer_list); - -out_release: - xfs_buf_relse(bp); - return 0; -} - /* * This routine is called to create an in-core extent free intent * item from the efi format structure which was logged on disk. @@ -2730,9 +2621,6 @@ xlog_recover_commit_pass2( return xlog_recover_bui_pass2(log, item, trans->r_lsn); case XFS_LI_BUD: return xlog_recover_bud_pass2(log, item); - case XFS_LI_DQUOT: - return xlog_recover_dquot_pass2(log, buffer_list, item, - trans->r_lsn); case XFS_LI_ICREATE: return xlog_recover_do_icreate_pass2(log, buffer_list, item); case XFS_LI_QUOTAOFF: From 3ec6efa703cf65887e681d1f97d38a63261d907e Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:48 -0700 Subject: [PATCH 0280/1043] xfs: refactor log recovery icreate item dispatch for pass2 commit functions Move the log icreate item pass2 commit code into the per-item source code files and use the dispatch function to call it. We do these one at a time because there's a lot of code to move. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_icreate_item.c | 132 ++++++++++++++++++++++++++++++++++++++ fs/xfs/xfs_log_recover.c | 126 ------------------------------------ 2 files changed, 132 insertions(+), 126 deletions(-) diff --git a/fs/xfs/xfs_icreate_item.c b/fs/xfs/xfs_icreate_item.c index 366c1e722a29..287a9e5c7d75 100644 --- a/fs/xfs/xfs_icreate_item.c +++ b/fs/xfs/xfs_icreate_item.c @@ -6,13 +6,19 @@ #include "xfs.h" #include "xfs_fs.h" #include "xfs_shared.h" +#include "xfs_format.h" #include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" #include "xfs_trans.h" #include "xfs_trans_priv.h" #include "xfs_icreate_item.h" #include "xfs_log.h" #include "xfs_log_priv.h" #include "xfs_log_recover.h" +#include "xfs_ialloc.h" +#include "xfs_trace.h" kmem_zone_t *xfs_icreate_zone; /* inode create item zone */ @@ -123,7 +129,133 @@ xlog_recover_icreate_reorder( return XLOG_REORDER_BUFFER_LIST; } +/* + * This routine is called when an inode create format structure is found in a + * committed transaction in the log. It's purpose is to initialise the inodes + * being allocated on disk. This requires us to get inode cluster buffers that + * match the range to be initialised, stamped with inode templates and written + * by delayed write so that subsequent modifications will hit the cached buffer + * and only need writing out at the end of recovery. + */ +STATIC int +xlog_recover_icreate_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_icreate_log *icl; + struct xfs_ino_geometry *igeo = M_IGEO(mp); + xfs_agnumber_t agno; + xfs_agblock_t agbno; + unsigned int count; + unsigned int isize; + xfs_agblock_t length; + int bb_per_cluster; + int cancel_count; + int nbufs; + int i; + + icl = (struct xfs_icreate_log *)item->ri_buf[0].i_addr; + if (icl->icl_type != XFS_LI_ICREATE) { + xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad type"); + return -EINVAL; + } + + if (icl->icl_size != 1) { + xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad icl size"); + return -EINVAL; + } + + agno = be32_to_cpu(icl->icl_ag); + if (agno >= mp->m_sb.sb_agcount) { + xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad agno"); + return -EINVAL; + } + agbno = be32_to_cpu(icl->icl_agbno); + if (!agbno || agbno == NULLAGBLOCK || agbno >= mp->m_sb.sb_agblocks) { + xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad agbno"); + return -EINVAL; + } + isize = be32_to_cpu(icl->icl_isize); + if (isize != mp->m_sb.sb_inodesize) { + xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad isize"); + return -EINVAL; + } + count = be32_to_cpu(icl->icl_count); + if (!count) { + xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad count"); + return -EINVAL; + } + length = be32_to_cpu(icl->icl_length); + if (!length || length >= mp->m_sb.sb_agblocks) { + xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad length"); + return -EINVAL; + } + + /* + * The inode chunk is either full or sparse and we only support + * m_ino_geo.ialloc_min_blks sized sparse allocations at this time. + */ + if (length != igeo->ialloc_blks && + length != igeo->ialloc_min_blks) { + xfs_warn(log->l_mp, + "%s: unsupported chunk length", __FUNCTION__); + return -EINVAL; + } + + /* verify inode count is consistent with extent length */ + if ((count >> mp->m_sb.sb_inopblog) != length) { + xfs_warn(log->l_mp, + "%s: inconsistent inode count and chunk length", + __FUNCTION__); + return -EINVAL; + } + + /* + * The icreate transaction can cover multiple cluster buffers and these + * buffers could have been freed and reused. Check the individual + * buffers for cancellation so we don't overwrite anything written after + * a cancellation. + */ + bb_per_cluster = XFS_FSB_TO_BB(mp, igeo->blocks_per_cluster); + nbufs = length / igeo->blocks_per_cluster; + for (i = 0, cancel_count = 0; i < nbufs; i++) { + xfs_daddr_t daddr; + + daddr = XFS_AGB_TO_DADDR(mp, agno, + agbno + i * igeo->blocks_per_cluster); + if (xlog_is_buffer_cancelled(log, daddr, bb_per_cluster)) + cancel_count++; + } + + /* + * We currently only use icreate for a single allocation at a time. This + * means we should expect either all or none of the buffers to be + * cancelled. Be conservative and skip replay if at least one buffer is + * cancelled, but warn the user that something is awry if the buffers + * are not consistent. + * + * XXX: This must be refined to only skip cancelled clusters once we use + * icreate for multiple chunk allocations. + */ + ASSERT(!cancel_count || cancel_count == nbufs); + if (cancel_count) { + if (cancel_count != nbufs) + xfs_warn(mp, + "WARNING: partial inode chunk cancellation, skipped icreate."); + trace_xfs_log_recover_icreate_cancel(log, icl); + return 0; + } + + trace_xfs_log_recover_icreate_recover(log, icl); + return xfs_ialloc_inode_init(mp, NULL, buffer_list, count, agno, agbno, + length, be32_to_cpu(icl->icl_gen)); +} + const struct xlog_recover_item_ops xlog_icreate_item_ops = { .item_type = XFS_LI_ICREATE, .reorder = xlog_recover_icreate_reorder, + .commit_pass2 = xlog_recover_icreate_commit_pass2, }; diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 1b96df783756..0a90ec9d0ca6 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2467,130 +2467,6 @@ xlog_recover_bud_pass2( return 0; } -/* - * This routine is called when an inode create format structure is found in a - * committed transaction in the log. It's purpose is to initialise the inodes - * being allocated on disk. This requires us to get inode cluster buffers that - * match the range to be initialised, stamped with inode templates and written - * by delayed write so that subsequent modifications will hit the cached buffer - * and only need writing out at the end of recovery. - */ -STATIC int -xlog_recover_do_icreate_pass2( - struct xlog *log, - struct list_head *buffer_list, - struct xlog_recover_item *item) -{ - struct xfs_mount *mp = log->l_mp; - struct xfs_icreate_log *icl; - struct xfs_ino_geometry *igeo = M_IGEO(mp); - xfs_agnumber_t agno; - xfs_agblock_t agbno; - unsigned int count; - unsigned int isize; - xfs_agblock_t length; - int bb_per_cluster; - int cancel_count; - int nbufs; - int i; - - icl = (struct xfs_icreate_log *)item->ri_buf[0].i_addr; - if (icl->icl_type != XFS_LI_ICREATE) { - xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad type"); - return -EINVAL; - } - - if (icl->icl_size != 1) { - xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad icl size"); - return -EINVAL; - } - - agno = be32_to_cpu(icl->icl_ag); - if (agno >= mp->m_sb.sb_agcount) { - xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad agno"); - return -EINVAL; - } - agbno = be32_to_cpu(icl->icl_agbno); - if (!agbno || agbno == NULLAGBLOCK || agbno >= mp->m_sb.sb_agblocks) { - xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad agbno"); - return -EINVAL; - } - isize = be32_to_cpu(icl->icl_isize); - if (isize != mp->m_sb.sb_inodesize) { - xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad isize"); - return -EINVAL; - } - count = be32_to_cpu(icl->icl_count); - if (!count) { - xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad count"); - return -EINVAL; - } - length = be32_to_cpu(icl->icl_length); - if (!length || length >= mp->m_sb.sb_agblocks) { - xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad length"); - return -EINVAL; - } - - /* - * The inode chunk is either full or sparse and we only support - * m_ino_geo.ialloc_min_blks sized sparse allocations at this time. - */ - if (length != igeo->ialloc_blks && - length != igeo->ialloc_min_blks) { - xfs_warn(log->l_mp, - "%s: unsupported chunk length", __FUNCTION__); - return -EINVAL; - } - - /* verify inode count is consistent with extent length */ - if ((count >> mp->m_sb.sb_inopblog) != length) { - xfs_warn(log->l_mp, - "%s: inconsistent inode count and chunk length", - __FUNCTION__); - return -EINVAL; - } - - /* - * The icreate transaction can cover multiple cluster buffers and these - * buffers could have been freed and reused. Check the individual - * buffers for cancellation so we don't overwrite anything written after - * a cancellation. - */ - bb_per_cluster = XFS_FSB_TO_BB(mp, igeo->blocks_per_cluster); - nbufs = length / igeo->blocks_per_cluster; - for (i = 0, cancel_count = 0; i < nbufs; i++) { - xfs_daddr_t daddr; - - daddr = XFS_AGB_TO_DADDR(mp, agno, - agbno + i * igeo->blocks_per_cluster); - if (xlog_is_buffer_cancelled(log, daddr, bb_per_cluster)) - cancel_count++; - } - - /* - * We currently only use icreate for a single allocation at a time. This - * means we should expect either all or none of the buffers to be - * cancelled. Be conservative and skip replay if at least one buffer is - * cancelled, but warn the user that something is awry if the buffers - * are not consistent. - * - * XXX: This must be refined to only skip cancelled clusters once we use - * icreate for multiple chunk allocations. - */ - ASSERT(!cancel_count || cancel_count == nbufs); - if (cancel_count) { - if (cancel_count != nbufs) - xfs_warn(mp, - "WARNING: partial inode chunk cancellation, skipped icreate."); - trace_xfs_log_recover_icreate_cancel(log, icl); - return 0; - } - - trace_xfs_log_recover_icreate_recover(log, icl); - return xfs_ialloc_inode_init(mp, NULL, buffer_list, count, agno, agbno, - length, be32_to_cpu(icl->icl_gen)); -} - STATIC int xlog_recover_commit_pass2( struct xlog *log, @@ -2621,8 +2497,6 @@ xlog_recover_commit_pass2( return xlog_recover_bui_pass2(log, item, trans->r_lsn); case XFS_LI_BUD: return xlog_recover_bud_pass2(log, item); - case XFS_LI_ICREATE: - return xlog_recover_do_icreate_pass2(log, buffer_list, item); case XFS_LI_QUOTAOFF: /* nothing to do in pass2 */ return 0; From 9817aa80dcdc0358226f41322e1ff35339d0d24b Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:48 -0700 Subject: [PATCH 0281/1043] xfs: refactor log recovery EFI item dispatch for pass2 commit functions Move the extent free intent and intent-done pass2 commit code into the per-item source code files and use dispatch functions to call them. We do these one at a time because there's a lot of code to move. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_extfree_item.c | 107 ++++++++++++++++++++++++++++++++++++-- fs/xfs/xfs_extfree_item.h | 4 -- fs/xfs/xfs_log_recover.c | 100 ----------------------------------- 3 files changed, 104 insertions(+), 107 deletions(-) diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index 163d01cb9f9f..69f7e75a747e 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -22,6 +22,7 @@ #include "xfs_bmap.h" #include "xfs_trace.h" #include "xfs_error.h" +#include "xfs_log_priv.h" #include "xfs_log_recover.h" kmem_zone_t *xfs_efi_zone; @@ -32,7 +33,7 @@ static inline struct xfs_efi_log_item *EFI_ITEM(struct xfs_log_item *lip) return container_of(lip, struct xfs_efi_log_item, efi_item); } -void +STATIC void xfs_efi_item_free( struct xfs_efi_log_item *efip) { @@ -151,7 +152,7 @@ static const struct xfs_item_ops xfs_efi_item_ops = { /* * Allocate and initialize an efi item with the given number of extents. */ -struct xfs_efi_log_item * +STATIC struct xfs_efi_log_item * xfs_efi_init( struct xfs_mount *mp, uint nextents) @@ -185,7 +186,7 @@ xfs_efi_init( * one of which will be the native format for this kernel. * It will handle the conversion of formats if necessary. */ -int +STATIC int xfs_efi_copy_format(xfs_log_iovec_t *buf, xfs_efi_log_format_t *dst_efi_fmt) { xfs_efi_log_format_t *src_efi_fmt = buf->i_addr; @@ -646,10 +647,110 @@ abort_error: return error; } +/* + * This routine is called to create an in-core extent free intent + * item from the efi format structure which was logged on disk. + * It allocates an in-core efi, copies the extents from the format + * structure into it, and adds the efi to the AIL with the given + * LSN. + */ +STATIC int +xlog_recover_efi_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_efi_log_item *efip; + struct xfs_efi_log_format *efi_formatp; + int error; + + efi_formatp = item->ri_buf[0].i_addr; + + efip = xfs_efi_init(mp, efi_formatp->efi_nextents); + error = xfs_efi_copy_format(&item->ri_buf[0], &efip->efi_format); + if (error) { + xfs_efi_item_free(efip); + return error; + } + atomic_set(&efip->efi_next_extent, efi_formatp->efi_nextents); + + spin_lock(&log->l_ailp->ail_lock); + /* + * The EFI has two references. One for the EFD and one for EFI to ensure + * it makes it into the AIL. Insert the EFI into the AIL directly and + * drop the EFI reference. Note that xfs_trans_ail_update() drops the + * AIL lock. + */ + xfs_trans_ail_update(log->l_ailp, &efip->efi_item, lsn); + xfs_efi_release(efip); + return 0; +} + const struct xlog_recover_item_ops xlog_efi_item_ops = { .item_type = XFS_LI_EFI, + .commit_pass2 = xlog_recover_efi_commit_pass2, }; +/* + * This routine is called when an EFD format structure is found in a committed + * transaction in the log. Its purpose is to cancel the corresponding EFI if it + * was still in the log. To do this it searches the AIL for the EFI with an id + * equal to that in the EFD format structure. If we find it we drop the EFD + * reference, which removes the EFI from the AIL and frees it. + */ +STATIC int +xlog_recover_efd_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_ail_cursor cur; + struct xfs_efd_log_format *efd_formatp; + struct xfs_efi_log_item *efip = NULL; + struct xfs_log_item *lip; + struct xfs_ail *ailp = log->l_ailp; + uint64_t efi_id; + + efd_formatp = item->ri_buf[0].i_addr; + ASSERT((item->ri_buf[0].i_len == (sizeof(xfs_efd_log_format_32_t) + + ((efd_formatp->efd_nextents - 1) * sizeof(xfs_extent_32_t)))) || + (item->ri_buf[0].i_len == (sizeof(xfs_efd_log_format_64_t) + + ((efd_formatp->efd_nextents - 1) * sizeof(xfs_extent_64_t))))); + efi_id = efd_formatp->efd_efi_id; + + /* + * Search for the EFI with the id in the EFD format structure in the + * AIL. + */ + spin_lock(&ailp->ail_lock); + lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); + while (lip != NULL) { + if (lip->li_type == XFS_LI_EFI) { + efip = (struct xfs_efi_log_item *)lip; + if (efip->efi_format.efi_id == efi_id) { + /* + * Drop the EFD reference to the EFI. This + * removes the EFI from the AIL and frees it. + */ + spin_unlock(&ailp->ail_lock); + xfs_efi_release(efip); + spin_lock(&ailp->ail_lock); + break; + } + } + lip = xfs_trans_ail_cursor_next(ailp, &cur); + } + + xfs_trans_ail_cursor_done(&cur); + spin_unlock(&ailp->ail_lock); + + return 0; +} + const struct xlog_recover_item_ops xlog_efd_item_ops = { .item_type = XFS_LI_EFD, + .commit_pass2 = xlog_recover_efd_commit_pass2, }; diff --git a/fs/xfs/xfs_extfree_item.h b/fs/xfs/xfs_extfree_item.h index a2a736a77fa9..876e3d237f48 100644 --- a/fs/xfs/xfs_extfree_item.h +++ b/fs/xfs/xfs_extfree_item.h @@ -78,10 +78,6 @@ struct xfs_efd_log_item { extern struct kmem_zone *xfs_efi_zone; extern struct kmem_zone *xfs_efd_zone; -struct xfs_efi_log_item *xfs_efi_init(struct xfs_mount *, uint); -int xfs_efi_copy_format(xfs_log_iovec_t *buf, - xfs_efi_log_format_t *dst_efi_fmt); -void xfs_efi_item_free(struct xfs_efi_log_item *); void xfs_efi_release(struct xfs_efi_log_item *); int xfs_efi_recover(struct xfs_mount *mp, diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 0a90ec9d0ca6..26f0f84f2528 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2034,102 +2034,6 @@ xlog_buf_readahead( xfs_buf_readahead(log->l_mp->m_ddev_targp, blkno, len, ops); } -/* - * This routine is called to create an in-core extent free intent - * item from the efi format structure which was logged on disk. - * It allocates an in-core efi, copies the extents from the format - * structure into it, and adds the efi to the AIL with the given - * LSN. - */ -STATIC int -xlog_recover_efi_pass2( - struct xlog *log, - struct xlog_recover_item *item, - xfs_lsn_t lsn) -{ - int error; - struct xfs_mount *mp = log->l_mp; - struct xfs_efi_log_item *efip; - struct xfs_efi_log_format *efi_formatp; - - efi_formatp = item->ri_buf[0].i_addr; - - efip = xfs_efi_init(mp, efi_formatp->efi_nextents); - error = xfs_efi_copy_format(&item->ri_buf[0], &efip->efi_format); - if (error) { - xfs_efi_item_free(efip); - return error; - } - atomic_set(&efip->efi_next_extent, efi_formatp->efi_nextents); - - spin_lock(&log->l_ailp->ail_lock); - /* - * The EFI has two references. One for the EFD and one for EFI to ensure - * it makes it into the AIL. Insert the EFI into the AIL directly and - * drop the EFI reference. Note that xfs_trans_ail_update() drops the - * AIL lock. - */ - xfs_trans_ail_update(log->l_ailp, &efip->efi_item, lsn); - xfs_efi_release(efip); - return 0; -} - - -/* - * This routine is called when an EFD format structure is found in a committed - * transaction in the log. Its purpose is to cancel the corresponding EFI if it - * was still in the log. To do this it searches the AIL for the EFI with an id - * equal to that in the EFD format structure. If we find it we drop the EFD - * reference, which removes the EFI from the AIL and frees it. - */ -STATIC int -xlog_recover_efd_pass2( - struct xlog *log, - struct xlog_recover_item *item) -{ - xfs_efd_log_format_t *efd_formatp; - struct xfs_efi_log_item *efip = NULL; - struct xfs_log_item *lip; - uint64_t efi_id; - struct xfs_ail_cursor cur; - struct xfs_ail *ailp = log->l_ailp; - - efd_formatp = item->ri_buf[0].i_addr; - ASSERT((item->ri_buf[0].i_len == (sizeof(xfs_efd_log_format_32_t) + - ((efd_formatp->efd_nextents - 1) * sizeof(xfs_extent_32_t)))) || - (item->ri_buf[0].i_len == (sizeof(xfs_efd_log_format_64_t) + - ((efd_formatp->efd_nextents - 1) * sizeof(xfs_extent_64_t))))); - efi_id = efd_formatp->efd_efi_id; - - /* - * Search for the EFI with the id in the EFD format structure in the - * AIL. - */ - spin_lock(&ailp->ail_lock); - lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); - while (lip != NULL) { - if (lip->li_type == XFS_LI_EFI) { - efip = (struct xfs_efi_log_item *)lip; - if (efip->efi_format.efi_id == efi_id) { - /* - * Drop the EFD reference to the EFI. This - * removes the EFI from the AIL and frees it. - */ - spin_unlock(&ailp->ail_lock); - xfs_efi_release(efip); - spin_lock(&ailp->ail_lock); - break; - } - } - lip = xfs_trans_ail_cursor_next(ailp, &cur); - } - - xfs_trans_ail_cursor_done(&cur); - spin_unlock(&ailp->ail_lock); - - return 0; -} - /* * This routine is called to create an in-core extent rmap update * item from the rui format structure which was logged on disk. @@ -2481,10 +2385,6 @@ xlog_recover_commit_pass2( trans->r_lsn); switch (ITEM_TYPE(item)) { - case XFS_LI_EFI: - return xlog_recover_efi_pass2(log, item, trans->r_lsn); - case XFS_LI_EFD: - return xlog_recover_efd_pass2(log, item); case XFS_LI_RUI: return xlog_recover_rui_pass2(log, item, trans->r_lsn); case XFS_LI_RUD: From 07590a9d38b8587076c175550743daca9e067f09 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:49 -0700 Subject: [PATCH 0282/1043] xfs: refactor log recovery RUI item dispatch for pass2 commit functions Move the rmap update intent and intent-done pass2 commit code into the per-item source code files and use dispatch functions to call them. We do these one at a time because there's a lot of code to move. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_log_recover.c | 97 ------------------------------------ fs/xfs/xfs_rmap_item.c | 104 +++++++++++++++++++++++++++++++++++++-- fs/xfs/xfs_rmap_item.h | 4 -- 3 files changed, 101 insertions(+), 104 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 26f0f84f2528..875d79d88018 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2034,99 +2034,6 @@ xlog_buf_readahead( xfs_buf_readahead(log->l_mp->m_ddev_targp, blkno, len, ops); } -/* - * This routine is called to create an in-core extent rmap update - * item from the rui format structure which was logged on disk. - * It allocates an in-core rui, copies the extents from the format - * structure into it, and adds the rui to the AIL with the given - * LSN. - */ -STATIC int -xlog_recover_rui_pass2( - struct xlog *log, - struct xlog_recover_item *item, - xfs_lsn_t lsn) -{ - int error; - struct xfs_mount *mp = log->l_mp; - struct xfs_rui_log_item *ruip; - struct xfs_rui_log_format *rui_formatp; - - rui_formatp = item->ri_buf[0].i_addr; - - ruip = xfs_rui_init(mp, rui_formatp->rui_nextents); - error = xfs_rui_copy_format(&item->ri_buf[0], &ruip->rui_format); - if (error) { - xfs_rui_item_free(ruip); - return error; - } - atomic_set(&ruip->rui_next_extent, rui_formatp->rui_nextents); - - spin_lock(&log->l_ailp->ail_lock); - /* - * The RUI has two references. One for the RUD and one for RUI to ensure - * it makes it into the AIL. Insert the RUI into the AIL directly and - * drop the RUI reference. Note that xfs_trans_ail_update() drops the - * AIL lock. - */ - xfs_trans_ail_update(log->l_ailp, &ruip->rui_item, lsn); - xfs_rui_release(ruip); - return 0; -} - - -/* - * This routine is called when an RUD format structure is found in a committed - * transaction in the log. Its purpose is to cancel the corresponding RUI if it - * was still in the log. To do this it searches the AIL for the RUI with an id - * equal to that in the RUD format structure. If we find it we drop the RUD - * reference, which removes the RUI from the AIL and frees it. - */ -STATIC int -xlog_recover_rud_pass2( - struct xlog *log, - struct xlog_recover_item *item) -{ - struct xfs_rud_log_format *rud_formatp; - struct xfs_rui_log_item *ruip = NULL; - struct xfs_log_item *lip; - uint64_t rui_id; - struct xfs_ail_cursor cur; - struct xfs_ail *ailp = log->l_ailp; - - rud_formatp = item->ri_buf[0].i_addr; - ASSERT(item->ri_buf[0].i_len == sizeof(struct xfs_rud_log_format)); - rui_id = rud_formatp->rud_rui_id; - - /* - * Search for the RUI with the id in the RUD format structure in the - * AIL. - */ - spin_lock(&ailp->ail_lock); - lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); - while (lip != NULL) { - if (lip->li_type == XFS_LI_RUI) { - ruip = (struct xfs_rui_log_item *)lip; - if (ruip->rui_format.rui_id == rui_id) { - /* - * Drop the RUD reference to the RUI. This - * removes the RUI from the AIL and frees it. - */ - spin_unlock(&ailp->ail_lock); - xfs_rui_release(ruip); - spin_lock(&ailp->ail_lock); - break; - } - } - lip = xfs_trans_ail_cursor_next(ailp, &cur); - } - - xfs_trans_ail_cursor_done(&cur); - spin_unlock(&ailp->ail_lock); - - return 0; -} - /* * Copy an CUI format buffer from the given buf, and into the destination * CUI format structure. The CUI/CUD items were designed not to need any @@ -2385,10 +2292,6 @@ xlog_recover_commit_pass2( trans->r_lsn); switch (ITEM_TYPE(item)) { - case XFS_LI_RUI: - return xlog_recover_rui_pass2(log, item, trans->r_lsn); - case XFS_LI_RUD: - return xlog_recover_rud_pass2(log, item); case XFS_LI_CUI: return xlog_recover_cui_pass2(log, item, trans->r_lsn); case XFS_LI_CUD: diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index 0f3af9f05764..44049dbdb161 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -18,6 +18,7 @@ #include "xfs_log.h" #include "xfs_rmap.h" #include "xfs_error.h" +#include "xfs_log_priv.h" #include "xfs_log_recover.h" kmem_zone_t *xfs_rui_zone; @@ -28,7 +29,7 @@ static inline struct xfs_rui_log_item *RUI_ITEM(struct xfs_log_item *lip) return container_of(lip, struct xfs_rui_log_item, rui_item); } -void +STATIC void xfs_rui_item_free( struct xfs_rui_log_item *ruip) { @@ -133,7 +134,7 @@ static const struct xfs_item_ops xfs_rui_item_ops = { /* * Allocate and initialize an rui item with the given number of extents. */ -struct xfs_rui_log_item * +STATIC struct xfs_rui_log_item * xfs_rui_init( struct xfs_mount *mp, uint nextents) @@ -161,7 +162,7 @@ xfs_rui_init( * RUI format structure. The RUI/RUD items were designed not to need any * special alignment handling. */ -int +STATIC int xfs_rui_copy_format( struct xfs_log_iovec *buf, struct xfs_rui_log_format *dst_rui_fmt) @@ -587,10 +588,107 @@ abort_error: return error; } +/* + * This routine is called to create an in-core extent rmap update + * item from the rui format structure which was logged on disk. + * It allocates an in-core rui, copies the extents from the format + * structure into it, and adds the rui to the AIL with the given + * LSN. + */ +STATIC int +xlog_recover_rui_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + int error; + struct xfs_mount *mp = log->l_mp; + struct xfs_rui_log_item *ruip; + struct xfs_rui_log_format *rui_formatp; + + rui_formatp = item->ri_buf[0].i_addr; + + ruip = xfs_rui_init(mp, rui_formatp->rui_nextents); + error = xfs_rui_copy_format(&item->ri_buf[0], &ruip->rui_format); + if (error) { + xfs_rui_item_free(ruip); + return error; + } + atomic_set(&ruip->rui_next_extent, rui_formatp->rui_nextents); + + spin_lock(&log->l_ailp->ail_lock); + /* + * The RUI has two references. One for the RUD and one for RUI to ensure + * it makes it into the AIL. Insert the RUI into the AIL directly and + * drop the RUI reference. Note that xfs_trans_ail_update() drops the + * AIL lock. + */ + xfs_trans_ail_update(log->l_ailp, &ruip->rui_item, lsn); + xfs_rui_release(ruip); + return 0; +} + const struct xlog_recover_item_ops xlog_rui_item_ops = { .item_type = XFS_LI_RUI, + .commit_pass2 = xlog_recover_rui_commit_pass2, }; +/* + * This routine is called when an RUD format structure is found in a committed + * transaction in the log. Its purpose is to cancel the corresponding RUI if it + * was still in the log. To do this it searches the AIL for the RUI with an id + * equal to that in the RUD format structure. If we find it we drop the RUD + * reference, which removes the RUI from the AIL and frees it. + */ +STATIC int +xlog_recover_rud_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_rud_log_format *rud_formatp; + struct xfs_rui_log_item *ruip = NULL; + struct xfs_log_item *lip; + uint64_t rui_id; + struct xfs_ail_cursor cur; + struct xfs_ail *ailp = log->l_ailp; + + rud_formatp = item->ri_buf[0].i_addr; + ASSERT(item->ri_buf[0].i_len == sizeof(struct xfs_rud_log_format)); + rui_id = rud_formatp->rud_rui_id; + + /* + * Search for the RUI with the id in the RUD format structure in the + * AIL. + */ + spin_lock(&ailp->ail_lock); + lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); + while (lip != NULL) { + if (lip->li_type == XFS_LI_RUI) { + ruip = (struct xfs_rui_log_item *)lip; + if (ruip->rui_format.rui_id == rui_id) { + /* + * Drop the RUD reference to the RUI. This + * removes the RUI from the AIL and frees it. + */ + spin_unlock(&ailp->ail_lock); + xfs_rui_release(ruip); + spin_lock(&ailp->ail_lock); + break; + } + } + lip = xfs_trans_ail_cursor_next(ailp, &cur); + } + + xfs_trans_ail_cursor_done(&cur); + spin_unlock(&ailp->ail_lock); + + return 0; +} + const struct xlog_recover_item_ops xlog_rud_item_ops = { .item_type = XFS_LI_RUD, + .commit_pass2 = xlog_recover_rud_commit_pass2, }; diff --git a/fs/xfs/xfs_rmap_item.h b/fs/xfs/xfs_rmap_item.h index 8708e4a5aa5c..89bd192779f8 100644 --- a/fs/xfs/xfs_rmap_item.h +++ b/fs/xfs/xfs_rmap_item.h @@ -77,10 +77,6 @@ struct xfs_rud_log_item { extern struct kmem_zone *xfs_rui_zone; extern struct kmem_zone *xfs_rud_zone; -struct xfs_rui_log_item *xfs_rui_init(struct xfs_mount *, uint); -int xfs_rui_copy_format(struct xfs_log_iovec *buf, - struct xfs_rui_log_format *dst_rui_fmt); -void xfs_rui_item_free(struct xfs_rui_log_item *); void xfs_rui_release(struct xfs_rui_log_item *); int xfs_rui_recover(struct xfs_mount *mp, struct xfs_rui_log_item *ruip); From 9b4467e9834058df5be27a7a903be84921977170 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:49 -0700 Subject: [PATCH 0283/1043] xfs: refactor log recovery CUI item dispatch for pass2 commit functions Move the refcount update intent and intent-done pass2 commit code into the per-item source code files and use dispatch functions to call them. We do these one at a time because there's a lot of code to move. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_log_recover.c | 124 ----------------------------------- fs/xfs/xfs_refcount_item.c | 129 ++++++++++++++++++++++++++++++++++++- fs/xfs/xfs_refcount_item.h | 2 - 3 files changed, 127 insertions(+), 128 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 875d79d88018..0bce57088693 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2034,126 +2034,6 @@ xlog_buf_readahead( xfs_buf_readahead(log->l_mp->m_ddev_targp, blkno, len, ops); } -/* - * Copy an CUI format buffer from the given buf, and into the destination - * CUI format structure. The CUI/CUD items were designed not to need any - * special alignment handling. - */ -static int -xfs_cui_copy_format( - struct xfs_log_iovec *buf, - struct xfs_cui_log_format *dst_cui_fmt) -{ - struct xfs_cui_log_format *src_cui_fmt; - uint len; - - src_cui_fmt = buf->i_addr; - len = xfs_cui_log_format_sizeof(src_cui_fmt->cui_nextents); - - if (buf->i_len == len) { - memcpy(dst_cui_fmt, src_cui_fmt, len); - return 0; - } - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL); - return -EFSCORRUPTED; -} - -/* - * This routine is called to create an in-core extent refcount update - * item from the cui format structure which was logged on disk. - * It allocates an in-core cui, copies the extents from the format - * structure into it, and adds the cui to the AIL with the given - * LSN. - */ -STATIC int -xlog_recover_cui_pass2( - struct xlog *log, - struct xlog_recover_item *item, - xfs_lsn_t lsn) -{ - int error; - struct xfs_mount *mp = log->l_mp; - struct xfs_cui_log_item *cuip; - struct xfs_cui_log_format *cui_formatp; - - cui_formatp = item->ri_buf[0].i_addr; - - cuip = xfs_cui_init(mp, cui_formatp->cui_nextents); - error = xfs_cui_copy_format(&item->ri_buf[0], &cuip->cui_format); - if (error) { - xfs_cui_item_free(cuip); - return error; - } - atomic_set(&cuip->cui_next_extent, cui_formatp->cui_nextents); - - spin_lock(&log->l_ailp->ail_lock); - /* - * The CUI has two references. One for the CUD and one for CUI to ensure - * it makes it into the AIL. Insert the CUI into the AIL directly and - * drop the CUI reference. Note that xfs_trans_ail_update() drops the - * AIL lock. - */ - xfs_trans_ail_update(log->l_ailp, &cuip->cui_item, lsn); - xfs_cui_release(cuip); - return 0; -} - - -/* - * This routine is called when an CUD format structure is found in a committed - * transaction in the log. Its purpose is to cancel the corresponding CUI if it - * was still in the log. To do this it searches the AIL for the CUI with an id - * equal to that in the CUD format structure. If we find it we drop the CUD - * reference, which removes the CUI from the AIL and frees it. - */ -STATIC int -xlog_recover_cud_pass2( - struct xlog *log, - struct xlog_recover_item *item) -{ - struct xfs_cud_log_format *cud_formatp; - struct xfs_cui_log_item *cuip = NULL; - struct xfs_log_item *lip; - uint64_t cui_id; - struct xfs_ail_cursor cur; - struct xfs_ail *ailp = log->l_ailp; - - cud_formatp = item->ri_buf[0].i_addr; - if (item->ri_buf[0].i_len != sizeof(struct xfs_cud_log_format)) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); - return -EFSCORRUPTED; - } - cui_id = cud_formatp->cud_cui_id; - - /* - * Search for the CUI with the id in the CUD format structure in the - * AIL. - */ - spin_lock(&ailp->ail_lock); - lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); - while (lip != NULL) { - if (lip->li_type == XFS_LI_CUI) { - cuip = (struct xfs_cui_log_item *)lip; - if (cuip->cui_format.cui_id == cui_id) { - /* - * Drop the CUD reference to the CUI. This - * removes the CUI from the AIL and frees it. - */ - spin_unlock(&ailp->ail_lock); - xfs_cui_release(cuip); - spin_lock(&ailp->ail_lock); - break; - } - } - lip = xfs_trans_ail_cursor_next(ailp, &cur); - } - - xfs_trans_ail_cursor_done(&cur); - spin_unlock(&ailp->ail_lock); - - return 0; -} - /* * Copy an BUI format buffer from the given buf, and into the destination * BUI format structure. The BUI/BUD items were designed not to need any @@ -2292,10 +2172,6 @@ xlog_recover_commit_pass2( trans->r_lsn); switch (ITEM_TYPE(item)) { - case XFS_LI_CUI: - return xlog_recover_cui_pass2(log, item, trans->r_lsn); - case XFS_LI_CUD: - return xlog_recover_cud_pass2(log, item); case XFS_LI_BUI: return xlog_recover_bui_pass2(log, item, trans->r_lsn); case XFS_LI_BUD: diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index 2a9465d9a77f..7ccdeafdb7e7 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -18,6 +18,7 @@ #include "xfs_log.h" #include "xfs_refcount.h" #include "xfs_error.h" +#include "xfs_log_priv.h" #include "xfs_log_recover.h" kmem_zone_t *xfs_cui_zone; @@ -28,7 +29,7 @@ static inline struct xfs_cui_log_item *CUI_ITEM(struct xfs_log_item *lip) return container_of(lip, struct xfs_cui_log_item, cui_item); } -void +STATIC void xfs_cui_item_free( struct xfs_cui_log_item *cuip) { @@ -134,7 +135,7 @@ static const struct xfs_item_ops xfs_cui_item_ops = { /* * Allocate and initialize an cui item with the given number of extents. */ -struct xfs_cui_log_item * +STATIC struct xfs_cui_log_item * xfs_cui_init( struct xfs_mount *mp, uint nextents) @@ -572,10 +573,134 @@ abort_error: return error; } +/* + * Copy an CUI format buffer from the given buf, and into the destination + * CUI format structure. The CUI/CUD items were designed not to need any + * special alignment handling. + */ +static int +xfs_cui_copy_format( + struct xfs_log_iovec *buf, + struct xfs_cui_log_format *dst_cui_fmt) +{ + struct xfs_cui_log_format *src_cui_fmt; + uint len; + + src_cui_fmt = buf->i_addr; + len = xfs_cui_log_format_sizeof(src_cui_fmt->cui_nextents); + + if (buf->i_len == len) { + memcpy(dst_cui_fmt, src_cui_fmt, len); + return 0; + } + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL); + return -EFSCORRUPTED; +} + +/* + * This routine is called to create an in-core extent refcount update + * item from the cui format structure which was logged on disk. + * It allocates an in-core cui, copies the extents from the format + * structure into it, and adds the cui to the AIL with the given + * LSN. + */ +STATIC int +xlog_recover_cui_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + int error; + struct xfs_mount *mp = log->l_mp; + struct xfs_cui_log_item *cuip; + struct xfs_cui_log_format *cui_formatp; + + cui_formatp = item->ri_buf[0].i_addr; + + cuip = xfs_cui_init(mp, cui_formatp->cui_nextents); + error = xfs_cui_copy_format(&item->ri_buf[0], &cuip->cui_format); + if (error) { + xfs_cui_item_free(cuip); + return error; + } + atomic_set(&cuip->cui_next_extent, cui_formatp->cui_nextents); + + spin_lock(&log->l_ailp->ail_lock); + /* + * The CUI has two references. One for the CUD and one for CUI to ensure + * it makes it into the AIL. Insert the CUI into the AIL directly and + * drop the CUI reference. Note that xfs_trans_ail_update() drops the + * AIL lock. + */ + xfs_trans_ail_update(log->l_ailp, &cuip->cui_item, lsn); + xfs_cui_release(cuip); + return 0; +} + const struct xlog_recover_item_ops xlog_cui_item_ops = { .item_type = XFS_LI_CUI, + .commit_pass2 = xlog_recover_cui_commit_pass2, }; +/* + * This routine is called when an CUD format structure is found in a committed + * transaction in the log. Its purpose is to cancel the corresponding CUI if it + * was still in the log. To do this it searches the AIL for the CUI with an id + * equal to that in the CUD format structure. If we find it we drop the CUD + * reference, which removes the CUI from the AIL and frees it. + */ +STATIC int +xlog_recover_cud_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_cud_log_format *cud_formatp; + struct xfs_cui_log_item *cuip = NULL; + struct xfs_log_item *lip; + uint64_t cui_id; + struct xfs_ail_cursor cur; + struct xfs_ail *ailp = log->l_ailp; + + cud_formatp = item->ri_buf[0].i_addr; + if (item->ri_buf[0].i_len != sizeof(struct xfs_cud_log_format)) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; + } + cui_id = cud_formatp->cud_cui_id; + + /* + * Search for the CUI with the id in the CUD format structure in the + * AIL. + */ + spin_lock(&ailp->ail_lock); + lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); + while (lip != NULL) { + if (lip->li_type == XFS_LI_CUI) { + cuip = (struct xfs_cui_log_item *)lip; + if (cuip->cui_format.cui_id == cui_id) { + /* + * Drop the CUD reference to the CUI. This + * removes the CUI from the AIL and frees it. + */ + spin_unlock(&ailp->ail_lock); + xfs_cui_release(cuip); + spin_lock(&ailp->ail_lock); + break; + } + } + lip = xfs_trans_ail_cursor_next(ailp, &cur); + } + + xfs_trans_ail_cursor_done(&cur); + spin_unlock(&ailp->ail_lock); + + return 0; +} + const struct xlog_recover_item_ops xlog_cud_item_ops = { .item_type = XFS_LI_CUD, + .commit_pass2 = xlog_recover_cud_commit_pass2, }; diff --git a/fs/xfs/xfs_refcount_item.h b/fs/xfs/xfs_refcount_item.h index e47530f30489..ebe12779eaac 100644 --- a/fs/xfs/xfs_refcount_item.h +++ b/fs/xfs/xfs_refcount_item.h @@ -77,8 +77,6 @@ struct xfs_cud_log_item { extern struct kmem_zone *xfs_cui_zone; extern struct kmem_zone *xfs_cud_zone; -struct xfs_cui_log_item *xfs_cui_init(struct xfs_mount *, uint); -void xfs_cui_item_free(struct xfs_cui_log_item *); void xfs_cui_release(struct xfs_cui_log_item *); int xfs_cui_recover(struct xfs_trans *parent_tp, struct xfs_cui_log_item *cuip); From 3c6ba3cf90c7233359a190c5230a553d19fbc8ef Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:50 -0700 Subject: [PATCH 0284/1043] xfs: refactor log recovery BUI item dispatch for pass2 commit functions Move the bmap update intent and intent-done pass2 commit code into the per-item source code files and use dispatch functions to call them. We do these one at a time because there's a lot of code to move. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_bmap_item.c | 133 ++++++++++++++++++++++++++++++++++++++- fs/xfs/xfs_bmap_item.h | 2 - fs/xfs/xfs_log_recover.c | 128 ------------------------------------- 3 files changed, 131 insertions(+), 132 deletions(-) diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index 508b48ca5ced..1537759b9ea8 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -22,6 +22,7 @@ #include "xfs_bmap_btree.h" #include "xfs_trans_space.h" #include "xfs_error.h" +#include "xfs_log_priv.h" #include "xfs_log_recover.h" kmem_zone_t *xfs_bui_zone; @@ -32,7 +33,7 @@ static inline struct xfs_bui_log_item *BUI_ITEM(struct xfs_log_item *lip) return container_of(lip, struct xfs_bui_log_item, bui_item); } -void +STATIC void xfs_bui_item_free( struct xfs_bui_log_item *buip) { @@ -135,7 +136,7 @@ static const struct xfs_item_ops xfs_bui_item_ops = { /* * Allocate and initialize an bui item with the given number of extents. */ -struct xfs_bui_log_item * +STATIC struct xfs_bui_log_item * xfs_bui_init( struct xfs_mount *mp) @@ -559,10 +560,138 @@ err_inode: return error; } +/* + * Copy an BUI format buffer from the given buf, and into the destination + * BUI format structure. The BUI/BUD items were designed not to need any + * special alignment handling. + */ +static int +xfs_bui_copy_format( + struct xfs_log_iovec *buf, + struct xfs_bui_log_format *dst_bui_fmt) +{ + struct xfs_bui_log_format *src_bui_fmt; + uint len; + + src_bui_fmt = buf->i_addr; + len = xfs_bui_log_format_sizeof(src_bui_fmt->bui_nextents); + + if (buf->i_len == len) { + memcpy(dst_bui_fmt, src_bui_fmt, len); + return 0; + } + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL); + return -EFSCORRUPTED; +} + +/* + * This routine is called to create an in-core extent bmap update + * item from the bui format structure which was logged on disk. + * It allocates an in-core bui, copies the extents from the format + * structure into it, and adds the bui to the AIL with the given + * LSN. + */ +STATIC int +xlog_recover_bui_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + int error; + struct xfs_mount *mp = log->l_mp; + struct xfs_bui_log_item *buip; + struct xfs_bui_log_format *bui_formatp; + + bui_formatp = item->ri_buf[0].i_addr; + + if (bui_formatp->bui_nextents != XFS_BUI_MAX_FAST_EXTENTS) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; + } + buip = xfs_bui_init(mp); + error = xfs_bui_copy_format(&item->ri_buf[0], &buip->bui_format); + if (error) { + xfs_bui_item_free(buip); + return error; + } + atomic_set(&buip->bui_next_extent, bui_formatp->bui_nextents); + + spin_lock(&log->l_ailp->ail_lock); + /* + * The RUI has two references. One for the RUD and one for RUI to ensure + * it makes it into the AIL. Insert the RUI into the AIL directly and + * drop the RUI reference. Note that xfs_trans_ail_update() drops the + * AIL lock. + */ + xfs_trans_ail_update(log->l_ailp, &buip->bui_item, lsn); + xfs_bui_release(buip); + return 0; +} + const struct xlog_recover_item_ops xlog_bui_item_ops = { .item_type = XFS_LI_BUI, + .commit_pass2 = xlog_recover_bui_commit_pass2, }; +/* + * This routine is called when an BUD format structure is found in a committed + * transaction in the log. Its purpose is to cancel the corresponding BUI if it + * was still in the log. To do this it searches the AIL for the BUI with an id + * equal to that in the BUD format structure. If we find it we drop the BUD + * reference, which removes the BUI from the AIL and frees it. + */ +STATIC int +xlog_recover_bud_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_bud_log_format *bud_formatp; + struct xfs_bui_log_item *buip = NULL; + struct xfs_log_item *lip; + uint64_t bui_id; + struct xfs_ail_cursor cur; + struct xfs_ail *ailp = log->l_ailp; + + bud_formatp = item->ri_buf[0].i_addr; + if (item->ri_buf[0].i_len != sizeof(struct xfs_bud_log_format)) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; + } + bui_id = bud_formatp->bud_bui_id; + + /* + * Search for the BUI with the id in the BUD format structure in the + * AIL. + */ + spin_lock(&ailp->ail_lock); + lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); + while (lip != NULL) { + if (lip->li_type == XFS_LI_BUI) { + buip = (struct xfs_bui_log_item *)lip; + if (buip->bui_format.bui_id == bui_id) { + /* + * Drop the BUD reference to the BUI. This + * removes the BUI from the AIL and frees it. + */ + spin_unlock(&ailp->ail_lock); + xfs_bui_release(buip); + spin_lock(&ailp->ail_lock); + break; + } + } + lip = xfs_trans_ail_cursor_next(ailp, &cur); + } + + xfs_trans_ail_cursor_done(&cur); + spin_unlock(&ailp->ail_lock); + + return 0; +} + const struct xlog_recover_item_ops xlog_bud_item_ops = { .item_type = XFS_LI_BUD, + .commit_pass2 = xlog_recover_bud_commit_pass2, }; diff --git a/fs/xfs/xfs_bmap_item.h b/fs/xfs/xfs_bmap_item.h index ad479cc73de8..515b1d5d6ab7 100644 --- a/fs/xfs/xfs_bmap_item.h +++ b/fs/xfs/xfs_bmap_item.h @@ -74,8 +74,6 @@ struct xfs_bud_log_item { extern struct kmem_zone *xfs_bui_zone; extern struct kmem_zone *xfs_bud_zone; -struct xfs_bui_log_item *xfs_bui_init(struct xfs_mount *); -void xfs_bui_item_free(struct xfs_bui_log_item *); void xfs_bui_release(struct xfs_bui_log_item *); int xfs_bui_recover(struct xfs_trans *parent_tp, struct xfs_bui_log_item *buip); diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 0bce57088693..1ed0bdabb9a4 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2034,130 +2034,6 @@ xlog_buf_readahead( xfs_buf_readahead(log->l_mp->m_ddev_targp, blkno, len, ops); } -/* - * Copy an BUI format buffer from the given buf, and into the destination - * BUI format structure. The BUI/BUD items were designed not to need any - * special alignment handling. - */ -static int -xfs_bui_copy_format( - struct xfs_log_iovec *buf, - struct xfs_bui_log_format *dst_bui_fmt) -{ - struct xfs_bui_log_format *src_bui_fmt; - uint len; - - src_bui_fmt = buf->i_addr; - len = xfs_bui_log_format_sizeof(src_bui_fmt->bui_nextents); - - if (buf->i_len == len) { - memcpy(dst_bui_fmt, src_bui_fmt, len); - return 0; - } - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL); - return -EFSCORRUPTED; -} - -/* - * This routine is called to create an in-core extent bmap update - * item from the bui format structure which was logged on disk. - * It allocates an in-core bui, copies the extents from the format - * structure into it, and adds the bui to the AIL with the given - * LSN. - */ -STATIC int -xlog_recover_bui_pass2( - struct xlog *log, - struct xlog_recover_item *item, - xfs_lsn_t lsn) -{ - int error; - struct xfs_mount *mp = log->l_mp; - struct xfs_bui_log_item *buip; - struct xfs_bui_log_format *bui_formatp; - - bui_formatp = item->ri_buf[0].i_addr; - - if (bui_formatp->bui_nextents != XFS_BUI_MAX_FAST_EXTENTS) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); - return -EFSCORRUPTED; - } - buip = xfs_bui_init(mp); - error = xfs_bui_copy_format(&item->ri_buf[0], &buip->bui_format); - if (error) { - xfs_bui_item_free(buip); - return error; - } - atomic_set(&buip->bui_next_extent, bui_formatp->bui_nextents); - - spin_lock(&log->l_ailp->ail_lock); - /* - * The RUI has two references. One for the RUD and one for RUI to ensure - * it makes it into the AIL. Insert the RUI into the AIL directly and - * drop the RUI reference. Note that xfs_trans_ail_update() drops the - * AIL lock. - */ - xfs_trans_ail_update(log->l_ailp, &buip->bui_item, lsn); - xfs_bui_release(buip); - return 0; -} - - -/* - * This routine is called when an BUD format structure is found in a committed - * transaction in the log. Its purpose is to cancel the corresponding BUI if it - * was still in the log. To do this it searches the AIL for the BUI with an id - * equal to that in the BUD format structure. If we find it we drop the BUD - * reference, which removes the BUI from the AIL and frees it. - */ -STATIC int -xlog_recover_bud_pass2( - struct xlog *log, - struct xlog_recover_item *item) -{ - struct xfs_bud_log_format *bud_formatp; - struct xfs_bui_log_item *buip = NULL; - struct xfs_log_item *lip; - uint64_t bui_id; - struct xfs_ail_cursor cur; - struct xfs_ail *ailp = log->l_ailp; - - bud_formatp = item->ri_buf[0].i_addr; - if (item->ri_buf[0].i_len != sizeof(struct xfs_bud_log_format)) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); - return -EFSCORRUPTED; - } - bui_id = bud_formatp->bud_bui_id; - - /* - * Search for the BUI with the id in the BUD format structure in the - * AIL. - */ - spin_lock(&ailp->ail_lock); - lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); - while (lip != NULL) { - if (lip->li_type == XFS_LI_BUI) { - buip = (struct xfs_bui_log_item *)lip; - if (buip->bui_format.bui_id == bui_id) { - /* - * Drop the BUD reference to the BUI. This - * removes the BUI from the AIL and frees it. - */ - spin_unlock(&ailp->ail_lock); - xfs_bui_release(buip); - spin_lock(&ailp->ail_lock); - break; - } - } - lip = xfs_trans_ail_cursor_next(ailp, &cur); - } - - xfs_trans_ail_cursor_done(&cur); - spin_unlock(&ailp->ail_lock); - - return 0; -} - STATIC int xlog_recover_commit_pass2( struct xlog *log, @@ -2172,10 +2048,6 @@ xlog_recover_commit_pass2( trans->r_lsn); switch (ITEM_TYPE(item)) { - case XFS_LI_BUI: - return xlog_recover_bui_pass2(log, item, trans->r_lsn); - case XFS_LI_BUD: - return xlog_recover_bud_pass2(log, item); case XFS_LI_QUOTAOFF: /* nothing to do in pass2 */ return 0; From 2565a11b224b68a222838d09623c6a398c4d5f6c Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:50 -0700 Subject: [PATCH 0285/1043] xfs: remove log recovery quotaoff item dispatch for pass2 commit functions Quotaoff doesn't actually do anything, so take advantage of the commit_pass2 pointer being optional and get rid of the switch statement clause. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_dquot_item_recover.c | 1 + fs/xfs/xfs_log_recover.c | 33 ++++++--------------------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/fs/xfs/xfs_dquot_item_recover.c b/fs/xfs/xfs_dquot_item_recover.c index 07ff943972a3..3400be4c88f0 100644 --- a/fs/xfs/xfs_dquot_item_recover.c +++ b/fs/xfs/xfs_dquot_item_recover.c @@ -197,4 +197,5 @@ xlog_recover_quotaoff_commit_pass1( const struct xlog_recover_item_ops xlog_quotaoff_item_ops = { .item_type = XFS_LI_QUOTAOFF, .commit_pass1 = xlog_recover_quotaoff_commit_pass1, + /* nothing to commit in pass2 */ }; diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 1ed0bdabb9a4..02148e341760 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2034,31 +2034,6 @@ xlog_buf_readahead( xfs_buf_readahead(log->l_mp->m_ddev_targp, blkno, len, ops); } -STATIC int -xlog_recover_commit_pass2( - struct xlog *log, - struct xlog_recover *trans, - struct list_head *buffer_list, - struct xlog_recover_item *item) -{ - trace_xfs_log_recover_item_recover(log, trans, item, XLOG_RECOVER_PASS2); - - if (item->ri_ops->commit_pass2) - return item->ri_ops->commit_pass2(log, buffer_list, item, - trans->r_lsn); - - switch (ITEM_TYPE(item)) { - case XFS_LI_QUOTAOFF: - /* nothing to do in pass2 */ - return 0; - default: - xfs_warn(log->l_mp, "%s: invalid item type (%d)", - __func__, ITEM_TYPE(item)); - ASSERT(0); - return -EFSCORRUPTED; - } -} - STATIC int xlog_recover_items_pass2( struct xlog *log, @@ -2070,8 +2045,12 @@ xlog_recover_items_pass2( int error = 0; list_for_each_entry(item, item_list, ri_list) { - error = xlog_recover_commit_pass2(log, trans, - buffer_list, item); + trace_xfs_log_recover_item_recover(log, trans, item, + XLOG_RECOVER_PASS2); + + if (item->ri_ops->commit_pass2) + error = item->ri_ops->commit_pass2(log, buffer_list, + item, trans->r_lsn); if (error) return error; } From 10d0c6e06fc87fe9e99d7c8df493f973b354fd9c Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:50 -0700 Subject: [PATCH 0286/1043] xfs: refactor recovered EFI log item playback Move the code that processes the log items created from the recovered log items into the per-item source code files and use dispatch functions to call them. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_extfree_item.c | 46 +++++++++++++++++++++++++-------- fs/xfs/xfs_extfree_item.h | 5 ---- fs/xfs/xfs_log_recover.c | 54 ++++++--------------------------------- fs/xfs/xfs_trans.h | 1 + 4 files changed, 45 insertions(+), 61 deletions(-) diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index 69f7e75a747e..307f71bdd398 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -28,6 +28,8 @@ kmem_zone_t *xfs_efi_zone; kmem_zone_t *xfs_efd_zone; +static const struct xfs_item_ops xfs_efi_item_ops; + static inline struct xfs_efi_log_item *EFI_ITEM(struct xfs_log_item *lip) { return container_of(lip, struct xfs_efi_log_item, efi_item); @@ -51,7 +53,7 @@ xfs_efi_item_free( * committed vs unpin operations in bulk insert operations. Hence the reference * count to ensure only the last caller frees the EFI. */ -void +STATIC void xfs_efi_release( struct xfs_efi_log_item *efip) { @@ -141,14 +143,6 @@ xfs_efi_item_release( xfs_efi_release(EFI_ITEM(lip)); } -static const struct xfs_item_ops xfs_efi_item_ops = { - .iop_size = xfs_efi_item_size, - .iop_format = xfs_efi_item_format, - .iop_unpin = xfs_efi_item_unpin, - .iop_release = xfs_efi_item_release, -}; - - /* * Allocate and initialize an efi item with the given number of extents. */ @@ -586,7 +580,7 @@ const struct xfs_defer_op_type xfs_agfl_free_defer_type = { * Process an extent free intent item that was recovered from * the log. We need to free the extents that it describes. */ -int +STATIC int xfs_efi_recover( struct xfs_mount *mp, struct xfs_efi_log_item *efip) @@ -647,6 +641,38 @@ abort_error: return error; } +/* Recover the EFI if necessary. */ +STATIC int +xfs_efi_item_recover( + struct xfs_log_item *lip, + struct xfs_trans *tp) +{ + struct xfs_ail *ailp = lip->li_ailp; + struct xfs_efi_log_item *efip; + int error; + + /* + * Skip EFIs that we've already processed. + */ + efip = container_of(lip, struct xfs_efi_log_item, efi_item); + if (test_bit(XFS_EFI_RECOVERED, &efip->efi_flags)) + return 0; + + spin_unlock(&ailp->ail_lock); + error = xfs_efi_recover(tp->t_mountp, efip); + spin_lock(&ailp->ail_lock); + + return error; +} + +static const struct xfs_item_ops xfs_efi_item_ops = { + .iop_size = xfs_efi_item_size, + .iop_format = xfs_efi_item_format, + .iop_unpin = xfs_efi_item_unpin, + .iop_release = xfs_efi_item_release, + .iop_recover = xfs_efi_item_recover, +}; + /* * This routine is called to create an in-core extent free intent * item from the efi format structure which was logged on disk. diff --git a/fs/xfs/xfs_extfree_item.h b/fs/xfs/xfs_extfree_item.h index 876e3d237f48..4b2c2c5c5985 100644 --- a/fs/xfs/xfs_extfree_item.h +++ b/fs/xfs/xfs_extfree_item.h @@ -78,9 +78,4 @@ struct xfs_efd_log_item { extern struct kmem_zone *xfs_efi_zone; extern struct kmem_zone *xfs_efd_zone; -void xfs_efi_release(struct xfs_efi_log_item *); - -int xfs_efi_recover(struct xfs_mount *mp, - struct xfs_efi_log_item *efip); - #endif /* __XFS_EXTFREE_ITEM_H__ */ diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 02148e341760..055a9c0c20b0 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2546,46 +2546,6 @@ xlog_recover_process_data( return 0; } -/* Recover the EFI if necessary. */ -STATIC int -xlog_recover_process_efi( - struct xfs_mount *mp, - struct xfs_ail *ailp, - struct xfs_log_item *lip) -{ - struct xfs_efi_log_item *efip; - int error; - - /* - * Skip EFIs that we've already processed. - */ - efip = container_of(lip, struct xfs_efi_log_item, efi_item); - if (test_bit(XFS_EFI_RECOVERED, &efip->efi_flags)) - return 0; - - spin_unlock(&ailp->ail_lock); - error = xfs_efi_recover(mp, efip); - spin_lock(&ailp->ail_lock); - - return error; -} - -/* Release the EFI since we're cancelling everything. */ -STATIC void -xlog_recover_cancel_efi( - struct xfs_mount *mp, - struct xfs_ail *ailp, - struct xfs_log_item *lip) -{ - struct xfs_efi_log_item *efip; - - efip = container_of(lip, struct xfs_efi_log_item, efi_item); - - spin_unlock(&ailp->ail_lock); - xfs_efi_release(efip); - spin_lock(&ailp->ail_lock); -} - /* Recover the RUI if necessary. */ STATIC int xlog_recover_process_rui( @@ -2829,9 +2789,6 @@ xlog_recover_process_intents( * replayed in the wrong order! */ switch (lip->li_type) { - case XFS_LI_EFI: - error = xlog_recover_process_efi(log->l_mp, ailp, lip); - break; case XFS_LI_RUI: error = xlog_recover_process_rui(log->l_mp, ailp, lip); break; @@ -2841,6 +2798,9 @@ xlog_recover_process_intents( case XFS_LI_BUI: error = xlog_recover_process_bui(parent_tp, ailp, lip); break; + default: + error = lip->li_ops->iop_recover(lip, parent_tp); + break; } if (error) goto out; @@ -2885,9 +2845,6 @@ xlog_recover_cancel_intents( } switch (lip->li_type) { - case XFS_LI_EFI: - xlog_recover_cancel_efi(log->l_mp, ailp, lip); - break; case XFS_LI_RUI: xlog_recover_cancel_rui(log->l_mp, ailp, lip); break; @@ -2897,6 +2854,11 @@ xlog_recover_cancel_intents( case XFS_LI_BUI: xlog_recover_cancel_bui(log->l_mp, ailp, lip); break; + default: + spin_unlock(&ailp->ail_lock); + lip->li_ops->iop_release(lip); + spin_lock(&ailp->ail_lock); + break; } lip = xfs_trans_ail_cursor_next(ailp, &cur); diff --git a/fs/xfs/xfs_trans.h b/fs/xfs/xfs_trans.h index 752c7fef9de7..3f6a79108991 100644 --- a/fs/xfs/xfs_trans.h +++ b/fs/xfs/xfs_trans.h @@ -77,6 +77,7 @@ struct xfs_item_ops { void (*iop_release)(struct xfs_log_item *); xfs_lsn_t (*iop_committed)(struct xfs_log_item *, xfs_lsn_t); void (*iop_error)(struct xfs_log_item *, xfs_buf_t *); + int (*iop_recover)(struct xfs_log_item *lip, struct xfs_trans *tp); }; /* From cba0ccac28a766968243cda597566983843f5be2 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:51 -0700 Subject: [PATCH 0287/1043] xfs: refactor recovered RUI log item playback Move the code that processes the log items created from the recovered log items into the per-item source code files and use dispatch functions to call them. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_log_recover.c | 46 ---------------------------------------- fs/xfs/xfs_rmap_item.c | 44 ++++++++++++++++++++++++++++++-------- fs/xfs/xfs_rmap_item.h | 3 --- 3 files changed, 35 insertions(+), 58 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 055a9c0c20b0..4eb837476e44 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2546,46 +2546,6 @@ xlog_recover_process_data( return 0; } -/* Recover the RUI if necessary. */ -STATIC int -xlog_recover_process_rui( - struct xfs_mount *mp, - struct xfs_ail *ailp, - struct xfs_log_item *lip) -{ - struct xfs_rui_log_item *ruip; - int error; - - /* - * Skip RUIs that we've already processed. - */ - ruip = container_of(lip, struct xfs_rui_log_item, rui_item); - if (test_bit(XFS_RUI_RECOVERED, &ruip->rui_flags)) - return 0; - - spin_unlock(&ailp->ail_lock); - error = xfs_rui_recover(mp, ruip); - spin_lock(&ailp->ail_lock); - - return error; -} - -/* Release the RUI since we're cancelling everything. */ -STATIC void -xlog_recover_cancel_rui( - struct xfs_mount *mp, - struct xfs_ail *ailp, - struct xfs_log_item *lip) -{ - struct xfs_rui_log_item *ruip; - - ruip = container_of(lip, struct xfs_rui_log_item, rui_item); - - spin_unlock(&ailp->ail_lock); - xfs_rui_release(ruip); - spin_lock(&ailp->ail_lock); -} - /* Recover the CUI if necessary. */ STATIC int xlog_recover_process_cui( @@ -2789,9 +2749,6 @@ xlog_recover_process_intents( * replayed in the wrong order! */ switch (lip->li_type) { - case XFS_LI_RUI: - error = xlog_recover_process_rui(log->l_mp, ailp, lip); - break; case XFS_LI_CUI: error = xlog_recover_process_cui(parent_tp, ailp, lip); break; @@ -2845,9 +2802,6 @@ xlog_recover_cancel_intents( } switch (lip->li_type) { - case XFS_LI_RUI: - xlog_recover_cancel_rui(log->l_mp, ailp, lip); - break; case XFS_LI_CUI: xlog_recover_cancel_cui(log->l_mp, ailp, lip); break; diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index 44049dbdb161..1b7c7e3db872 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -24,6 +24,8 @@ kmem_zone_t *xfs_rui_zone; kmem_zone_t *xfs_rud_zone; +static const struct xfs_item_ops xfs_rui_item_ops; + static inline struct xfs_rui_log_item *RUI_ITEM(struct xfs_log_item *lip) { return container_of(lip, struct xfs_rui_log_item, rui_item); @@ -46,7 +48,7 @@ xfs_rui_item_free( * committed vs unpin operations in bulk insert operations. Hence the reference * count to ensure only the last caller frees the RUI. */ -void +STATIC void xfs_rui_release( struct xfs_rui_log_item *ruip) { @@ -124,13 +126,6 @@ xfs_rui_item_release( xfs_rui_release(RUI_ITEM(lip)); } -static const struct xfs_item_ops xfs_rui_item_ops = { - .iop_size = xfs_rui_item_size, - .iop_format = xfs_rui_item_format, - .iop_unpin = xfs_rui_item_unpin, - .iop_release = xfs_rui_item_release, -}; - /* * Allocate and initialize an rui item with the given number of extents. */ @@ -468,7 +463,7 @@ const struct xfs_defer_op_type xfs_rmap_update_defer_type = { * Process an rmap update intent item that was recovered from the log. * We need to update the rmapbt. */ -int +STATIC int xfs_rui_recover( struct xfs_mount *mp, struct xfs_rui_log_item *ruip) @@ -588,6 +583,37 @@ abort_error: return error; } +/* Recover the RUI if necessary. */ +STATIC int +xfs_rui_item_recover( + struct xfs_log_item *lip, + struct xfs_trans *tp) +{ + struct xfs_ail *ailp = lip->li_ailp; + struct xfs_rui_log_item *ruip = RUI_ITEM(lip); + int error; + + /* + * Skip RUIs that we've already processed. + */ + if (test_bit(XFS_RUI_RECOVERED, &ruip->rui_flags)) + return 0; + + spin_unlock(&ailp->ail_lock); + error = xfs_rui_recover(tp->t_mountp, ruip); + spin_lock(&ailp->ail_lock); + + return error; +} + +static const struct xfs_item_ops xfs_rui_item_ops = { + .iop_size = xfs_rui_item_size, + .iop_format = xfs_rui_item_format, + .iop_unpin = xfs_rui_item_unpin, + .iop_release = xfs_rui_item_release, + .iop_recover = xfs_rui_item_recover, +}; + /* * This routine is called to create an in-core extent rmap update * item from the rui format structure which was logged on disk. diff --git a/fs/xfs/xfs_rmap_item.h b/fs/xfs/xfs_rmap_item.h index 89bd192779f8..48a77a6f5c94 100644 --- a/fs/xfs/xfs_rmap_item.h +++ b/fs/xfs/xfs_rmap_item.h @@ -77,7 +77,4 @@ struct xfs_rud_log_item { extern struct kmem_zone *xfs_rui_zone; extern struct kmem_zone *xfs_rud_zone; -void xfs_rui_release(struct xfs_rui_log_item *); -int xfs_rui_recover(struct xfs_mount *mp, struct xfs_rui_log_item *ruip); - #endif /* __XFS_RMAP_ITEM_H__ */ From c57ed2f5a2ffa2e2009ffa638efd06c6e86e7ebd Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:51 -0700 Subject: [PATCH 0288/1043] xfs: refactor recovered CUI log item playback Move the code that processes the log items created from the recovered log items into the per-item source code files and use dispatch functions to call them. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_log_recover.c | 46 -------------------------------------- fs/xfs/xfs_refcount_item.c | 44 ++++++++++++++++++++++++++++-------- fs/xfs/xfs_refcount_item.h | 3 --- 3 files changed, 35 insertions(+), 58 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 4eb837476e44..7d3f7be05395 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2546,46 +2546,6 @@ xlog_recover_process_data( return 0; } -/* Recover the CUI if necessary. */ -STATIC int -xlog_recover_process_cui( - struct xfs_trans *parent_tp, - struct xfs_ail *ailp, - struct xfs_log_item *lip) -{ - struct xfs_cui_log_item *cuip; - int error; - - /* - * Skip CUIs that we've already processed. - */ - cuip = container_of(lip, struct xfs_cui_log_item, cui_item); - if (test_bit(XFS_CUI_RECOVERED, &cuip->cui_flags)) - return 0; - - spin_unlock(&ailp->ail_lock); - error = xfs_cui_recover(parent_tp, cuip); - spin_lock(&ailp->ail_lock); - - return error; -} - -/* Release the CUI since we're cancelling everything. */ -STATIC void -xlog_recover_cancel_cui( - struct xfs_mount *mp, - struct xfs_ail *ailp, - struct xfs_log_item *lip) -{ - struct xfs_cui_log_item *cuip; - - cuip = container_of(lip, struct xfs_cui_log_item, cui_item); - - spin_unlock(&ailp->ail_lock); - xfs_cui_release(cuip); - spin_lock(&ailp->ail_lock); -} - /* Recover the BUI if necessary. */ STATIC int xlog_recover_process_bui( @@ -2749,9 +2709,6 @@ xlog_recover_process_intents( * replayed in the wrong order! */ switch (lip->li_type) { - case XFS_LI_CUI: - error = xlog_recover_process_cui(parent_tp, ailp, lip); - break; case XFS_LI_BUI: error = xlog_recover_process_bui(parent_tp, ailp, lip); break; @@ -2802,9 +2759,6 @@ xlog_recover_cancel_intents( } switch (lip->li_type) { - case XFS_LI_CUI: - xlog_recover_cancel_cui(log->l_mp, ailp, lip); - break; case XFS_LI_BUI: xlog_recover_cancel_bui(log->l_mp, ailp, lip); break; diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index 7ccdeafdb7e7..4eee8add4cd5 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -24,6 +24,8 @@ kmem_zone_t *xfs_cui_zone; kmem_zone_t *xfs_cud_zone; +static const struct xfs_item_ops xfs_cui_item_ops; + static inline struct xfs_cui_log_item *CUI_ITEM(struct xfs_log_item *lip) { return container_of(lip, struct xfs_cui_log_item, cui_item); @@ -46,7 +48,7 @@ xfs_cui_item_free( * committed vs unpin operations in bulk insert operations. Hence the reference * count to ensure only the last caller frees the CUI. */ -void +STATIC void xfs_cui_release( struct xfs_cui_log_item *cuip) { @@ -125,13 +127,6 @@ xfs_cui_item_release( xfs_cui_release(CUI_ITEM(lip)); } -static const struct xfs_item_ops xfs_cui_item_ops = { - .iop_size = xfs_cui_item_size, - .iop_format = xfs_cui_item_format, - .iop_unpin = xfs_cui_item_unpin, - .iop_release = xfs_cui_item_release, -}; - /* * Allocate and initialize an cui item with the given number of extents. */ @@ -425,7 +420,7 @@ const struct xfs_defer_op_type xfs_refcount_update_defer_type = { * Process a refcount update intent item that was recovered from the log. * We need to update the refcountbt. */ -int +STATIC int xfs_cui_recover( struct xfs_trans *parent_tp, struct xfs_cui_log_item *cuip) @@ -573,6 +568,37 @@ abort_error: return error; } +/* Recover the CUI if necessary. */ +STATIC int +xfs_cui_item_recover( + struct xfs_log_item *lip, + struct xfs_trans *tp) +{ + struct xfs_ail *ailp = lip->li_ailp; + struct xfs_cui_log_item *cuip = CUI_ITEM(lip); + int error; + + /* + * Skip CUIs that we've already processed. + */ + if (test_bit(XFS_CUI_RECOVERED, &cuip->cui_flags)) + return 0; + + spin_unlock(&ailp->ail_lock); + error = xfs_cui_recover(tp, cuip); + spin_lock(&ailp->ail_lock); + + return error; +} + +static const struct xfs_item_ops xfs_cui_item_ops = { + .iop_size = xfs_cui_item_size, + .iop_format = xfs_cui_item_format, + .iop_unpin = xfs_cui_item_unpin, + .iop_release = xfs_cui_item_release, + .iop_recover = xfs_cui_item_recover, +}; + /* * Copy an CUI format buffer from the given buf, and into the destination * CUI format structure. The CUI/CUD items were designed not to need any diff --git a/fs/xfs/xfs_refcount_item.h b/fs/xfs/xfs_refcount_item.h index ebe12779eaac..cfaa857673a6 100644 --- a/fs/xfs/xfs_refcount_item.h +++ b/fs/xfs/xfs_refcount_item.h @@ -77,7 +77,4 @@ struct xfs_cud_log_item { extern struct kmem_zone *xfs_cui_zone; extern struct kmem_zone *xfs_cud_zone; -void xfs_cui_release(struct xfs_cui_log_item *); -int xfs_cui_recover(struct xfs_trans *parent_tp, struct xfs_cui_log_item *cuip); - #endif /* __XFS_REFCOUNT_ITEM_H__ */ From 9329ba89cbb1f261decfedfd83e67d89d9d6c591 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:52 -0700 Subject: [PATCH 0289/1043] xfs: refactor recovered BUI log item playback Move the code that processes the log items created from the recovered log items into the per-item source code files and use dispatch functions to call them. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_bmap_item.c | 44 ++++++++++++++++++++------ fs/xfs/xfs_bmap_item.h | 3 -- fs/xfs/xfs_log_recover.c | 67 ++++------------------------------------ 3 files changed, 41 insertions(+), 73 deletions(-) diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index 1537759b9ea8..b08015caed32 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -28,6 +28,8 @@ kmem_zone_t *xfs_bui_zone; kmem_zone_t *xfs_bud_zone; +static const struct xfs_item_ops xfs_bui_item_ops; + static inline struct xfs_bui_log_item *BUI_ITEM(struct xfs_log_item *lip) { return container_of(lip, struct xfs_bui_log_item, bui_item); @@ -47,7 +49,7 @@ xfs_bui_item_free( * committed vs unpin operations in bulk insert operations. Hence the reference * count to ensure only the last caller frees the BUI. */ -void +STATIC void xfs_bui_release( struct xfs_bui_log_item *buip) { @@ -126,13 +128,6 @@ xfs_bui_item_release( xfs_bui_release(BUI_ITEM(lip)); } -static const struct xfs_item_ops xfs_bui_item_ops = { - .iop_size = xfs_bui_item_size, - .iop_format = xfs_bui_item_format, - .iop_unpin = xfs_bui_item_unpin, - .iop_release = xfs_bui_item_release, -}; - /* * Allocate and initialize an bui item with the given number of extents. */ @@ -425,7 +420,7 @@ const struct xfs_defer_op_type xfs_bmap_update_defer_type = { * Process a bmap update intent item that was recovered from the log. * We need to update some inode's bmbt. */ -int +STATIC int xfs_bui_recover( struct xfs_trans *parent_tp, struct xfs_bui_log_item *buip) @@ -560,6 +555,37 @@ err_inode: return error; } +/* Recover the BUI if necessary. */ +STATIC int +xfs_bui_item_recover( + struct xfs_log_item *lip, + struct xfs_trans *tp) +{ + struct xfs_ail *ailp = lip->li_ailp; + struct xfs_bui_log_item *buip = BUI_ITEM(lip); + int error; + + /* + * Skip BUIs that we've already processed. + */ + if (test_bit(XFS_BUI_RECOVERED, &buip->bui_flags)) + return 0; + + spin_unlock(&ailp->ail_lock); + error = xfs_bui_recover(tp, buip); + spin_lock(&ailp->ail_lock); + + return error; +} + +static const struct xfs_item_ops xfs_bui_item_ops = { + .iop_size = xfs_bui_item_size, + .iop_format = xfs_bui_item_format, + .iop_unpin = xfs_bui_item_unpin, + .iop_release = xfs_bui_item_release, + .iop_recover = xfs_bui_item_recover, +}; + /* * Copy an BUI format buffer from the given buf, and into the destination * BUI format structure. The BUI/BUD items were designed not to need any diff --git a/fs/xfs/xfs_bmap_item.h b/fs/xfs/xfs_bmap_item.h index 515b1d5d6ab7..44d06e62f8f9 100644 --- a/fs/xfs/xfs_bmap_item.h +++ b/fs/xfs/xfs_bmap_item.h @@ -74,7 +74,4 @@ struct xfs_bud_log_item { extern struct kmem_zone *xfs_bui_zone; extern struct kmem_zone *xfs_bud_zone; -void xfs_bui_release(struct xfs_bui_log_item *); -int xfs_bui_recover(struct xfs_trans *parent_tp, struct xfs_bui_log_item *buip); - #endif /* __XFS_BMAP_ITEM_H__ */ diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 7d3f7be05395..65081a3efeff 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2546,46 +2546,6 @@ xlog_recover_process_data( return 0; } -/* Recover the BUI if necessary. */ -STATIC int -xlog_recover_process_bui( - struct xfs_trans *parent_tp, - struct xfs_ail *ailp, - struct xfs_log_item *lip) -{ - struct xfs_bui_log_item *buip; - int error; - - /* - * Skip BUIs that we've already processed. - */ - buip = container_of(lip, struct xfs_bui_log_item, bui_item); - if (test_bit(XFS_BUI_RECOVERED, &buip->bui_flags)) - return 0; - - spin_unlock(&ailp->ail_lock); - error = xfs_bui_recover(parent_tp, buip); - spin_lock(&ailp->ail_lock); - - return error; -} - -/* Release the BUI since we're cancelling everything. */ -STATIC void -xlog_recover_cancel_bui( - struct xfs_mount *mp, - struct xfs_ail *ailp, - struct xfs_log_item *lip) -{ - struct xfs_bui_log_item *buip; - - buip = container_of(lip, struct xfs_bui_log_item, bui_item); - - spin_unlock(&ailp->ail_lock); - xfs_bui_release(buip); - spin_lock(&ailp->ail_lock); -} - /* Is this log item a deferred action intent? */ static inline bool xlog_item_is_intent(struct xfs_log_item *lip) { @@ -2704,18 +2664,11 @@ xlog_recover_process_intents( /* * NOTE: If your intent processing routine can create more - * deferred ops, you /must/ attach them to the dfops in this - * routine or else those subsequent intents will get + * deferred ops, you /must/ attach them to the transaction in + * this routine or else those subsequent intents will get * replayed in the wrong order! */ - switch (lip->li_type) { - case XFS_LI_BUI: - error = xlog_recover_process_bui(parent_tp, ailp, lip); - break; - default: - error = lip->li_ops->iop_recover(lip, parent_tp); - break; - } + error = lip->li_ops->iop_recover(lip, parent_tp); if (error) goto out; lip = xfs_trans_ail_cursor_next(ailp, &cur); @@ -2758,17 +2711,9 @@ xlog_recover_cancel_intents( break; } - switch (lip->li_type) { - case XFS_LI_BUI: - xlog_recover_cancel_bui(log->l_mp, ailp, lip); - break; - default: - spin_unlock(&ailp->ail_lock); - lip->li_ops->iop_release(lip); - spin_lock(&ailp->ail_lock); - break; - } - + spin_unlock(&ailp->ail_lock); + lip->li_ops->iop_release(lip); + spin_lock(&ailp->ail_lock); lip = xfs_trans_ail_cursor_next(ailp, &cur); } From bba7b1644a25809c299684a325e766263ad15c62 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Wed, 6 May 2020 12:07:25 -0700 Subject: [PATCH 0290/1043] xfs: refactor xlog_item_is_intent now that we're done converting Now that we've finished converting all types of log intent items to provide an ->iop_recover function, we can convert the "is this an intent item?" predicate to look for a non-null iop_recover pointer. Move the predicate closer to the functions that use it. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_log_recover.c | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 65081a3efeff..e21cb9c33faa 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2546,20 +2546,6 @@ xlog_recover_process_data( return 0; } -/* Is this log item a deferred action intent? */ -static inline bool xlog_item_is_intent(struct xfs_log_item *lip) -{ - switch (lip->li_type) { - case XFS_LI_EFI: - case XFS_LI_RUI: - case XFS_LI_CUI: - case XFS_LI_BUI: - return true; - default: - return false; - } -} - /* Take all the collected deferred ops and finish them in order. */ static int xlog_finish_defer_ops( @@ -2594,6 +2580,12 @@ xlog_finish_defer_ops( return xfs_trans_commit(tp); } +/* Is this log item a deferred action intent? */ +static inline bool xlog_item_is_intent(struct xfs_log_item *lip) +{ + return lip->li_ops->iop_recover != NULL; +} + /* * When this is called, all of the log intent items which did not have * corresponding log done items should be in the AIL. What we do now From 154c733a33d9cdaabec42ae76ca1189044d0447e Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:54 -0700 Subject: [PATCH 0291/1043] xfs: refactor releasing finished intents during log recovery Replace the open-coded AIL item walking with a proper helper when we're trying to release an intent item that has been finished. We add a new ->iop_match method to decide if an intent item matches a supplied ID. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_log_recover.h | 3 +++ fs/xfs/xfs_bmap_item.c | 42 ++++++++------------------------- fs/xfs/xfs_extfree_item.c | 42 ++++++++------------------------- fs/xfs/xfs_log_recover.c | 35 ++++++++++++++++++++++++++- fs/xfs/xfs_refcount_item.c | 42 ++++++++------------------------- fs/xfs/xfs_rmap_item.c | 42 ++++++++------------------------- fs/xfs/xfs_trans.h | 1 + 7 files changed, 78 insertions(+), 129 deletions(-) diff --git a/fs/xfs/libxfs/xfs_log_recover.h b/fs/xfs/libxfs/xfs_log_recover.h index 91fe954a796c..929366d58c35 100644 --- a/fs/xfs/libxfs/xfs_log_recover.h +++ b/fs/xfs/libxfs/xfs_log_recover.h @@ -125,4 +125,7 @@ bool xlog_is_buffer_cancelled(struct xlog *log, xfs_daddr_t blkno, uint len); bool xlog_put_buffer_cancelled(struct xlog *log, xfs_daddr_t blkno, uint len); void xlog_recover_iodone(struct xfs_buf *bp); +void xlog_recover_release_intent(struct xlog *log, unsigned short intent_type, + uint64_t intent_id); + #endif /* __XFS_LOG_RECOVER_H__ */ diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index b08015caed32..b3996f361b87 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -578,12 +578,21 @@ xfs_bui_item_recover( return error; } +STATIC bool +xfs_bui_item_match( + struct xfs_log_item *lip, + uint64_t intent_id) +{ + return BUI_ITEM(lip)->bui_format.bui_id == intent_id; +} + static const struct xfs_item_ops xfs_bui_item_ops = { .iop_size = xfs_bui_item_size, .iop_format = xfs_bui_item_format, .iop_unpin = xfs_bui_item_unpin, .iop_release = xfs_bui_item_release, .iop_recover = xfs_bui_item_recover, + .iop_match = xfs_bui_item_match, }; /* @@ -675,45 +684,14 @@ xlog_recover_bud_commit_pass2( xfs_lsn_t lsn) { struct xfs_bud_log_format *bud_formatp; - struct xfs_bui_log_item *buip = NULL; - struct xfs_log_item *lip; - uint64_t bui_id; - struct xfs_ail_cursor cur; - struct xfs_ail *ailp = log->l_ailp; bud_formatp = item->ri_buf[0].i_addr; if (item->ri_buf[0].i_len != sizeof(struct xfs_bud_log_format)) { XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); return -EFSCORRUPTED; } - bui_id = bud_formatp->bud_bui_id; - - /* - * Search for the BUI with the id in the BUD format structure in the - * AIL. - */ - spin_lock(&ailp->ail_lock); - lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); - while (lip != NULL) { - if (lip->li_type == XFS_LI_BUI) { - buip = (struct xfs_bui_log_item *)lip; - if (buip->bui_format.bui_id == bui_id) { - /* - * Drop the BUD reference to the BUI. This - * removes the BUI from the AIL and frees it. - */ - spin_unlock(&ailp->ail_lock); - xfs_bui_release(buip); - spin_lock(&ailp->ail_lock); - break; - } - } - lip = xfs_trans_ail_cursor_next(ailp, &cur); - } - - xfs_trans_ail_cursor_done(&cur); - spin_unlock(&ailp->ail_lock); + xlog_recover_release_intent(log, XFS_LI_BUI, bud_formatp->bud_bui_id); return 0; } diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index 307f71bdd398..3855e30109bf 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -665,12 +665,21 @@ xfs_efi_item_recover( return error; } +STATIC bool +xfs_efi_item_match( + struct xfs_log_item *lip, + uint64_t intent_id) +{ + return EFI_ITEM(lip)->efi_format.efi_id == intent_id; +} + static const struct xfs_item_ops xfs_efi_item_ops = { .iop_size = xfs_efi_item_size, .iop_format = xfs_efi_item_format, .iop_unpin = xfs_efi_item_unpin, .iop_release = xfs_efi_item_release, .iop_recover = xfs_efi_item_recover, + .iop_match = xfs_efi_item_match, }; /* @@ -733,46 +742,15 @@ xlog_recover_efd_commit_pass2( struct xlog_recover_item *item, xfs_lsn_t lsn) { - struct xfs_ail_cursor cur; struct xfs_efd_log_format *efd_formatp; - struct xfs_efi_log_item *efip = NULL; - struct xfs_log_item *lip; - struct xfs_ail *ailp = log->l_ailp; - uint64_t efi_id; efd_formatp = item->ri_buf[0].i_addr; ASSERT((item->ri_buf[0].i_len == (sizeof(xfs_efd_log_format_32_t) + ((efd_formatp->efd_nextents - 1) * sizeof(xfs_extent_32_t)))) || (item->ri_buf[0].i_len == (sizeof(xfs_efd_log_format_64_t) + ((efd_formatp->efd_nextents - 1) * sizeof(xfs_extent_64_t))))); - efi_id = efd_formatp->efd_efi_id; - - /* - * Search for the EFI with the id in the EFD format structure in the - * AIL. - */ - spin_lock(&ailp->ail_lock); - lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); - while (lip != NULL) { - if (lip->li_type == XFS_LI_EFI) { - efip = (struct xfs_efi_log_item *)lip; - if (efip->efi_format.efi_id == efi_id) { - /* - * Drop the EFD reference to the EFI. This - * removes the EFI from the AIL and frees it. - */ - spin_unlock(&ailp->ail_lock); - xfs_efi_release(efip); - spin_lock(&ailp->ail_lock); - break; - } - } - lip = xfs_trans_ail_cursor_next(ailp, &cur); - } - - xfs_trans_ail_cursor_done(&cur); - spin_unlock(&ailp->ail_lock); + xlog_recover_release_intent(log, XFS_LI_EFI, efd_formatp->efd_efi_id); return 0; } diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index e21cb9c33faa..8a397566b7bb 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -1779,6 +1779,38 @@ xlog_clear_stale_blocks( return 0; } +/* + * Release the recovered intent item in the AIL that matches the given intent + * type and intent id. + */ +void +xlog_recover_release_intent( + struct xlog *log, + unsigned short intent_type, + uint64_t intent_id) +{ + struct xfs_ail_cursor cur; + struct xfs_log_item *lip; + struct xfs_ail *ailp = log->l_ailp; + + spin_lock(&ailp->ail_lock); + for (lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); lip != NULL; + lip = xfs_trans_ail_cursor_next(ailp, &cur)) { + if (lip->li_type != intent_type) + continue; + if (!lip->li_ops->iop_match(lip, intent_id)) + continue; + + spin_unlock(&ailp->ail_lock); + lip->li_ops->iop_release(lip); + spin_lock(&ailp->ail_lock); + break; + } + + xfs_trans_ail_cursor_done(&cur); + spin_unlock(&ailp->ail_lock); +} + /****************************************************************************** * * Log recover routines @@ -2583,7 +2615,8 @@ xlog_finish_defer_ops( /* Is this log item a deferred action intent? */ static inline bool xlog_item_is_intent(struct xfs_log_item *lip) { - return lip->li_ops->iop_recover != NULL; + return lip->li_ops->iop_recover != NULL && + lip->li_ops->iop_match != NULL; } /* diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index 4eee8add4cd5..c03836e1a6d7 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -591,12 +591,21 @@ xfs_cui_item_recover( return error; } +STATIC bool +xfs_cui_item_match( + struct xfs_log_item *lip, + uint64_t intent_id) +{ + return CUI_ITEM(lip)->cui_format.cui_id == intent_id; +} + static const struct xfs_item_ops xfs_cui_item_ops = { .iop_size = xfs_cui_item_size, .iop_format = xfs_cui_item_format, .iop_unpin = xfs_cui_item_unpin, .iop_release = xfs_cui_item_release, .iop_recover = xfs_cui_item_recover, + .iop_match = xfs_cui_item_match, }; /* @@ -684,45 +693,14 @@ xlog_recover_cud_commit_pass2( xfs_lsn_t lsn) { struct xfs_cud_log_format *cud_formatp; - struct xfs_cui_log_item *cuip = NULL; - struct xfs_log_item *lip; - uint64_t cui_id; - struct xfs_ail_cursor cur; - struct xfs_ail *ailp = log->l_ailp; cud_formatp = item->ri_buf[0].i_addr; if (item->ri_buf[0].i_len != sizeof(struct xfs_cud_log_format)) { XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); return -EFSCORRUPTED; } - cui_id = cud_formatp->cud_cui_id; - - /* - * Search for the CUI with the id in the CUD format structure in the - * AIL. - */ - spin_lock(&ailp->ail_lock); - lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); - while (lip != NULL) { - if (lip->li_type == XFS_LI_CUI) { - cuip = (struct xfs_cui_log_item *)lip; - if (cuip->cui_format.cui_id == cui_id) { - /* - * Drop the CUD reference to the CUI. This - * removes the CUI from the AIL and frees it. - */ - spin_unlock(&ailp->ail_lock); - xfs_cui_release(cuip); - spin_lock(&ailp->ail_lock); - break; - } - } - lip = xfs_trans_ail_cursor_next(ailp, &cur); - } - - xfs_trans_ail_cursor_done(&cur); - spin_unlock(&ailp->ail_lock); + xlog_recover_release_intent(log, XFS_LI_CUI, cud_formatp->cud_cui_id); return 0; } diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index 1b7c7e3db872..31d35de518d1 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -606,12 +606,21 @@ xfs_rui_item_recover( return error; } +STATIC bool +xfs_rui_item_match( + struct xfs_log_item *lip, + uint64_t intent_id) +{ + return RUI_ITEM(lip)->rui_format.rui_id == intent_id; +} + static const struct xfs_item_ops xfs_rui_item_ops = { .iop_size = xfs_rui_item_size, .iop_format = xfs_rui_item_format, .iop_unpin = xfs_rui_item_unpin, .iop_release = xfs_rui_item_release, .iop_recover = xfs_rui_item_recover, + .iop_match = xfs_rui_item_match, }; /* @@ -675,42 +684,11 @@ xlog_recover_rud_commit_pass2( xfs_lsn_t lsn) { struct xfs_rud_log_format *rud_formatp; - struct xfs_rui_log_item *ruip = NULL; - struct xfs_log_item *lip; - uint64_t rui_id; - struct xfs_ail_cursor cur; - struct xfs_ail *ailp = log->l_ailp; rud_formatp = item->ri_buf[0].i_addr; ASSERT(item->ri_buf[0].i_len == sizeof(struct xfs_rud_log_format)); - rui_id = rud_formatp->rud_rui_id; - - /* - * Search for the RUI with the id in the RUD format structure in the - * AIL. - */ - spin_lock(&ailp->ail_lock); - lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); - while (lip != NULL) { - if (lip->li_type == XFS_LI_RUI) { - ruip = (struct xfs_rui_log_item *)lip; - if (ruip->rui_format.rui_id == rui_id) { - /* - * Drop the RUD reference to the RUI. This - * removes the RUI from the AIL and frees it. - */ - spin_unlock(&ailp->ail_lock); - xfs_rui_release(ruip); - spin_lock(&ailp->ail_lock); - break; - } - } - lip = xfs_trans_ail_cursor_next(ailp, &cur); - } - - xfs_trans_ail_cursor_done(&cur); - spin_unlock(&ailp->ail_lock); + xlog_recover_release_intent(log, XFS_LI_RUI, rud_formatp->rud_rui_id); return 0; } diff --git a/fs/xfs/xfs_trans.h b/fs/xfs/xfs_trans.h index 3f6a79108991..3e8808bb07c5 100644 --- a/fs/xfs/xfs_trans.h +++ b/fs/xfs/xfs_trans.h @@ -78,6 +78,7 @@ struct xfs_item_ops { xfs_lsn_t (*iop_committed)(struct xfs_log_item *, xfs_lsn_t); void (*iop_error)(struct xfs_log_item *, xfs_buf_t *); int (*iop_recover)(struct xfs_log_item *lip, struct xfs_trans *tp); + bool (*iop_match)(struct xfs_log_item *item, uint64_t id); }; /* From 86a37174138621a44c38621b69595e2cd67e5956 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:54 -0700 Subject: [PATCH 0292/1043] xfs: refactor adding recovered intent items to the log During recovery, every intent that we recover from the log has to be added to the AIL. Replace the open-coded addition with a helper. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Chandan Babu R --- fs/xfs/xfs_bmap_item.c | 10 +++------- fs/xfs/xfs_extfree_item.c | 10 +++------- fs/xfs/xfs_refcount_item.c | 10 +++------- fs/xfs/xfs_rmap_item.c | 10 +++------- fs/xfs/xfs_trans_ail.c | 11 +++++++++++ fs/xfs/xfs_trans_priv.h | 3 +++ 6 files changed, 26 insertions(+), 28 deletions(-) diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index b3996f361b87..1e9bc8d15f51 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -651,15 +651,11 @@ xlog_recover_bui_commit_pass2( return error; } atomic_set(&buip->bui_next_extent, bui_formatp->bui_nextents); - - spin_lock(&log->l_ailp->ail_lock); /* - * The RUI has two references. One for the RUD and one for RUI to ensure - * it makes it into the AIL. Insert the RUI into the AIL directly and - * drop the RUI reference. Note that xfs_trans_ail_update() drops the - * AIL lock. + * Insert the intent into the AIL directly and drop one reference so + * that finishing or canceling the work will drop the other. */ - xfs_trans_ail_update(log->l_ailp, &buip->bui_item, lsn); + xfs_trans_ail_insert(log->l_ailp, &buip->bui_item, lsn); xfs_bui_release(buip); return 0; } diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index 3855e30109bf..99c4643d0ae8 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -710,15 +710,11 @@ xlog_recover_efi_commit_pass2( return error; } atomic_set(&efip->efi_next_extent, efi_formatp->efi_nextents); - - spin_lock(&log->l_ailp->ail_lock); /* - * The EFI has two references. One for the EFD and one for EFI to ensure - * it makes it into the AIL. Insert the EFI into the AIL directly and - * drop the EFI reference. Note that xfs_trans_ail_update() drops the - * AIL lock. + * Insert the intent into the AIL directly and drop one reference so + * that finishing or canceling the work will drop the other. */ - xfs_trans_ail_update(log->l_ailp, &efip->efi_item, lsn); + xfs_trans_ail_insert(log->l_ailp, &efip->efi_item, lsn); xfs_efi_release(efip); return 0; } diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index c03836e1a6d7..a9c513338ddc 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -660,15 +660,11 @@ xlog_recover_cui_commit_pass2( return error; } atomic_set(&cuip->cui_next_extent, cui_formatp->cui_nextents); - - spin_lock(&log->l_ailp->ail_lock); /* - * The CUI has two references. One for the CUD and one for CUI to ensure - * it makes it into the AIL. Insert the CUI into the AIL directly and - * drop the CUI reference. Note that xfs_trans_ail_update() drops the - * AIL lock. + * Insert the intent into the AIL directly and drop one reference so + * that finishing or canceling the work will drop the other. */ - xfs_trans_ail_update(log->l_ailp, &cuip->cui_item, lsn); + xfs_trans_ail_insert(log->l_ailp, &cuip->cui_item, lsn); xfs_cui_release(cuip); return 0; } diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index 31d35de518d1..ee0be4310c7c 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -651,15 +651,11 @@ xlog_recover_rui_commit_pass2( return error; } atomic_set(&ruip->rui_next_extent, rui_formatp->rui_nextents); - - spin_lock(&log->l_ailp->ail_lock); /* - * The RUI has two references. One for the RUD and one for RUI to ensure - * it makes it into the AIL. Insert the RUI into the AIL directly and - * drop the RUI reference. Note that xfs_trans_ail_update() drops the - * AIL lock. + * Insert the intent into the AIL directly and drop one reference so + * that finishing or canceling the work will drop the other. */ - xfs_trans_ail_update(log->l_ailp, &ruip->rui_item, lsn); + xfs_trans_ail_insert(log->l_ailp, &ruip->rui_item, lsn); xfs_rui_release(ruip); return 0; } diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c index bf09d4b4df58..ac5019361a13 100644 --- a/fs/xfs/xfs_trans_ail.c +++ b/fs/xfs/xfs_trans_ail.c @@ -815,6 +815,17 @@ xfs_trans_ail_update_bulk( xfs_ail_update_finish(ailp, tail_lsn); } +/* Insert a log item into the AIL. */ +void +xfs_trans_ail_insert( + struct xfs_ail *ailp, + struct xfs_log_item *lip, + xfs_lsn_t lsn) +{ + spin_lock(&ailp->ail_lock); + xfs_trans_ail_update_bulk(ailp, NULL, &lip, 1, lsn); +} + /* * Delete one log item from the AIL. * diff --git a/fs/xfs/xfs_trans_priv.h b/fs/xfs/xfs_trans_priv.h index cc046d9557ae..3004aeac9110 100644 --- a/fs/xfs/xfs_trans_priv.h +++ b/fs/xfs/xfs_trans_priv.h @@ -91,6 +91,9 @@ xfs_trans_ail_update( xfs_trans_ail_update_bulk(ailp, NULL, &lip, 1, lsn); } +void xfs_trans_ail_insert(struct xfs_ail *ailp, struct xfs_log_item *lip, + xfs_lsn_t lsn); + xfs_lsn_t xfs_ail_delete_one(struct xfs_ail *ailp, struct xfs_log_item *lip); void xfs_ail_update_finish(struct xfs_ail *ailp, xfs_lsn_t old_lsn) __releases(ailp->ail_lock); From 889eb55dd68f97729adccb8c06834b35e8bd9590 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:55 -0700 Subject: [PATCH 0293/1043] xfs: refactor intent item RECOVERED flag into the log item Rename XFS_{EFI,BUI,RUI,CUI}_RECOVERED to XFS_LI_RECOVERED so that we track recovery status in the log item, then get rid of the now unused flags fields in each of those log item types. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_bmap_item.c | 10 +++++----- fs/xfs/xfs_bmap_item.h | 6 ------ fs/xfs/xfs_extfree_item.c | 8 ++++---- fs/xfs/xfs_extfree_item.h | 6 ------ fs/xfs/xfs_refcount_item.c | 8 ++++---- fs/xfs/xfs_refcount_item.h | 6 ------ fs/xfs/xfs_rmap_item.c | 8 ++++---- fs/xfs/xfs_rmap_item.h | 6 ------ fs/xfs/xfs_trans.h | 4 +++- 9 files changed, 20 insertions(+), 42 deletions(-) diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index 1e9bc8d15f51..8a5ac8cfd5f2 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -441,11 +441,11 @@ xfs_bui_recover( struct xfs_bmbt_irec irec; struct xfs_mount *mp = parent_tp->t_mountp; - ASSERT(!test_bit(XFS_BUI_RECOVERED, &buip->bui_flags)); + ASSERT(!test_bit(XFS_LI_RECOVERED, &buip->bui_item.li_flags)); /* Only one mapping operation per BUI... */ if (buip->bui_format.bui_nextents != XFS_BUI_MAX_FAST_EXTENTS) { - set_bit(XFS_BUI_RECOVERED, &buip->bui_flags); + set_bit(XFS_LI_RECOVERED, &buip->bui_item.li_flags); xfs_bui_release(buip); return -EFSCORRUPTED; } @@ -479,7 +479,7 @@ xfs_bui_recover( * This will pull the BUI from the AIL and * free the memory associated with it. */ - set_bit(XFS_BUI_RECOVERED, &buip->bui_flags); + set_bit(XFS_LI_RECOVERED, &buip->bui_item.li_flags); xfs_bui_release(buip); return -EFSCORRUPTED; } @@ -537,7 +537,7 @@ xfs_bui_recover( xfs_bmap_unmap_extent(tp, ip, &irec); } - set_bit(XFS_BUI_RECOVERED, &buip->bui_flags); + set_bit(XFS_LI_RECOVERED, &buip->bui_item.li_flags); xfs_defer_move(parent_tp, tp); error = xfs_trans_commit(tp); xfs_iunlock(ip, XFS_ILOCK_EXCL); @@ -568,7 +568,7 @@ xfs_bui_item_recover( /* * Skip BUIs that we've already processed. */ - if (test_bit(XFS_BUI_RECOVERED, &buip->bui_flags)) + if (test_bit(XFS_LI_RECOVERED, &buip->bui_item.li_flags)) return 0; spin_unlock(&ailp->ail_lock); diff --git a/fs/xfs/xfs_bmap_item.h b/fs/xfs/xfs_bmap_item.h index 44d06e62f8f9..b9be62f8bd52 100644 --- a/fs/xfs/xfs_bmap_item.h +++ b/fs/xfs/xfs_bmap_item.h @@ -32,11 +32,6 @@ struct kmem_zone; */ #define XFS_BUI_MAX_FAST_EXTENTS 1 -/* - * Define BUI flag bits. Manipulated by set/clear/test_bit operators. - */ -#define XFS_BUI_RECOVERED 1 - /* * This is the "bmap update intent" log item. It is used to log the fact that * some reverse mappings need to change. It is used in conjunction with the @@ -49,7 +44,6 @@ struct xfs_bui_log_item { struct xfs_log_item bui_item; atomic_t bui_refcount; atomic_t bui_next_extent; - unsigned long bui_flags; /* misc flags */ struct xfs_bui_log_format bui_format; }; diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index 99c4643d0ae8..ffa15bcaea33 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -592,7 +592,7 @@ xfs_efi_recover( xfs_extent_t *extp; xfs_fsblock_t startblock_fsb; - ASSERT(!test_bit(XFS_EFI_RECOVERED, &efip->efi_flags)); + ASSERT(!test_bit(XFS_LI_RECOVERED, &efip->efi_item.li_flags)); /* * First check the validity of the extents described by the @@ -611,7 +611,7 @@ xfs_efi_recover( * This will pull the EFI from the AIL and * free the memory associated with it. */ - set_bit(XFS_EFI_RECOVERED, &efip->efi_flags); + set_bit(XFS_LI_RECOVERED, &efip->efi_item.li_flags); xfs_efi_release(efip); return -EFSCORRUPTED; } @@ -632,7 +632,7 @@ xfs_efi_recover( } - set_bit(XFS_EFI_RECOVERED, &efip->efi_flags); + set_bit(XFS_LI_RECOVERED, &efip->efi_item.li_flags); error = xfs_trans_commit(tp); return error; @@ -655,7 +655,7 @@ xfs_efi_item_recover( * Skip EFIs that we've already processed. */ efip = container_of(lip, struct xfs_efi_log_item, efi_item); - if (test_bit(XFS_EFI_RECOVERED, &efip->efi_flags)) + if (test_bit(XFS_LI_RECOVERED, &efip->efi_item.li_flags)) return 0; spin_unlock(&ailp->ail_lock); diff --git a/fs/xfs/xfs_extfree_item.h b/fs/xfs/xfs_extfree_item.h index 4b2c2c5c5985..cd2860c875bf 100644 --- a/fs/xfs/xfs_extfree_item.h +++ b/fs/xfs/xfs_extfree_item.h @@ -16,11 +16,6 @@ struct kmem_zone; */ #define XFS_EFI_MAX_FAST_EXTENTS 16 -/* - * Define EFI flag bits. Manipulated by set/clear/test_bit operators. - */ -#define XFS_EFI_RECOVERED 1 - /* * This is the "extent free intention" log item. It is used to log the fact * that some extents need to be free. It is used in conjunction with the @@ -54,7 +49,6 @@ struct xfs_efi_log_item { struct xfs_log_item efi_item; atomic_t efi_refcount; atomic_t efi_next_extent; - unsigned long efi_flags; /* misc flags */ xfs_efi_log_format_t efi_format; }; diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index a9c513338ddc..c7d584b99508 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -441,7 +441,7 @@ xfs_cui_recover( bool requeue_only = false; struct xfs_mount *mp = parent_tp->t_mountp; - ASSERT(!test_bit(XFS_CUI_RECOVERED, &cuip->cui_flags)); + ASSERT(!test_bit(XFS_LI_RECOVERED, &cuip->cui_item.li_flags)); /* * First check the validity of the extents described by the @@ -472,7 +472,7 @@ xfs_cui_recover( * This will pull the CUI from the AIL and * free the memory associated with it. */ - set_bit(XFS_CUI_RECOVERED, &cuip->cui_flags); + set_bit(XFS_LI_RECOVERED, &cuip->cui_item.li_flags); xfs_cui_release(cuip); return -EFSCORRUPTED; } @@ -556,7 +556,7 @@ xfs_cui_recover( } xfs_refcount_finish_one_cleanup(tp, rcur, error); - set_bit(XFS_CUI_RECOVERED, &cuip->cui_flags); + set_bit(XFS_LI_RECOVERED, &cuip->cui_item.li_flags); xfs_defer_move(parent_tp, tp); error = xfs_trans_commit(tp); return error; @@ -581,7 +581,7 @@ xfs_cui_item_recover( /* * Skip CUIs that we've already processed. */ - if (test_bit(XFS_CUI_RECOVERED, &cuip->cui_flags)) + if (test_bit(XFS_LI_RECOVERED, &cuip->cui_item.li_flags)) return 0; spin_unlock(&ailp->ail_lock); diff --git a/fs/xfs/xfs_refcount_item.h b/fs/xfs/xfs_refcount_item.h index cfaa857673a6..f4f2e836540b 100644 --- a/fs/xfs/xfs_refcount_item.h +++ b/fs/xfs/xfs_refcount_item.h @@ -32,11 +32,6 @@ struct kmem_zone; */ #define XFS_CUI_MAX_FAST_EXTENTS 16 -/* - * Define CUI flag bits. Manipulated by set/clear/test_bit operators. - */ -#define XFS_CUI_RECOVERED 1 - /* * This is the "refcount update intent" log item. It is used to log * the fact that some reverse mappings need to change. It is used in @@ -51,7 +46,6 @@ struct xfs_cui_log_item { struct xfs_log_item cui_item; atomic_t cui_refcount; atomic_t cui_next_extent; - unsigned long cui_flags; /* misc flags */ struct xfs_cui_log_format cui_format; }; diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index ee0be4310c7c..45cc7bfe82b4 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -480,7 +480,7 @@ xfs_rui_recover( struct xfs_trans *tp; struct xfs_btree_cur *rcur = NULL; - ASSERT(!test_bit(XFS_RUI_RECOVERED, &ruip->rui_flags)); + ASSERT(!test_bit(XFS_LI_RECOVERED, &ruip->rui_item.li_flags)); /* * First check the validity of the extents described by the @@ -515,7 +515,7 @@ xfs_rui_recover( * This will pull the RUI from the AIL and * free the memory associated with it. */ - set_bit(XFS_RUI_RECOVERED, &ruip->rui_flags); + set_bit(XFS_LI_RECOVERED, &ruip->rui_item.li_flags); xfs_rui_release(ruip); return -EFSCORRUPTED; } @@ -573,7 +573,7 @@ xfs_rui_recover( } xfs_rmap_finish_one_cleanup(tp, rcur, error); - set_bit(XFS_RUI_RECOVERED, &ruip->rui_flags); + set_bit(XFS_LI_RECOVERED, &ruip->rui_item.li_flags); error = xfs_trans_commit(tp); return error; @@ -596,7 +596,7 @@ xfs_rui_item_recover( /* * Skip RUIs that we've already processed. */ - if (test_bit(XFS_RUI_RECOVERED, &ruip->rui_flags)) + if (test_bit(XFS_LI_RECOVERED, &ruip->rui_item.li_flags)) return 0; spin_unlock(&ailp->ail_lock); diff --git a/fs/xfs/xfs_rmap_item.h b/fs/xfs/xfs_rmap_item.h index 48a77a6f5c94..31e6cdfff71f 100644 --- a/fs/xfs/xfs_rmap_item.h +++ b/fs/xfs/xfs_rmap_item.h @@ -35,11 +35,6 @@ struct kmem_zone; */ #define XFS_RUI_MAX_FAST_EXTENTS 16 -/* - * Define RUI flag bits. Manipulated by set/clear/test_bit operators. - */ -#define XFS_RUI_RECOVERED 1 - /* * This is the "rmap update intent" log item. It is used to log the fact that * some reverse mappings need to change. It is used in conjunction with the @@ -52,7 +47,6 @@ struct xfs_rui_log_item { struct xfs_log_item rui_item; atomic_t rui_refcount; atomic_t rui_next_extent; - unsigned long rui_flags; /* misc flags */ struct xfs_rui_log_format rui_format; }; diff --git a/fs/xfs/xfs_trans.h b/fs/xfs/xfs_trans.h index 3e8808bb07c5..8308bf6d7e40 100644 --- a/fs/xfs/xfs_trans.h +++ b/fs/xfs/xfs_trans.h @@ -59,12 +59,14 @@ struct xfs_log_item { #define XFS_LI_ABORTED 1 #define XFS_LI_FAILED 2 #define XFS_LI_DIRTY 3 /* log item dirty in transaction */ +#define XFS_LI_RECOVERED 4 /* log intent item has been recovered */ #define XFS_LI_FLAGS \ { (1 << XFS_LI_IN_AIL), "IN_AIL" }, \ { (1 << XFS_LI_ABORTED), "ABORTED" }, \ { (1 << XFS_LI_FAILED), "FAILED" }, \ - { (1 << XFS_LI_DIRTY), "DIRTY" } + { (1 << XFS_LI_DIRTY), "DIRTY" }, \ + { (1 << XFS_LI_RECOVERED), "RECOVERED" } struct xfs_item_ops { unsigned flags; From 96b60f826713767f222254418c995fb6fd7596b3 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:55 -0700 Subject: [PATCH 0294/1043] xfs: refactor intent item iop_recover calls Now that we've made the recovered item tests all the same, we can hoist the test and the ail locking code to the ->iop_recover caller and call the recovery function directly. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_bmap_item.c | 50 +++++++++++--------------------------- fs/xfs/xfs_extfree_item.c | 44 +++++++++------------------------ fs/xfs/xfs_log_recover.c | 8 ++++-- fs/xfs/xfs_refcount_item.c | 46 +++++++++-------------------------- fs/xfs/xfs_rmap_item.c | 45 +++++++++------------------------- 5 files changed, 55 insertions(+), 138 deletions(-) diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index 8a5ac8cfd5f2..3b8ca4409aa5 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -421,25 +421,26 @@ const struct xfs_defer_op_type xfs_bmap_update_defer_type = { * We need to update some inode's bmbt. */ STATIC int -xfs_bui_recover( - struct xfs_trans *parent_tp, - struct xfs_bui_log_item *buip) +xfs_bui_item_recover( + struct xfs_log_item *lip, + struct xfs_trans *parent_tp) { - int error = 0; - unsigned int bui_type; + struct xfs_bmbt_irec irec; + struct xfs_bui_log_item *buip = BUI_ITEM(lip); + struct xfs_trans *tp; + struct xfs_inode *ip = NULL; + struct xfs_mount *mp = parent_tp->t_mountp; struct xfs_map_extent *bmap; + struct xfs_bud_log_item *budp; xfs_fsblock_t startblock_fsb; xfs_fsblock_t inode_fsb; xfs_filblks_t count; - bool op_ok; - struct xfs_bud_log_item *budp; - enum xfs_bmap_intent_type type; - int whichfork; xfs_exntst_t state; - struct xfs_trans *tp; - struct xfs_inode *ip = NULL; - struct xfs_bmbt_irec irec; - struct xfs_mount *mp = parent_tp->t_mountp; + enum xfs_bmap_intent_type type; + bool op_ok; + unsigned int bui_type; + int whichfork; + int error = 0; ASSERT(!test_bit(XFS_LI_RECOVERED, &buip->bui_item.li_flags)); @@ -555,29 +556,6 @@ err_inode: return error; } -/* Recover the BUI if necessary. */ -STATIC int -xfs_bui_item_recover( - struct xfs_log_item *lip, - struct xfs_trans *tp) -{ - struct xfs_ail *ailp = lip->li_ailp; - struct xfs_bui_log_item *buip = BUI_ITEM(lip); - int error; - - /* - * Skip BUIs that we've already processed. - */ - if (test_bit(XFS_LI_RECOVERED, &buip->bui_item.li_flags)) - return 0; - - spin_unlock(&ailp->ail_lock); - error = xfs_bui_recover(tp, buip); - spin_lock(&ailp->ail_lock); - - return error; -} - STATIC bool xfs_bui_item_match( struct xfs_log_item *lip, diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index ffa15bcaea33..a8ee9aaef50d 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -581,16 +581,18 @@ const struct xfs_defer_op_type xfs_agfl_free_defer_type = { * the log. We need to free the extents that it describes. */ STATIC int -xfs_efi_recover( - struct xfs_mount *mp, - struct xfs_efi_log_item *efip) +xfs_efi_item_recover( + struct xfs_log_item *lip, + struct xfs_trans *parent_tp) { - struct xfs_efd_log_item *efdp; - struct xfs_trans *tp; - int i; - int error = 0; - xfs_extent_t *extp; - xfs_fsblock_t startblock_fsb; + struct xfs_efi_log_item *efip = EFI_ITEM(lip); + struct xfs_mount *mp = parent_tp->t_mountp; + struct xfs_efd_log_item *efdp; + struct xfs_trans *tp; + struct xfs_extent *extp; + xfs_fsblock_t startblock_fsb; + int i; + int error = 0; ASSERT(!test_bit(XFS_LI_RECOVERED, &efip->efi_item.li_flags)); @@ -641,30 +643,6 @@ abort_error: return error; } -/* Recover the EFI if necessary. */ -STATIC int -xfs_efi_item_recover( - struct xfs_log_item *lip, - struct xfs_trans *tp) -{ - struct xfs_ail *ailp = lip->li_ailp; - struct xfs_efi_log_item *efip; - int error; - - /* - * Skip EFIs that we've already processed. - */ - efip = container_of(lip, struct xfs_efi_log_item, efi_item); - if (test_bit(XFS_LI_RECOVERED, &efip->efi_item.li_flags)) - return 0; - - spin_unlock(&ailp->ail_lock); - error = xfs_efi_recover(tp->t_mountp, efip); - spin_lock(&ailp->ail_lock); - - return error; -} - STATIC bool xfs_efi_item_match( struct xfs_log_item *lip, diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 8a397566b7bb..60e98e48d04b 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2643,7 +2643,7 @@ xlog_recover_process_intents( struct xfs_ail_cursor cur; struct xfs_log_item *lip; struct xfs_ail *ailp; - int error; + int error = 0; #if defined(DEBUG) || defined(XFS_WARN) xfs_lsn_t last_lsn; #endif @@ -2693,7 +2693,11 @@ xlog_recover_process_intents( * this routine or else those subsequent intents will get * replayed in the wrong order! */ - error = lip->li_ops->iop_recover(lip, parent_tp); + if (!test_bit(XFS_LI_RECOVERED, &lip->li_flags)) { + spin_unlock(&ailp->ail_lock); + error = lip->li_ops->iop_recover(lip, parent_tp); + spin_lock(&ailp->ail_lock); + } if (error) goto out; lip = xfs_trans_ail_cursor_next(ailp, &cur); diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index c7d584b99508..b256eafd30d3 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -421,25 +421,26 @@ const struct xfs_defer_op_type xfs_refcount_update_defer_type = { * We need to update the refcountbt. */ STATIC int -xfs_cui_recover( - struct xfs_trans *parent_tp, - struct xfs_cui_log_item *cuip) +xfs_cui_item_recover( + struct xfs_log_item *lip, + struct xfs_trans *parent_tp) { - int i; - int error = 0; - unsigned int refc_type; + struct xfs_bmbt_irec irec; + struct xfs_cui_log_item *cuip = CUI_ITEM(lip); struct xfs_phys_extent *refc; - xfs_fsblock_t startblock_fsb; - bool op_ok; struct xfs_cud_log_item *cudp; struct xfs_trans *tp; struct xfs_btree_cur *rcur = NULL; - enum xfs_refcount_intent_type type; + struct xfs_mount *mp = parent_tp->t_mountp; + xfs_fsblock_t startblock_fsb; xfs_fsblock_t new_fsb; xfs_extlen_t new_len; - struct xfs_bmbt_irec irec; + unsigned int refc_type; + bool op_ok; bool requeue_only = false; - struct xfs_mount *mp = parent_tp->t_mountp; + enum xfs_refcount_intent_type type; + int i; + int error = 0; ASSERT(!test_bit(XFS_LI_RECOVERED, &cuip->cui_item.li_flags)); @@ -568,29 +569,6 @@ abort_error: return error; } -/* Recover the CUI if necessary. */ -STATIC int -xfs_cui_item_recover( - struct xfs_log_item *lip, - struct xfs_trans *tp) -{ - struct xfs_ail *ailp = lip->li_ailp; - struct xfs_cui_log_item *cuip = CUI_ITEM(lip); - int error; - - /* - * Skip CUIs that we've already processed. - */ - if (test_bit(XFS_LI_RECOVERED, &cuip->cui_item.li_flags)) - return 0; - - spin_unlock(&ailp->ail_lock); - error = xfs_cui_recover(tp, cuip); - spin_lock(&ailp->ail_lock); - - return error; -} - STATIC bool xfs_cui_item_match( struct xfs_log_item *lip, diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index 45cc7bfe82b4..d190060729a3 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -464,21 +464,23 @@ const struct xfs_defer_op_type xfs_rmap_update_defer_type = { * We need to update the rmapbt. */ STATIC int -xfs_rui_recover( - struct xfs_mount *mp, - struct xfs_rui_log_item *ruip) +xfs_rui_item_recover( + struct xfs_log_item *lip, + struct xfs_trans *parent_tp) { - int i; - int error = 0; + struct xfs_rui_log_item *ruip = RUI_ITEM(lip); struct xfs_map_extent *rmap; - xfs_fsblock_t startblock_fsb; - bool op_ok; struct xfs_rud_log_item *rudp; - enum xfs_rmap_intent_type type; - int whichfork; - xfs_exntst_t state; struct xfs_trans *tp; struct xfs_btree_cur *rcur = NULL; + struct xfs_mount *mp = parent_tp->t_mountp; + xfs_fsblock_t startblock_fsb; + enum xfs_rmap_intent_type type; + xfs_exntst_t state; + bool op_ok; + int i; + int whichfork; + int error = 0; ASSERT(!test_bit(XFS_LI_RECOVERED, &ruip->rui_item.li_flags)); @@ -583,29 +585,6 @@ abort_error: return error; } -/* Recover the RUI if necessary. */ -STATIC int -xfs_rui_item_recover( - struct xfs_log_item *lip, - struct xfs_trans *tp) -{ - struct xfs_ail *ailp = lip->li_ailp; - struct xfs_rui_log_item *ruip = RUI_ITEM(lip); - int error; - - /* - * Skip RUIs that we've already processed. - */ - if (test_bit(XFS_LI_RECOVERED, &ruip->rui_item.li_flags)) - return 0; - - spin_unlock(&ailp->ail_lock); - error = xfs_rui_recover(tp->t_mountp, ruip); - spin_lock(&ailp->ail_lock); - - return error; -} - STATIC bool xfs_rui_item_match( struct xfs_log_item *lip, From cc560a5a9540be2d907c0c170e29ebde98d13d63 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:55 -0700 Subject: [PATCH 0295/1043] xfs: hoist setting of XFS_LI_RECOVERED to caller The only purpose of XFS_LI_RECOVERED is to prevent log recovery from trying to replay recovered intents more than once. Therefore, we can move the bit setting up to the ->iop_recover caller. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_bmap_item.c | 5 ----- fs/xfs/xfs_extfree_item.c | 4 ---- fs/xfs/xfs_log_recover.c | 4 ++-- fs/xfs/xfs_refcount_item.c | 4 ---- fs/xfs/xfs_rmap_item.c | 4 ---- 5 files changed, 2 insertions(+), 19 deletions(-) diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index 3b8ca4409aa5..6736c5ab188f 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -442,11 +442,8 @@ xfs_bui_item_recover( int whichfork; int error = 0; - ASSERT(!test_bit(XFS_LI_RECOVERED, &buip->bui_item.li_flags)); - /* Only one mapping operation per BUI... */ if (buip->bui_format.bui_nextents != XFS_BUI_MAX_FAST_EXTENTS) { - set_bit(XFS_LI_RECOVERED, &buip->bui_item.li_flags); xfs_bui_release(buip); return -EFSCORRUPTED; } @@ -480,7 +477,6 @@ xfs_bui_item_recover( * This will pull the BUI from the AIL and * free the memory associated with it. */ - set_bit(XFS_LI_RECOVERED, &buip->bui_item.li_flags); xfs_bui_release(buip); return -EFSCORRUPTED; } @@ -538,7 +534,6 @@ xfs_bui_item_recover( xfs_bmap_unmap_extent(tp, ip, &irec); } - set_bit(XFS_LI_RECOVERED, &buip->bui_item.li_flags); xfs_defer_move(parent_tp, tp); error = xfs_trans_commit(tp); xfs_iunlock(ip, XFS_ILOCK_EXCL); diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index a8ee9aaef50d..b9c333bae0a1 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -594,8 +594,6 @@ xfs_efi_item_recover( int i; int error = 0; - ASSERT(!test_bit(XFS_LI_RECOVERED, &efip->efi_item.li_flags)); - /* * First check the validity of the extents described by the * EFI. If any are bad, then assume that all are bad and @@ -613,7 +611,6 @@ xfs_efi_item_recover( * This will pull the EFI from the AIL and * free the memory associated with it. */ - set_bit(XFS_LI_RECOVERED, &efip->efi_item.li_flags); xfs_efi_release(efip); return -EFSCORRUPTED; } @@ -634,7 +631,6 @@ xfs_efi_item_recover( } - set_bit(XFS_LI_RECOVERED, &efip->efi_item.li_flags); error = xfs_trans_commit(tp); return error; diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 60e98e48d04b..fa1b63bd9031 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2643,7 +2643,7 @@ xlog_recover_process_intents( struct xfs_ail_cursor cur; struct xfs_log_item *lip; struct xfs_ail *ailp; - int error = 0; + int error; #if defined(DEBUG) || defined(XFS_WARN) xfs_lsn_t last_lsn; #endif @@ -2693,7 +2693,7 @@ xlog_recover_process_intents( * this routine or else those subsequent intents will get * replayed in the wrong order! */ - if (!test_bit(XFS_LI_RECOVERED, &lip->li_flags)) { + if (!test_and_set_bit(XFS_LI_RECOVERED, &lip->li_flags)) { spin_unlock(&ailp->ail_lock); error = lip->li_ops->iop_recover(lip, parent_tp); spin_lock(&ailp->ail_lock); diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index b256eafd30d3..c81639891e29 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -442,8 +442,6 @@ xfs_cui_item_recover( int i; int error = 0; - ASSERT(!test_bit(XFS_LI_RECOVERED, &cuip->cui_item.li_flags)); - /* * First check the validity of the extents described by the * CUI. If any are bad, then assume that all are bad and @@ -473,7 +471,6 @@ xfs_cui_item_recover( * This will pull the CUI from the AIL and * free the memory associated with it. */ - set_bit(XFS_LI_RECOVERED, &cuip->cui_item.li_flags); xfs_cui_release(cuip); return -EFSCORRUPTED; } @@ -557,7 +554,6 @@ xfs_cui_item_recover( } xfs_refcount_finish_one_cleanup(tp, rcur, error); - set_bit(XFS_LI_RECOVERED, &cuip->cui_item.li_flags); xfs_defer_move(parent_tp, tp); error = xfs_trans_commit(tp); return error; diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index d190060729a3..a86599db20a6 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -482,8 +482,6 @@ xfs_rui_item_recover( int whichfork; int error = 0; - ASSERT(!test_bit(XFS_LI_RECOVERED, &ruip->rui_item.li_flags)); - /* * First check the validity of the extents described by the * RUI. If any are bad, then assume that all are bad and @@ -517,7 +515,6 @@ xfs_rui_item_recover( * This will pull the RUI from the AIL and * free the memory associated with it. */ - set_bit(XFS_LI_RECOVERED, &ruip->rui_item.li_flags); xfs_rui_release(ruip); return -EFSCORRUPTED; } @@ -575,7 +572,6 @@ xfs_rui_item_recover( } xfs_rmap_finish_one_cleanup(tp, rcur, error); - set_bit(XFS_LI_RECOVERED, &ruip->rui_item.li_flags); error = xfs_trans_commit(tp); return error; From 17d29bf271ea48b253c93969a590a11a51c19c1f Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:56 -0700 Subject: [PATCH 0296/1043] xfs: move log recovery buffer cancellation code to xfs_buf_item_recover.c Move the helpers that handle incore buffer cancellation records to xfs_buf_item_recover.c since they're not directly related to the main log recovery machinery. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_log_recover.h | 2 - fs/xfs/xfs_buf_item_recover.c | 104 ++++++++++++++++++++++++++++++++ fs/xfs/xfs_log_recover.c | 102 ------------------------------- 3 files changed, 104 insertions(+), 104 deletions(-) diff --git a/fs/xfs/libxfs/xfs_log_recover.h b/fs/xfs/libxfs/xfs_log_recover.h index 929366d58c35..641132d0e39d 100644 --- a/fs/xfs/libxfs/xfs_log_recover.h +++ b/fs/xfs/libxfs/xfs_log_recover.h @@ -120,9 +120,7 @@ struct xlog_recover { void xlog_buf_readahead(struct xlog *log, xfs_daddr_t blkno, uint len, const struct xfs_buf_ops *ops); -bool xlog_add_buffer_cancelled(struct xlog *log, xfs_daddr_t blkno, uint len); bool xlog_is_buffer_cancelled(struct xlog *log, xfs_daddr_t blkno, uint len); -bool xlog_put_buffer_cancelled(struct xlog *log, xfs_daddr_t blkno, uint len); void xlog_recover_iodone(struct xfs_buf *bp); void xlog_recover_release_intent(struct xlog *log, unsigned short intent_type, diff --git a/fs/xfs/xfs_buf_item_recover.c b/fs/xfs/xfs_buf_item_recover.c index 4ba2e27a15ca..04faa7310c4f 100644 --- a/fs/xfs/xfs_buf_item_recover.c +++ b/fs/xfs/xfs_buf_item_recover.c @@ -23,6 +23,110 @@ #include "xfs_dir2.h" #include "xfs_quota.h" +/* + * This structure is used during recovery to record the buf log items which + * have been canceled and should not be replayed. + */ +struct xfs_buf_cancel { + xfs_daddr_t bc_blkno; + uint bc_len; + int bc_refcount; + struct list_head bc_list; +}; + +static struct xfs_buf_cancel * +xlog_find_buffer_cancelled( + struct xlog *log, + xfs_daddr_t blkno, + uint len) +{ + struct list_head *bucket; + struct xfs_buf_cancel *bcp; + + if (!log->l_buf_cancel_table) + return NULL; + + bucket = XLOG_BUF_CANCEL_BUCKET(log, blkno); + list_for_each_entry(bcp, bucket, bc_list) { + if (bcp->bc_blkno == blkno && bcp->bc_len == len) + return bcp; + } + + return NULL; +} + +static bool +xlog_add_buffer_cancelled( + struct xlog *log, + xfs_daddr_t blkno, + uint len) +{ + struct xfs_buf_cancel *bcp; + + /* + * If we find an existing cancel record, this indicates that the buffer + * was cancelled multiple times. To ensure that during pass 2 we keep + * the record in the table until we reach its last occurrence in the + * log, a reference count is kept to tell how many times we expect to + * see this record during the second pass. + */ + bcp = xlog_find_buffer_cancelled(log, blkno, len); + if (bcp) { + bcp->bc_refcount++; + return false; + } + + bcp = kmem_alloc(sizeof(struct xfs_buf_cancel), 0); + bcp->bc_blkno = blkno; + bcp->bc_len = len; + bcp->bc_refcount = 1; + list_add_tail(&bcp->bc_list, XLOG_BUF_CANCEL_BUCKET(log, blkno)); + return true; +} + +/* + * Check if there is and entry for blkno, len in the buffer cancel record table. + */ +bool +xlog_is_buffer_cancelled( + struct xlog *log, + xfs_daddr_t blkno, + uint len) +{ + return xlog_find_buffer_cancelled(log, blkno, len) != NULL; +} + +/* + * Check if there is and entry for blkno, len in the buffer cancel record table, + * and decremented the reference count on it if there is one. + * + * Remove the cancel record once the refcount hits zero, so that if the same + * buffer is re-used again after its last cancellation we actually replay the + * changes made at that point. + */ +static bool +xlog_put_buffer_cancelled( + struct xlog *log, + xfs_daddr_t blkno, + uint len) +{ + struct xfs_buf_cancel *bcp; + + bcp = xlog_find_buffer_cancelled(log, blkno, len); + if (!bcp) { + ASSERT(0); + return false; + } + + if (--bcp->bc_refcount == 0) { + list_del(&bcp->bc_list); + kmem_free(bcp); + } + return true; +} + +/* log buffer item recovery */ + /* * Sort buffer items for log recovery. Most buffer items should end up on the * buffer list and are recovered first, with the following exceptions: diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index fa1b63bd9031..572e6707362a 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -55,17 +55,6 @@ STATIC int xlog_do_recovery_pass( struct xlog *, xfs_daddr_t, xfs_daddr_t, int, xfs_daddr_t *); -/* - * This structure is used during recovery to record the buf log items which - * have been canceled and should not be replayed. - */ -struct xfs_buf_cancel { - xfs_daddr_t bc_blkno; - uint bc_len; - int bc_refcount; - struct list_head bc_list; -}; - /* * Sector aligned buffer routines for buffer create/read/write/access */ @@ -1964,97 +1953,6 @@ xlog_recover_reorder_trans( return error; } -static struct xfs_buf_cancel * -xlog_find_buffer_cancelled( - struct xlog *log, - xfs_daddr_t blkno, - uint len) -{ - struct list_head *bucket; - struct xfs_buf_cancel *bcp; - - if (!log->l_buf_cancel_table) - return NULL; - - bucket = XLOG_BUF_CANCEL_BUCKET(log, blkno); - list_for_each_entry(bcp, bucket, bc_list) { - if (bcp->bc_blkno == blkno && bcp->bc_len == len) - return bcp; - } - - return NULL; -} - -bool -xlog_add_buffer_cancelled( - struct xlog *log, - xfs_daddr_t blkno, - uint len) -{ - struct xfs_buf_cancel *bcp; - - /* - * If we find an existing cancel record, this indicates that the buffer - * was cancelled multiple times. To ensure that during pass 2 we keep - * the record in the table until we reach its last occurrence in the - * log, a reference count is kept to tell how many times we expect to - * see this record during the second pass. - */ - bcp = xlog_find_buffer_cancelled(log, blkno, len); - if (bcp) { - bcp->bc_refcount++; - return false; - } - - bcp = kmem_alloc(sizeof(struct xfs_buf_cancel), 0); - bcp->bc_blkno = blkno; - bcp->bc_len = len; - bcp->bc_refcount = 1; - list_add_tail(&bcp->bc_list, XLOG_BUF_CANCEL_BUCKET(log, blkno)); - return true; -} - -/* - * Check if there is and entry for blkno, len in the buffer cancel record table. - */ -bool -xlog_is_buffer_cancelled( - struct xlog *log, - xfs_daddr_t blkno, - uint len) -{ - return xlog_find_buffer_cancelled(log, blkno, len) != NULL; -} - -/* - * Check if there is and entry for blkno, len in the buffer cancel record table, - * and decremented the reference count on it if there is one. - * - * Remove the cancel record once the refcount hits zero, so that if the same - * buffer is re-used again after its last cancellation we actually replay the - * changes made at that point. - */ -bool -xlog_put_buffer_cancelled( - struct xlog *log, - xfs_daddr_t blkno, - uint len) -{ - struct xfs_buf_cancel *bcp; - - bcp = xlog_find_buffer_cancelled(log, blkno, len); - if (!bcp) { - ASSERT(0); - return false; - } - - if (--bcp->bc_refcount == 0) { - list_del(&bcp->bc_list); - kmem_free(bcp); - } - return true; -} - void xlog_buf_readahead( struct xlog *log, From 6ea670ade207232d7863933bc6a834defe8b0269 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 1 May 2020 16:00:56 -0700 Subject: [PATCH 0297/1043] xfs: remove unnecessary includes from xfs_log_recover.c Remove unnecessary includes from the log recovery code. Suggested-by: Christoph Hellwig Signed-off-by: Darrick J. Wong Reviewed-by: Chandan Babu R --- fs/xfs/xfs_log_recover.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 572e6707362a..ec015df55b77 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -18,21 +18,13 @@ #include "xfs_log.h" #include "xfs_log_priv.h" #include "xfs_log_recover.h" -#include "xfs_inode_item.h" -#include "xfs_extfree_item.h" #include "xfs_trans_priv.h" #include "xfs_alloc.h" #include "xfs_ialloc.h" -#include "xfs_quota.h" #include "xfs_trace.h" #include "xfs_icache.h" -#include "xfs_bmap_btree.h" #include "xfs_error.h" -#include "xfs_dir2.h" -#include "xfs_rmap_item.h" #include "xfs_buf_item.h" -#include "xfs_refcount_item.h" -#include "xfs_bmap_item.h" #define BLK_AVG(blk1, blk2) ((blk1+blk2) >> 1) From 43dc0aa84ef74e3acd908b6d5eaa8ae15d7ecc8a Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Fri, 8 May 2020 08:50:52 -0700 Subject: [PATCH 0298/1043] xfs: fix unused variable warning in buffer completion on !DEBUG The random buffer write failure errortag patch introduced a local mount pointer variable for the test macro, but the macro is compiled out on !DEBUG kernels. This results in an unused variable warning. Access the mount structure through the buffer pointer and remove the local mount pointer to address the warning. Fixes: 7376d745473 ("xfs: random buffer write failure errortag") Signed-off-by: Brian Foster Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_buf.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 9d8841ac7375..9c2fbb6bbf89 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -1289,11 +1289,10 @@ xfs_buf_bio_end_io( struct bio *bio) { struct xfs_buf *bp = (struct xfs_buf *)bio->bi_private; - struct xfs_mount *mp = bp->b_mount; if (!bio->bi_status && (bp->b_flags & XBF_WRITE) && (bp->b_flags & XBF_ASYNC) && - XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BUF_IOERROR)) + XFS_TEST_ERROR(false, bp->b_mount, XFS_ERRTAG_BUF_IOERROR)) bio->bi_status = BLK_STS_IOERR; /* From 3d60548b216b5486a541a06c167df3dfb6df9c2e Mon Sep 17 00:00:00 2001 From: Chen Zhou Date: Fri, 8 May 2020 08:51:34 -0700 Subject: [PATCH 0299/1043] xfs: remove duplicate headers Remove duplicate headers which are included twice. Signed-off-by: Chen Zhou Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_xattr.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c index fc5d7276026e..bca48b308c02 100644 --- a/fs/xfs/xfs_xattr.c +++ b/fs/xfs/xfs_xattr.c @@ -12,7 +12,6 @@ #include "xfs_inode.h" #include "xfs_attr.h" #include "xfs_acl.h" -#include "xfs_da_format.h" #include "xfs_da_btree.h" #include From 904fbcb115c85090484dfdffaf7f461d96fe8e53 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 8 May 2020 21:27:24 -0600 Subject: [PATCH 0300/1043] io_uring: remove 'fd is io_uring' from close path The attempt protecting us from closing the ring itself wasn't really complete, and we actually don't need it. The referencing of requests themselve, and the references they hold on the ring, ensures that the life time of the ring is sane. With the check removed, we can also remove the need to have the close operation fget() the file. Reported-by: Al Viro Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 979d9f977409..9fd1257c8404 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -786,7 +786,6 @@ static const struct io_op_def io_op_defs[] = { .needs_fs = 1, }, [IORING_OP_CLOSE] = { - .needs_file = 1, .file_table = 1, }, [IORING_OP_FILES_UPDATE] = { @@ -3399,10 +3398,6 @@ static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EBADF; req->close.fd = READ_ONCE(sqe->fd); - if (req->file->f_op == &io_uring_fops || - req->close.fd == req->ctx->ring_fd) - return -EBADF; - return 0; } @@ -3434,8 +3429,11 @@ static int io_close(struct io_kiocb *req, bool force_nonblock) req->close.put_file = NULL; ret = __close_fd_get_file(req->close.fd, &req->close.put_file); - if (ret < 0) + if (ret < 0) { + if (ret == -ENOENT) + ret = -EBADF; return ret; + } /* if the file has a flush method, be safe and punt to async */ if (req->close.put_file->f_op->flush && force_nonblock) { From 7d01bd745a8f52ff2883f661235139ab6e7d23e6 Mon Sep 17 00:00:00 2001 From: Xiaoguang Wang Date: Fri, 8 May 2020 21:19:30 +0800 Subject: [PATCH 0301/1043] io_uring: remove obsolete 'state' parameter The "struct io_submit_state *state" parameter is not used, remove it. Signed-off-by: Xiaoguang Wang Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 9fd1257c8404..94870d63b16a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5629,7 +5629,7 @@ static inline void io_queue_link_head(struct io_kiocb *req) } static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, - struct io_submit_state *state, struct io_kiocb **link) + struct io_kiocb **link) { struct io_ring_ctx *ctx = req->ctx; int ret; @@ -5893,7 +5893,7 @@ fail_req: trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, true, async); - err = io_submit_sqe(req, sqe, statep, &link); + err = io_submit_sqe(req, sqe, &link); if (err) goto fail_req; } From f2a8d52e0a4db968c346c4332630a71cba377567 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Tue, 5 May 2020 16:04:30 +0200 Subject: [PATCH 0302/1043] nsproxy: add struct nsset Add a simple struct nsset. It holds all necessary pieces to switch to a new set of namespaces without leaving a task in a half-switched state which we will make use of in the next patch. This patch switches the existing setns logic over without causing a change in setns() behavior. This brings setns() closer to how unshare() works(). The prepare_ns() function is responsible to prepare all necessary information. This has two reasons. First it minimizes dependencies between individual namespaces, i.e. all install handler can expect that all fields are properly initialized independent in what order they are called in. Second, this makes the code easier to maintain and easier to follow if it needs to be changed. The prepare_ns() helper will only be switched over to use a flags argument in the next patch. Here it will still use nstype as a simple integer argument which was argued would be clearer. I'm not particularly opinionated about this if it really helps or not. The struct nsset itself already contains the flags field since its name already indicates that it can contain information required by different namespaces. None of this should have functional consequences. Signed-off-by: Christian Brauner Reviewed-by: Serge Hallyn Cc: Eric W. Biederman Cc: Serge Hallyn Cc: Jann Horn Cc: Michael Kerrisk Cc: Aleksa Sarai Link: https://lore.kernel.org/r/20200505140432.181565-2-christian.brauner@ubuntu.com --- fs/namespace.c | 10 ++-- include/linux/mnt_namespace.h | 1 + include/linux/nsproxy.h | 24 ++++++++++ include/linux/proc_ns.h | 4 +- ipc/namespace.c | 7 ++- kernel/cgroup/namespace.c | 5 +- kernel/nsproxy.c | 90 ++++++++++++++++++++++++++++++----- kernel/pid_namespace.c | 5 +- kernel/time/namespace.c | 5 +- kernel/user_namespace.c | 8 ++-- kernel/utsname.c | 5 +- net/core/net_namespace.c | 5 +- 12 files changed, 132 insertions(+), 37 deletions(-) diff --git a/fs/namespace.c b/fs/namespace.c index a28e4db075ed..62899fad4a04 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -3954,16 +3954,18 @@ static void mntns_put(struct ns_common *ns) put_mnt_ns(to_mnt_ns(ns)); } -static int mntns_install(struct nsproxy *nsproxy, struct ns_common *ns) +static int mntns_install(struct nsset *nsset, struct ns_common *ns) { - struct fs_struct *fs = current->fs; + struct nsproxy *nsproxy = nsset->nsproxy; + struct fs_struct *fs = nsset->fs; struct mnt_namespace *mnt_ns = to_mnt_ns(ns), *old_mnt_ns; + struct user_namespace *user_ns = nsset->cred->user_ns; struct path root; int err; if (!ns_capable(mnt_ns->user_ns, CAP_SYS_ADMIN) || - !ns_capable(current_user_ns(), CAP_SYS_CHROOT) || - !ns_capable(current_user_ns(), CAP_SYS_ADMIN)) + !ns_capable(user_ns, CAP_SYS_CHROOT) || + !ns_capable(user_ns, CAP_SYS_ADMIN)) return -EPERM; if (is_anon_ns(mnt_ns)) diff --git a/include/linux/mnt_namespace.h b/include/linux/mnt_namespace.h index 35942084cd40..007cfa52efb2 100644 --- a/include/linux/mnt_namespace.h +++ b/include/linux/mnt_namespace.h @@ -6,6 +6,7 @@ struct mnt_namespace; struct fs_struct; struct user_namespace; +struct ns_common; extern struct mnt_namespace *copy_mnt_ns(unsigned long, struct mnt_namespace *, struct user_namespace *, struct fs_struct *); diff --git a/include/linux/nsproxy.h b/include/linux/nsproxy.h index 074f395b9ad2..cdb171efc7cb 100644 --- a/include/linux/nsproxy.h +++ b/include/linux/nsproxy.h @@ -41,6 +41,30 @@ struct nsproxy { }; extern struct nsproxy init_nsproxy; +/* + * A structure to encompass all bits needed to install + * a partial or complete new set of namespaces. + * + * If a new user namespace is requested cred will + * point to a modifiable set of credentials. If a pointer + * to a modifiable set is needed nsset_cred() must be + * used and tested. + */ +struct nsset { + unsigned flags; + struct nsproxy *nsproxy; + struct fs_struct *fs; + const struct cred *cred; +}; + +static inline struct cred *nsset_cred(struct nsset *set) +{ + if (set->flags & CLONE_NEWUSER) + return (struct cred *)set->cred; + + return NULL; +} + /* * the namespaces access rules are: * diff --git a/include/linux/proc_ns.h b/include/linux/proc_ns.h index 6abe85c34681..75807ecef880 100644 --- a/include/linux/proc_ns.h +++ b/include/linux/proc_ns.h @@ -8,7 +8,7 @@ #include struct pid_namespace; -struct nsproxy; +struct nsset; struct path; struct task_struct; struct inode; @@ -19,7 +19,7 @@ struct proc_ns_operations { int type; struct ns_common *(*get)(struct task_struct *task); void (*put)(struct ns_common *ns); - int (*install)(struct nsproxy *nsproxy, struct ns_common *ns); + int (*install)(struct nsset *nsset, struct ns_common *ns); struct user_namespace *(*owner)(struct ns_common *ns); struct ns_common *(*get_parent)(struct ns_common *ns); } __randomize_layout; diff --git a/ipc/namespace.c b/ipc/namespace.c index b3ca1476ca51..fdc3b5f3f53a 100644 --- a/ipc/namespace.c +++ b/ipc/namespace.c @@ -177,15 +177,14 @@ static void ipcns_put(struct ns_common *ns) return put_ipc_ns(to_ipc_ns(ns)); } -static int ipcns_install(struct nsproxy *nsproxy, struct ns_common *new) +static int ipcns_install(struct nsset *nsset, struct ns_common *new) { + struct nsproxy *nsproxy = nsset->nsproxy; struct ipc_namespace *ns = to_ipc_ns(new); if (!ns_capable(ns->user_ns, CAP_SYS_ADMIN) || - !ns_capable(current_user_ns(), CAP_SYS_ADMIN)) + !ns_capable(nsset->cred->user_ns, CAP_SYS_ADMIN)) return -EPERM; - /* Ditch state from the old ipc namespace */ - exit_sem(current); put_ipc_ns(nsproxy->ipc_ns); nsproxy->ipc_ns = get_ipc_ns(ns); return 0; diff --git a/kernel/cgroup/namespace.c b/kernel/cgroup/namespace.c index b05f1dd58a62..812a61afd538 100644 --- a/kernel/cgroup/namespace.c +++ b/kernel/cgroup/namespace.c @@ -95,11 +95,12 @@ static inline struct cgroup_namespace *to_cg_ns(struct ns_common *ns) return container_of(ns, struct cgroup_namespace, ns); } -static int cgroupns_install(struct nsproxy *nsproxy, struct ns_common *ns) +static int cgroupns_install(struct nsset *nsset, struct ns_common *ns) { + struct nsproxy *nsproxy = nsset->nsproxy; struct cgroup_namespace *cgroup_ns = to_cg_ns(ns); - if (!ns_capable(current_user_ns(), CAP_SYS_ADMIN) || + if (!ns_capable(nsset->cred->user_ns, CAP_SYS_ADMIN) || !ns_capable(cgroup_ns->user_ns, CAP_SYS_ADMIN)) return -EPERM; diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c index ed9882108cd2..b7954fd60475 100644 --- a/kernel/nsproxy.c +++ b/kernel/nsproxy.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -257,12 +258,79 @@ void exit_task_namespaces(struct task_struct *p) switch_task_namespaces(p, NULL); } +static void put_nsset(struct nsset *nsset) +{ + unsigned flags = nsset->flags; + + if (flags & CLONE_NEWUSER) + put_cred(nsset_cred(nsset)); + if (nsset->nsproxy) + free_nsproxy(nsset->nsproxy); +} + +static int prepare_nsset(int nstype, struct nsset *nsset) +{ + struct task_struct *me = current; + + nsset->nsproxy = create_new_namespaces(0, me, current_user_ns(), me->fs); + if (IS_ERR(nsset->nsproxy)) + return PTR_ERR(nsset->nsproxy); + + if (nstype == CLONE_NEWUSER) + nsset->cred = prepare_creds(); + else + nsset->cred = current_cred(); + if (!nsset->cred) + goto out; + + if (nstype == CLONE_NEWNS) + nsset->fs = me->fs; + + nsset->flags = nstype; + return 0; + +out: + put_nsset(nsset); + return -ENOMEM; +} + +/* + * This is the point of no return. There are just a few namespaces + * that do some actual work here and it's sufficiently minimal that + * a separate ns_common operation seems unnecessary for now. + * Unshare is doing the same thing. If we'll end up needing to do + * more in a given namespace or a helper here is ultimately not + * exported anymore a simple commit handler for each namespace + * should be added to ns_common. + */ +static void commit_nsset(struct nsset *nsset) +{ + unsigned flags = nsset->flags; + struct task_struct *me = current; + +#ifdef CONFIG_USER_NS + if (flags & CLONE_NEWUSER) { + /* transfer ownership */ + commit_creds(nsset_cred(nsset)); + nsset->cred = NULL; + } +#endif + +#ifdef CONFIG_IPC_NS + if (flags & CLONE_NEWIPC) + exit_sem(me); +#endif + + /* transfer ownership */ + switch_task_namespaces(me, nsset->nsproxy); + nsset->nsproxy = NULL; +} + SYSCALL_DEFINE2(setns, int, fd, int, nstype) { - struct task_struct *tsk = current; - struct nsproxy *new_nsproxy; struct file *file; struct ns_common *ns; + struct nsset nsset = {}; int err; file = proc_ns_fget(fd); @@ -274,20 +342,16 @@ SYSCALL_DEFINE2(setns, int, fd, int, nstype) if (nstype && (ns->ops->type != nstype)) goto out; - new_nsproxy = create_new_namespaces(0, tsk, current_user_ns(), tsk->fs); - if (IS_ERR(new_nsproxy)) { - err = PTR_ERR(new_nsproxy); + err = prepare_nsset(ns->ops->type, &nsset); + if (err) goto out; - } - err = ns->ops->install(new_nsproxy, ns); - if (err) { - free_nsproxy(new_nsproxy); - goto out; + err = ns->ops->install(&nsset, ns); + if (!err) { + commit_nsset(&nsset); + perf_event_namespaces(current); } - switch_task_namespaces(tsk, new_nsproxy); - - perf_event_namespaces(tsk); + put_nsset(&nsset); out: fput(file); return err; diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c index 01f8ba32cc0c..11db2bdbb41e 100644 --- a/kernel/pid_namespace.c +++ b/kernel/pid_namespace.c @@ -378,13 +378,14 @@ static void pidns_put(struct ns_common *ns) put_pid_ns(to_pid_ns(ns)); } -static int pidns_install(struct nsproxy *nsproxy, struct ns_common *ns) +static int pidns_install(struct nsset *nsset, struct ns_common *ns) { + struct nsproxy *nsproxy = nsset->nsproxy; struct pid_namespace *active = task_active_pid_ns(current); struct pid_namespace *ancestor, *new = to_pid_ns(ns); if (!ns_capable(new->user_ns, CAP_SYS_ADMIN) || - !ns_capable(current_user_ns(), CAP_SYS_ADMIN)) + !ns_capable(nsset->cred->user_ns, CAP_SYS_ADMIN)) return -EPERM; /* diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c index 53bce347cd50..5d9fc22d836a 100644 --- a/kernel/time/namespace.c +++ b/kernel/time/namespace.c @@ -280,8 +280,9 @@ static void timens_put(struct ns_common *ns) put_time_ns(to_time_ns(ns)); } -static int timens_install(struct nsproxy *nsproxy, struct ns_common *new) +static int timens_install(struct nsset *nsset, struct ns_common *new) { + struct nsproxy *nsproxy = nsset->nsproxy; struct time_namespace *ns = to_time_ns(new); int err; @@ -289,7 +290,7 @@ static int timens_install(struct nsproxy *nsproxy, struct ns_common *new) return -EUSERS; if (!ns_capable(ns->user_ns, CAP_SYS_ADMIN) || - !ns_capable(current_user_ns(), CAP_SYS_ADMIN)) + !ns_capable(nsset->cred->user_ns, CAP_SYS_ADMIN)) return -EPERM; timens_set_vvar_page(current, ns); diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c index 8eadadc478f9..87804e0371fe 100644 --- a/kernel/user_namespace.c +++ b/kernel/user_namespace.c @@ -1253,7 +1253,7 @@ static void userns_put(struct ns_common *ns) put_user_ns(to_user_ns(ns)); } -static int userns_install(struct nsproxy *nsproxy, struct ns_common *ns) +static int userns_install(struct nsset *nsset, struct ns_common *ns) { struct user_namespace *user_ns = to_user_ns(ns); struct cred *cred; @@ -1274,14 +1274,14 @@ static int userns_install(struct nsproxy *nsproxy, struct ns_common *ns) if (!ns_capable(user_ns, CAP_SYS_ADMIN)) return -EPERM; - cred = prepare_creds(); + cred = nsset_cred(nsset); if (!cred) - return -ENOMEM; + return -EINVAL; put_user_ns(cred->user_ns); set_cred_user_ns(cred, get_user_ns(user_ns)); - return commit_creds(cred); + return 0; } struct ns_common *ns_get_owner(struct ns_common *ns) diff --git a/kernel/utsname.c b/kernel/utsname.c index f0e491193009..e488d0e2ab45 100644 --- a/kernel/utsname.c +++ b/kernel/utsname.c @@ -140,12 +140,13 @@ static void utsns_put(struct ns_common *ns) put_uts_ns(to_uts_ns(ns)); } -static int utsns_install(struct nsproxy *nsproxy, struct ns_common *new) +static int utsns_install(struct nsset *nsset, struct ns_common *new) { + struct nsproxy *nsproxy = nsset->nsproxy; struct uts_namespace *ns = to_uts_ns(new); if (!ns_capable(ns->user_ns, CAP_SYS_ADMIN) || - !ns_capable(current_user_ns(), CAP_SYS_ADMIN)) + !ns_capable(nsset->cred->user_ns, CAP_SYS_ADMIN)) return -EPERM; get_uts_ns(ns); diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index 190ca66a383b..dcd61aca343e 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -1353,12 +1353,13 @@ static void netns_put(struct ns_common *ns) put_net(to_net_ns(ns)); } -static int netns_install(struct nsproxy *nsproxy, struct ns_common *ns) +static int netns_install(struct nsset *nsset, struct ns_common *ns) { + struct nsproxy *nsproxy = nsset->nsproxy; struct net *net = to_net_ns(ns); if (!ns_capable(net->user_ns, CAP_SYS_ADMIN) || - !ns_capable(current_user_ns(), CAP_SYS_ADMIN)) + !ns_capable(nsset->cred->user_ns, CAP_SYS_ADMIN)) return -EPERM; put_net(nsproxy->net_ns); From 10760dde9be317a1abb426b2db9d6a698086cac9 Mon Sep 17 00:00:00 2001 From: Thomas Bogendoerfer Date: Mon, 20 Apr 2020 14:22:29 +0200 Subject: [PATCH 0303/1043] MIPS: Remove support for LASAT All LASAT has probably gone bad, so let's remove Linux support. Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kbuild.platforms | 1 - arch/mips/Kconfig | 21 -- arch/mips/Makefile | 7 - arch/mips/configs/lasat_defconfig | 55 ---- arch/mips/include/asm/lasat/ds1603.h | 19 -- arch/mips/include/asm/lasat/eeprom.h | 18 - arch/mips/include/asm/lasat/head.h | 23 -- arch/mips/include/asm/lasat/lasat.h | 245 -------------- arch/mips/include/asm/lasat/lasatint.h | 15 - arch/mips/include/asm/lasat/picvue.h | 16 - arch/mips/include/asm/lasat/serial.h | 14 - arch/mips/include/asm/mach-lasat/irq.h | 14 - .../include/asm/mach-lasat/mach-gt64120.h | 28 -- arch/mips/include/asm/nile4.h | 310 ------------------ arch/mips/lasat/Kconfig | 16 - arch/mips/lasat/Makefile | 15 - arch/mips/lasat/Platform | 7 - arch/mips/lasat/at93c.c | 148 --------- arch/mips/lasat/at93c.h | 19 -- arch/mips/lasat/ds1603.c | 190 ----------- arch/mips/lasat/ds1603.h | 32 -- arch/mips/lasat/image/Makefile | 53 --- arch/mips/lasat/image/head.S | 32 -- arch/mips/lasat/image/romscript.normal | 23 -- arch/mips/lasat/interrupt.c | 119 ------- arch/mips/lasat/lasat_board.c | 268 --------------- arch/mips/lasat/lasat_models.h | 68 ---- arch/mips/lasat/picvue.c | 242 -------------- arch/mips/lasat/picvue.h | 45 --- arch/mips/lasat/picvue_proc.c | 208 ------------ arch/mips/lasat/prom.c | 126 ------- arch/mips/lasat/prom.h | 8 - arch/mips/lasat/reset.c | 48 --- arch/mips/lasat/serial.c | 80 ----- arch/mips/lasat/setup.c | 141 -------- arch/mips/lasat/sysctl.c | 268 --------------- arch/mips/pci/Makefile | 2 - arch/mips/pci/ops-nile4.c | 136 -------- arch/mips/pci/pci-lasat.c | 88 ----- 39 files changed, 3168 deletions(-) delete mode 100644 arch/mips/configs/lasat_defconfig delete mode 100644 arch/mips/include/asm/lasat/ds1603.h delete mode 100644 arch/mips/include/asm/lasat/eeprom.h delete mode 100644 arch/mips/include/asm/lasat/head.h delete mode 100644 arch/mips/include/asm/lasat/lasat.h delete mode 100644 arch/mips/include/asm/lasat/lasatint.h delete mode 100644 arch/mips/include/asm/lasat/picvue.h delete mode 100644 arch/mips/include/asm/lasat/serial.h delete mode 100644 arch/mips/include/asm/mach-lasat/irq.h delete mode 100644 arch/mips/include/asm/mach-lasat/mach-gt64120.h delete mode 100644 arch/mips/include/asm/nile4.h delete mode 100644 arch/mips/lasat/Kconfig delete mode 100644 arch/mips/lasat/Makefile delete mode 100644 arch/mips/lasat/Platform delete mode 100644 arch/mips/lasat/at93c.c delete mode 100644 arch/mips/lasat/at93c.h delete mode 100644 arch/mips/lasat/ds1603.c delete mode 100644 arch/mips/lasat/ds1603.h delete mode 100644 arch/mips/lasat/image/Makefile delete mode 100644 arch/mips/lasat/image/head.S delete mode 100644 arch/mips/lasat/image/romscript.normal delete mode 100644 arch/mips/lasat/interrupt.c delete mode 100644 arch/mips/lasat/lasat_board.c delete mode 100644 arch/mips/lasat/lasat_models.h delete mode 100644 arch/mips/lasat/picvue.c delete mode 100644 arch/mips/lasat/picvue.h delete mode 100644 arch/mips/lasat/picvue_proc.c delete mode 100644 arch/mips/lasat/prom.c delete mode 100644 arch/mips/lasat/prom.h delete mode 100644 arch/mips/lasat/reset.c delete mode 100644 arch/mips/lasat/serial.c delete mode 100644 arch/mips/lasat/setup.c delete mode 100644 arch/mips/lasat/sysctl.c delete mode 100644 arch/mips/pci/ops-nile4.c delete mode 100644 arch/mips/pci/pci-lasat.c diff --git a/arch/mips/Kbuild.platforms b/arch/mips/Kbuild.platforms index a69b272a3ab0..9f9b4164d4c9 100644 --- a/arch/mips/Kbuild.platforms +++ b/arch/mips/Kbuild.platforms @@ -16,7 +16,6 @@ platforms += generic platforms += jazz platforms += jz4740 platforms += lantiq -platforms += lasat platforms += loongson2ef platforms += loongson32 platforms += loongson64 diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 9f15539a6342..0aaab09bdea6 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -431,23 +431,6 @@ config LANTIQ select ARCH_HAS_RESET_CONTROLLER select RESET_CONTROLLER -config LASAT - bool "LASAT Networks platforms" - select CEVT_R4K - select CRC32 - select CSRC_R4K - select DMA_NONCOHERENT - select SYS_HAS_EARLY_PRINTK - select HAVE_PCI - select IRQ_MIPS_CPU - select PCI_GT64XXX_PCI0 - select MIPS_NILE4 - select R5000_CPU_SCACHE - select SYS_HAS_CPU_R5000 - select SYS_SUPPORTS_32BIT_KERNEL - select SYS_SUPPORTS_64BIT_KERNEL if BROKEN - select SYS_SUPPORTS_LITTLE_ENDIAN - config MACH_LOONGSON32 bool "Loongson 32-bit family of machines" select SYS_SUPPORTS_ZBOOT @@ -1091,7 +1074,6 @@ source "arch/mips/generic/Kconfig" source "arch/mips/jazz/Kconfig" source "arch/mips/jz4740/Kconfig" source "arch/mips/lantiq/Kconfig" -source "arch/mips/lasat/Kconfig" source "arch/mips/pic32/Kconfig" source "arch/mips/pistachio/Kconfig" source "arch/mips/pmcs-msp71xx/Kconfig" @@ -1215,9 +1197,6 @@ config MIPS_BONITO64 config MIPS_MSC bool -config MIPS_NILE4 - bool - config SYNC_R4K bool diff --git a/arch/mips/Makefile b/arch/mips/Makefile index b6ee29e4565a..ef15caa4d320 100644 --- a/arch/mips/Makefile +++ b/arch/mips/Makefile @@ -370,12 +370,6 @@ ifeq ($(shell expr $(zload-y) \< 0xffffffff80000000 2> /dev/null), 0) bootz-y += uzImage.bin endif -ifdef CONFIG_LASAT -rom.bin rom.sw: vmlinux - $(Q)$(MAKE) $(build)=arch/mips/lasat/image \ - $(bootvars-y) $@ -endif - # # Some machines like the Indy need 32-bit ELF binaries for booting purposes. # Other need ECOFF, so we build a 32-bit ELF binary for them which we then @@ -441,7 +435,6 @@ archclean: $(Q)$(MAKE) $(clean)=arch/mips/boot $(Q)$(MAKE) $(clean)=arch/mips/boot/compressed $(Q)$(MAKE) $(clean)=arch/mips/boot/tools - $(Q)$(MAKE) $(clean)=arch/mips/lasat archheaders: $(Q)$(MAKE) $(build)=arch/mips/kernel/syscalls all diff --git a/arch/mips/configs/lasat_defconfig b/arch/mips/configs/lasat_defconfig deleted file mode 100644 index 00cf461db971..000000000000 --- a/arch/mips/configs/lasat_defconfig +++ /dev/null @@ -1,55 +0,0 @@ -CONFIG_SYSVIPC=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_EXPERT=y -# CONFIG_EPOLL is not set -# CONFIG_SIGNALFD is not set -# CONFIG_TIMERFD is not set -# CONFIG_EVENTFD is not set -# CONFIG_KALLSYMS is not set -CONFIG_SLAB=y -CONFIG_LASAT=y -CONFIG_PICVUE=y -CONFIG_PICVUE_PROC=y -CONFIG_DS1603=y -CONFIG_LASAT_SYSCTL=y -CONFIG_HZ_1000=y -# CONFIG_SECCOMP is not set -CONFIG_PCI=y -# CONFIG_BLK_DEV_BSG is not set -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_UNIX=y -CONFIG_INET=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set -# CONFIG_INET_DIAG is not set -# CONFIG_IPV6 is not set -CONFIG_MTD=y -CONFIG_MTD_BLOCK=y -CONFIG_MTD_CFI=y -CONFIG_MTD_CFI_AMDSTD=y -CONFIG_BLK_DEV_SD=y -CONFIG_ATA=y -CONFIG_PATA_CMD64X=y -CONFIG_ATA_GENERIC=y -CONFIG_PATA_LEGACY=y -CONFIG_NETDEVICES=y -CONFIG_PCNET32=y -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set -CONFIG_SERIO_RAW=y -# CONFIG_VT is not set -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -# CONFIG_SERIAL_8250_PCI is not set -# CONFIG_HW_RANDOM is not set -# CONFIG_HWMON is not set -# CONFIG_USB_SUPPORT is not set -CONFIG_EXT2_FS=y -CONFIG_EXT3_FS=y -# CONFIG_DNOTIFY is not set -CONFIG_PROC_KCORE=y -CONFIG_TMPFS=y -CONFIG_CONFIGFS_FS=y -CONFIG_MAGIC_SYSRQ=y diff --git a/arch/mips/include/asm/lasat/ds1603.h b/arch/mips/include/asm/lasat/ds1603.h deleted file mode 100644 index ab833be9637d..000000000000 --- a/arch/mips/include/asm/lasat/ds1603.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include - -/* Lasat 100 */ -#define DS1603_REG_100 (KSEG1ADDR(0x1c810000)) -#define DS1603_RST_100 (1 << 2) -#define DS1603_CLK_100 (1 << 0) -#define DS1603_DATA_SHIFT_100 1 -#define DS1603_DATA_100 (1 << DS1603_DATA_SHIFT_100) - -/* Lasat 200 */ -#define DS1603_REG_200 (KSEG1ADDR(0x11000000)) -#define DS1603_RST_200 (1 << 3) -#define DS1603_CLK_200 (1 << 4) -#define DS1603_DATA_200 (1 << 5) - -#define DS1603_DATA_REG_200 (DS1603_REG_200 + 0x10000) -#define DS1603_DATA_READ_SHIFT_200 9 -#define DS1603_DATA_READ_200 (1 << DS1603_DATA_READ_SHIFT_200) diff --git a/arch/mips/include/asm/lasat/eeprom.h b/arch/mips/include/asm/lasat/eeprom.h deleted file mode 100644 index 24001a5cbb11..000000000000 --- a/arch/mips/include/asm/lasat/eeprom.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include - -/* lasat 100 */ -#define AT93C_REG_100 KSEG1ADDR(0x1c810000) -#define AT93C_RDATA_REG_100 AT93C_REG_100 -#define AT93C_RDATA_SHIFT_100 4 -#define AT93C_WDATA_SHIFT_100 4 -#define AT93C_CS_M_100 (1 << 5) -#define AT93C_CLK_M_100 (1 << 3) - -/* lasat 200 */ -#define AT93C_REG_200 KSEG1ADDR(0x11000000) -#define AT93C_RDATA_REG_200 (AT93C_REG_200+0x10000) -#define AT93C_RDATA_SHIFT_200 8 -#define AT93C_WDATA_SHIFT_200 2 -#define AT93C_CS_M_200 (1 << 0) -#define AT93C_CLK_M_200 (1 << 1) diff --git a/arch/mips/include/asm/lasat/head.h b/arch/mips/include/asm/lasat/head.h deleted file mode 100644 index 20b0ecedd4b5..000000000000 --- a/arch/mips/include/asm/lasat/head.h +++ /dev/null @@ -1,23 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Image header stuff - */ -#ifndef _HEAD_H -#define _HEAD_H - -#define LASAT_K_MAGIC0_VAL 0xfedeabba -#define LASAT_K_MAGIC1_VAL 0x00bedead - -#ifndef _LANGUAGE_ASSEMBLY -#include -struct bootloader_header { - u32 magic[2]; - u32 version; - u32 image_start; - u32 image_size; - u32 kernel_start; - u32 kernel_entry; -}; -#endif - -#endif /* _HEAD_H */ diff --git a/arch/mips/include/asm/lasat/lasat.h b/arch/mips/include/asm/lasat/lasat.h deleted file mode 100644 index 483be606960d..000000000000 --- a/arch/mips/include/asm/lasat/lasat.h +++ /dev/null @@ -1,245 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * lasat.h - * - * Thomas Horsten - * Copyright (C) 2000 LASAT Networks A/S. - * - * Configuration for LASAT boards, loads the appropriate include files. - */ -#ifndef _LASAT_H -#define _LASAT_H - -#ifndef _LANGUAGE_ASSEMBLY - -extern struct lasat_misc { - volatile u32 *reset_reg; - volatile u32 *flash_wp_reg; - u32 flash_wp_bit; -} *lasat_misc; - -enum lasat_mtdparts { - LASAT_MTD_BOOTLOADER, - LASAT_MTD_SERVICE, - LASAT_MTD_NORMAL, - LASAT_MTD_CONFIG, - LASAT_MTD_FS, - LASAT_MTD_LAST -}; - -/* - * The format of the data record in the EEPROM. - * See the LASAT Hardware Configuration field specification for a detailed - * description of the config field. - */ -#include - -#define LASAT_EEPROM_VERSION 7 -struct lasat_eeprom_struct { - unsigned int version; - unsigned int cfg[3]; - unsigned char hwaddr[6]; - unsigned char print_partno[12]; - unsigned char term0; - unsigned char print_serial[14]; - unsigned char term1; - unsigned char prod_partno[12]; - unsigned char term2; - unsigned char prod_serial[14]; - unsigned char term3; - unsigned char passwd_hash[16]; - unsigned char pwdnull; - unsigned char vendid; - unsigned char ts_ref; - unsigned char ts_signoff; - unsigned char reserved[11]; - unsigned char debugaccess; - unsigned short prid; - unsigned int serviceflag; - unsigned int ipaddr; - unsigned int netmask; - unsigned int crc32; -}; - -struct lasat_eeprom_struct_pre7 { - unsigned int version; - unsigned int flags[3]; - unsigned char hwaddr0[6]; - unsigned char hwaddr1[6]; - unsigned char print_partno[9]; - unsigned char term0; - unsigned char print_serial[14]; - unsigned char term1; - unsigned char prod_partno[9]; - unsigned char term2; - unsigned char prod_serial[14]; - unsigned char term3; - unsigned char passwd_hash[24]; - unsigned char pwdnull; - unsigned char vendor; - unsigned char ts_ref; - unsigned char ts_signoff; - unsigned char reserved[6]; - unsigned int writecount; - unsigned int ipaddr; - unsigned int netmask; - unsigned int crc32; -}; - -/* Configuration descriptor encoding - see the doc for details */ - -#define LASAT_W0_DSCTYPE(v) (((v)) & 0xf) -#define LASAT_W0_BMID(v) (((v) >> 0x04) & 0xf) -#define LASAT_W0_CPUTYPE(v) (((v) >> 0x08) & 0xf) -#define LASAT_W0_BUSSPEED(v) (((v) >> 0x0c) & 0xf) -#define LASAT_W0_CPUCLK(v) (((v) >> 0x10) & 0xf) -#define LASAT_W0_SDRAMBANKSZ(v) (((v) >> 0x14) & 0xf) -#define LASAT_W0_SDRAMBANKS(v) (((v) >> 0x18) & 0xf) -#define LASAT_W0_L2CACHE(v) (((v) >> 0x1c) & 0xf) - -#define LASAT_W1_EDHAC(v) (((v)) & 0xf) -#define LASAT_W1_HIFN(v) (((v) >> 0x04) & 0x1) -#define LASAT_W1_ISDN(v) (((v) >> 0x05) & 0x1) -#define LASAT_W1_IDE(v) (((v) >> 0x06) & 0x1) -#define LASAT_W1_HDLC(v) (((v) >> 0x07) & 0x1) -#define LASAT_W1_USVERSION(v) (((v) >> 0x08) & 0x1) -#define LASAT_W1_4MACS(v) (((v) >> 0x09) & 0x1) -#define LASAT_W1_EXTSERIAL(v) (((v) >> 0x0a) & 0x1) -#define LASAT_W1_FLASHSIZE(v) (((v) >> 0x0c) & 0xf) -#define LASAT_W1_PCISLOTS(v) (((v) >> 0x10) & 0xf) -#define LASAT_W1_PCI1OPT(v) (((v) >> 0x14) & 0xf) -#define LASAT_W1_PCI2OPT(v) (((v) >> 0x18) & 0xf) -#define LASAT_W1_PCI3OPT(v) (((v) >> 0x1c) & 0xf) - -/* Routines specific to LASAT boards */ - -#define LASAT_BMID_MASQUERADE2 0 -#define LASAT_BMID_MASQUERADEPRO 1 -#define LASAT_BMID_SAFEPIPE25 2 -#define LASAT_BMID_SAFEPIPE50 3 -#define LASAT_BMID_SAFEPIPE100 4 -#define LASAT_BMID_SAFEPIPE5000 5 -#define LASAT_BMID_SAFEPIPE7000 6 -#define LASAT_BMID_SAFEPIPE1000 7 -#if 0 -#define LASAT_BMID_SAFEPIPE30 7 -#define LASAT_BMID_SAFEPIPE5100 8 -#define LASAT_BMID_SAFEPIPE7100 9 -#endif -#define LASAT_BMID_UNKNOWN 0xf -#define LASAT_MAX_BMID_NAMES 9 /* no larger than 15! */ - -#define LASAT_HAS_EDHAC (1 << 0) -#define LASAT_EDHAC_FAST (1 << 1) -#define LASAT_HAS_EADI (1 << 2) -#define LASAT_HAS_HIFN (1 << 3) -#define LASAT_HAS_ISDN (1 << 4) -#define LASAT_HAS_LEASEDLINE_IF (1 << 5) -#define LASAT_HAS_HDC (1 << 6) - -#define LASAT_PRID_MASQUERADE2 0 -#define LASAT_PRID_MASQUERADEPRO 1 -#define LASAT_PRID_SAFEPIPE25 2 -#define LASAT_PRID_SAFEPIPE50 3 -#define LASAT_PRID_SAFEPIPE100 4 -#define LASAT_PRID_SAFEPIPE5000 5 -#define LASAT_PRID_SAFEPIPE7000 6 -#define LASAT_PRID_SAFEPIPE30 7 -#define LASAT_PRID_SAFEPIPE5100 8 -#define LASAT_PRID_SAFEPIPE7100 9 - -#define LASAT_PRID_SAFEPIPE1110 10 -#define LASAT_PRID_SAFEPIPE3020 11 -#define LASAT_PRID_SAFEPIPE3030 12 -#define LASAT_PRID_SAFEPIPE5020 13 -#define LASAT_PRID_SAFEPIPE5030 14 -#define LASAT_PRID_SAFEPIPE1120 15 -#define LASAT_PRID_SAFEPIPE1130 16 -#define LASAT_PRID_SAFEPIPE6010 17 -#define LASAT_PRID_SAFEPIPE6110 18 -#define LASAT_PRID_SAFEPIPE6210 19 -#define LASAT_PRID_SAFEPIPE1020 20 -#define LASAT_PRID_SAFEPIPE1040 21 -#define LASAT_PRID_SAFEPIPE1060 22 - -struct lasat_info { - unsigned int li_cpu_hz; - unsigned int li_bus_hz; - unsigned int li_bmid; - unsigned int li_memsize; - unsigned int li_flash_size; - unsigned int li_prid; - unsigned char li_bmstr[16]; - unsigned char li_namestr[32]; - unsigned char li_typestr[16]; - /* Info on the Flash layout */ - unsigned int li_flash_base; - unsigned long li_flashpart_base[LASAT_MTD_LAST]; - unsigned long li_flashpart_size[LASAT_MTD_LAST]; - struct lasat_eeprom_struct li_eeprom_info; - unsigned int li_eeprom_upgrade_version; - unsigned int li_debugaccess; -}; - -extern struct lasat_info lasat_board_info; - -static inline unsigned long lasat_flash_partition_start(int partno) -{ - if (partno < 0 || partno >= LASAT_MTD_LAST) - return 0; - - return lasat_board_info.li_flashpart_base[partno]; -} - -static inline unsigned long lasat_flash_partition_size(int partno) -{ - if (partno < 0 || partno >= LASAT_MTD_LAST) - return 0; - - return lasat_board_info.li_flashpart_size[partno]; -} - -/* Called from setup() to initialize the global board_info struct */ -extern int lasat_init_board_info(void); - -/* Write the modified EEPROM info struct */ -extern void lasat_write_eeprom_info(void); - -#define N_MACHTYPES 2 -/* for calibration of delays */ - -/* the lasat_ndelay function is necessary because it is used at an - * early stage of the boot process where ndelay is not calibrated. - * It is used for the bit-banging rtc and eeprom drivers */ - -#include -#include - -/* calculating with the slowest board with 100 MHz clock */ -#define LASAT_100_DIVIDER 20 -/* All 200's run at 250 MHz clock */ -#define LASAT_200_DIVIDER 8 - -extern unsigned int lasat_ndelay_divider; - -static inline void lasat_ndelay(unsigned int ns) -{ - __delay(ns / lasat_ndelay_divider); -} - -#define IS_LASAT_200() (current_cpu_data.cputype == CPU_R5000) - -#endif /* !defined (_LANGUAGE_ASSEMBLY) */ - -#define LASAT_SERVICEMODE_MAGIC_1 0xdeadbeef -#define LASAT_SERVICEMODE_MAGIC_2 0xfedeabba - -/* Lasat 100 boards */ -#define LASAT_GT_BASE (KSEG1ADDR(0x14000000)) - -/* Lasat 200 boards */ -#define Vrc5074_PHYS_BASE 0x1fa00000 -#define Vrc5074_BASE (KSEG1ADDR(Vrc5074_PHYS_BASE)) -#define PCI_WINDOW1 0x1a000000 - -#endif /* _LASAT_H */ diff --git a/arch/mips/include/asm/lasat/lasatint.h b/arch/mips/include/asm/lasat/lasatint.h deleted file mode 100644 index b2b346e0ca38..000000000000 --- a/arch/mips/include/asm/lasat/lasatint.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __ASM_LASAT_LASATINT_H -#define __ASM_LASAT_LASATINT_H - -/* lasat 100 */ -#define LASAT_INT_STATUS_REG_100 (KSEG1ADDR(0x1c880000)) -#define LASAT_INT_MASK_REG_100 (KSEG1ADDR(0x1c890000)) -#define LASATINT_MASK_SHIFT_100 0 - -/* lasat 200 */ -#define LASAT_INT_STATUS_REG_200 (KSEG1ADDR(0x1104003c)) -#define LASAT_INT_MASK_REG_200 (KSEG1ADDR(0x1104003c)) -#define LASATINT_MASK_SHIFT_200 16 - -#endif /* __ASM_LASAT_LASATINT_H */ diff --git a/arch/mips/include/asm/lasat/picvue.h b/arch/mips/include/asm/lasat/picvue.h deleted file mode 100644 index 99987c5a4b83..000000000000 --- a/arch/mips/include/asm/lasat/picvue.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Lasat 100 */ -#define PVC_REG_100 KSEG1ADDR(0x1c820000) -#define PVC_DATA_SHIFT_100 0 -#define PVC_DATA_M_100 0xFF -#define PVC_E_100 (1 << 8) -#define PVC_RW_100 (1 << 9) -#define PVC_RS_100 (1 << 10) - -/* Lasat 200 */ -#define PVC_REG_200 KSEG1ADDR(0x11000000) -#define PVC_DATA_SHIFT_200 24 -#define PVC_DATA_M_200 (0xFF << PVC_DATA_SHIFT_200) -#define PVC_E_200 (1 << 16) -#define PVC_RW_200 (1 << 17) -#define PVC_RS_200 (1 << 18) diff --git a/arch/mips/include/asm/lasat/serial.h b/arch/mips/include/asm/lasat/serial.h deleted file mode 100644 index 7b43d74089d1..000000000000 --- a/arch/mips/include/asm/lasat/serial.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include - -/* Lasat 100 boards serial configuration */ -#define LASAT_BASE_BAUD_100 (7372800 / 16) -#define LASAT_UART_REGS_BASE_100 0x1c8b0000 -#define LASAT_UART_REGS_SHIFT_100 2 -#define LASATINT_UART_100 16 - -/* * LASAT 200 boards serial configuration */ -#define LASAT_BASE_BAUD_200 (100000000 / 16 / 12) -#define LASAT_UART_REGS_BASE_200 (Vrc5074_PHYS_BASE + 0x0300) -#define LASAT_UART_REGS_SHIFT_200 3 -#define LASATINT_UART_200 21 diff --git a/arch/mips/include/asm/mach-lasat/irq.h b/arch/mips/include/asm/mach-lasat/irq.h deleted file mode 100644 index e8994921779e..000000000000 --- a/arch/mips/include/asm/mach-lasat/irq.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_MACH_LASAT_IRQ_H -#define _ASM_MACH_LASAT_IRQ_H - -#define LASAT_CASCADE_IRQ (MIPS_CPU_IRQ_BASE + 2) - -#define LASAT_IRQ_BASE 8 -#define LASAT_IRQ_END 23 - -#define NR_IRQS 24 - -#include - -#endif /* _ASM_MACH_LASAT_IRQ_H */ diff --git a/arch/mips/include/asm/mach-lasat/mach-gt64120.h b/arch/mips/include/asm/mach-lasat/mach-gt64120.h deleted file mode 100644 index 6666a8871a23..000000000000 --- a/arch/mips/include/asm/mach-lasat/mach-gt64120.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * This is a direct copy of the ev96100.h file, with a global - * search and replace. The numbers are the same. - * - * The reason I'm duplicating this is so that the 64120/96100 - * defines won't be confusing in the source code. - */ -#ifndef _ASM_GT64120_LASAT_GT64120_DEP_H -#define _ASM_GT64120_LASAT_GT64120_DEP_H - -/* - * GT64120 config space base address on Lasat 100 - */ -#define GT64120_BASE (KSEG1ADDR(0x14000000)) - -/* - * PCI Bus allocation - * - * (Guessing ...) - */ -#define GT_PCI_MEM_BASE 0x12000000UL -#define GT_PCI_MEM_SIZE 0x02000000UL -#define GT_PCI_IO_BASE 0x10000000UL -#define GT_PCI_IO_SIZE 0x02000000UL -#define GT_ISA_IO_BASE PCI_IO_BASE - -#endif /* _ASM_GT64120_LASAT_GT64120_DEP_H */ diff --git a/arch/mips/include/asm/nile4.h b/arch/mips/include/asm/nile4.h deleted file mode 100644 index 9d36b7823603..000000000000 --- a/arch/mips/include/asm/nile4.h +++ /dev/null @@ -1,310 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * asm-mips/nile4.h -- NEC Vrc-5074 Nile 4 definitions - * - * Copyright (C) 2000 Geert Uytterhoeven - * Sony Software Development Center Europe (SDCE), Brussels - * - * This file is based on the following documentation: - * - * NEC Vrc 5074 System Controller Data Sheet, June 1998 - */ - -#ifndef _ASM_NILE4_H -#define _ASM_NILE4_H - -#define NILE4_BASE 0xbfa00000 -#define NILE4_SIZE 0x00200000 /* 2 MB */ - - - /* - * Physical Device Address Registers (PDARs) - */ - -#define NILE4_SDRAM0 0x0000 /* SDRAM Bank 0 [R/W] */ -#define NILE4_SDRAM1 0x0008 /* SDRAM Bank 1 [R/W] */ -#define NILE4_DCS2 0x0010 /* Device Chip-Select 2 [R/W] */ -#define NILE4_DCS3 0x0018 /* Device Chip-Select 3 [R/W] */ -#define NILE4_DCS4 0x0020 /* Device Chip-Select 4 [R/W] */ -#define NILE4_DCS5 0x0028 /* Device Chip-Select 5 [R/W] */ -#define NILE4_DCS6 0x0030 /* Device Chip-Select 6 [R/W] */ -#define NILE4_DCS7 0x0038 /* Device Chip-Select 7 [R/W] */ -#define NILE4_DCS8 0x0040 /* Device Chip-Select 8 [R/W] */ -#define NILE4_PCIW0 0x0060 /* PCI Address Window 0 [R/W] */ -#define NILE4_PCIW1 0x0068 /* PCI Address Window 1 [R/W] */ -#define NILE4_INTCS 0x0070 /* Controller Internal Registers and Devices */ - /* [R/W] */ -#define NILE4_BOOTCS 0x0078 /* Boot ROM Chip-Select [R/W] */ - - - /* - * CPU Interface Registers - */ - -#define NILE4_CPUSTAT 0x0080 /* CPU Status [R/W] */ -#define NILE4_INTCTRL 0x0088 /* Interrupt Control [R/W] */ -#define NILE4_INTSTAT0 0x0090 /* Interrupt Status 0 [R] */ -#define NILE4_INTSTAT1 0x0098 /* Interrupt Status 1 and CPU Interrupt */ - /* Enable [R/W] */ -#define NILE4_INTCLR 0x00A0 /* Interrupt Clear [R/W] */ -#define NILE4_INTPPES 0x00A8 /* PCI Interrupt Control [R/W] */ - - - /* - * Memory-Interface Registers - */ - -#define NILE4_MEMCTRL 0x00C0 /* Memory Control */ -#define NILE4_ACSTIME 0x00C8 /* Memory Access Timing [R/W] */ -#define NILE4_CHKERR 0x00D0 /* Memory Check Error Status [R] */ - - - /* - * PCI-Bus Registers - */ - -#define NILE4_PCICTRL 0x00E0 /* PCI Control [R/W] */ -#define NILE4_PCIARB 0x00E8 /* PCI Arbiter [R/W] */ -#define NILE4_PCIINIT0 0x00F0 /* PCI Master (Initiator) 0 [R/W] */ -#define NILE4_PCIINIT1 0x00F8 /* PCI Master (Initiator) 1 [R/W] */ -#define NILE4_PCIERR 0x00B8 /* PCI Error [R/W] */ - - - /* - * Local-Bus Registers - */ - -#define NILE4_LCNFG 0x0100 /* Local Bus Configuration [R/W] */ -#define NILE4_LCST2 0x0110 /* Local Bus Chip-Select Timing 2 [R/W] */ -#define NILE4_LCST3 0x0118 /* Local Bus Chip-Select Timing 3 [R/W] */ -#define NILE4_LCST4 0x0120 /* Local Bus Chip-Select Timing 4 [R/W] */ -#define NILE4_LCST5 0x0128 /* Local Bus Chip-Select Timing 5 [R/W] */ -#define NILE4_LCST6 0x0130 /* Local Bus Chip-Select Timing 6 [R/W] */ -#define NILE4_LCST7 0x0138 /* Local Bus Chip-Select Timing 7 [R/W] */ -#define NILE4_LCST8 0x0140 /* Local Bus Chip-Select Timing 8 [R/W] */ -#define NILE4_DCSFN 0x0150 /* Device Chip-Select Muxing and Output */ - /* Enables [R/W] */ -#define NILE4_DCSIO 0x0158 /* Device Chip-Selects As I/O Bits [R/W] */ -#define NILE4_BCST 0x0178 /* Local Boot Chip-Select Timing [R/W] */ - - - /* - * DMA Registers - */ - -#define NILE4_DMACTRL0 0x0180 /* DMA Control 0 [R/W] */ -#define NILE4_DMASRCA0 0x0188 /* DMA Source Address 0 [R/W] */ -#define NILE4_DMADESA0 0x0190 /* DMA Destination Address 0 [R/W] */ -#define NILE4_DMACTRL1 0x0198 /* DMA Control 1 [R/W] */ -#define NILE4_DMASRCA1 0x01A0 /* DMA Source Address 1 [R/W] */ -#define NILE4_DMADESA1 0x01A8 /* DMA Destination Address 1 [R/W] */ - - - /* - * Timer Registers - */ - -#define NILE4_T0CTRL 0x01C0 /* SDRAM Refresh Control [R/W] */ -#define NILE4_T0CNTR 0x01C8 /* SDRAM Refresh Counter [R/W] */ -#define NILE4_T1CTRL 0x01D0 /* CPU-Bus Read Time-Out Control [R/W] */ -#define NILE4_T1CNTR 0x01D8 /* CPU-Bus Read Time-Out Counter [R/W] */ -#define NILE4_T2CTRL 0x01E0 /* General-Purpose Timer Control [R/W] */ -#define NILE4_T2CNTR 0x01E8 /* General-Purpose Timer Counter [R/W] */ -#define NILE4_T3CTRL 0x01F0 /* Watchdog Timer Control [R/W] */ -#define NILE4_T3CNTR 0x01F8 /* Watchdog Timer Counter [R/W] */ - - - /* - * PCI Configuration Space Registers - */ - -#define NILE4_PCI_BASE 0x0200 - -#define NILE4_VID 0x0200 /* PCI Vendor ID [R] */ -#define NILE4_DID 0x0202 /* PCI Device ID [R] */ -#define NILE4_PCICMD 0x0204 /* PCI Command [R/W] */ -#define NILE4_PCISTS 0x0206 /* PCI Status [R/W] */ -#define NILE4_REVID 0x0208 /* PCI Revision ID [R] */ -#define NILE4_CLASS 0x0209 /* PCI Class Code [R] */ -#define NILE4_CLSIZ 0x020C /* PCI Cache Line Size [R/W] */ -#define NILE4_MLTIM 0x020D /* PCI Latency Timer [R/W] */ -#define NILE4_HTYPE 0x020E /* PCI Header Type [R] */ -#define NILE4_BIST 0x020F /* BIST [R] (unimplemented) */ -#define NILE4_BARC 0x0210 /* PCI Base Address Register Control [R/W] */ -#define NILE4_BAR0 0x0218 /* PCI Base Address Register 0 [R/W] */ -#define NILE4_BAR1 0x0220 /* PCI Base Address Register 1 [R/W] */ -#define NILE4_CIS 0x0228 /* PCI Cardbus CIS Pointer [R] */ - /* (unimplemented) */ -#define NILE4_SSVID 0x022C /* PCI Sub-System Vendor ID [R/W] */ -#define NILE4_SSID 0x022E /* PCI Sub-System ID [R/W] */ -#define NILE4_ROM 0x0230 /* Expansion ROM Base Address [R] */ - /* (unimplemented) */ -#define NILE4_INTLIN 0x023C /* PCI Interrupt Line [R/W] */ -#define NILE4_INTPIN 0x023D /* PCI Interrupt Pin [R] */ -#define NILE4_MINGNT 0x023E /* PCI Min_Gnt [R] (unimplemented) */ -#define NILE4_MAXLAT 0x023F /* PCI Max_Lat [R] (unimplemented) */ -#define NILE4_BAR2 0x0240 /* PCI Base Address Register 2 [R/W] */ -#define NILE4_BAR3 0x0248 /* PCI Base Address Register 3 [R/W] */ -#define NILE4_BAR4 0x0250 /* PCI Base Address Register 4 [R/W] */ -#define NILE4_BAR5 0x0258 /* PCI Base Address Register 5 [R/W] */ -#define NILE4_BAR6 0x0260 /* PCI Base Address Register 6 [R/W] */ -#define NILE4_BAR7 0x0268 /* PCI Base Address Register 7 [R/W] */ -#define NILE4_BAR8 0x0270 /* PCI Base Address Register 8 [R/W] */ -#define NILE4_BARB 0x0278 /* PCI Base Address Register BOOT [R/W] */ - - - /* - * Serial-Port Registers - */ - -#define NILE4_UART_BASE 0x0300 - -#define NILE4_UARTRBR 0x0300 /* UART Receiver Data Buffer [R] */ -#define NILE4_UARTTHR 0x0300 /* UART Transmitter Data Holding [W] */ -#define NILE4_UARTIER 0x0308 /* UART Interrupt Enable [R/W] */ -#define NILE4_UARTDLL 0x0300 /* UART Divisor Latch LSB [R/W] */ -#define NILE4_UARTDLM 0x0308 /* UART Divisor Latch MSB [R/W] */ -#define NILE4_UARTIIR 0x0310 /* UART Interrupt ID [R] */ -#define NILE4_UARTFCR 0x0310 /* UART FIFO Control [W] */ -#define NILE4_UARTLCR 0x0318 /* UART Line Control [R/W] */ -#define NILE4_UARTMCR 0x0320 /* UART Modem Control [R/W] */ -#define NILE4_UARTLSR 0x0328 /* UART Line Status [R/W] */ -#define NILE4_UARTMSR 0x0330 /* UART Modem Status [R/W] */ -#define NILE4_UARTSCR 0x0338 /* UART Scratch [R/W] */ - -#define NILE4_UART_BASE_BAUD 520833 /* 100 MHz / 12 / 16 */ - - - /* - * Interrupt Lines - */ - -#define NILE4_INT_CPCE 0 /* CPU-Interface Parity-Error Interrupt */ -#define NILE4_INT_CNTD 1 /* CPU No-Target Decode Interrupt */ -#define NILE4_INT_MCE 2 /* Memory-Check Error Interrupt */ -#define NILE4_INT_DMA 3 /* DMA Controller Interrupt */ -#define NILE4_INT_UART 4 /* UART Interrupt */ -#define NILE4_INT_WDOG 5 /* Watchdog Timer Interrupt */ -#define NILE4_INT_GPT 6 /* General-Purpose Timer Interrupt */ -#define NILE4_INT_LBRTD 7 /* Local-Bus Ready Timer Interrupt */ -#define NILE4_INT_INTA 8 /* PCI Interrupt Signal INTA# */ -#define NILE4_INT_INTB 9 /* PCI Interrupt Signal INTB# */ -#define NILE4_INT_INTC 10 /* PCI Interrupt Signal INTC# */ -#define NILE4_INT_INTD 11 /* PCI Interrupt Signal INTD# */ -#define NILE4_INT_INTE 12 /* PCI Interrupt Signal INTE# (ISA cascade) */ -#define NILE4_INT_RESV 13 /* Reserved */ -#define NILE4_INT_PCIS 14 /* PCI SERR# Interrupt */ -#define NILE4_INT_PCIE 15 /* PCI Internal Error Interrupt */ - - - /* - * Nile 4 Register Access - */ - -static inline void nile4_sync(void) -{ - volatile u32 *p = (volatile u32 *)0xbfc00000; - (void)(*p); -} - -static inline void nile4_out32(u32 offset, u32 val) -{ - *(volatile u32 *)(NILE4_BASE+offset) = val; - nile4_sync(); -} - -static inline u32 nile4_in32(u32 offset) -{ - u32 val = *(volatile u32 *)(NILE4_BASE+offset); - nile4_sync(); - return val; -} - -static inline void nile4_out16(u32 offset, u16 val) -{ - *(volatile u16 *)(NILE4_BASE+offset) = val; - nile4_sync(); -} - -static inline u16 nile4_in16(u32 offset) -{ - u16 val = *(volatile u16 *)(NILE4_BASE+offset); - nile4_sync(); - return val; -} - -static inline void nile4_out8(u32 offset, u8 val) -{ - *(volatile u8 *)(NILE4_BASE+offset) = val; - nile4_sync(); -} - -static inline u8 nile4_in8(u32 offset) -{ - u8 val = *(volatile u8 *)(NILE4_BASE+offset); - nile4_sync(); - return val; -} - - - /* - * Physical Device Address Registers - */ - -extern void nile4_set_pdar(u32 pdar, u32 phys, u32 size, int width, - int on_memory_bus, int visible); - - - /* - * PCI Master Registers - */ - -#define NILE4_PCICMD_IACK 0 /* PCI Interrupt Acknowledge */ -#define NILE4_PCICMD_IO 1 /* PCI I/O Space */ -#define NILE4_PCICMD_MEM 3 /* PCI Memory Space */ -#define NILE4_PCICMD_CFG 5 /* PCI Configuration Space */ - - - /* - * PCI Address Spaces - * - * Note that these are multiplexed using PCIINIT[01]! - */ - -#define NILE4_PCI_IO_BASE 0xa6000000 -#define NILE4_PCI_MEM_BASE 0xa8000000 -#define NILE4_PCI_CFG_BASE NILE4_PCI_MEM_BASE -#define NILE4_PCI_IACK_BASE NILE4_PCI_IO_BASE - - -extern void nile4_set_pmr(u32 pmr, u32 type, u32 addr); - - - /* - * Interrupt Programming - */ - -#define NUM_I8259_INTERRUPTS 16 -#define NUM_NILE4_INTERRUPTS 16 - -#define IRQ_I8259_CASCADE NILE4_INT_INTE -#define is_i8259_irq(irq) ((irq) < NUM_I8259_INTERRUPTS) -#define nile4_to_irq(n) ((n)+NUM_I8259_INTERRUPTS) -#define irq_to_nile4(n) ((n)-NUM_I8259_INTERRUPTS) - -extern void nile4_map_irq(int nile4_irq, int cpu_irq); -extern void nile4_map_irq_all(int cpu_irq); -extern void nile4_enable_irq(unsigned int nile4_irq); -extern void nile4_disable_irq(unsigned int nile4_irq); -extern void nile4_disable_irq_all(void); -extern u16 nile4_get_irq_stat(int cpu_irq); -extern void nile4_enable_irq_output(int cpu_irq); -extern void nile4_disable_irq_output(int cpu_irq); -extern void nile4_set_pci_irq_polarity(int pci_irq, int high); -extern void nile4_set_pci_irq_level_or_edge(int pci_irq, int level); -extern void nile4_clear_irq(int nile4_irq); -extern void nile4_clear_irq_mask(u32 mask); -extern u8 nile4_i8259_iack(void); -extern void nile4_dump_irq_status(void); /* Debug */ - -#endif diff --git a/arch/mips/lasat/Kconfig b/arch/mips/lasat/Kconfig deleted file mode 100644 index 11b89e94b835..000000000000 --- a/arch/mips/lasat/Kconfig +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -config PICVUE - tristate "PICVUE LCD display driver" - depends on LASAT - -config PICVUE_PROC - tristate "PICVUE LCD display driver /proc interface" - depends on PICVUE && PROC_FS - -config DS1603 - bool "DS1603 RTC driver" - depends on LASAT - -config LASAT_SYSCTL - bool "LASAT sysctl interface" - depends on LASAT diff --git a/arch/mips/lasat/Makefile b/arch/mips/lasat/Makefile deleted file mode 100644 index 1789b227ef20..000000000000 --- a/arch/mips/lasat/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for the LASAT specific kernel interface routines under Linux. -# - -obj-y += reset.o setup.o prom.o lasat_board.o \ - at93c.o interrupt.o serial.o - -obj-$(CONFIG_LASAT_SYSCTL) += sysctl.o -obj-$(CONFIG_DS1603) += ds1603.o -obj-$(CONFIG_PICVUE) += picvue.o -obj-$(CONFIG_PICVUE_PROC) += picvue_proc.o - -clean: - make -C image clean diff --git a/arch/mips/lasat/Platform b/arch/mips/lasat/Platform deleted file mode 100644 index 760252828bf1..000000000000 --- a/arch/mips/lasat/Platform +++ /dev/null @@ -1,7 +0,0 @@ -# -# LASAT platforms -# -platform-$(CONFIG_LASAT) += lasat/ -cflags-$(CONFIG_LASAT) += \ - -I$(srctree)/arch/mips/include/asm/mach-lasat -load-$(CONFIG_LASAT) += 0xffffffff80000000 diff --git a/arch/mips/lasat/at93c.c b/arch/mips/lasat/at93c.c deleted file mode 100644 index f895fe94b937..000000000000 --- a/arch/mips/lasat/at93c.c +++ /dev/null @@ -1,148 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Atmel AT93C46 serial eeprom driver - * - * Brian Murphy - * - */ -#include -#include -#include - -#include "at93c.h" - -#define AT93C_ADDR_SHIFT 7 -#define AT93C_ADDR_MAX ((1 << AT93C_ADDR_SHIFT) - 1) -#define AT93C_RCMD (0x6 << AT93C_ADDR_SHIFT) -#define AT93C_WCMD (0x5 << AT93C_ADDR_SHIFT) -#define AT93C_WENCMD 0x260 -#define AT93C_WDSCMD 0x200 - -struct at93c_defs *at93c; - -static void at93c_reg_write(u32 val) -{ - *at93c->reg = val; -} - -static u32 at93c_reg_read(void) -{ - u32 tmp = *at93c->reg; - return tmp; -} - -static u32 at93c_datareg_read(void) -{ - u32 tmp = *at93c->rdata_reg; - return tmp; -} - -static void at93c_cycle_clk(u32 data) -{ - at93c_reg_write(data | at93c->clk); - lasat_ndelay(250); - at93c_reg_write(data & ~at93c->clk); - lasat_ndelay(250); -} - -static void at93c_write_databit(u8 bit) -{ - u32 data = at93c_reg_read(); - if (bit) - data |= 1 << at93c->wdata_shift; - else - data &= ~(1 << at93c->wdata_shift); - - at93c_reg_write(data); - lasat_ndelay(100); - at93c_cycle_clk(data); -} - -static unsigned int at93c_read_databit(void) -{ - u32 data; - - at93c_cycle_clk(at93c_reg_read()); - data = (at93c_datareg_read() >> at93c->rdata_shift) & 1; - return data; -} - -static u8 at93c_read_byte(void) -{ - int i; - u8 data = 0; - - for (i = 0; i <= 7; i++) { - data <<= 1; - data |= at93c_read_databit(); - } - return data; -} - -static void at93c_write_bits(u32 data, int size) -{ - int i; - int shift = size - 1; - u32 mask = (1 << shift); - - for (i = 0; i < size; i++) { - at93c_write_databit((data & mask) >> shift); - data <<= 1; - } -} - -static void at93c_init_op(void) -{ - at93c_reg_write((at93c_reg_read() | at93c->cs) & - ~at93c->clk & ~(1 << at93c->rdata_shift)); - lasat_ndelay(50); -} - -static void at93c_end_op(void) -{ - at93c_reg_write(at93c_reg_read() & ~at93c->cs); - lasat_ndelay(250); -} - -static void at93c_wait(void) -{ - at93c_init_op(); - while (!at93c_read_databit()) - ; - at93c_end_op(); -}; - -static void at93c_disable_wp(void) -{ - at93c_init_op(); - at93c_write_bits(AT93C_WENCMD, 10); - at93c_end_op(); -} - -static void at93c_enable_wp(void) -{ - at93c_init_op(); - at93c_write_bits(AT93C_WDSCMD, 10); - at93c_end_op(); -} - -u8 at93c_read(u8 addr) -{ - u8 byte; - at93c_init_op(); - at93c_write_bits((addr & AT93C_ADDR_MAX)|AT93C_RCMD, 10); - byte = at93c_read_byte(); - at93c_end_op(); - return byte; -} - -void at93c_write(u8 addr, u8 data) -{ - at93c_disable_wp(); - at93c_init_op(); - at93c_write_bits((addr & AT93C_ADDR_MAX)|AT93C_WCMD, 10); - at93c_write_bits(data, 8); - at93c_end_op(); - at93c_wait(); - at93c_enable_wp(); -} diff --git a/arch/mips/lasat/at93c.h b/arch/mips/lasat/at93c.h deleted file mode 100644 index 7a99a02d81d0..000000000000 --- a/arch/mips/lasat/at93c.h +++ /dev/null @@ -1,19 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Atmel AT93C46 serial eeprom driver - * - * Brian Murphy - * - */ - -extern struct at93c_defs { - volatile u32 *reg; - volatile u32 *rdata_reg; - int rdata_shift; - int wdata_shift; - u32 cs; - u32 clk; -} *at93c; - -u8 at93c_read(u8 addr); -void at93c_write(u8 addr, u8 data); diff --git a/arch/mips/lasat/ds1603.c b/arch/mips/lasat/ds1603.c deleted file mode 100644 index e6ce39fefa78..000000000000 --- a/arch/mips/lasat/ds1603.c +++ /dev/null @@ -1,190 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Dallas Semiconductors 1603 RTC driver - * - * Brian Murphy - * - */ -#include -#include -#include -#include -#include - -#include "ds1603.h" - -#define READ_TIME_CMD 0x81 -#define SET_TIME_CMD 0x80 -#define TRIMMER_SET_CMD 0xC0 -#define TRIMMER_VALUE_MASK 0x38 -#define TRIMMER_SHIFT 3 - -struct ds_defs *ds1603; - -/* HW specific register functions */ -static void rtc_reg_write(unsigned long val) -{ - *ds1603->reg = val; -} - -static unsigned long rtc_reg_read(void) -{ - unsigned long tmp = *ds1603->reg; - return tmp; -} - -static unsigned long rtc_datareg_read(void) -{ - unsigned long tmp = *ds1603->data_reg; - return tmp; -} - -static void rtc_nrst_high(void) -{ - rtc_reg_write(rtc_reg_read() | ds1603->rst); -} - -static void rtc_nrst_low(void) -{ - rtc_reg_write(rtc_reg_read() & ~ds1603->rst); -} - -static void rtc_cycle_clock(unsigned long data) -{ - data |= ds1603->clk; - rtc_reg_write(data); - lasat_ndelay(250); - if (ds1603->data_reversed) - data &= ~ds1603->data; - else - data |= ds1603->data; - data &= ~ds1603->clk; - rtc_reg_write(data); - lasat_ndelay(250 + ds1603->huge_delay); -} - -static void rtc_write_databit(unsigned int bit) -{ - unsigned long data = rtc_reg_read(); - if (ds1603->data_reversed) - bit = !bit; - if (bit) - data |= ds1603->data; - else - data &= ~ds1603->data; - - rtc_reg_write(data); - lasat_ndelay(50 + ds1603->huge_delay); - rtc_cycle_clock(data); -} - -static unsigned int rtc_read_databit(void) -{ - unsigned int data; - - data = (rtc_datareg_read() & (1 << ds1603->data_read_shift)) - >> ds1603->data_read_shift; - rtc_cycle_clock(rtc_reg_read()); - return data; -} - -static void rtc_write_byte(unsigned int byte) -{ - int i; - - for (i = 0; i <= 7; i++) { - rtc_write_databit(byte & 1L); - byte >>= 1; - } -} - -static void rtc_write_word(unsigned long word) -{ - int i; - - for (i = 0; i <= 31; i++) { - rtc_write_databit(word & 1L); - word >>= 1; - } -} - -static unsigned long rtc_read_word(void) -{ - int i; - unsigned long word = 0; - unsigned long shift = 0; - - for (i = 0; i <= 31; i++) { - word |= rtc_read_databit() << shift; - shift++; - } - return word; -} - -static void rtc_init_op(void) -{ - rtc_nrst_high(); - - rtc_reg_write(rtc_reg_read() & ~ds1603->clk); - - lasat_ndelay(50); -} - -static void rtc_end_op(void) -{ - rtc_nrst_low(); - lasat_ndelay(1000); -} - -void read_persistent_clock64(struct timespec64 *ts) -{ - unsigned long word; - unsigned long flags; - - spin_lock_irqsave(&rtc_lock, flags); - rtc_init_op(); - rtc_write_byte(READ_TIME_CMD); - word = rtc_read_word(); - rtc_end_op(); - spin_unlock_irqrestore(&rtc_lock, flags); - - ts->tv_sec = word; - ts->tv_nsec = 0; -} - -int update_persistent_clock64(struct timespec64 now) -{ - time64_t time = now.tv_sec; - unsigned long flags; - - spin_lock_irqsave(&rtc_lock, flags); - rtc_init_op(); - rtc_write_byte(SET_TIME_CMD); - /* - * Due to the hardware limitation, we cast to 'unsigned long' type, - * so it will overflow in year 2106 on 32-bit machine. - */ - rtc_write_word((unsigned long)time); - rtc_end_op(); - spin_unlock_irqrestore(&rtc_lock, flags); - - return 0; -} - -void ds1603_set_trimmer(unsigned int trimval) -{ - rtc_init_op(); - rtc_write_byte(((trimval << TRIMMER_SHIFT) & TRIMMER_VALUE_MASK) - | (TRIMMER_SET_CMD)); - rtc_end_op(); -} - -void ds1603_disable(void) -{ - ds1603_set_trimmer(TRIMMER_DISABLE_RTC); -} - -void ds1603_enable(void) -{ - ds1603_set_trimmer(TRIMMER_DEFAULT); -} diff --git a/arch/mips/lasat/ds1603.h b/arch/mips/lasat/ds1603.h deleted file mode 100644 index 00987d3bdc21..000000000000 --- a/arch/mips/lasat/ds1603.h +++ /dev/null @@ -1,32 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Dallas Semiconductors 1603 RTC driver - * - * Brian Murphy - * - */ -#ifndef __DS1603_H -#define __DS1603_H - -struct ds_defs { - volatile u32 *reg; - volatile u32 *data_reg; - u32 rst; - u32 clk; - u32 data; - u32 data_read_shift; - char data_reversed; - u32 huge_delay; -}; - -extern struct ds_defs *ds1603; - -void ds1603_set_trimmer(unsigned int); -void ds1603_enable(void); -void ds1603_disable(void); -void ds1603_init(struct ds_defs *); - -#define TRIMMER_DEFAULT 3 -#define TRIMMER_DISABLE_RTC 0 - -#endif diff --git a/arch/mips/lasat/image/Makefile b/arch/mips/lasat/image/Makefile deleted file mode 100644 index 78ce4cff1012..000000000000 --- a/arch/mips/lasat/image/Makefile +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# MAKEFILE FOR THE MIPS LINUX BOOTLOADER AND ROM DEBUGGER -# -# i-data Networks -# -# Author: Thomas Horsten -# - -ifndef Version - Version = "$(USER)-test" -endif - -MKLASATIMG = mklasatimg -MKLASATIMG_ARCH = mq2,mqpro,sp100,sp200 -KERNEL_IMAGE = vmlinux - -LDSCRIPT= -L$(srctree)/$(src) -Tromscript.normal - -HEAD_DEFINES := -D_kernel_start=$(VMLINUX_LOAD_ADDRESS) \ - -D_kernel_entry=$(VMLINUX_ENTRY_ADDRESS) \ - -D VERSION="\"$(Version)\"" \ - -D TIMESTAMP=$(shell date +%s) - -$(obj)/head.o: $(obj)/head.S $(KERNEL_IMAGE) - $(CC) -fno-pic $(HEAD_DEFINES) $(LINUXINCLUDE) -c -o $@ $< - -OBJECTS = head.o kImage.o - -rom.sw: $(obj)/rom.sw -rom.bin: $(obj)/rom.bin - -$(obj)/rom.sw: $(obj)/rom.bin - $(MKLASATIMG) -o $@ -k $^ -m $(MKLASATIMG_ARCH) - -$(obj)/rom.bin: $(obj)/rom - $(OBJCOPY) -O binary -S $^ $@ - -# Rule to make the bootloader -$(obj)/rom: $(addprefix $(obj)/,$(OBJECTS)) - $(LD) $(KBUILD_LDFLAGS) $(LDSCRIPT) -o $@ $^ - -$(obj)/%.o: $(obj)/%.gz - $(LD) -r -o $@ -b binary $< - -$(obj)/%.gz: $(obj)/%.bin - gzip -cf -9 $< > $@ - -$(obj)/kImage.bin: $(KERNEL_IMAGE) - $(OBJCOPY) -O binary -S $^ $@ - -clean: - rm -f rom rom.bin rom.sw kImage.bin kImage.o diff --git a/arch/mips/lasat/image/head.S b/arch/mips/lasat/image/head.S deleted file mode 100644 index 1a27312d4c2e..000000000000 --- a/arch/mips/lasat/image/head.S +++ /dev/null @@ -1,32 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include - - .text - .section .text..start, "ax" - .set noreorder - .set mips3 - - /* Magic words identifying a software image */ - .word LASAT_K_MAGIC0_VAL - .word LASAT_K_MAGIC1_VAL - - /* Image header version */ - .word 0x00000002 - - /* image start and size */ - .word _image_start - .word _image_size - - /* start of kernel and entrypoint in uncompressed image */ - .word _kernel_start - .word _kernel_entry - - /* Here we have room for future flags */ - - .org 0x40 -reldate: - .word TIMESTAMP - - .org 0x50 -release: - .string VERSION diff --git a/arch/mips/lasat/image/romscript.normal b/arch/mips/lasat/image/romscript.normal deleted file mode 100644 index 0864c963e188..000000000000 --- a/arch/mips/lasat/image/romscript.normal +++ /dev/null @@ -1,23 +0,0 @@ -OUTPUT_ARCH(mips) - -SECTIONS -{ - .text : - { - *(.text..start) - } - - /* Data in ROM */ - - .data ALIGN(0x10) : - { - *(.data) - } - _image_start = ADDR(.data); - _image_size = SIZEOF(.data); - - .other : - { - *(.*) - } -} diff --git a/arch/mips/lasat/interrupt.c b/arch/mips/lasat/interrupt.c deleted file mode 100644 index 7965bbd0d319..000000000000 --- a/arch/mips/lasat/interrupt.c +++ /dev/null @@ -1,119 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Carsten Langgaard, carstenl@mips.com - * Copyright (C) 1999,2000 MIPS Technologies, Inc. All rights reserved. - * - * Routines for generic manipulation of the interrupts found on the - * Lasat boards. - */ -#include -#include -#include - -#include -#include -#include - -#include - -static volatile int *lasat_int_status; -static volatile int *lasat_int_mask; -static volatile int lasat_int_mask_shift; - -void disable_lasat_irq(struct irq_data *d) -{ - unsigned int irq_nr = d->irq - LASAT_IRQ_BASE; - - *lasat_int_mask &= ~(1 << irq_nr) << lasat_int_mask_shift; -} - -void enable_lasat_irq(struct irq_data *d) -{ - unsigned int irq_nr = d->irq - LASAT_IRQ_BASE; - - *lasat_int_mask |= (1 << irq_nr) << lasat_int_mask_shift; -} - -static struct irq_chip lasat_irq_type = { - .name = "Lasat", - .irq_mask = disable_lasat_irq, - .irq_unmask = enable_lasat_irq, -}; - -static inline int ls1bit32(unsigned int x) -{ - int b = 31, s; - - s = 16; if (x << 16 == 0) s = 0; b -= s; x <<= s; - s = 8; if (x << 8 == 0) s = 0; b -= s; x <<= s; - s = 4; if (x << 4 == 0) s = 0; b -= s; x <<= s; - s = 2; if (x << 2 == 0) s = 0; b -= s; x <<= s; - s = 1; if (x << 1 == 0) s = 0; b -= s; - - return b; -} - -static unsigned long (*get_int_status)(void); - -static unsigned long get_int_status_100(void) -{ - return *lasat_int_status & *lasat_int_mask; -} - -static unsigned long get_int_status_200(void) -{ - unsigned long int_status; - - int_status = *lasat_int_status; - int_status &= (int_status >> LASATINT_MASK_SHIFT_200) & 0xffff; - return int_status; -} - -asmlinkage void plat_irq_dispatch(void) -{ - unsigned long int_status; - unsigned int cause = read_c0_cause(); - int irq; - - if (cause & CAUSEF_IP7) { /* R4000 count / compare IRQ */ - do_IRQ(7); - return; - } - - int_status = get_int_status(); - - /* if int_status == 0, then the interrupt has already been cleared */ - if (int_status) { - irq = LASAT_IRQ_BASE + ls1bit32(int_status); - - do_IRQ(irq); - } -} - -void __init arch_init_irq(void) -{ - int irq = LASAT_CASCADE_IRQ; - int i; - - if (IS_LASAT_200()) { - lasat_int_status = (void *)LASAT_INT_STATUS_REG_200; - lasat_int_mask = (void *)LASAT_INT_MASK_REG_200; - lasat_int_mask_shift = LASATINT_MASK_SHIFT_200; - get_int_status = get_int_status_200; - *lasat_int_mask &= 0xffff; - } else { - lasat_int_status = (void *)LASAT_INT_STATUS_REG_100; - lasat_int_mask = (void *)LASAT_INT_MASK_REG_100; - lasat_int_mask_shift = LASATINT_MASK_SHIFT_100; - get_int_status = get_int_status_100; - *lasat_int_mask = 0; - } - - mips_cpu_irq_init(); - - for (i = LASAT_IRQ_BASE; i <= LASAT_IRQ_END; i++) - irq_set_chip_and_handler(i, &lasat_irq_type, handle_level_irq); - - if (request_irq(irq, no_action, IRQF_NO_THREAD, "cascade", NULL)) - pr_err("Failed to request irq %d (cascade)\n", irq); -} diff --git a/arch/mips/lasat/lasat_board.c b/arch/mips/lasat/lasat_board.c deleted file mode 100644 index 80e1ba541148..000000000000 --- a/arch/mips/lasat/lasat_board.c +++ /dev/null @@ -1,268 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Thomas Horsten - * Copyright (C) 2000 LASAT Networks A/S. - * - * Routines specific to the LASAT boards - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include "at93c.h" -/* New model description table */ -#include "lasat_models.h" - -static DEFINE_MUTEX(lasat_eeprom_mutex); - -#define EEPROM_CRC(data, len) (~crc32(~0, data, len)) - -struct lasat_info lasat_board_info; - -int EEPROMRead(unsigned int pos, unsigned char *data, int len) -{ - int i; - - for (i = 0; i < len; i++) - *data++ = at93c_read(pos++); - - return 0; -} - -int EEPROMWrite(unsigned int pos, unsigned char *data, int len) -{ - int i; - - for (i = 0; i < len; i++) - at93c_write(pos++, *data++); - - return 0; -} - -static void init_flash_sizes(void) -{ - unsigned long *lb = lasat_board_info.li_flashpart_base; - unsigned long *ls = lasat_board_info.li_flashpart_size; - int i; - - ls[LASAT_MTD_BOOTLOADER] = 0x40000; - ls[LASAT_MTD_SERVICE] = 0xC0000; - ls[LASAT_MTD_NORMAL] = 0x100000; - - if (!IS_LASAT_200()) { - lasat_board_info.li_flash_base = 0x1e000000; - - lb[LASAT_MTD_BOOTLOADER] = 0x1e400000; - - if (lasat_board_info.li_flash_size > 0x200000) { - ls[LASAT_MTD_CONFIG] = 0x100000; - ls[LASAT_MTD_FS] = 0x500000; - } - } else { - lasat_board_info.li_flash_base = 0x10000000; - - if (lasat_board_info.li_flash_size < 0x1000000) { - lb[LASAT_MTD_BOOTLOADER] = 0x10000000; - ls[LASAT_MTD_CONFIG] = 0x100000; - if (lasat_board_info.li_flash_size >= 0x400000) - ls[LASAT_MTD_FS] = - lasat_board_info.li_flash_size - 0x300000; - } - } - - for (i = 1; i < LASAT_MTD_LAST; i++) - lb[i] = lb[i-1] + ls[i-1]; -} - -int lasat_init_board_info(void) -{ - int c; - unsigned long crc; - unsigned long cfg0, cfg1; - const struct product_info *ppi; - int i_n_base_models = N_BASE_MODELS; - const char * const * i_txt_base_models = txt_base_models; - int i_n_prids = N_PRIDS; - - memset(&lasat_board_info, 0, sizeof(lasat_board_info)); - - /* First read the EEPROM info */ - EEPROMRead(0, (unsigned char *)&lasat_board_info.li_eeprom_info, - sizeof(struct lasat_eeprom_struct)); - - /* Check the CRC */ - crc = EEPROM_CRC((unsigned char *)(&lasat_board_info.li_eeprom_info), - sizeof(struct lasat_eeprom_struct) - 4); - - if (crc != lasat_board_info.li_eeprom_info.crc32) { - printk(KERN_WARNING "WARNING...\nWARNING...\nEEPROM CRC does " - "not match calculated, attempting to soldier on...\n"); - } - - if (lasat_board_info.li_eeprom_info.version != LASAT_EEPROM_VERSION) { - printk(KERN_WARNING "WARNING...\nWARNING...\nEEPROM version " - "%d, wanted version %d, attempting to soldier on...\n", - (unsigned int)lasat_board_info.li_eeprom_info.version, - LASAT_EEPROM_VERSION); - } - - cfg0 = lasat_board_info.li_eeprom_info.cfg[0]; - cfg1 = lasat_board_info.li_eeprom_info.cfg[1]; - - if (LASAT_W0_DSCTYPE(cfg0) != 1) { - printk(KERN_WARNING "WARNING...\nWARNING...\n" - "Invalid configuration read from EEPROM, attempting to " - "soldier on..."); - } - /* We have a valid configuration */ - - switch (LASAT_W0_SDRAMBANKSZ(cfg0)) { - case 0: - lasat_board_info.li_memsize = 0x0800000; - break; - case 1: - lasat_board_info.li_memsize = 0x1000000; - break; - case 2: - lasat_board_info.li_memsize = 0x2000000; - break; - case 3: - lasat_board_info.li_memsize = 0x4000000; - break; - case 4: - lasat_board_info.li_memsize = 0x8000000; - break; - default: - lasat_board_info.li_memsize = 0; - } - - switch (LASAT_W0_SDRAMBANKS(cfg0)) { - case 0: - break; - case 1: - lasat_board_info.li_memsize *= 2; - break; - default: - break; - } - - switch (LASAT_W0_BUSSPEED(cfg0)) { - case 0x0: - lasat_board_info.li_bus_hz = 60000000; - break; - case 0x1: - lasat_board_info.li_bus_hz = 66000000; - break; - case 0x2: - lasat_board_info.li_bus_hz = 66666667; - break; - case 0x3: - lasat_board_info.li_bus_hz = 80000000; - break; - case 0x4: - lasat_board_info.li_bus_hz = 83333333; - break; - case 0x5: - lasat_board_info.li_bus_hz = 100000000; - break; - } - - switch (LASAT_W0_CPUCLK(cfg0)) { - case 0x0: - lasat_board_info.li_cpu_hz = - lasat_board_info.li_bus_hz; - break; - case 0x1: - lasat_board_info.li_cpu_hz = - lasat_board_info.li_bus_hz + - (lasat_board_info.li_bus_hz >> 1); - break; - case 0x2: - lasat_board_info.li_cpu_hz = - lasat_board_info.li_bus_hz + - lasat_board_info.li_bus_hz; - break; - case 0x3: - lasat_board_info.li_cpu_hz = - lasat_board_info.li_bus_hz + - lasat_board_info.li_bus_hz + - (lasat_board_info.li_bus_hz >> 1); - break; - case 0x4: - lasat_board_info.li_cpu_hz = - lasat_board_info.li_bus_hz + - lasat_board_info.li_bus_hz + - lasat_board_info.li_bus_hz; - break; - } - - /* Flash size */ - switch (LASAT_W1_FLASHSIZE(cfg1)) { - case 0: - lasat_board_info.li_flash_size = 0x200000; - break; - case 1: - lasat_board_info.li_flash_size = 0x400000; - break; - case 2: - lasat_board_info.li_flash_size = 0x800000; - break; - case 3: - lasat_board_info.li_flash_size = 0x1000000; - break; - case 4: - lasat_board_info.li_flash_size = 0x2000000; - break; - } - - init_flash_sizes(); - - lasat_board_info.li_bmid = LASAT_W0_BMID(cfg0); - lasat_board_info.li_prid = lasat_board_info.li_eeprom_info.prid; - if (lasat_board_info.li_prid == 0xffff || lasat_board_info.li_prid == 0) - lasat_board_info.li_prid = lasat_board_info.li_bmid; - - /* Base model stuff */ - if (lasat_board_info.li_bmid > i_n_base_models) - lasat_board_info.li_bmid = i_n_base_models; - strcpy(lasat_board_info.li_bmstr, - i_txt_base_models[lasat_board_info.li_bmid]); - - /* Product ID dependent values */ - c = lasat_board_info.li_prid; - if (c >= i_n_prids) { - strcpy(lasat_board_info.li_namestr, "Unknown Model"); - strcpy(lasat_board_info.li_typestr, "Unknown Type"); - } else { - ppi = &vendor_info_table[0].vi_product_info[c]; - strcpy(lasat_board_info.li_namestr, ppi->pi_name); - if (ppi->pi_type) - strcpy(lasat_board_info.li_typestr, ppi->pi_type); - else - sprintf(lasat_board_info.li_typestr, "%d", 10 * c); - } - - return 0; -} - -void lasat_write_eeprom_info(void) -{ - unsigned long crc; - - mutex_lock(&lasat_eeprom_mutex); - - /* Generate the CRC */ - crc = EEPROM_CRC((unsigned char *)(&lasat_board_info.li_eeprom_info), - sizeof(struct lasat_eeprom_struct) - 4); - lasat_board_info.li_eeprom_info.crc32 = crc; - - /* Write the EEPROM info */ - EEPROMWrite(0, (unsigned char *)&lasat_board_info.li_eeprom_info, - sizeof(struct lasat_eeprom_struct)); - - mutex_unlock(&lasat_eeprom_mutex); -} diff --git a/arch/mips/lasat/lasat_models.h b/arch/mips/lasat/lasat_models.h deleted file mode 100644 index 474e57342484..000000000000 --- a/arch/mips/lasat/lasat_models.h +++ /dev/null @@ -1,68 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Model description tables - */ -#include - -struct product_info { - const char *pi_name; - const char *pi_type; -}; - -struct vendor_info { - const char *vi_name; - const struct product_info *vi_product_info; -}; - -/* - * Base models - */ -static const char * const txt_base_models[] = { - "MQ 2", "MQ Pro", "SP 25", "SP 50", "SP 100", "SP 5000", "SP 7000", - "SP 1000", "Unknown" -}; -#define N_BASE_MODELS (ARRAY_SIZE(txt_base_models) - 1) - -/* - * Eicon Networks - */ -static const char txt_en_mq[] = "Masquerade"; -static const char txt_en_sp[] = "Safepipe"; - -static const struct product_info product_info_eicon[] = { - { txt_en_mq, "II" }, /* 0 */ - { txt_en_mq, "Pro" }, /* 1 */ - { txt_en_sp, "25" }, /* 2 */ - { txt_en_sp, "50" }, /* 3 */ - { txt_en_sp, "100" }, /* 4 */ - { txt_en_sp, "5000" }, /* 5 */ - { txt_en_sp, "7000" }, /* 6 */ - { txt_en_sp, "30" }, /* 7 */ - { txt_en_sp, "5100" }, /* 8 */ - { txt_en_sp, "7100" }, /* 9 */ - { txt_en_sp, "1110" }, /* 10 */ - { txt_en_sp, "3020" }, /* 11 */ - { txt_en_sp, "3030" }, /* 12 */ - { txt_en_sp, "5020" }, /* 13 */ - { txt_en_sp, "5030" }, /* 14 */ - { txt_en_sp, "1120" }, /* 15 */ - { txt_en_sp, "1130" }, /* 16 */ - { txt_en_sp, "6010" }, /* 17 */ - { txt_en_sp, "6110" }, /* 18 */ - { txt_en_sp, "6210" }, /* 19 */ - { txt_en_sp, "1020" }, /* 20 */ - { txt_en_sp, "1040" }, /* 21 */ - { txt_en_sp, "1050" }, /* 22 */ - { txt_en_sp, "1060" }, /* 23 */ -}; - -#define N_PRIDS ARRAY_SIZE(product_info_eicon) - -/* - * The vendor table - */ -static struct vendor_info const vendor_info_table[] = { - { "Eicon Networks", product_info_eicon }, -}; - -#define N_VENDORS ARRAY_SIZE(vendor_info_table) diff --git a/arch/mips/lasat/picvue.c b/arch/mips/lasat/picvue.c deleted file mode 100644 index 08298ccf5ccf..000000000000 --- a/arch/mips/lasat/picvue.c +++ /dev/null @@ -1,242 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Picvue PVC160206 display driver - * - * Brian Murphy - * - */ -#include -#include -#include -#include -#include -#include -#include - -#include "picvue.h" - -#define PVC_BUSY 0x80 -#define PVC_NLINES 2 -#define PVC_DISPMEM 80 -#define PVC_LINELEN PVC_DISPMEM / PVC_NLINES - -struct pvc_defs *picvue; - -static void pvc_reg_write(u32 val) -{ - *picvue->reg = val; -} - -static u32 pvc_reg_read(void) -{ - u32 tmp = *picvue->reg; - return tmp; -} - -static void pvc_write_byte(u32 data, u8 byte) -{ - data |= picvue->e; - pvc_reg_write(data); - data &= ~picvue->data_mask; - data |= byte << picvue->data_shift; - pvc_reg_write(data); - ndelay(220); - pvc_reg_write(data & ~picvue->e); - ndelay(220); -} - -static u8 pvc_read_byte(u32 data) -{ - u8 byte; - - data |= picvue->e; - pvc_reg_write(data); - ndelay(220); - byte = (pvc_reg_read() & picvue->data_mask) >> picvue->data_shift; - data &= ~picvue->e; - pvc_reg_write(data); - ndelay(220); - return byte; -} - -static u8 pvc_read_data(void) -{ - u32 data = pvc_reg_read(); - u8 byte; - data |= picvue->rw; - data &= ~picvue->rs; - pvc_reg_write(data); - ndelay(40); - byte = pvc_read_byte(data); - data |= picvue->rs; - pvc_reg_write(data); - return byte; -} - -#define TIMEOUT 1000 -static int pvc_wait(void) -{ - int i = TIMEOUT; - int err = 0; - - while ((pvc_read_data() & PVC_BUSY) && i) - i--; - if (i == 0) - err = -ETIME; - - return err; -} - -#define MODE_INST 0 -#define MODE_DATA 1 -static void pvc_write(u8 byte, int mode) -{ - u32 data = pvc_reg_read(); - data &= ~picvue->rw; - if (mode == MODE_DATA) - data |= picvue->rs; - else - data &= ~picvue->rs; - pvc_reg_write(data); - ndelay(40); - pvc_write_byte(data, byte); - if (mode == MODE_DATA) - data &= ~picvue->rs; - else - data |= picvue->rs; - pvc_reg_write(data); - pvc_wait(); -} - -void pvc_write_string(const unsigned char *str, u8 addr, int line) -{ - int i = 0; - - if (line > 0 && (PVC_NLINES > 1)) - addr += 0x40 * line; - pvc_write(0x80 | addr, MODE_INST); - - while (*str != 0 && i < PVC_LINELEN) { - pvc_write(*str++, MODE_DATA); - i++; - } -} - -void pvc_write_string_centered(const unsigned char *str, int line) -{ - int len = strlen(str); - u8 addr; - - if (len > PVC_VISIBLE_CHARS) - addr = 0; - else - addr = (PVC_VISIBLE_CHARS - strlen(str))/2; - - pvc_write_string(str, addr, line); -} - -void pvc_dump_string(const unsigned char *str) -{ - int len = strlen(str); - - pvc_write_string(str, 0, 0); - if (len > PVC_VISIBLE_CHARS) - pvc_write_string(&str[PVC_VISIBLE_CHARS], 0, 1); -} - -#define BM_SIZE 8 -#define MAX_PROGRAMMABLE_CHARS 8 -int pvc_program_cg(int charnum, u8 bitmap[BM_SIZE]) -{ - int i; - int addr; - - if (charnum > MAX_PROGRAMMABLE_CHARS) - return -ENOENT; - - addr = charnum * 8; - pvc_write(0x40 | addr, MODE_INST); - - for (i = 0; i < BM_SIZE; i++) - pvc_write(bitmap[i], MODE_DATA); - return 0; -} - -#define FUNC_SET_CMD 0x20 -#define EIGHT_BYTE (1 << 4) -#define FOUR_BYTE 0 -#define TWO_LINES (1 << 3) -#define ONE_LINE 0 -#define LARGE_FONT (1 << 2) -#define SMALL_FONT 0 - -static void pvc_funcset(u8 cmd) -{ - pvc_write(FUNC_SET_CMD | (cmd & (EIGHT_BYTE|TWO_LINES|LARGE_FONT)), - MODE_INST); -} - -#define ENTRYMODE_CMD 0x4 -#define AUTO_INC (1 << 1) -#define AUTO_DEC 0 -#define CURSOR_FOLLOWS_DISP (1 << 0) - -static void pvc_entrymode(u8 cmd) -{ - pvc_write(ENTRYMODE_CMD | (cmd & (AUTO_INC|CURSOR_FOLLOWS_DISP)), - MODE_INST); -} - -#define DISP_CNT_CMD 0x08 -#define DISP_OFF 0 -#define DISP_ON (1 << 2) -#define CUR_ON (1 << 1) -#define CUR_BLINK (1 << 0) -void pvc_dispcnt(u8 cmd) -{ - pvc_write(DISP_CNT_CMD | (cmd & (DISP_ON|CUR_ON|CUR_BLINK)), MODE_INST); -} - -#define MOVE_CMD 0x10 -#define DISPLAY (1 << 3) -#define CURSOR 0 -#define RIGHT (1 << 2) -#define LEFT 0 -void pvc_move(u8 cmd) -{ - pvc_write(MOVE_CMD | (cmd & (DISPLAY|RIGHT)), MODE_INST); -} - -#define CLEAR_CMD 0x1 -void pvc_clear(void) -{ - pvc_write(CLEAR_CMD, MODE_INST); -} - -#define HOME_CMD 0x2 -void pvc_home(void) -{ - pvc_write(HOME_CMD, MODE_INST); -} - -int pvc_init(void) -{ - u8 cmd = EIGHT_BYTE; - - if (PVC_NLINES == 2) - cmd |= (SMALL_FONT|TWO_LINES); - else - cmd |= (LARGE_FONT|ONE_LINE); - pvc_funcset(cmd); - pvc_dispcnt(DISP_ON); - pvc_entrymode(AUTO_INC); - - pvc_clear(); - pvc_write_string_centered("Display", 0); - pvc_write_string_centered("Initialized", 1); - - return 0; -} - -module_init(pvc_init); -MODULE_LICENSE("GPL"); diff --git a/arch/mips/lasat/picvue.h b/arch/mips/lasat/picvue.h deleted file mode 100644 index 161d3bf50811..000000000000 --- a/arch/mips/lasat/picvue.h +++ /dev/null @@ -1,45 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Picvue PVC160206 display driver - * - * Brian Murphy - * - */ -struct pvc_defs { - volatile u32 *reg; - u32 data_shift; - u32 data_mask; - u32 e; - u32 rw; - u32 rs; -}; - -extern struct pvc_defs *picvue; - -#define PVC_NLINES 2 -#define PVC_DISPMEM 80 -#define PVC_LINELEN PVC_DISPMEM / PVC_NLINES -#define PVC_VISIBLE_CHARS 16 - -void pvc_write_string(const unsigned char *str, u8 addr, int line); -void pvc_write_string_centered(const unsigned char *str, int line); -void pvc_dump_string(const unsigned char *str); - -#define BM_SIZE 8 -#define MAX_PROGRAMMABLE_CHARS 8 -int pvc_program_cg(int charnum, u8 bitmap[BM_SIZE]); - -void pvc_dispcnt(u8 cmd); -#define DISP_OFF 0 -#define DISP_ON (1 << 2) -#define CUR_ON (1 << 1) -#define CUR_BLINK (1 << 0) - -void pvc_move(u8 cmd); -#define DISPLAY (1 << 3) -#define CURSOR 0 -#define RIGHT (1 << 2) -#define LEFT 0 - -void pvc_clear(void); -void pvc_home(void); diff --git a/arch/mips/lasat/picvue_proc.c b/arch/mips/lasat/picvue_proc.c deleted file mode 100644 index 61c033494af5..000000000000 --- a/arch/mips/lasat/picvue_proc.c +++ /dev/null @@ -1,208 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Picvue PVC160206 display driver - * - * Brian Murphy - * - */ -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -#include "picvue.h" - -static DEFINE_MUTEX(pvc_mutex); -static char pvc_lines[PVC_NLINES][PVC_LINELEN+1]; -static int pvc_linedata[PVC_NLINES]; -static char *pvc_linename[PVC_NLINES] = {"line1", "line2"}; -#define DISPLAY_DIR_NAME "display" -static int scroll_dir, scroll_interval; - -static struct timer_list timer; - -static void pvc_display(unsigned long data) -{ - int i; - - pvc_clear(); - for (i = 0; i < PVC_NLINES; i++) - pvc_write_string(pvc_lines[i], 0, i); -} - -static DECLARE_TASKLET(pvc_display_tasklet, &pvc_display, 0); - -static int pvc_line_proc_show(struct seq_file *m, void *v) -{ - int lineno = *(int *)m->private; - - if (lineno < 0 || lineno >= PVC_NLINES) { - printk(KERN_WARNING "proc_read_line: invalid lineno %d\n", lineno); - return 0; - } - - mutex_lock(&pvc_mutex); - seq_printf(m, "%s\n", pvc_lines[lineno]); - mutex_unlock(&pvc_mutex); - - return 0; -} - -static int pvc_line_proc_open(struct inode *inode, struct file *file) -{ - return single_open(file, pvc_line_proc_show, PDE_DATA(inode)); -} - -static ssize_t pvc_line_proc_write(struct file *file, const char __user *buf, - size_t count, loff_t *pos) -{ - int lineno = *(int *)PDE_DATA(file_inode(file)); - char kbuf[PVC_LINELEN]; - size_t len; - - BUG_ON(lineno < 0 || lineno >= PVC_NLINES); - - len = min(count, sizeof(kbuf) - 1); - if (copy_from_user(kbuf, buf, len)) - return -EFAULT; - kbuf[len] = '\0'; - - if (len > 0 && kbuf[len - 1] == '\n') - len--; - - mutex_lock(&pvc_mutex); - strncpy(pvc_lines[lineno], kbuf, len); - pvc_lines[lineno][len] = '\0'; - mutex_unlock(&pvc_mutex); - - tasklet_schedule(&pvc_display_tasklet); - - return count; -} - -static const struct proc_ops pvc_line_proc_ops = { - .proc_open = pvc_line_proc_open, - .proc_read = seq_read, - .proc_lseek = seq_lseek, - .proc_release = single_release, - .proc_write = pvc_line_proc_write, -}; - -static ssize_t pvc_scroll_proc_write(struct file *file, const char __user *buf, - size_t count, loff_t *pos) -{ - char kbuf[42]; - size_t len; - int cmd; - - len = min(count, sizeof(kbuf) - 1); - if (copy_from_user(kbuf, buf, len)) - return -EFAULT; - kbuf[len] = '\0'; - - cmd = simple_strtol(kbuf, NULL, 10); - - mutex_lock(&pvc_mutex); - if (scroll_interval != 0) - del_timer(&timer); - - if (cmd == 0) { - scroll_dir = 0; - scroll_interval = 0; - } else { - if (cmd < 0) { - scroll_dir = -1; - scroll_interval = -cmd; - } else { - scroll_dir = 1; - scroll_interval = cmd; - } - add_timer(&timer); - } - mutex_unlock(&pvc_mutex); - - return count; -} - -static int pvc_scroll_proc_show(struct seq_file *m, void *v) -{ - mutex_lock(&pvc_mutex); - seq_printf(m, "%d\n", scroll_dir * scroll_interval); - mutex_unlock(&pvc_mutex); - - return 0; -} - -static int pvc_scroll_proc_open(struct inode *inode, struct file *file) -{ - return single_open(file, pvc_scroll_proc_show, NULL); -} - -static const struct proc_ops pvc_scroll_proc_ops = { - .proc_open = pvc_scroll_proc_open, - .proc_read = seq_read, - .proc_lseek = seq_lseek, - .proc_release = single_release, - .proc_write = pvc_scroll_proc_write, -}; - -void pvc_proc_timerfunc(struct timer_list *unused) -{ - if (scroll_dir < 0) - pvc_move(DISPLAY|RIGHT); - else if (scroll_dir > 0) - pvc_move(DISPLAY|LEFT); - - timer.expires = jiffies + scroll_interval; - add_timer(&timer); -} - -static void pvc_proc_cleanup(void) -{ - remove_proc_subtree(DISPLAY_DIR_NAME, NULL); - del_timer_sync(&timer); -} - -static int __init pvc_proc_init(void) -{ - struct proc_dir_entry *dir, *proc_entry; - int i; - - dir = proc_mkdir(DISPLAY_DIR_NAME, NULL); - if (dir == NULL) - goto error; - - for (i = 0; i < PVC_NLINES; i++) { - strcpy(pvc_lines[i], ""); - pvc_linedata[i] = i; - } - for (i = 0; i < PVC_NLINES; i++) { - proc_entry = proc_create_data(pvc_linename[i], 0644, dir, - &pvc_line_proc_ops, &pvc_linedata[i]); - if (proc_entry == NULL) - goto error; - } - proc_entry = proc_create("scroll", 0644, dir, &pvc_scroll_proc_ops); - if (proc_entry == NULL) - goto error; - - timer_setup(&timer, pvc_proc_timerfunc, 0); - - return 0; -error: - pvc_proc_cleanup(); - return -ENOMEM; -} - -module_init(pvc_proc_init); -module_exit(pvc_proc_cleanup); -MODULE_LICENSE("GPL"); diff --git a/arch/mips/lasat/prom.c b/arch/mips/lasat/prom.c deleted file mode 100644 index 5ce1407de2d5..000000000000 --- a/arch/mips/lasat/prom.c +++ /dev/null @@ -1,126 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * PROM interface routines. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "at93c.h" -#include -#include "prom.h" - -#define RESET_VECTOR 0xbfc00000 -#define PROM_JUMP_TABLE_ENTRY(n) (*((u32 *)(RESET_VECTOR + 0x20) + n)) -#define PROM_DISPLAY_ADDR PROM_JUMP_TABLE_ENTRY(0) -#define PROM_PUTC_ADDR PROM_JUMP_TABLE_ENTRY(1) -#define PROM_MONITOR_ADDR PROM_JUMP_TABLE_ENTRY(2) - -static void null_prom_display(const char *string, int pos, int clear) -{ -} - -static void null_prom_monitor(void) -{ -} - -static void null_prom_putc(char c) -{ -} - -/* these are functions provided by the bootloader */ -static void (*__prom_putc)(char c) = null_prom_putc; - -void prom_putchar(char c) -{ - __prom_putc(c); -} - -void (*prom_display)(const char *string, int pos, int clear) = - null_prom_display; -void (*prom_monitor)(void) = null_prom_monitor; - -unsigned int lasat_ndelay_divider; - -static void setup_prom_vectors(void) -{ - u32 version = *(u32 *)(RESET_VECTOR + 0x90); - - if (version >= 307) { - prom_display = (void *)PROM_DISPLAY_ADDR; - __prom_putc = (void *)PROM_PUTC_ADDR; - prom_monitor = (void *)PROM_MONITOR_ADDR; - } - printk(KERN_DEBUG "prom vectors set up\n"); -} - -static struct at93c_defs at93c_defs[N_MACHTYPES] = { - { - .reg = (void *)AT93C_REG_100, - .rdata_reg = (void *)AT93C_RDATA_REG_100, - .rdata_shift = AT93C_RDATA_SHIFT_100, - .wdata_shift = AT93C_WDATA_SHIFT_100, - .cs = AT93C_CS_M_100, - .clk = AT93C_CLK_M_100 - }, { - .reg = (void *)AT93C_REG_200, - .rdata_reg = (void *)AT93C_RDATA_REG_200, - .rdata_shift = AT93C_RDATA_SHIFT_200, - .wdata_shift = AT93C_WDATA_SHIFT_200, - .cs = AT93C_CS_M_200, - .clk = AT93C_CLK_M_200 - }, -}; - -void __init prom_init(void) -{ - int argc = fw_arg0; - char **argv = (char **) fw_arg1; - - setup_prom_vectors(); - - if (IS_LASAT_200()) { - printk(KERN_INFO "LASAT 200 board\n"); - lasat_ndelay_divider = LASAT_200_DIVIDER; - at93c = &at93c_defs[1]; - } else { - printk(KERN_INFO "LASAT 100 board\n"); - lasat_ndelay_divider = LASAT_100_DIVIDER; - at93c = &at93c_defs[0]; - } - - lasat_init_board_info(); /* Read info from EEPROM */ - - /* Get the command line */ - if (argc > 0) { - strncpy(arcs_cmdline, argv[0], COMMAND_LINE_SIZE-1); - arcs_cmdline[COMMAND_LINE_SIZE-1] = '\0'; - } - - /* Set the I/O base address */ - set_io_port_base(KSEG1); - - /* Set memory regions */ - ioport_resource.start = 0; - ioport_resource.end = 0xffffffff; /* Wrong, fixme. */ - - add_memory_region(0, lasat_board_info.li_memsize, BOOT_MEM_RAM); -} - -void __init prom_free_prom_memory(void) -{ -} - -const char *get_system_type(void) -{ - return lasat_board_info.li_bmstr; -} diff --git a/arch/mips/lasat/prom.h b/arch/mips/lasat/prom.h deleted file mode 100644 index 3d1df853e9d3..000000000000 --- a/arch/mips/lasat/prom.h +++ /dev/null @@ -1,8 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __PROM_H -#define __PROM_H - -extern void (*prom_display)(const char *string, int pos, int clear); -extern void (*prom_monitor)(void); - -#endif /* __PROM_H */ diff --git a/arch/mips/lasat/reset.c b/arch/mips/lasat/reset.c deleted file mode 100644 index 7c516ed9af15..000000000000 --- a/arch/mips/lasat/reset.c +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Thomas Horsten - * Copyright (C) 2000 LASAT Networks A/S. - * - * Reset the LASAT board. - */ -#include -#include - -#include -#include - -#include "picvue.h" -#include "prom.h" - -static void lasat_machine_restart(char *command); -static void lasat_machine_halt(void); - -/* Used to set machine to boot in service mode via /proc interface */ -int lasat_boot_to_service; - -static void lasat_machine_restart(char *command) -{ - local_irq_disable(); - - if (lasat_boot_to_service) { - *(volatile unsigned int *)0xa0000024 = 0xdeadbeef; - *(volatile unsigned int *)0xa00000fc = 0xfedeabba; - } - *lasat_misc->reset_reg = 0xbedead; - for (;;) ; -} - -static void lasat_machine_halt(void) -{ - local_irq_disable(); - - prom_monitor(); - for (;;) ; -} - -void lasat_reboot_setup(void) -{ - _machine_restart = lasat_machine_restart; - _machine_halt = lasat_machine_halt; - pm_power_off = lasat_machine_halt; -} diff --git a/arch/mips/lasat/serial.c b/arch/mips/lasat/serial.c deleted file mode 100644 index 16b242713420..000000000000 --- a/arch/mips/lasat/serial.c +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Registration of Lasat UART platform device. - * - * Copyright (C) 2007 Brian Murphy - */ -#include -#include -#include -#include -#include - -#include -#include - -static struct resource lasat_serial_res[2] __initdata; - -static struct plat_serial8250_port lasat_serial8250_port[] = { - { - .iotype = UPIO_MEM, - .flags = UPF_IOREMAP | UPF_BOOT_AUTOCONF | - UPF_SKIP_TEST, - }, - {}, -}; - -static __init int lasat_uart_add(void) -{ - struct platform_device *pdev; - int retval; - - pdev = platform_device_alloc("serial8250", -1); - if (!pdev) - return -ENOMEM; - - if (!IS_LASAT_200()) { - lasat_serial_res[0].start = KSEG1ADDR(LASAT_UART_REGS_BASE_100); - lasat_serial_res[0].end = lasat_serial_res[0].start + LASAT_UART_REGS_SHIFT_100 * 8 - 1; - lasat_serial_res[0].flags = IORESOURCE_MEM; - lasat_serial_res[1].start = LASATINT_UART_100; - lasat_serial_res[1].end = LASATINT_UART_100; - lasat_serial_res[1].flags = IORESOURCE_IRQ; - - lasat_serial8250_port[0].mapbase = LASAT_UART_REGS_BASE_100; - lasat_serial8250_port[0].uartclk = LASAT_BASE_BAUD_100 * 16; - lasat_serial8250_port[0].regshift = LASAT_UART_REGS_SHIFT_100; - lasat_serial8250_port[0].irq = LASATINT_UART_100; - } else { - lasat_serial_res[0].start = KSEG1ADDR(LASAT_UART_REGS_BASE_200); - lasat_serial_res[0].end = lasat_serial_res[0].start + LASAT_UART_REGS_SHIFT_200 * 8 - 1; - lasat_serial_res[0].flags = IORESOURCE_MEM; - lasat_serial_res[1].start = LASATINT_UART_200; - lasat_serial_res[1].end = LASATINT_UART_200; - lasat_serial_res[1].flags = IORESOURCE_IRQ; - - lasat_serial8250_port[0].mapbase = LASAT_UART_REGS_BASE_200; - lasat_serial8250_port[0].uartclk = LASAT_BASE_BAUD_200 * 16; - lasat_serial8250_port[0].regshift = LASAT_UART_REGS_SHIFT_200; - lasat_serial8250_port[0].irq = LASATINT_UART_200; - } - - pdev->id = PLAT8250_DEV_PLATFORM; - pdev->dev.platform_data = lasat_serial8250_port; - - retval = platform_device_add_resources(pdev, lasat_serial_res, ARRAY_SIZE(lasat_serial_res)); - if (retval) - goto err_free_device; - - retval = platform_device_add(pdev); - if (retval) - goto err_free_device; - - return 0; - -err_free_device: - platform_device_put(pdev); - - return retval; -} -device_initcall(lasat_uart_add); diff --git a/arch/mips/lasat/setup.c b/arch/mips/lasat/setup.c deleted file mode 100644 index 0743243fd86d..000000000000 --- a/arch/mips/lasat/setup.c +++ /dev/null @@ -1,141 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Carsten Langgaard, carstenl@mips.com - * Copyright (C) 1999 MIPS Technologies, Inc. All rights reserved. - * - * Thomas Horsten - * Copyright (C) 2000 LASAT Networks A/S. - * - * Brian Murphy - * - * Lasat specific setup. - */ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#ifdef CONFIG_PICVUE -#include -#endif - -#include "ds1603.h" -#include -#include -#include - -#include "prom.h" - -int lasat_command_line; -void lasatint_init(void); - -extern void lasat_reboot_setup(void); -extern void pcisetup(void); -extern void edhac_init(void *, void *, void *); -extern void addrflt_init(void); - -struct lasat_misc lasat_misc_info[N_MACHTYPES] = { - { - .reset_reg = (void *)KSEG1ADDR(0x1c840000), - .flash_wp_reg = (void *)KSEG1ADDR(0x1c800000), 2 - }, { - .reset_reg = (void *)KSEG1ADDR(0x11080000), - .flash_wp_reg = (void *)KSEG1ADDR(0x11000000), 6 - } -}; - -struct lasat_misc *lasat_misc; - -#ifdef CONFIG_DS1603 -static struct ds_defs ds_defs[N_MACHTYPES] = { - { (void *)DS1603_REG_100, (void *)DS1603_REG_100, - DS1603_RST_100, DS1603_CLK_100, DS1603_DATA_100, - DS1603_DATA_SHIFT_100, 0, 0 }, - { (void *)DS1603_REG_200, (void *)DS1603_DATA_REG_200, - DS1603_RST_200, DS1603_CLK_200, DS1603_DATA_200, - DS1603_DATA_READ_SHIFT_200, 1, 2000 } -}; -#endif - -#ifdef CONFIG_PICVUE -#include "picvue.h" -static struct pvc_defs pvc_defs[N_MACHTYPES] = { - { (void *)PVC_REG_100, PVC_DATA_SHIFT_100, PVC_DATA_M_100, - PVC_E_100, PVC_RW_100, PVC_RS_100 }, - { (void *)PVC_REG_200, PVC_DATA_SHIFT_200, PVC_DATA_M_200, - PVC_E_200, PVC_RW_200, PVC_RS_200 } -}; -#endif - -static int lasat_panic_display(struct notifier_block *this, - unsigned long event, void *ptr) -{ -#ifdef CONFIG_PICVUE - unsigned char *string = ptr; - if (string == NULL) - string = "Kernel Panic"; - pvc_dump_string(string); -#endif - return NOTIFY_DONE; -} - -static int lasat_panic_prom_monitor(struct notifier_block *this, - unsigned long event, void *ptr) -{ - prom_monitor(); - return NOTIFY_DONE; -} - -static struct notifier_block lasat_panic_block[] = -{ - { - .notifier_call = lasat_panic_display, - .priority = INT_MAX - }, { - .notifier_call = lasat_panic_prom_monitor, - .priority = INT_MIN - } -}; - -void __init plat_time_init(void) -{ - mips_hpt_frequency = lasat_board_info.li_cpu_hz / 2; - - change_c0_status(ST0_IM, IE_IRQ0); -} - -void __init plat_mem_setup(void) -{ - int i; - int lasat_type = IS_LASAT_200() ? 1 : 0; - - lasat_misc = &lasat_misc_info[lasat_type]; -#ifdef CONFIG_PICVUE - picvue = &pvc_defs[lasat_type]; -#endif - - /* Set up panic notifier */ - for (i = 0; i < ARRAY_SIZE(lasat_panic_block); i++) - atomic_notifier_chain_register(&panic_notifier_list, - &lasat_panic_block[i]); - - lasat_reboot_setup(); - -#ifdef CONFIG_DS1603 - ds1603 = &ds_defs[lasat_type]; -#endif - -#ifdef DYNAMIC_SERIAL_INIT - serial_init(); -#endif - - pr_info("Lasat specific initialization complete\n"); -} diff --git a/arch/mips/lasat/sysctl.c b/arch/mips/lasat/sysctl.c deleted file mode 100644 index e666fe26c50d..000000000000 --- a/arch/mips/lasat/sysctl.c +++ /dev/null @@ -1,268 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Thomas Horsten - * Copyright (C) 2000 LASAT Networks A/S. - * - * Routines specific to the LASAT boards - */ -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#ifdef CONFIG_DS1603 -#include "ds1603.h" -#endif - - -/* And the same for proc */ -int proc_dolasatstring(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - int r; - - r = proc_dostring(table, write, buffer, lenp, ppos); - if ((!write) || r) - return r; - - lasat_write_eeprom_info(); - - return 0; -} - -#ifdef CONFIG_DS1603 -static int rtctmp; - -/* proc function to read/write RealTime Clock */ -int proc_dolasatrtc(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - struct timespec64 ts; - int r; - - if (!write) { - read_persistent_clock64(&ts); - rtctmp = ts.tv_sec; - /* check for time < 0 and set to 0 */ - if (rtctmp < 0) - rtctmp = 0; - } - r = proc_dointvec(table, write, buffer, lenp, ppos); - if (r) - return r; - - if (write) { - /* - * Due to the RTC hardware limitation, we can not actually - * use the full 64-bit range here. - */ - ts.tv_sec = rtctmp; - ts.tv_nsec = 0; - - update_persistent_clock64(ts); - } - - return 0; -} -#endif - -#ifdef CONFIG_INET -int proc_lasat_ip(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - unsigned int ip; - char *p, c; - int len; - char ipbuf[32]; - - if (!table->data || !table->maxlen || !*lenp || - (*ppos && !write)) { - *lenp = 0; - return 0; - } - - if (write) { - len = 0; - p = buffer; - while (len < *lenp) { - if (get_user(c, p++)) - return -EFAULT; - if (c == 0 || c == '\n') - break; - len++; - } - if (len >= sizeof(ipbuf)-1) - len = sizeof(ipbuf) - 1; - if (copy_from_user(ipbuf, buffer, len)) - return -EFAULT; - ipbuf[len] = 0; - *ppos += *lenp; - /* Now see if we can convert it to a valid IP */ - ip = in_aton(ipbuf); - *(unsigned int *)(table->data) = ip; - lasat_write_eeprom_info(); - } else { - ip = *(unsigned int *)(table->data); - sprintf(ipbuf, "%d.%d.%d.%d", - (ip) & 0xff, - (ip >> 8) & 0xff, - (ip >> 16) & 0xff, - (ip >> 24) & 0xff); - len = strlen(ipbuf); - if (len > *lenp) - len = *lenp; - if (len) - if (copy_to_user(buffer, ipbuf, len)) - return -EFAULT; - if (len < *lenp) { - if (put_user('\n', ((char *) buffer) + len)) - return -EFAULT; - len++; - } - *lenp = len; - *ppos += len; - } - - return 0; -} -#endif - -int proc_lasat_prid(struct ctl_table *table, int write, - void *buffer, size_t *lenp, loff_t *ppos) -{ - int r; - - r = proc_dointvec(table, write, buffer, lenp, ppos); - if (r < 0) - return r; - if (write) { - lasat_board_info.li_eeprom_info.prid = - lasat_board_info.li_prid; - lasat_write_eeprom_info(); - lasat_init_board_info(); - } - return 0; -} - -extern int lasat_boot_to_service; - -static struct ctl_table lasat_table[] = { - { - .procname = "cpu-hz", - .data = &lasat_board_info.li_cpu_hz, - .maxlen = sizeof(int), - .mode = 0444, - .proc_handler = proc_dointvec, - }, - { - .procname = "bus-hz", - .data = &lasat_board_info.li_bus_hz, - .maxlen = sizeof(int), - .mode = 0444, - .proc_handler = proc_dointvec, - }, - { - .procname = "bmid", - .data = &lasat_board_info.li_bmid, - .maxlen = sizeof(int), - .mode = 0444, - .proc_handler = proc_dointvec, - }, - { - .procname = "prid", - .data = &lasat_board_info.li_prid, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_lasat_prid, - }, -#ifdef CONFIG_INET - { - .procname = "ipaddr", - .data = &lasat_board_info.li_eeprom_info.ipaddr, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_lasat_ip, - }, - { - .procname = "netmask", - .data = &lasat_board_info.li_eeprom_info.netmask, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_lasat_ip, - }, -#endif - { - .procname = "passwd_hash", - .data = &lasat_board_info.li_eeprom_info.passwd_hash, - .maxlen = - sizeof(lasat_board_info.li_eeprom_info.passwd_hash), - .mode = 0600, - .proc_handler = proc_dolasatstring, - }, - { - .procname = "boot-service", - .data = &lasat_boot_to_service, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, -#ifdef CONFIG_DS1603 - { - .procname = "rtc", - .data = &rtctmp, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dolasatrtc, - }, -#endif - { - .procname = "namestr", - .data = &lasat_board_info.li_namestr, - .maxlen = sizeof(lasat_board_info.li_namestr), - .mode = 0444, - .proc_handler = proc_dostring, - }, - { - .procname = "typestr", - .data = &lasat_board_info.li_typestr, - .maxlen = sizeof(lasat_board_info.li_typestr), - .mode = 0444, - .proc_handler = proc_dostring, - }, - {} -}; - -static struct ctl_table lasat_root_table[] = { - { - .procname = "lasat", - .mode = 0555, - .child = lasat_table - }, - {} -}; - -static int __init lasat_register_sysctl(void) -{ - struct ctl_table_header *lasat_table_header; - - lasat_table_header = - register_sysctl_table(lasat_root_table); - if (!lasat_table_header) { - printk(KERN_ERR "Unable to register LASAT sysctl\n"); - return -ENOMEM; - } - - return 0; -} - -arch_initcall(lasat_register_sysctl); diff --git a/arch/mips/pci/Makefile b/arch/mips/pci/Makefile index 342ce10ef593..c6142b289086 100644 --- a/arch/mips/pci/Makefile +++ b/arch/mips/pci/Makefile @@ -13,7 +13,6 @@ obj-$(CONFIG_PCI_DRIVERS_GENERIC)+= pci-generic.o obj-$(CONFIG_MIPS_BONITO64) += ops-bonito64.o obj-$(CONFIG_PCI_GT64XXX_PCI0) += ops-gt64xxx_pci0.o obj-$(CONFIG_MIPS_MSC) += ops-msc.o -obj-$(CONFIG_MIPS_NILE4) += ops-nile4.o obj-$(CONFIG_SOC_TX3927) += ops-tx3927.o obj-$(CONFIG_PCI_VR41XX) += ops-vr41xx.o pci-vr41xx.o obj-$(CONFIG_NEC_MARKEINS) += ops-emma2rh.o pci-emma2rh.o fixup-emma2rh.o @@ -31,7 +30,6 @@ obj-$(CONFIG_PCI_XTALK_BRIDGE) += pci-xtalk-bridge.o # These are still pretty much in the old state, watch, go blind. # obj-$(CONFIG_ATH79) += fixup-ath79.o -obj-$(CONFIG_LASAT) += pci-lasat.o obj-$(CONFIG_MIPS_COBALT) += fixup-cobalt.o obj-$(CONFIG_LEMOTE_FULOONG2E) += fixup-fuloong2e.o ops-loongson2.o obj-$(CONFIG_LEMOTE_MACH2F) += fixup-lemote2f.o ops-loongson2.o diff --git a/arch/mips/pci/ops-nile4.c b/arch/mips/pci/ops-nile4.c deleted file mode 100644 index b00658d19116..000000000000 --- a/arch/mips/pci/ops-nile4.c +++ /dev/null @@ -1,136 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include - -#include -#include - -#define PCI_ACCESS_READ 0 -#define PCI_ACCESS_WRITE 1 - -#define LO(reg) (reg / 4) -#define HI(reg) (reg / 4 + 1) - -volatile unsigned long *const vrc_pciregs = (void *) Vrc5074_BASE; - -static int nile4_pcibios_config_access(unsigned char access_type, - struct pci_bus *bus, unsigned int devfn, int where, u32 *val) -{ - unsigned char busnum = bus->number; - u32 adr, mask, err; - - if ((busnum == 0) && (PCI_SLOT(devfn) > 8)) - /* The addressing scheme chosen leaves room for just - * 8 devices on the first busnum (besides the PCI - * controller itself) */ - return PCIBIOS_DEVICE_NOT_FOUND; - - if ((busnum == 0) && (devfn == PCI_DEVFN(0, 0))) { - /* Access controller registers directly */ - if (access_type == PCI_ACCESS_WRITE) { - vrc_pciregs[(0x200 + where) >> 2] = *val; - } else { - *val = vrc_pciregs[(0x200 + where) >> 2]; - } - return PCIBIOS_SUCCESSFUL; - } - - /* Temporarily map PCI Window 1 to config space */ - mask = vrc_pciregs[LO(NILE4_PCIINIT1)]; - vrc_pciregs[LO(NILE4_PCIINIT1)] = 0x0000001a | (busnum ? 0x200 : 0); - - /* Clear PCI Error register. This also clears the Error Type - * bits in the Control register */ - vrc_pciregs[LO(NILE4_PCIERR)] = 0; - vrc_pciregs[HI(NILE4_PCIERR)] = 0; - - /* Setup address */ - if (busnum == 0) - adr = - KSEG1ADDR(PCI_WINDOW1) + - ((1 << (PCI_SLOT(devfn) + 15)) | (PCI_FUNC(devfn) << 8) - | (where & ~3)); - else - adr = KSEG1ADDR(PCI_WINDOW1) | (busnum << 16) | (devfn << 8) | - (where & ~3); - - if (access_type == PCI_ACCESS_WRITE) - *(u32 *) adr = *val; - else - *val = *(u32 *) adr; - - /* Check for master or target abort */ - err = (vrc_pciregs[HI(NILE4_PCICTRL)] >> 5) & 0x7; - - /* Restore PCI Window 1 */ - vrc_pciregs[LO(NILE4_PCIINIT1)] = mask; - - if (err) - return PCIBIOS_DEVICE_NOT_FOUND; - - return PCIBIOS_SUCCESSFUL; -} - -static int nile4_pcibios_read(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 *val) -{ - u32 data = 0; - int err; - - if ((size == 2) && (where & 1)) - return PCIBIOS_BAD_REGISTER_NUMBER; - else if ((size == 4) && (where & 3)) - return PCIBIOS_BAD_REGISTER_NUMBER; - - err = nile4_pcibios_config_access(PCI_ACCESS_READ, bus, devfn, where, - &data); - if (err) - return err; - - if (size == 1) - *val = (data >> ((where & 3) << 3)) & 0xff; - else if (size == 2) - *val = (data >> ((where & 3) << 3)) & 0xffff; - else - *val = data; - - return PCIBIOS_SUCCESSFUL; -} - -static int nile4_pcibios_write(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 val) -{ - u32 data = 0; - int err; - - if ((size == 2) && (where & 1)) - return PCIBIOS_BAD_REGISTER_NUMBER; - else if ((size == 4) && (where & 3)) - return PCIBIOS_BAD_REGISTER_NUMBER; - - err = nile4_pcibios_config_access(PCI_ACCESS_READ, bus, devfn, where, - &data); - if (err) - return err; - - if (size == 1) - data = (data & ~(0xff << ((where & 3) << 3))) | - (val << ((where & 3) << 3)); - else if (size == 2) - data = (data & ~(0xffff << ((where & 3) << 3))) | - (val << ((where & 3) << 3)); - else - data = val; - - if (nile4_pcibios_config_access - (PCI_ACCESS_WRITE, bus, devfn, where, &data)) - return -1; - - return PCIBIOS_SUCCESSFUL; -} - -struct pci_ops nile4_pci_ops = { - .read = nile4_pcibios_read, - .write = nile4_pcibios_write, -}; diff --git a/arch/mips/pci/pci-lasat.c b/arch/mips/pci/pci-lasat.c deleted file mode 100644 index 47f4ee6bbb3b..000000000000 --- a/arch/mips/pci/pci-lasat.c +++ /dev/null @@ -1,88 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2000, 2001, 04 Keith M Wesolowski - */ -#include -#include -#include -#include - -#include - -#include - -extern struct pci_ops nile4_pci_ops; -extern struct pci_ops gt64xxx_pci0_ops; -static struct resource lasat_pci_mem_resource = { - .name = "LASAT PCI MEM", - .start = 0x18000000, - .end = 0x19ffffff, - .flags = IORESOURCE_MEM, -}; - -static struct resource lasat_pci_io_resource = { - .name = "LASAT PCI IO", - .start = 0x1a000000, - .end = 0x1bffffff, - .flags = IORESOURCE_IO, -}; - -static struct pci_controller lasat_pci_controller = { - .mem_resource = &lasat_pci_mem_resource, - .io_resource = &lasat_pci_io_resource, -}; - -static int __init lasat_pci_setup(void) -{ - printk(KERN_DEBUG "PCI: starting\n"); - - if (IS_LASAT_200()) - lasat_pci_controller.pci_ops = &nile4_pci_ops; - else - lasat_pci_controller.pci_ops = >64xxx_pci0_ops; - - register_pci_controller(&lasat_pci_controller); - - return 0; -} - -arch_initcall(lasat_pci_setup); - -#define LASAT_IRQ_ETH1 (LASAT_IRQ_BASE + 0) -#define LASAT_IRQ_ETH0 (LASAT_IRQ_BASE + 1) -#define LASAT_IRQ_HDC (LASAT_IRQ_BASE + 2) -#define LASAT_IRQ_COMP (LASAT_IRQ_BASE + 3) -#define LASAT_IRQ_HDLC (LASAT_IRQ_BASE + 4) -#define LASAT_IRQ_PCIA (LASAT_IRQ_BASE + 5) -#define LASAT_IRQ_PCIB (LASAT_IRQ_BASE + 6) -#define LASAT_IRQ_PCIC (LASAT_IRQ_BASE + 7) -#define LASAT_IRQ_PCID (LASAT_IRQ_BASE + 8) - -int pcibios_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) -{ - switch (slot) { - case 1: - case 2: - case 3: - return LASAT_IRQ_PCIA + (((slot-1) + (pin-1)) % 4); - case 4: - return LASAT_IRQ_ETH1; /* Ethernet 1 (LAN 2) */ - case 5: - return LASAT_IRQ_ETH0; /* Ethernet 0 (LAN 1) */ - case 6: - return LASAT_IRQ_HDC; /* IDE controller */ - default: - return 0xff; /* Illegal */ - } - - return -1; -} - -/* Do platform specific device initialization at pci_enable_device() time */ -int pcibios_plat_dev_init(struct pci_dev *dev) -{ - return 0; -} From 1b00767fd8e1b49685ee7237c3875b8aa998ae3b Mon Sep 17 00:00:00 2001 From: Thomas Bogendoerfer Date: Mon, 20 Apr 2020 14:30:35 +0200 Subject: [PATCH 0304/1043] MIPS: Remove PMC MSP71xx platform No (active) developer owns this hardware, so let's remove Linux support. Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kbuild.platforms | 1 - arch/mips/Kconfig | 25 - arch/mips/configs/msp71xx_defconfig | 77 -- arch/mips/include/asm/bootinfo.h | 11 - .../mach-pmcs-msp71xx/cpu-feature-overrides.h | 22 - .../asm/mach-pmcs-msp71xx/msp_cic_int.h | 139 --- .../asm/mach-pmcs-msp71xx/msp_gpio_macros.h | 343 ------- .../include/asm/mach-pmcs-msp71xx/msp_int.h | 31 - .../include/asm/mach-pmcs-msp71xx/msp_pci.h | 189 ---- .../include/asm/mach-pmcs-msp71xx/msp_prom.h | 159 --- .../asm/mach-pmcs-msp71xx/msp_regops.h | 237 ----- .../include/asm/mach-pmcs-msp71xx/msp_regs.h | 652 ------------ .../asm/mach-pmcs-msp71xx/msp_slp_int.h | 129 --- .../include/asm/mach-pmcs-msp71xx/msp_usb.h | 124 --- arch/mips/include/asm/mach-pmcs-msp71xx/war.h | 28 - arch/mips/pci/Makefile | 3 - arch/mips/pci/fixup-pmcmsp.c | 216 ---- arch/mips/pci/ops-pmcmsp.c | 944 ------------------ arch/mips/pmcs-msp71xx/Kconfig | 50 - arch/mips/pmcs-msp71xx/Makefile | 13 - arch/mips/pmcs-msp71xx/Platform | 7 - arch/mips/pmcs-msp71xx/msp_elb.c | 46 - arch/mips/pmcs-msp71xx/msp_eth.c | 111 -- arch/mips/pmcs-msp71xx/msp_hwbutton.c | 165 --- arch/mips/pmcs-msp71xx/msp_irq.c | 155 --- arch/mips/pmcs-msp71xx/msp_irq_cic.c | 208 ---- arch/mips/pmcs-msp71xx/msp_irq_per.c | 127 --- arch/mips/pmcs-msp71xx/msp_irq_slp.c | 102 -- arch/mips/pmcs-msp71xx/msp_pci.c | 50 - arch/mips/pmcs-msp71xx/msp_prom.c | 513 ---------- arch/mips/pmcs-msp71xx/msp_serial.c | 154 --- arch/mips/pmcs-msp71xx/msp_setup.c | 228 ----- arch/mips/pmcs-msp71xx/msp_smp.c | 56 -- arch/mips/pmcs-msp71xx/msp_time.c | 90 -- arch/mips/pmcs-msp71xx/msp_usb.c | 173 ---- 35 files changed, 5578 deletions(-) delete mode 100644 arch/mips/configs/msp71xx_defconfig delete mode 100644 arch/mips/include/asm/mach-pmcs-msp71xx/cpu-feature-overrides.h delete mode 100644 arch/mips/include/asm/mach-pmcs-msp71xx/msp_cic_int.h delete mode 100644 arch/mips/include/asm/mach-pmcs-msp71xx/msp_gpio_macros.h delete mode 100644 arch/mips/include/asm/mach-pmcs-msp71xx/msp_int.h delete mode 100644 arch/mips/include/asm/mach-pmcs-msp71xx/msp_pci.h delete mode 100644 arch/mips/include/asm/mach-pmcs-msp71xx/msp_prom.h delete mode 100644 arch/mips/include/asm/mach-pmcs-msp71xx/msp_regops.h delete mode 100644 arch/mips/include/asm/mach-pmcs-msp71xx/msp_regs.h delete mode 100644 arch/mips/include/asm/mach-pmcs-msp71xx/msp_slp_int.h delete mode 100644 arch/mips/include/asm/mach-pmcs-msp71xx/msp_usb.h delete mode 100644 arch/mips/include/asm/mach-pmcs-msp71xx/war.h delete mode 100644 arch/mips/pci/fixup-pmcmsp.c delete mode 100644 arch/mips/pci/ops-pmcmsp.c delete mode 100644 arch/mips/pmcs-msp71xx/Kconfig delete mode 100644 arch/mips/pmcs-msp71xx/Makefile delete mode 100644 arch/mips/pmcs-msp71xx/Platform delete mode 100644 arch/mips/pmcs-msp71xx/msp_elb.c delete mode 100644 arch/mips/pmcs-msp71xx/msp_eth.c delete mode 100644 arch/mips/pmcs-msp71xx/msp_hwbutton.c delete mode 100644 arch/mips/pmcs-msp71xx/msp_irq.c delete mode 100644 arch/mips/pmcs-msp71xx/msp_irq_cic.c delete mode 100644 arch/mips/pmcs-msp71xx/msp_irq_per.c delete mode 100644 arch/mips/pmcs-msp71xx/msp_irq_slp.c delete mode 100644 arch/mips/pmcs-msp71xx/msp_pci.c delete mode 100644 arch/mips/pmcs-msp71xx/msp_prom.c delete mode 100644 arch/mips/pmcs-msp71xx/msp_serial.c delete mode 100644 arch/mips/pmcs-msp71xx/msp_setup.c delete mode 100644 arch/mips/pmcs-msp71xx/msp_smp.c delete mode 100644 arch/mips/pmcs-msp71xx/msp_time.c delete mode 100644 arch/mips/pmcs-msp71xx/msp_usb.c diff --git a/arch/mips/Kbuild.platforms b/arch/mips/Kbuild.platforms index 9f9b4164d4c9..65c119432e06 100644 --- a/arch/mips/Kbuild.platforms +++ b/arch/mips/Kbuild.platforms @@ -24,7 +24,6 @@ platforms += netlogic platforms += paravirt platforms += pic32 platforms += pistachio -platforms += pmcs-msp71xx platforms += pnx833x platforms += ralink platforms += rb532 diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 0aaab09bdea6..f2565a88e086 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -608,30 +608,6 @@ config NXP_STB225 help Support for NXP Semiconductors STB225 Development Board. -config PMC_MSP - bool "PMC-Sierra MSP chipsets" - select CEVT_R4K - select CSRC_R4K - select DMA_NONCOHERENT - select SWAP_IO_SPACE - select NO_EXCEPT_FILL - select BOOT_RAW - select SYS_HAS_CPU_MIPS32_R1 - select SYS_HAS_CPU_MIPS32_R2 - select SYS_SUPPORTS_32BIT_KERNEL - select SYS_SUPPORTS_BIG_ENDIAN - select SYS_SUPPORTS_MIPS16 - select IRQ_MIPS_CPU - select SERIAL_8250 - select SERIAL_8250_CONSOLE - select USB_EHCI_BIG_ENDIAN_MMIO - select USB_EHCI_BIG_ENDIAN_DESC - help - This adds support for the PMC-Sierra family of Multi-Service - Processor System-On-A-Chips. These parts include a number - of integrated peripherals, interfaces and DSPs in addition to - a variety of MIPS cores. - config RALINK bool "Ralink based machines" select CEVT_R4K @@ -1076,7 +1052,6 @@ source "arch/mips/jz4740/Kconfig" source "arch/mips/lantiq/Kconfig" source "arch/mips/pic32/Kconfig" source "arch/mips/pistachio/Kconfig" -source "arch/mips/pmcs-msp71xx/Kconfig" source "arch/mips/ralink/Kconfig" source "arch/mips/sgi-ip27/Kconfig" source "arch/mips/sibyte/Kconfig" diff --git a/arch/mips/configs/msp71xx_defconfig b/arch/mips/configs/msp71xx_defconfig deleted file mode 100644 index 6ad1a2381226..000000000000 --- a/arch/mips/configs/msp71xx_defconfig +++ /dev/null @@ -1,77 +0,0 @@ -CONFIG_LOCALVERSION="-pmc" -# CONFIG_SWAP is not set -CONFIG_SYSVIPC=y -CONFIG_PREEMPT=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_EXPERT=y -# CONFIG_SHMEM is not set -CONFIG_SLAB=y -CONFIG_PMC_MSP=y -CONFIG_PMC_MSP7120_GW=y -CONFIG_CPU_MIPS32_R2=y -CONFIG_NR_CPUS=2 -CONFIG_PCI=y -CONFIG_MODULES=y -CONFIG_MODULE_UNLOAD=y -CONFIG_MODVERSIONS=y -CONFIG_NET=y -CONFIG_UNIX=y -CONFIG_XFRM_USER=y -CONFIG_NET_KEY=y -CONFIG_INET=y -CONFIG_IP_MULTICAST=y -CONFIG_IP_PNP=y -CONFIG_IP_PNP_DHCP=y -CONFIG_IP_PNP_BOOTP=y -CONFIG_INET_AH=y -CONFIG_INET_ESP=y -CONFIG_INET_IPCOMP=y -# CONFIG_IPV6 is not set -CONFIG_NETFILTER=y -CONFIG_IP_NF_IPTABLES=y -CONFIG_IP_NF_FILTER=y -CONFIG_IP_NF_TARGET_REJECT=y -CONFIG_BRIDGE=y -# CONFIG_PREVENT_FIRMWARE_BUILD is not set -# CONFIG_FW_LOADER is not set -CONFIG_MTD=y -CONFIG_MTD_BLOCK=y -CONFIG_MTD_CFI=y -CONFIG_MTD_CFI_AMDSTD=y -CONFIG_MTD_RAM=y -CONFIG_MTD_PMC_MSP_EVM=y -CONFIG_BLK_DEV_RAM=y -CONFIG_SCSI=y -CONFIG_BLK_DEV_SD=y -CONFIG_NETDEVICES=y -CONFIG_DUMMY=y -CONFIG_PPP=y -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set -# CONFIG_SERIO is not set -# CONFIG_VT is not set -# CONFIG_LEGACY_PTYS is not set -# CONFIG_SERIAL_8250_PCI is not set -CONFIG_SERIAL_8250_NR_UARTS=2 -CONFIG_SERIAL_8250_RUNTIME_UARTS=2 -# CONFIG_HW_RANDOM is not set -CONFIG_I2C=y -CONFIG_I2C_CHARDEV=y -CONFIG_I2C_PMCMSP=y -# CONFIG_USB_HID is not set -CONFIG_USB=y -CONFIG_USB_MON=y -CONFIG_USB_EHCI_HCD=y -CONFIG_USB_EHCI_ROOT_HUB_TT=y -# CONFIG_USB_EHCI_TT_NEWSCHED is not set -CONFIG_USB_STORAGE=y -CONFIG_EXT2_FS=y -# CONFIG_DNOTIFY is not set -CONFIG_MSDOS_FS=y -CONFIG_VFAT_FS=y -CONFIG_JFFS2_FS=y -CONFIG_SQUASHFS=y -CONFIG_SQUASHFS_EMBEDDED=y -CONFIG_NLS_CODEPAGE_437=y -CONFIG_NLS_ISO8859_1=y -CONFIG_MAGIC_SYSRQ=y diff --git a/arch/mips/include/asm/bootinfo.h b/arch/mips/include/asm/bootinfo.h index dcd6a75277d1..c3bd9b2d66e4 100644 --- a/arch/mips/include/asm/bootinfo.h +++ b/arch/mips/include/asm/bootinfo.h @@ -41,17 +41,6 @@ #define MACH_DS5800 9 /* DECsystem 5800 */ #define MACH_DS5900 10 /* DECsystem 5900 */ -/* - * Valid machtype for group PMC-MSP - */ -#define MACH_MSP4200_EVAL 0 /* PMC-Sierra MSP4200 Evaluation */ -#define MACH_MSP4200_GW 1 /* PMC-Sierra MSP4200 Gateway demo */ -#define MACH_MSP4200_FPGA 2 /* PMC-Sierra MSP4200 Emulation */ -#define MACH_MSP7120_EVAL 3 /* PMC-Sierra MSP7120 Evaluation */ -#define MACH_MSP7120_GW 4 /* PMC-Sierra MSP7120 Residential GW */ -#define MACH_MSP7120_FPGA 5 /* PMC-Sierra MSP7120 Emulation */ -#define MACH_MSP_OTHER 255 /* PMC-Sierra unknown board type */ - /* * Valid machtype for group Mikrotik */ diff --git a/arch/mips/include/asm/mach-pmcs-msp71xx/cpu-feature-overrides.h b/arch/mips/include/asm/mach-pmcs-msp71xx/cpu-feature-overrides.h deleted file mode 100644 index 016fa9446ba9..000000000000 --- a/arch/mips/include/asm/mach-pmcs-msp71xx/cpu-feature-overrides.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2003, 04, 07 Ralf Baechle (ralf@linux-mips.org) - */ -#ifndef __ASM_MACH_MSP71XX_CPU_FEATURE_OVERRIDES_H -#define __ASM_MACH_MSP71XX_CPU_FEATURE_OVERRIDES_H - -#define cpu_has_mips16 1 -#define cpu_has_dsp 1 -/* #define cpu_has_dsp2 ??? - do runtime detection */ -#define cpu_has_mipsmt 1 -#define cpu_has_fpu 0 - -#define cpu_has_mips32r1 0 -#define cpu_has_mips32r2 1 -#define cpu_has_mips64r1 0 -#define cpu_has_mips64r2 0 - -#endif /* __ASM_MACH_MSP71XX_CPU_FEATURE_OVERRIDES_H */ diff --git a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_cic_int.h b/arch/mips/include/asm/mach-pmcs-msp71xx/msp_cic_int.h deleted file mode 100644 index 50de6876e1c9..000000000000 --- a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_cic_int.h +++ /dev/null @@ -1,139 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Defines for the MSP interrupt controller. - * - * Copyright (C) 1999 MIPS Technologies, Inc. All rights reserved. - * Author: Carsten Langgaard, carstenl@mips.com - * - * ######################################################################## - * - * ######################################################################## - */ - -#ifndef _MSP_CIC_INT_H -#define _MSP_CIC_INT_H - -/* - * The PMC-Sierra CIC interrupts are all centrally managed by the - * CIC sub-system. - * We attempt to keep the interrupt numbers as consistent as possible - * across all of the MSP devices, but some differences will creep in ... - * The interrupts which are directly forwarded to the MIPS core interrupts - * are assigned interrupts in the range 0-7, interrupts cascaded through - * the CIC are assigned interrupts 8-39. The cascade occurs on C_IRQ4 - * (MSP_INT_CIC). Currently we don't really distinguish between VPE1 - * and VPE0 (or thread contexts for that matter). Will have to fix. - * The PER interrupts are assigned interrupts in the range 40-71. -*/ - - -/* - * IRQs directly forwarded to the CPU - */ -#define MSP_MIPS_INTBASE 0 -#define MSP_INT_SW0 0 /* IRQ for swint0, C_SW0 */ -#define MSP_INT_SW1 1 /* IRQ for swint1, C_SW1 */ -#define MSP_INT_MAC0 2 /* IRQ for MAC 0, C_IRQ0 */ -#define MSP_INT_MAC1 3 /* IRQ for MAC 1, C_IRQ1 */ -#define MSP_INT_USB 4 /* IRQ for USB, C_IRQ2 */ -#define MSP_INT_SAR 5 /* IRQ for ADSL2+ SAR, C_IRQ3 */ -#define MSP_INT_CIC 6 /* IRQ for CIC block, C_IRQ4 */ -#define MSP_INT_SEC 7 /* IRQ for Sec engine, C_IRQ5 */ - -/* - * IRQs cascaded on CPU interrupt 4 (CAUSE bit 12, C_IRQ4) - * These defines should be tied to the register definitions for the CIC - * interrupt routine. For now, just use hard-coded values. - */ -#define MSP_CIC_INTBASE (MSP_MIPS_INTBASE + 8) -#define MSP_INT_EXT0 (MSP_CIC_INTBASE + 0) - /* External interrupt 0 */ -#define MSP_INT_EXT1 (MSP_CIC_INTBASE + 1) - /* External interrupt 1 */ -#define MSP_INT_EXT2 (MSP_CIC_INTBASE + 2) - /* External interrupt 2 */ -#define MSP_INT_EXT3 (MSP_CIC_INTBASE + 3) - /* External interrupt 3 */ -#define MSP_INT_CPUIF (MSP_CIC_INTBASE + 4) - /* CPU interface interrupt */ -#define MSP_INT_EXT4 (MSP_CIC_INTBASE + 5) - /* External interrupt 4 */ -#define MSP_INT_CIC_USB (MSP_CIC_INTBASE + 6) - /* Cascaded IRQ for USB */ -#define MSP_INT_MBOX (MSP_CIC_INTBASE + 7) - /* Sec engine mailbox IRQ */ -#define MSP_INT_EXT5 (MSP_CIC_INTBASE + 8) - /* External interrupt 5 */ -#define MSP_INT_TDM (MSP_CIC_INTBASE + 9) - /* TDM interrupt */ -#define MSP_INT_CIC_MAC0 (MSP_CIC_INTBASE + 10) - /* Cascaded IRQ for MAC 0 */ -#define MSP_INT_CIC_MAC1 (MSP_CIC_INTBASE + 11) - /* Cascaded IRQ for MAC 1 */ -#define MSP_INT_CIC_SEC (MSP_CIC_INTBASE + 12) - /* Cascaded IRQ for sec engine */ -#define MSP_INT_PER (MSP_CIC_INTBASE + 13) - /* Peripheral interrupt */ -#define MSP_INT_TIMER0 (MSP_CIC_INTBASE + 14) - /* SLP timer 0 */ -#define MSP_INT_TIMER1 (MSP_CIC_INTBASE + 15) - /* SLP timer 1 */ -#define MSP_INT_TIMER2 (MSP_CIC_INTBASE + 16) - /* SLP timer 2 */ -#define MSP_INT_VPE0_TIMER (MSP_CIC_INTBASE + 17) - /* VPE0 MIPS timer */ -#define MSP_INT_BLKCP (MSP_CIC_INTBASE + 18) - /* Block Copy */ -#define MSP_INT_UART0 (MSP_CIC_INTBASE + 19) - /* UART 0 */ -#define MSP_INT_PCI (MSP_CIC_INTBASE + 20) - /* PCI subsystem */ -#define MSP_INT_EXT6 (MSP_CIC_INTBASE + 21) - /* External interrupt 5 */ -#define MSP_INT_PCI_MSI (MSP_CIC_INTBASE + 22) - /* PCI Message Signal */ -#define MSP_INT_CIC_SAR (MSP_CIC_INTBASE + 23) - /* Cascaded ADSL2+ SAR IRQ */ -#define MSP_INT_DSL (MSP_CIC_INTBASE + 24) - /* ADSL2+ IRQ */ -#define MSP_INT_CIC_ERR (MSP_CIC_INTBASE + 25) - /* SLP error condition */ -#define MSP_INT_VPE1_TIMER (MSP_CIC_INTBASE + 26) - /* VPE1 MIPS timer */ -#define MSP_INT_VPE0_PC (MSP_CIC_INTBASE + 27) - /* VPE0 Performance counter */ -#define MSP_INT_VPE1_PC (MSP_CIC_INTBASE + 28) - /* VPE1 Performance counter */ -#define MSP_INT_EXT7 (MSP_CIC_INTBASE + 29) - /* External interrupt 5 */ -#define MSP_INT_VPE0_SW (MSP_CIC_INTBASE + 30) - /* VPE0 Software interrupt */ -#define MSP_INT_VPE1_SW (MSP_CIC_INTBASE + 31) - /* VPE0 Software interrupt */ - -/* - * IRQs cascaded on CIC PER interrupt (MSP_INT_PER) - */ -#define MSP_PER_INTBASE (MSP_CIC_INTBASE + 32) -/* Reserved 0-1 */ -#define MSP_INT_UART1 (MSP_PER_INTBASE + 2) - /* UART 1 */ -/* Reserved 3-5 */ -#define MSP_INT_2WIRE (MSP_PER_INTBASE + 6) - /* 2-wire */ -#define MSP_INT_TM0 (MSP_PER_INTBASE + 7) - /* Peripheral timer block out 0 */ -#define MSP_INT_TM1 (MSP_PER_INTBASE + 8) - /* Peripheral timer block out 1 */ -/* Reserved 9 */ -#define MSP_INT_SPRX (MSP_PER_INTBASE + 10) - /* SPI RX complete */ -#define MSP_INT_SPTX (MSP_PER_INTBASE + 11) - /* SPI TX complete */ -#define MSP_INT_GPIO (MSP_PER_INTBASE + 12) - /* GPIO */ -#define MSP_INT_PER_ERR (MSP_PER_INTBASE + 13) - /* Peripheral error */ -/* Reserved 14-31 */ - -#endif /* !_MSP_CIC_INT_H */ diff --git a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_gpio_macros.h b/arch/mips/include/asm/mach-pmcs-msp71xx/msp_gpio_macros.h deleted file mode 100644 index daacebb047c2..000000000000 --- a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_gpio_macros.h +++ /dev/null @@ -1,343 +0,0 @@ -/* - * - * Macros for external SMP-safe access to the PMC MSP71xx reference - * board GPIO pins - * - * Copyright 2010 PMC-Sierra, Inc. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#ifndef __MSP_GPIO_MACROS_H__ -#define __MSP_GPIO_MACROS_H__ - -#include -#include - -#ifdef CONFIG_PMC_MSP7120_GW -#define MSP_NUM_GPIOS 20 -#else -#define MSP_NUM_GPIOS 28 -#endif - -/* -- GPIO Enumerations -- */ -enum msp_gpio_data { - MSP_GPIO_LO = 0, - MSP_GPIO_HI = 1, - MSP_GPIO_NONE, /* Special - Means pin is out of range */ - MSP_GPIO_TOGGLE, /* Special - Sets pin to opposite */ -}; - -enum msp_gpio_mode { - MSP_GPIO_INPUT = 0x0, - /* MSP_GPIO_ INTERRUPT = 0x1, Not supported yet */ - MSP_GPIO_UART_INPUT = 0x2, /* Only GPIO 4 or 5 */ - MSP_GPIO_OUTPUT = 0x8, - MSP_GPIO_UART_OUTPUT = 0x9, /* Only GPIO 2 or 3 */ - MSP_GPIO_PERIF_TIMERA = 0x9, /* Only GPIO 0 or 1 */ - MSP_GPIO_PERIF_TIMERB = 0xa, /* Only GPIO 0 or 1 */ - MSP_GPIO_UNKNOWN = 0xb, /* No such GPIO or mode */ -}; - -/* -- Static Tables -- */ - -/* Maps pins to data register */ -static volatile u32 * const MSP_GPIO_DATA_REGISTER[] = { - /* GPIO 0 and 1 on the first register */ - GPIO_DATA1_REG, GPIO_DATA1_REG, - /* GPIO 2, 3, 4, and 5 on the second register */ - GPIO_DATA2_REG, GPIO_DATA2_REG, GPIO_DATA2_REG, GPIO_DATA2_REG, - /* GPIO 6, 7, 8, and 9 on the third register */ - GPIO_DATA3_REG, GPIO_DATA3_REG, GPIO_DATA3_REG, GPIO_DATA3_REG, - /* GPIO 10, 11, 12, 13, 14, and 15 on the fourth register */ - GPIO_DATA4_REG, GPIO_DATA4_REG, GPIO_DATA4_REG, GPIO_DATA4_REG, - GPIO_DATA4_REG, GPIO_DATA4_REG, - /* GPIO 16 - 23 on the first strange EXTENDED register */ - EXTENDED_GPIO1_REG, EXTENDED_GPIO1_REG, EXTENDED_GPIO1_REG, - EXTENDED_GPIO1_REG, EXTENDED_GPIO1_REG, EXTENDED_GPIO1_REG, - EXTENDED_GPIO1_REG, EXTENDED_GPIO1_REG, - /* GPIO 24 - 27 on the second strange EXTENDED register */ - EXTENDED_GPIO2_REG, EXTENDED_GPIO2_REG, EXTENDED_GPIO2_REG, - EXTENDED_GPIO2_REG, -}; - -/* Maps pins to mode register */ -static volatile u32 * const MSP_GPIO_MODE_REGISTER[] = { - /* GPIO 0 and 1 on the first register */ - GPIO_CFG1_REG, GPIO_CFG1_REG, - /* GPIO 2, 3, 4, and 5 on the second register */ - GPIO_CFG2_REG, GPIO_CFG2_REG, GPIO_CFG2_REG, GPIO_CFG2_REG, - /* GPIO 6, 7, 8, and 9 on the third register */ - GPIO_CFG3_REG, GPIO_CFG3_REG, GPIO_CFG3_REG, GPIO_CFG3_REG, - /* GPIO 10, 11, 12, 13, 14, and 15 on the fourth register */ - GPIO_CFG4_REG, GPIO_CFG4_REG, GPIO_CFG4_REG, GPIO_CFG4_REG, - GPIO_CFG4_REG, GPIO_CFG4_REG, - /* GPIO 16 - 23 on the first strange EXTENDED register */ - EXTENDED_GPIO1_REG, EXTENDED_GPIO1_REG, EXTENDED_GPIO1_REG, - EXTENDED_GPIO1_REG, EXTENDED_GPIO1_REG, EXTENDED_GPIO1_REG, - EXTENDED_GPIO1_REG, EXTENDED_GPIO1_REG, - /* GPIO 24 - 27 on the second strange EXTENDED register */ - EXTENDED_GPIO2_REG, EXTENDED_GPIO2_REG, EXTENDED_GPIO2_REG, - EXTENDED_GPIO2_REG, -}; - -/* Maps 'basic' pins to relative offset from 0 per register */ -static int MSP_GPIO_OFFSET[] = { - /* GPIO 0 and 1 on the first register */ - 0, 0, - /* GPIO 2, 3, 4, and 5 on the second register */ - 2, 2, 2, 2, - /* GPIO 6, 7, 8, and 9 on the third register */ - 6, 6, 6, 6, - /* GPIO 10, 11, 12, 13, 14, and 15 on the fourth register */ - 10, 10, 10, 10, 10, 10, -}; - -/* Maps MODE to allowed pin mask */ -static unsigned int MSP_GPIO_MODE_ALLOWED[] = { - 0xffffffff, /* Mode 0 - INPUT */ - 0x00000, /* Mode 1 - INTERRUPT */ - 0x00030, /* Mode 2 - UART_INPUT (GPIO 4, 5)*/ - 0, 0, 0, 0, 0, /* Modes 3, 4, 5, 6, and 7 are reserved */ - 0xffffffff, /* Mode 8 - OUTPUT */ - 0x0000f, /* Mode 9 - UART_OUTPUT/ - PERF_TIMERA (GPIO 0, 1, 2, 3) */ - 0x00003, /* Mode a - PERF_TIMERB (GPIO 0, 1) */ - 0x00000, /* Mode b - Not really a mode! */ -}; - -/* -- Bit masks -- */ - -/* This gives you the 'register relative offset gpio' number */ -#define OFFSET_GPIO_NUMBER(gpio) (gpio - MSP_GPIO_OFFSET[gpio]) - -/* These take the 'register relative offset gpio' number */ -#define BASIC_DATA_REG_MASK(ogpio) (1 << ogpio) -#define BASIC_MODE_REG_VALUE(mode, ogpio) \ - (mode << BASIC_MODE_REG_SHIFT(ogpio)) -#define BASIC_MODE_REG_MASK(ogpio) \ - BASIC_MODE_REG_VALUE(0xf, ogpio) -#define BASIC_MODE_REG_SHIFT(ogpio) (ogpio * 4) -#define BASIC_MODE_REG_FROM_REG(data, ogpio) \ - ((data & BASIC_MODE_REG_MASK(ogpio)) >> BASIC_MODE_REG_SHIFT(ogpio)) - -/* These take the actual GPIO number (0 through 15) */ -#define BASIC_DATA_MASK(gpio) \ - BASIC_DATA_REG_MASK(OFFSET_GPIO_NUMBER(gpio)) -#define BASIC_MODE_MASK(gpio) \ - BASIC_MODE_REG_MASK(OFFSET_GPIO_NUMBER(gpio)) -#define BASIC_MODE(mode, gpio) \ - BASIC_MODE_REG_VALUE(mode, OFFSET_GPIO_NUMBER(gpio)) -#define BASIC_MODE_SHIFT(gpio) \ - BASIC_MODE_REG_SHIFT(OFFSET_GPIO_NUMBER(gpio)) -#define BASIC_MODE_FROM_REG(data, gpio) \ - BASIC_MODE_REG_FROM_REG(data, OFFSET_GPIO_NUMBER(gpio)) - -/* - * Each extended GPIO register is 32 bits long and is responsible for up to - * eight GPIOs. The least significant 16 bits contain the set and clear bit - * pair for each of the GPIOs. The most significant 16 bits contain the - * disable and enable bit pair for each of the GPIOs. For example, the - * extended GPIO reg for GPIOs 16-23 is as follows: - * - * 31: GPIO23_DISABLE - * ... - * 19: GPIO17_DISABLE - * 18: GPIO17_ENABLE - * 17: GPIO16_DISABLE - * 16: GPIO16_ENABLE - * ... - * 3: GPIO17_SET - * 2: GPIO17_CLEAR - * 1: GPIO16_SET - * 0: GPIO16_CLEAR - */ - -/* This gives the 'register relative offset gpio' number */ -#define EXTENDED_OFFSET_GPIO(gpio) (gpio < 24 ? gpio - 16 : gpio - 24) - -/* These take the 'register relative offset gpio' number */ -#define EXTENDED_REG_DISABLE(ogpio) (0x2 << ((ogpio * 2) + 16)) -#define EXTENDED_REG_ENABLE(ogpio) (0x1 << ((ogpio * 2) + 16)) -#define EXTENDED_REG_SET(ogpio) (0x2 << (ogpio * 2)) -#define EXTENDED_REG_CLR(ogpio) (0x1 << (ogpio * 2)) - -/* These take the actual GPIO number (16 through 27) */ -#define EXTENDED_DISABLE(gpio) \ - EXTENDED_REG_DISABLE(EXTENDED_OFFSET_GPIO(gpio)) -#define EXTENDED_ENABLE(gpio) \ - EXTENDED_REG_ENABLE(EXTENDED_OFFSET_GPIO(gpio)) -#define EXTENDED_SET(gpio) \ - EXTENDED_REG_SET(EXTENDED_OFFSET_GPIO(gpio)) -#define EXTENDED_CLR(gpio) \ - EXTENDED_REG_CLR(EXTENDED_OFFSET_GPIO(gpio)) - -#define EXTENDED_FULL_MASK (0xffffffff) - -/* -- API inline-functions -- */ - -/* - * Gets the current value of the specified pin - */ -static inline enum msp_gpio_data msp_gpio_pin_get(unsigned int gpio) -{ - u32 pinhi_mask = 0, pinhi_mask2 = 0; - - if (gpio >= MSP_NUM_GPIOS) - return MSP_GPIO_NONE; - - if (gpio < 16) { - pinhi_mask = BASIC_DATA_MASK(gpio); - } else { - /* - * Two cases are possible with the EXTENDED register: - * - In output mode (ENABLED flag set), check the CLR bit - * - In input mode (ENABLED flag not set), check the SET bit - */ - pinhi_mask = EXTENDED_ENABLE(gpio) | EXTENDED_CLR(gpio); - pinhi_mask2 = EXTENDED_SET(gpio); - } - if (((*MSP_GPIO_DATA_REGISTER[gpio] & pinhi_mask) == pinhi_mask) || - (*MSP_GPIO_DATA_REGISTER[gpio] & pinhi_mask2)) - return MSP_GPIO_HI; - else - return MSP_GPIO_LO; -} - -/* Sets the specified pin to the specified value */ -static inline void msp_gpio_pin_set(enum msp_gpio_data data, unsigned int gpio) -{ - if (gpio >= MSP_NUM_GPIOS) - return; - - if (gpio < 16) { - if (data == MSP_GPIO_TOGGLE) - toggle_reg32(MSP_GPIO_DATA_REGISTER[gpio], - BASIC_DATA_MASK(gpio)); - else if (data == MSP_GPIO_HI) - set_reg32(MSP_GPIO_DATA_REGISTER[gpio], - BASIC_DATA_MASK(gpio)); - else - clear_reg32(MSP_GPIO_DATA_REGISTER[gpio], - BASIC_DATA_MASK(gpio)); - } else { - if (data == MSP_GPIO_TOGGLE) { - /* Special ugly case: - * We have to read the CLR bit. - * If set, we write the CLR bit. - * If not, we write the SET bit. - */ - u32 tmpdata; - - custom_read_reg32(MSP_GPIO_DATA_REGISTER[gpio], - tmpdata); - if (tmpdata & EXTENDED_CLR(gpio)) - tmpdata = EXTENDED_CLR(gpio); - else - tmpdata = EXTENDED_SET(gpio); - custom_write_reg32(MSP_GPIO_DATA_REGISTER[gpio], - tmpdata); - } else { - u32 newdata; - - if (data == MSP_GPIO_HI) - newdata = EXTENDED_SET(gpio); - else - newdata = EXTENDED_CLR(gpio); - set_value_reg32(MSP_GPIO_DATA_REGISTER[gpio], - EXTENDED_FULL_MASK, newdata); - } - } -} - -/* Sets the specified pin to the specified value */ -static inline void msp_gpio_pin_hi(unsigned int gpio) -{ - msp_gpio_pin_set(MSP_GPIO_HI, gpio); -} - -/* Sets the specified pin to the specified value */ -static inline void msp_gpio_pin_lo(unsigned int gpio) -{ - msp_gpio_pin_set(MSP_GPIO_LO, gpio); -} - -/* Sets the specified pin to the opposite value */ -static inline void msp_gpio_pin_toggle(unsigned int gpio) -{ - msp_gpio_pin_set(MSP_GPIO_TOGGLE, gpio); -} - -/* Gets the mode of the specified pin */ -static inline enum msp_gpio_mode msp_gpio_pin_get_mode(unsigned int gpio) -{ - enum msp_gpio_mode retval = MSP_GPIO_UNKNOWN; - uint32_t data; - - if (gpio >= MSP_NUM_GPIOS) - return retval; - - data = *MSP_GPIO_MODE_REGISTER[gpio]; - - if (gpio < 16) { - retval = BASIC_MODE_FROM_REG(data, gpio); - } else { - /* Extended pins can only be either INPUT or OUTPUT */ - if (data & EXTENDED_ENABLE(gpio)) - retval = MSP_GPIO_OUTPUT; - else - retval = MSP_GPIO_INPUT; - } - - return retval; -} - -/* - * Sets the specified mode on the requested pin - * Returns 0 on success, or -1 if that mode is not allowed on this pin - */ -static inline int msp_gpio_pin_mode(enum msp_gpio_mode mode, unsigned int gpio) -{ - u32 modemask, newmode; - - if ((1 << gpio) & ~MSP_GPIO_MODE_ALLOWED[mode]) - return -1; - - if (gpio >= MSP_NUM_GPIOS) - return -1; - - if (gpio < 16) { - modemask = BASIC_MODE_MASK(gpio); - newmode = BASIC_MODE(mode, gpio); - } else { - modemask = EXTENDED_FULL_MASK; - if (mode == MSP_GPIO_INPUT) - newmode = EXTENDED_DISABLE(gpio); - else - newmode = EXTENDED_ENABLE(gpio); - } - /* Do the set atomically */ - set_value_reg32(MSP_GPIO_MODE_REGISTER[gpio], modemask, newmode); - - return 0; -} - -#endif /* __MSP_GPIO_MACROS_H__ */ diff --git a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_int.h b/arch/mips/include/asm/mach-pmcs-msp71xx/msp_int.h deleted file mode 100644 index 55078b40f5b5..000000000000 --- a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_int.h +++ /dev/null @@ -1,31 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Defines for the MSP interrupt handlers. - * - * Copyright (C) 2005, PMC-Sierra, Inc. All rights reserved. - * Author: Andrew Hughes, Andrew_Hughes@pmc-sierra.com - * - * ######################################################################## - * - * ######################################################################## - */ - -#ifndef _MSP_INT_H -#define _MSP_INT_H - -/* - * The PMC-Sierra MSP product line has at least two different interrupt - * controllers, the SLP register based scheme and the CIC interrupt - * controller block mechanism. This file distinguishes between them - * so that devices see a uniform interface. - */ - -#if defined(CONFIG_IRQ_MSP_SLP) - #include "msp_slp_int.h" -#elif defined(CONFIG_IRQ_MSP_CIC) - #include "msp_cic_int.h" -#else - #error "What sort of interrupt controller does *your* MSP have?" -#endif - -#endif /* !_MSP_INT_H */ diff --git a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_pci.h b/arch/mips/include/asm/mach-pmcs-msp71xx/msp_pci.h deleted file mode 100644 index 5b2535efceb2..000000000000 --- a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_pci.h +++ /dev/null @@ -1,189 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (c) 2000-2006 PMC-Sierra INC. - * - * PMC-SIERRA INC. DISCLAIMS ANY LIABILITY OF ANY KIND - * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS - * SOFTWARE. - */ - -#ifndef _MSP_PCI_H_ -#define _MSP_PCI_H_ - -#define MSP_HAS_PCI(ID) (((u32)(ID) <= 0x4236) && ((u32)(ID) >= 0x4220)) - -/* - * It is convenient to program the OATRAN register so that - * Athena virtual address space and PCI address space are - * the same. This is not a requirement, just a convenience. - * - * The only hard restrictions on the value of OATRAN is that - * OATRAN must not be programmed to allow translated memory - * addresses to fall within the lowest 512MB of - * PCI address space. This region is hardcoded - * for use as Athena PCI Host Controller target - * access memory space to the Athena's SDRAM. - * - * Note that OATRAN applies only to memory accesses, not - * to I/O accesses. - * - * To program OATRAN to make Athena virtual address space - * and PCI address space have the same values, OATRAN - * is to be programmed to 0xB8000000. The top seven - * bits of the value mimic the seven bits clipped off - * by the PCI Host controller. - * - * With OATRAN at the said value, when the CPU does - * an access to its virtual address at, say 0xB900_5000, - * the address appearing on the PCI bus will be - * 0xB900_5000. - * - Michael Penner - */ -#define MSP_PCI_OATRAN 0xB8000000UL - -#define MSP_PCI_SPACE_BASE (MSP_PCI_OATRAN + 0x1002000UL) -#define MSP_PCI_SPACE_SIZE (0x3000000UL - 0x2000) -#define MSP_PCI_SPACE_END \ - (MSP_PCI_SPACE_BASE + MSP_PCI_SPACE_SIZE - 1) -#define MSP_PCI_IOSPACE_BASE (MSP_PCI_OATRAN + 0x1001000UL) -#define MSP_PCI_IOSPACE_SIZE 0x1000 -#define MSP_PCI_IOSPACE_END \ - (MSP_PCI_IOSPACE_BASE + MSP_PCI_IOSPACE_SIZE - 1) - -/* IRQ for PCI status interrupts */ -#define PCI_STAT_IRQ 20 - -#define QFLUSH_REG_1 0xB7F40000 - -typedef volatile unsigned int pcireg; -typedef void * volatile ppcireg; - -struct pci_block_copy -{ - pcireg unused1; /* +0x00 */ - pcireg unused2; /* +0x04 */ - ppcireg unused3; /* +0x08 */ - ppcireg unused4; /* +0x0C */ - pcireg unused5; /* +0x10 */ - pcireg unused6; /* +0x14 */ - pcireg unused7; /* +0x18 */ - ppcireg unused8; /* +0x1C */ - ppcireg unused9; /* +0x20 */ - pcireg unusedA; /* +0x24 */ - ppcireg unusedB; /* +0x28 */ - ppcireg unusedC; /* +0x2C */ -}; - -enum -{ - config_device_vendor, /* 0 */ - config_status_command, /* 1 */ - config_class_revision, /* 2 */ - config_BIST_header_latency_cache, /* 3 */ - config_BAR0, /* 4 */ - config_BAR1, /* 5 */ - config_BAR2, /* 6 */ - config_not_used7, /* 7 */ - config_not_used8, /* 8 */ - config_not_used9, /* 9 */ - config_CIS, /* 10 */ - config_subsystem, /* 11 */ - config_not_used12, /* 12 */ - config_capabilities, /* 13 */ - config_not_used14, /* 14 */ - config_lat_grant_irq, /* 15 */ - config_message_control,/* 16 */ - config_message_addr, /* 17 */ - config_message_data, /* 18 */ - config_VPD_addr, /* 19 */ - config_VPD_data, /* 20 */ - config_maxregs /* 21 - number of registers */ -}; - -struct msp_pci_regs -{ - pcireg hop_unused_00; /* +0x00 */ - pcireg hop_unused_04; /* +0x04 */ - pcireg hop_unused_08; /* +0x08 */ - pcireg hop_unused_0C; /* +0x0C */ - pcireg hop_unused_10; /* +0x10 */ - pcireg hop_unused_14; /* +0x14 */ - pcireg hop_unused_18; /* +0x18 */ - pcireg hop_unused_1C; /* +0x1C */ - pcireg hop_unused_20; /* +0x20 */ - pcireg hop_unused_24; /* +0x24 */ - pcireg hop_unused_28; /* +0x28 */ - pcireg hop_unused_2C; /* +0x2C */ - pcireg hop_unused_30; /* +0x30 */ - pcireg hop_unused_34; /* +0x34 */ - pcireg if_control; /* +0x38 */ - pcireg oatran; /* +0x3C */ - pcireg reset_ctl; /* +0x40 */ - pcireg config_addr; /* +0x44 */ - pcireg hop_unused_48; /* +0x48 */ - pcireg msg_signaled_int_status; /* +0x4C */ - pcireg msg_signaled_int_mask; /* +0x50 */ - pcireg if_status; /* +0x54 */ - pcireg if_mask; /* +0x58 */ - pcireg hop_unused_5C; /* +0x5C */ - pcireg hop_unused_60; /* +0x60 */ - pcireg hop_unused_64; /* +0x64 */ - pcireg hop_unused_68; /* +0x68 */ - pcireg hop_unused_6C; /* +0x6C */ - pcireg hop_unused_70; /* +0x70 */ - - struct pci_block_copy pci_bc[2] __attribute__((aligned(64))); - - pcireg error_hdr1; /* +0xE0 */ - pcireg error_hdr2; /* +0xE4 */ - - pcireg config[config_maxregs] __attribute__((aligned(256))); - -}; - -#define BPCI_CFGADDR_BUSNUM_SHF 16 -#define BPCI_CFGADDR_FUNCTNUM_SHF 8 -#define BPCI_CFGADDR_REGNUM_SHF 2 -#define BPCI_CFGADDR_ENABLE (1<<31) - -#define BPCI_IFCONTROL_RTO (1<<20) /* Retry timeout */ -#define BPCI_IFCONTROL_HCE (1<<16) /* Host configuration enable */ -#define BPCI_IFCONTROL_CTO_SHF 12 /* Shift count for CTO bits */ -#define BPCI_IFCONTROL_SE (1<<5) /* Enable exceptions on errors */ -#define BPCI_IFCONTROL_BIST (1<<4) /* Use BIST in per. mode */ -#define BPCI_IFCONTROL_CAP (1<<3) /* Enable capabilities */ -#define BPCI_IFCONTROL_MMC_SHF 0 /* Shift count for MMC bits */ - -#define BPCI_IFSTATUS_MGT (1<<8) /* Master Grant timeout */ -#define BPCI_IFSTATUS_MTT (1<<9) /* Master TRDY timeout */ -#define BPCI_IFSTATUS_MRT (1<<10) /* Master retry timeout */ -#define BPCI_IFSTATUS_BC0F (1<<13) /* Block copy 0 fault */ -#define BPCI_IFSTATUS_BC1F (1<<14) /* Block copy 1 fault */ -#define BPCI_IFSTATUS_PCIU (1<<15) /* PCI unable to respond */ -#define BPCI_IFSTATUS_BSIZ (1<<16) /* PCI access with illegal size */ -#define BPCI_IFSTATUS_BADD (1<<17) /* PCI access with illegal addr */ -#define BPCI_IFSTATUS_RTO (1<<18) /* Retry time out */ -#define BPCI_IFSTATUS_SER (1<<19) /* System error */ -#define BPCI_IFSTATUS_PER (1<<20) /* Parity error */ -#define BPCI_IFSTATUS_LCA (1<<21) /* Local CPU abort */ -#define BPCI_IFSTATUS_MEM (1<<22) /* Memory prot. violation */ -#define BPCI_IFSTATUS_ARB (1<<23) /* Arbiter timed out */ -#define BPCI_IFSTATUS_STA (1<<27) /* Signaled target abort */ -#define BPCI_IFSTATUS_TA (1<<28) /* Target abort */ -#define BPCI_IFSTATUS_MA (1<<29) /* Master abort */ -#define BPCI_IFSTATUS_PEI (1<<30) /* Parity error as initiator */ -#define BPCI_IFSTATUS_PET (1<<31) /* Parity error as target */ - -#define BPCI_RESETCTL_PR (1<<0) /* True if reset asserted */ -#define BPCI_RESETCTL_RT (1<<4) /* Release time */ -#define BPCI_RESETCTL_CT (1<<8) /* Config time */ -#define BPCI_RESETCTL_PE (1<<12) /* PCI enabled */ -#define BPCI_RESETCTL_HM (1<<13) /* PCI host mode */ -#define BPCI_RESETCTL_RI (1<<14) /* PCI reset in */ - -extern struct msp_pci_regs msp_pci_regs - __attribute__((section(".register"))); -extern unsigned long msp_pci_config_space - __attribute__((section(".register"))); - -#endif /* !_MSP_PCI_H_ */ diff --git a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_prom.h b/arch/mips/include/asm/mach-pmcs-msp71xx/msp_prom.h deleted file mode 100644 index 4120a01c30a9..000000000000 --- a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_prom.h +++ /dev/null @@ -1,159 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * MIPS boards bootprom interface for the Linux kernel. - * - * Copyright (C) 2000 MIPS Technologies, Inc. All rights reserved. - * Author: Carsten Langgaard, carstenl@mips.com - * - * ######################################################################## - * - * ######################################################################## - */ - -#ifndef _ASM_MSP_PROM_H -#define _ASM_MSP_PROM_H - -#include - -#define DEVICEID "deviceid" -#define FEATURES "features" -#define PROM_ENV "prom_env" -#define PROM_ENV_FILE "/proc/"PROM_ENV -#define PROM_ENV_SIZE 256 - -#define CPU_DEVID_FAMILY 0x0000ff00 -#define CPU_DEVID_REVISION 0x000000ff - -#define FPGA_IS_POLO(revision) \ - (((revision >= 0xb0) && (revision < 0xd0))) -#define FPGA_IS_5000(revision) \ - ((revision >= 0x80) && (revision <= 0x90)) -#define FPGA_IS_ZEUS(revision) ((revision < 0x7f)) -#define FPGA_IS_DUET(revision) \ - (((revision >= 0xa0) && (revision < 0xb0))) -#define FPGA_IS_MSP4200(revision) ((revision >= 0xd0)) -#define FPGA_IS_MSP7100(revision) ((revision >= 0xd0)) - -#define MACHINE_TYPE_POLO "POLO" -#define MACHINE_TYPE_DUET "DUET" -#define MACHINE_TYPE_ZEUS "ZEUS" -#define MACHINE_TYPE_MSP2000REVB "MSP2000REVB" -#define MACHINE_TYPE_MSP5000 "MSP5000" -#define MACHINE_TYPE_MSP4200 "MSP4200" -#define MACHINE_TYPE_MSP7120 "MSP7120" -#define MACHINE_TYPE_MSP7130 "MSP7130" -#define MACHINE_TYPE_OTHER "OTHER" - -#define MACHINE_TYPE_POLO_FPGA "POLO-FPGA" -#define MACHINE_TYPE_DUET_FPGA "DUET-FPGA" -#define MACHINE_TYPE_ZEUS_FPGA "ZEUS_FPGA" -#define MACHINE_TYPE_MSP2000REVB_FPGA "MSP2000REVB-FPGA" -#define MACHINE_TYPE_MSP5000_FPGA "MSP5000-FPGA" -#define MACHINE_TYPE_MSP4200_FPGA "MSP4200-FPGA" -#define MACHINE_TYPE_MSP7100_FPGA "MSP7100-FPGA" -#define MACHINE_TYPE_OTHER_FPGA "OTHER-FPGA" - -/* Device Family definitions */ -#define FAMILY_FPGA 0x0000 -#define FAMILY_ZEUS 0x1000 -#define FAMILY_POLO 0x2000 -#define FAMILY_DUET 0x4000 -#define FAMILY_TRIAD 0x5000 -#define FAMILY_MSP4200 0x4200 -#define FAMILY_MSP4200_FPGA 0x4f00 -#define FAMILY_MSP7100 0x7100 -#define FAMILY_MSP7100_FPGA 0x7f00 - -/* Device Type definitions */ -#define TYPE_MSP7120 0x7120 -#define TYPE_MSP7130 0x7130 - -#define ENET_KEY 'E' -#define ENETTXD_KEY 'e' -#define PCI_KEY 'P' -#define PCIMUX_KEY 'p' -#define SEC_KEY 'S' -#define SPAD_KEY 'D' -#define TDM_KEY 'T' -#define ZSP_KEY 'Z' - -#define FEATURE_NOEXIST '-' -#define FEATURE_EXIST '+' - -#define ENET_MII 'M' -#define ENET_RMII 'R' - -#define ENETTXD_FALLING 'F' -#define ENETTXD_RISING 'R' - -#define PCI_HOST 'H' -#define PCI_PERIPHERAL 'P' - -#define PCIMUX_FULL 'F' -#define PCIMUX_SINGLE 'S' - -#define SEC_DUET 'D' -#define SEC_POLO 'P' -#define SEC_SLOW 'S' -#define SEC_TRIAD 'T' - -#define SPAD_POLO 'P' - -#define TDM_DUET 'D' /* DUET TDMs might exist */ -#define TDM_POLO 'P' /* POLO TDMs might exist */ -#define TDM_TRIAD 'T' /* TRIAD TDMs might exist */ - -#define ZSP_DUET 'D' /* one DUET zsp engine */ -#define ZSP_TRIAD 'T' /* two TRIAD zsp engines */ - -extern char *prom_getenv(char *name); -extern void prom_init_cmdline(void); -extern void prom_meminit(void); -extern void prom_fixup_mem_map(unsigned long start_mem, - unsigned long end_mem); - -extern int get_ethernet_addr(char *ethaddr_name, char *ethernet_addr); -extern unsigned long get_deviceid(void); -extern char identify_enet(unsigned long interface_num); -extern char identify_enetTxD(unsigned long interface_num); -extern char identify_pci(void); -extern char identify_sec(void); -extern char identify_spad(void); -extern char identify_sec(void); -extern char identify_tdm(void); -extern char identify_zsp(void); -extern unsigned long identify_family(void); -extern unsigned long identify_revision(void); - -/* - * The following macro calls prom_printf and puts the format string - * into an init section so it can be reclaimed. - */ -#define ppfinit(f, x...) \ - do { \ - static char _f[] __initdata = KERN_INFO f; \ - printk(_f, ## x); \ - } while (0) - -/* Memory descriptor management. */ -#define PROM_MAX_PMEMBLOCKS 7 /* 6 used */ - -enum yamon_memtypes { - yamon_dontuse, - yamon_prom, - yamon_free, -}; - -struct prom_pmemblock { - unsigned long base; /* Within KSEG0. */ - unsigned int size; /* In bytes. */ - unsigned int type; /* free or prom memory */ -}; - -extern int prom_argc; -extern char **prom_argv; -extern char **prom_envp; -extern int *prom_vec; -extern struct prom_pmemblock *prom_getmdesc(void); - -#endif /* !_ASM_MSP_PROM_H */ diff --git a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_regops.h b/arch/mips/include/asm/mach-pmcs-msp71xx/msp_regops.h deleted file mode 100644 index 90dbe43c8d27..000000000000 --- a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_regops.h +++ /dev/null @@ -1,237 +0,0 @@ -/* - * SMP/VPE-safe functions to access "registers" (see note). - * - * NOTES: -* - These macros use ll/sc instructions, so it is your responsibility to - * ensure these are available on your platform before including this file. - * - The MIPS32 spec states that ll/sc results are undefined for uncached - * accesses. This means they can't be used on HW registers accessed - * through kseg1. Code which requires these macros for this purpose must - * front-end the registers with cached memory "registers" and have a single - * thread update the actual HW registers. - * - A maximum of 2k of code can be inserted between ll and sc. Every - * memory accesses between the instructions will increase the chance of - * sc failing and having to loop. - * - When using custom_read_reg32/custom_write_reg32 only perform the - * necessary logical operations on the register value in between these - * two calls. All other logic should be performed before the first call. - * - There is a bug on the R10000 chips which has a workaround. If you - * are affected by this bug, make sure to define the symbol 'R10000_LLSC_WAR' - * to be non-zero. If you are using this header from within linux, you may - * include before including this file to have this defined - * appropriately for you. - * - * Copyright 2005-2007 PMC-Sierra, Inc. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO - * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., 675 - * Mass Ave, Cambridge, MA 02139, USA. - */ - -#ifndef __ASM_REGOPS_H__ -#define __ASM_REGOPS_H__ - -#include - -#include -#include - -#ifndef R10000_LLSC_WAR -#define R10000_LLSC_WAR 0 -#endif - -#if R10000_LLSC_WAR == 1 -#define __beqz "beqzl " -#else -#define __beqz "beqz " -#endif - -#ifndef _LINUX_TYPES_H -typedef unsigned int u32; -#endif - -/* - * Sets all the masked bits to the corresponding value bits - */ -static inline void set_value_reg32(volatile u32 *const addr, - u32 const mask, - u32 const value) -{ - u32 temp; - - __asm__ __volatile__( - " .set push \n" - " .set arch=r4000 \n" - "1: ll %0, %1 # set_value_reg32 \n" - " and %0, %2 \n" - " or %0, %3 \n" - " sc %0, %1 \n" - " "__beqz"%0, 1b \n" - " nop \n" - " .set pop \n" - : "=&r" (temp), "=" GCC_OFF_SMALL_ASM() (*addr) - : "ir" (~mask), "ir" (value), GCC_OFF_SMALL_ASM() (*addr)); -} - -/* - * Sets all the masked bits to '1' - */ -static inline void set_reg32(volatile u32 *const addr, - u32 const mask) -{ - u32 temp; - - __asm__ __volatile__( - " .set push \n" - " .set arch=r4000 \n" - "1: ll %0, %1 # set_reg32 \n" - " or %0, %2 \n" - " sc %0, %1 \n" - " "__beqz"%0, 1b \n" - " nop \n" - " .set pop \n" - : "=&r" (temp), "=" GCC_OFF_SMALL_ASM() (*addr) - : "ir" (mask), GCC_OFF_SMALL_ASM() (*addr)); -} - -/* - * Sets all the masked bits to '0' - */ -static inline void clear_reg32(volatile u32 *const addr, - u32 const mask) -{ - u32 temp; - - __asm__ __volatile__( - " .set push \n" - " .set arch=r4000 \n" - "1: ll %0, %1 # clear_reg32 \n" - " and %0, %2 \n" - " sc %0, %1 \n" - " "__beqz"%0, 1b \n" - " nop \n" - " .set pop \n" - : "=&r" (temp), "=" GCC_OFF_SMALL_ASM() (*addr) - : "ir" (~mask), GCC_OFF_SMALL_ASM() (*addr)); -} - -/* - * Toggles all masked bits from '0' to '1' and '1' to '0' - */ -static inline void toggle_reg32(volatile u32 *const addr, - u32 const mask) -{ - u32 temp; - - __asm__ __volatile__( - " .set push \n" - " .set arch=r4000 \n" - "1: ll %0, %1 # toggle_reg32 \n" - " xor %0, %2 \n" - " sc %0, %1 \n" - " "__beqz"%0, 1b \n" - " nop \n" - " .set pop \n" - : "=&r" (temp), "=" GCC_OFF_SMALL_ASM() (*addr) - : "ir" (mask), GCC_OFF_SMALL_ASM() (*addr)); -} - -/* - * Read all masked bits others are returned as '0' - */ -static inline u32 read_reg32(volatile u32 *const addr, - u32 const mask) -{ - u32 temp; - - __asm__ __volatile__( - " .set push \n" - " .set noreorder \n" - " lw %0, %1 # read \n" - " and %0, %2 # mask \n" - " .set pop \n" - : "=&r" (temp) - : "m" (*addr), "ir" (mask)); - - return temp; -} - -/* - * blocking_read_reg32 - Read address with blocking load - * - * Uncached writes need to be read back to ensure they reach RAM. - * The returned value must be 'used' to prevent from becoming a - * non-blocking load. - */ -static inline u32 blocking_read_reg32(volatile u32 *const addr) -{ - u32 temp; - - __asm__ __volatile__( - " .set push \n" - " .set noreorder \n" - " lw %0, %1 # read \n" - " move %0, %0 # block \n" - " .set pop \n" - : "=&r" (temp) - : "m" (*addr)); - - return temp; -} - -/* - * For special strange cases only: - * - * If you need custom processing within a ll/sc loop, use the following macros - * VERY CAREFULLY: - * - * u32 tmp; <-- Define a variable to hold the data - * - * custom_read_reg32(address, tmp); <-- Reads the address and put the value - * in the 'tmp' variable given - * - * From here on out, you are (basically) atomic, so don't do anything too - * fancy! - * Also, this code may loop if the end of this block fails to write - * everything back safely due do the other CPU, so do NOT do anything - * with side-effects! - * - * custom_write_reg32(address, tmp); <-- Writes back 'tmp' safely. - */ -#define custom_read_reg32(address, tmp) \ - __asm__ __volatile__( \ - " .set push \n" \ - " .set arch=r4000 \n" \ - "1: ll %0, %1 #custom_read_reg32 \n" \ - " .set pop \n" \ - : "=r" (tmp), "=" GCC_OFF_SMALL_ASM() (*address) \ - : GCC_OFF_SMALL_ASM() (*address)) - -#define custom_write_reg32(address, tmp) \ - __asm__ __volatile__( \ - " .set push \n" \ - " .set arch=r4000 \n" \ - " sc %0, %1 #custom_write_reg32 \n" \ - " "__beqz"%0, 1b \n" \ - " nop \n" \ - " .set pop \n" \ - : "=&r" (tmp), "=" GCC_OFF_SMALL_ASM() (*address) \ - : "0" (tmp), GCC_OFF_SMALL_ASM() (*address)) - -#endif /* __ASM_REGOPS_H__ */ diff --git a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_regs.h b/arch/mips/include/asm/mach-pmcs-msp71xx/msp_regs.h deleted file mode 100644 index e2ce9be51f3f..000000000000 --- a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_regs.h +++ /dev/null @@ -1,652 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Defines for the address space, registers and register configuration - * (bit masks, access macros etc) for the PMC-Sierra line of MSP products. - * This file contains addess maps for all the devices in the line of - * products but only has register definitions and configuration masks for - * registers which aren't definitely associated with any device. Things - * like clock settings, reset access, the ELB etc. Individual device - * drivers will reference the appropriate XXX_BASE value defined here - * and have individual registers offset from that. - * - * Copyright (C) 2005-2007 PMC-Sierra, Inc. All rights reserved. - * Author: Andrew Hughes, Andrew_Hughes@pmc-sierra.com - * - * ######################################################################## - * - * ######################################################################## - */ - -#include -#include - -#ifndef _ASM_MSP_REGS_H -#define _ASM_MSP_REGS_H - -/* - ######################################################################## - # Address space and device base definitions # - ######################################################################## - */ - -/* - *************************************************************************** - * System Logic and Peripherals (ELB, UART0, etc) device address space * - *************************************************************************** - */ -#define MSP_SLP_BASE 0x1c000000 - /* System Logic and Peripherals */ -#define MSP_RST_BASE (MSP_SLP_BASE + 0x10) - /* System reset register base */ -#define MSP_RST_SIZE 0x0C /* System reset register space */ - -#define MSP_WTIMER_BASE (MSP_SLP_BASE + 0x04C) - /* watchdog timer base */ -#define MSP_ITIMER_BASE (MSP_SLP_BASE + 0x054) - /* internal timer base */ -#define MSP_UART0_BASE (MSP_SLP_BASE + 0x100) - /* UART0 controller base */ -#define MSP_BCPY_CTRL_BASE (MSP_SLP_BASE + 0x120) - /* Block Copy controller base */ -#define MSP_BCPY_DESC_BASE (MSP_SLP_BASE + 0x160) - /* Block Copy descriptor base */ - -/* - *************************************************************************** - * PCI address space * - *************************************************************************** - */ -#define MSP_PCI_BASE 0x19000000 - -/* - *************************************************************************** - * MSbus device address space * - *************************************************************************** - */ -#define MSP_MSB_BASE 0x18000000 - /* MSbus address start */ -#define MSP_PER_BASE (MSP_MSB_BASE + 0x400000) - /* Peripheral device registers */ -#define MSP_MAC0_BASE (MSP_MSB_BASE + 0x600000) - /* MAC A device registers */ -#define MSP_MAC1_BASE (MSP_MSB_BASE + 0x700000) - /* MAC B device registers */ -#define MSP_MAC_SIZE 0xE0 /* MAC register space */ - -#define MSP_SEC_BASE (MSP_MSB_BASE + 0x800000) - /* Security Engine registers */ -#define MSP_MAC2_BASE (MSP_MSB_BASE + 0x900000) - /* MAC C device registers */ -#define MSP_ADSL2_BASE (MSP_MSB_BASE + 0xA80000) - /* ADSL2 device registers */ -#define MSP_USB0_BASE (MSP_MSB_BASE + 0xB00000) - /* USB0 device registers */ -#define MSP_USB1_BASE (MSP_MSB_BASE + 0x300000) - /* USB1 device registers */ -#define MSP_CPUIF_BASE (MSP_MSB_BASE + 0xC00000) - /* CPU interface registers */ - -/* Devices within the MSbus peripheral block */ -#define MSP_UART1_BASE (MSP_PER_BASE + 0x030) - /* UART1 controller base */ -#define MSP_SPI_BASE (MSP_PER_BASE + 0x058) - /* SPI/MPI control registers */ -#define MSP_TWI_BASE (MSP_PER_BASE + 0x090) - /* Two-wire control registers */ -#define MSP_PTIMER_BASE (MSP_PER_BASE + 0x0F0) - /* Programmable timer control */ - -/* - *************************************************************************** - * Physical Memory configuration address space * - *************************************************************************** - */ -#define MSP_MEM_CFG_BASE 0x17f00000 - -#define MSP_MEM_INDIRECT_CTL_10 0x10 - -/* - * Notes: - * 1) The SPI registers are split into two blocks, one offset from the - * MSP_SPI_BASE by 0x00 and the other offset from the MSP_SPI_BASE by - * 0x68. The SPI driver definitions for the register must be aware - * of this. - * 2) The block copy engine register are divided into two regions, one - * for the control/configuration of the engine proper and one for the - * values of the descriptors used in the copy process. These have - * different base defines (CTRL_BASE vs DESC_BASE) - * 3) These constants are for physical addresses which means that they - * work correctly with "ioremap" and friends. This means that device - * drivers will need to remap these addresses using ioremap and perhaps - * the readw/writew macros. Or they could use the regptr() macro - * defined below, but the readw/writew calls are the correct thing. - * 4) The UARTs have an additional status register offset from the base - * address. This register isn't used in the standard 8250 driver but - * may be used in other software. Consult the hardware datasheet for - * offset details. - * 5) For some unknown reason the security engine (MSP_SEC_BASE) registers - * start at an offset of 0x84 from the base address but the block of - * registers before this is reserved for the security engine. The - * driver will have to be aware of this but it makes the register - * definitions line up better with the documentation. - */ - -/* - ######################################################################## - # System register definitions. Not associated with a specific device # - ######################################################################## - */ - -/* - * This macro maps the physical register number into uncached space - * and (for C code) casts it into a u32 pointer so it can be dereferenced - * Normally these would be accessed with ioremap and readX/writeX, but - * these are convenient for a lot of internal kernel code. - */ -#ifdef __ASSEMBLER__ - #define regptr(addr) (KSEG1ADDR(addr)) -#else - #define regptr(addr) ((volatile u32 *const)(KSEG1ADDR(addr))) -#endif - -/* - *************************************************************************** - * System Logic and Peripherals (RESET, ELB, etc) registers * - *************************************************************************** - */ - -/* System Control register definitions */ -#define DEV_ID_REG regptr(MSP_SLP_BASE + 0x00) - /* Device-ID RO */ -#define FWR_ID_REG regptr(MSP_SLP_BASE + 0x04) - /* Firmware-ID Register RW */ -#define SYS_ID_REG0 regptr(MSP_SLP_BASE + 0x08) - /* System-ID Register-0 RW */ -#define SYS_ID_REG1 regptr(MSP_SLP_BASE + 0x0C) - /* System-ID Register-1 RW */ - -/* System Reset register definitions */ -#define RST_STS_REG regptr(MSP_SLP_BASE + 0x10) - /* System Reset Status RO */ -#define RST_SET_REG regptr(MSP_SLP_BASE + 0x14) - /* System Set Reset WO */ -#define RST_CLR_REG regptr(MSP_SLP_BASE + 0x18) - /* System Clear Reset WO */ - -/* System Clock Registers */ -#define PCI_SLP_REG regptr(MSP_SLP_BASE + 0x1C) - /* PCI clock generator RW */ -#define URT_SLP_REG regptr(MSP_SLP_BASE + 0x20) - /* UART clock generator RW */ -/* reserved (MSP_SLP_BASE + 0x24) */ -/* reserved (MSP_SLP_BASE + 0x28) */ -#define PLL1_SLP_REG regptr(MSP_SLP_BASE + 0x2C) - /* PLL1 clock generator RW */ -#define PLL0_SLP_REG regptr(MSP_SLP_BASE + 0x30) - /* PLL0 clock generator RW */ -#define MIPS_SLP_REG regptr(MSP_SLP_BASE + 0x34) - /* MIPS clock generator RW */ -#define VE_SLP_REG regptr(MSP_SLP_BASE + 0x38) - /* Voice Eng clock generator RW */ -/* reserved (MSP_SLP_BASE + 0x3C) */ -#define MSB_SLP_REG regptr(MSP_SLP_BASE + 0x40) - /* MS-Bus clock generator RW */ -#define SMAC_SLP_REG regptr(MSP_SLP_BASE + 0x44) - /* Sec & MAC clock generator RW */ -#define PERF_SLP_REG regptr(MSP_SLP_BASE + 0x48) - /* Per & TDM clock generator RW */ - -/* Interrupt Controller Registers */ -#define SLP_INT_STS_REG regptr(MSP_SLP_BASE + 0x70) - /* Interrupt status register RW */ -#define SLP_INT_MSK_REG regptr(MSP_SLP_BASE + 0x74) - /* Interrupt enable/mask RW */ -#define SE_MBOX_REG regptr(MSP_SLP_BASE + 0x78) - /* Security Engine mailbox RW */ -#define VE_MBOX_REG regptr(MSP_SLP_BASE + 0x7C) - /* Voice Engine mailbox RW */ - -/* ELB Controller Registers */ -#define CS0_CNFG_REG regptr(MSP_SLP_BASE + 0x80) - /* ELB CS0 Configuration Reg */ -#define CS0_ADDR_REG regptr(MSP_SLP_BASE + 0x84) - /* ELB CS0 Base Address Reg */ -#define CS0_MASK_REG regptr(MSP_SLP_BASE + 0x88) - /* ELB CS0 Mask Register */ -#define CS0_ACCESS_REG regptr(MSP_SLP_BASE + 0x8C) - /* ELB CS0 access register */ - -#define CS1_CNFG_REG regptr(MSP_SLP_BASE + 0x90) - /* ELB CS1 Configuration Reg */ -#define CS1_ADDR_REG regptr(MSP_SLP_BASE + 0x94) - /* ELB CS1 Base Address Reg */ -#define CS1_MASK_REG regptr(MSP_SLP_BASE + 0x98) - /* ELB CS1 Mask Register */ -#define CS1_ACCESS_REG regptr(MSP_SLP_BASE + 0x9C) - /* ELB CS1 access register */ - -#define CS2_CNFG_REG regptr(MSP_SLP_BASE + 0xA0) - /* ELB CS2 Configuration Reg */ -#define CS2_ADDR_REG regptr(MSP_SLP_BASE + 0xA4) - /* ELB CS2 Base Address Reg */ -#define CS2_MASK_REG regptr(MSP_SLP_BASE + 0xA8) - /* ELB CS2 Mask Register */ -#define CS2_ACCESS_REG regptr(MSP_SLP_BASE + 0xAC) - /* ELB CS2 access register */ - -#define CS3_CNFG_REG regptr(MSP_SLP_BASE + 0xB0) - /* ELB CS3 Configuration Reg */ -#define CS3_ADDR_REG regptr(MSP_SLP_BASE + 0xB4) - /* ELB CS3 Base Address Reg */ -#define CS3_MASK_REG regptr(MSP_SLP_BASE + 0xB8) - /* ELB CS3 Mask Register */ -#define CS3_ACCESS_REG regptr(MSP_SLP_BASE + 0xBC) - /* ELB CS3 access register */ - -#define CS4_CNFG_REG regptr(MSP_SLP_BASE + 0xC0) - /* ELB CS4 Configuration Reg */ -#define CS4_ADDR_REG regptr(MSP_SLP_BASE + 0xC4) - /* ELB CS4 Base Address Reg */ -#define CS4_MASK_REG regptr(MSP_SLP_BASE + 0xC8) - /* ELB CS4 Mask Register */ -#define CS4_ACCESS_REG regptr(MSP_SLP_BASE + 0xCC) - /* ELB CS4 access register */ - -#define CS5_CNFG_REG regptr(MSP_SLP_BASE + 0xD0) - /* ELB CS5 Configuration Reg */ -#define CS5_ADDR_REG regptr(MSP_SLP_BASE + 0xD4) - /* ELB CS5 Base Address Reg */ -#define CS5_MASK_REG regptr(MSP_SLP_BASE + 0xD8) - /* ELB CS5 Mask Register */ -#define CS5_ACCESS_REG regptr(MSP_SLP_BASE + 0xDC) - /* ELB CS5 access register */ - -/* reserved 0xE0 - 0xE8 */ -#define ELB_1PC_EN_REG regptr(MSP_SLP_BASE + 0xEC) - /* ELB single PC card detect */ - -/* reserved 0xF0 - 0xF8 */ -#define ELB_CLK_CFG_REG regptr(MSP_SLP_BASE + 0xFC) - /* SDRAM read/ELB timing Reg */ - -/* Extended UART status registers */ -#define UART0_STATUS_REG regptr(MSP_UART0_BASE + 0x0c0) - /* UART Status Register 0 */ -#define UART1_STATUS_REG regptr(MSP_UART1_BASE + 0x170) - /* UART Status Register 1 */ - -/* Performance monitoring registers */ -#define PERF_MON_CTRL_REG regptr(MSP_SLP_BASE + 0x140) - /* Performance monitor control */ -#define PERF_MON_CLR_REG regptr(MSP_SLP_BASE + 0x144) - /* Performance monitor clear */ -#define PERF_MON_CNTH_REG regptr(MSP_SLP_BASE + 0x148) - /* Perf monitor counter high */ -#define PERF_MON_CNTL_REG regptr(MSP_SLP_BASE + 0x14C) - /* Perf monitor counter low */ - -/* System control registers */ -#define SYS_CTRL_REG regptr(MSP_SLP_BASE + 0x150) - /* System control register */ -#define SYS_ERR1_REG regptr(MSP_SLP_BASE + 0x154) - /* System Error status 1 */ -#define SYS_ERR2_REG regptr(MSP_SLP_BASE + 0x158) - /* System Error status 2 */ -#define SYS_INT_CFG_REG regptr(MSP_SLP_BASE + 0x15C) - /* System Interrupt config */ - -/* Voice Engine Memory configuration */ -#define VE_MEM_REG regptr(MSP_SLP_BASE + 0x17C) - /* Voice engine memory config */ - -/* CPU/SLP Error Status registers */ -#define CPU_ERR1_REG regptr(MSP_SLP_BASE + 0x180) - /* CPU/SLP Error status 1 */ -#define CPU_ERR2_REG regptr(MSP_SLP_BASE + 0x184) - /* CPU/SLP Error status 1 */ - -/* Extended GPIO registers */ -#define EXTENDED_GPIO1_REG regptr(MSP_SLP_BASE + 0x188) -#define EXTENDED_GPIO2_REG regptr(MSP_SLP_BASE + 0x18c) -#define EXTENDED_GPIO_REG EXTENDED_GPIO1_REG - /* Backward-compatibility */ - -/* System Error registers */ -#define SLP_ERR_STS_REG regptr(MSP_SLP_BASE + 0x190) - /* Int status for SLP errors */ -#define SLP_ERR_MSK_REG regptr(MSP_SLP_BASE + 0x194) - /* Int mask for SLP errors */ -#define SLP_ELB_ERST_REG regptr(MSP_SLP_BASE + 0x198) - /* External ELB reset */ -#define SLP_BOOT_STS_REG regptr(MSP_SLP_BASE + 0x19C) - /* Boot Status */ - -/* Extended ELB addressing */ -#define CS0_EXT_ADDR_REG regptr(MSP_SLP_BASE + 0x1A0) - /* CS0 Extended address */ -#define CS1_EXT_ADDR_REG regptr(MSP_SLP_BASE + 0x1A4) - /* CS1 Extended address */ -#define CS2_EXT_ADDR_REG regptr(MSP_SLP_BASE + 0x1A8) - /* CS2 Extended address */ -#define CS3_EXT_ADDR_REG regptr(MSP_SLP_BASE + 0x1AC) - /* CS3 Extended address */ -/* reserved 0x1B0 */ -#define CS5_EXT_ADDR_REG regptr(MSP_SLP_BASE + 0x1B4) - /* CS5 Extended address */ - -/* PLL Adjustment registers */ -#define PLL_LOCK_REG regptr(MSP_SLP_BASE + 0x200) - /* PLL0 lock status */ -#define PLL_ARST_REG regptr(MSP_SLP_BASE + 0x204) - /* PLL Analog reset status */ -#define PLL0_ADJ_REG regptr(MSP_SLP_BASE + 0x208) - /* PLL0 Adjustment value */ -#define PLL1_ADJ_REG regptr(MSP_SLP_BASE + 0x20C) - /* PLL1 Adjustment value */ - -/* - *************************************************************************** - * Peripheral Register definitions * - *************************************************************************** - */ - -/* Peripheral status */ -#define PER_CTRL_REG regptr(MSP_PER_BASE + 0x50) - /* Peripheral control register */ -#define PER_STS_REG regptr(MSP_PER_BASE + 0x54) - /* Peripheral status register */ - -/* SPI/MPI Registers */ -#define SMPI_TX_SZ_REG regptr(MSP_PER_BASE + 0x58) - /* SPI/MPI Tx Size register */ -#define SMPI_RX_SZ_REG regptr(MSP_PER_BASE + 0x5C) - /* SPI/MPI Rx Size register */ -#define SMPI_CTL_REG regptr(MSP_PER_BASE + 0x60) - /* SPI/MPI Control register */ -#define SMPI_MS_REG regptr(MSP_PER_BASE + 0x64) - /* SPI/MPI Chip Select reg */ -#define SMPI_CORE_DATA_REG regptr(MSP_PER_BASE + 0xC0) - /* SPI/MPI Core Data reg */ -#define SMPI_CORE_CTRL_REG regptr(MSP_PER_BASE + 0xC4) - /* SPI/MPI Core Control reg */ -#define SMPI_CORE_STAT_REG regptr(MSP_PER_BASE + 0xC8) - /* SPI/MPI Core Status reg */ -#define SMPI_CORE_SSEL_REG regptr(MSP_PER_BASE + 0xCC) - /* SPI/MPI Core Ssel reg */ -#define SMPI_FIFO_REG regptr(MSP_PER_BASE + 0xD0) - /* SPI/MPI Data FIFO reg */ - -/* Peripheral Block Error Registers */ -#define PER_ERR_STS_REG regptr(MSP_PER_BASE + 0x70) - /* Error Bit Status Register */ -#define PER_ERR_MSK_REG regptr(MSP_PER_BASE + 0x74) - /* Error Bit Mask Register */ -#define PER_HDR1_REG regptr(MSP_PER_BASE + 0x78) - /* Error Header 1 Register */ -#define PER_HDR2_REG regptr(MSP_PER_BASE + 0x7C) - /* Error Header 2 Register */ - -/* Peripheral Block Interrupt Registers */ -#define PER_INT_STS_REG regptr(MSP_PER_BASE + 0x80) - /* Interrupt status register */ -#define PER_INT_MSK_REG regptr(MSP_PER_BASE + 0x84) - /* Interrupt Mask Register */ -#define GPIO_INT_STS_REG regptr(MSP_PER_BASE + 0x88) - /* GPIO interrupt status reg */ -#define GPIO_INT_MSK_REG regptr(MSP_PER_BASE + 0x8C) - /* GPIO interrupt MASK Reg */ - -/* POLO GPIO registers */ -#define POLO_GPIO_DAT1_REG regptr(MSP_PER_BASE + 0x0E0) - /* Polo GPIO[8:0] data reg */ -#define POLO_GPIO_CFG1_REG regptr(MSP_PER_BASE + 0x0E4) - /* Polo GPIO[7:0] config reg */ -#define POLO_GPIO_CFG2_REG regptr(MSP_PER_BASE + 0x0E8) - /* Polo GPIO[15:8] config reg */ -#define POLO_GPIO_OD1_REG regptr(MSP_PER_BASE + 0x0EC) - /* Polo GPIO[31:0] output drive */ -#define POLO_GPIO_CFG3_REG regptr(MSP_PER_BASE + 0x170) - /* Polo GPIO[23:16] config reg */ -#define POLO_GPIO_DAT2_REG regptr(MSP_PER_BASE + 0x174) - /* Polo GPIO[15:9] data reg */ -#define POLO_GPIO_DAT3_REG regptr(MSP_PER_BASE + 0x178) - /* Polo GPIO[23:16] data reg */ -#define POLO_GPIO_DAT4_REG regptr(MSP_PER_BASE + 0x17C) - /* Polo GPIO[31:24] data reg */ -#define POLO_GPIO_DAT5_REG regptr(MSP_PER_BASE + 0x180) - /* Polo GPIO[39:32] data reg */ -#define POLO_GPIO_DAT6_REG regptr(MSP_PER_BASE + 0x184) - /* Polo GPIO[47:40] data reg */ -#define POLO_GPIO_DAT7_REG regptr(MSP_PER_BASE + 0x188) - /* Polo GPIO[54:48] data reg */ -#define POLO_GPIO_CFG4_REG regptr(MSP_PER_BASE + 0x18C) - /* Polo GPIO[31:24] config reg */ -#define POLO_GPIO_CFG5_REG regptr(MSP_PER_BASE + 0x190) - /* Polo GPIO[39:32] config reg */ -#define POLO_GPIO_CFG6_REG regptr(MSP_PER_BASE + 0x194) - /* Polo GPIO[47:40] config reg */ -#define POLO_GPIO_CFG7_REG regptr(MSP_PER_BASE + 0x198) - /* Polo GPIO[54:48] config reg */ -#define POLO_GPIO_OD2_REG regptr(MSP_PER_BASE + 0x19C) - /* Polo GPIO[54:32] output drive */ - -/* Generic GPIO registers */ -#define GPIO_DATA1_REG regptr(MSP_PER_BASE + 0x170) - /* GPIO[1:0] data register */ -#define GPIO_DATA2_REG regptr(MSP_PER_BASE + 0x174) - /* GPIO[5:2] data register */ -#define GPIO_DATA3_REG regptr(MSP_PER_BASE + 0x178) - /* GPIO[9:6] data register */ -#define GPIO_DATA4_REG regptr(MSP_PER_BASE + 0x17C) - /* GPIO[15:10] data register */ -#define GPIO_CFG1_REG regptr(MSP_PER_BASE + 0x180) - /* GPIO[1:0] config register */ -#define GPIO_CFG2_REG regptr(MSP_PER_BASE + 0x184) - /* GPIO[5:2] config register */ -#define GPIO_CFG3_REG regptr(MSP_PER_BASE + 0x188) - /* GPIO[9:6] config register */ -#define GPIO_CFG4_REG regptr(MSP_PER_BASE + 0x18C) - /* GPIO[15:10] config register */ -#define GPIO_OD_REG regptr(MSP_PER_BASE + 0x190) - /* GPIO[15:0] output drive */ - -/* - *************************************************************************** - * CPU Interface register definitions * - *************************************************************************** - */ -#define PCI_FLUSH_REG regptr(MSP_CPUIF_BASE + 0x00) - /* PCI-SDRAM queue flush trigger */ -#define OCP_ERR1_REG regptr(MSP_CPUIF_BASE + 0x04) - /* OCP Error Attribute 1 */ -#define OCP_ERR2_REG regptr(MSP_CPUIF_BASE + 0x08) - /* OCP Error Attribute 2 */ -#define OCP_STS_REG regptr(MSP_CPUIF_BASE + 0x0C) - /* OCP Error Status */ -#define CPUIF_PM_REG regptr(MSP_CPUIF_BASE + 0x10) - /* CPU policy configuration */ -#define CPUIF_CFG_REG regptr(MSP_CPUIF_BASE + 0x10) - /* Misc configuration options */ - -/* Central Interrupt Controller Registers */ -#define MSP_CIC_BASE (MSP_CPUIF_BASE + 0x8000) - /* Central Interrupt registers */ -#define CIC_EXT_CFG_REG regptr(MSP_CIC_BASE + 0x00) - /* External interrupt config */ -#define CIC_STS_REG regptr(MSP_CIC_BASE + 0x04) - /* CIC Interrupt Status */ -#define CIC_VPE0_MSK_REG regptr(MSP_CIC_BASE + 0x08) - /* VPE0 Interrupt Mask */ -#define CIC_VPE1_MSK_REG regptr(MSP_CIC_BASE + 0x0C) - /* VPE1 Interrupt Mask */ -#define CIC_TC0_MSK_REG regptr(MSP_CIC_BASE + 0x10) - /* Thread Context 0 Int Mask */ -#define CIC_TC1_MSK_REG regptr(MSP_CIC_BASE + 0x14) - /* Thread Context 1 Int Mask */ -#define CIC_TC2_MSK_REG regptr(MSP_CIC_BASE + 0x18) - /* Thread Context 2 Int Mask */ -#define CIC_TC3_MSK_REG regptr(MSP_CIC_BASE + 0x18) - /* Thread Context 3 Int Mask */ -#define CIC_TC4_MSK_REG regptr(MSP_CIC_BASE + 0x18) - /* Thread Context 4 Int Mask */ -#define CIC_PCIMSI_STS_REG regptr(MSP_CIC_BASE + 0x18) -#define CIC_PCIMSI_MSK_REG regptr(MSP_CIC_BASE + 0x18) -#define CIC_PCIFLSH_REG regptr(MSP_CIC_BASE + 0x18) -#define CIC_VPE0_SWINT_REG regptr(MSP_CIC_BASE + 0x08) - - -/* - *************************************************************************** - * Memory controller registers * - *************************************************************************** - */ -#define MEM_CFG1_REG regptr(MSP_MEM_CFG_BASE + 0x00) -#define MEM_SS_ADDR regptr(MSP_MEM_CFG_BASE + 0x00) -#define MEM_SS_DATA regptr(MSP_MEM_CFG_BASE + 0x04) -#define MEM_SS_WRITE regptr(MSP_MEM_CFG_BASE + 0x08) - -/* - *************************************************************************** - * PCI controller registers * - *************************************************************************** - */ -#define PCI_BASE_REG regptr(MSP_PCI_BASE + 0x00) -#define PCI_CONFIG_SPACE_REG regptr(MSP_PCI_BASE + 0x800) -#define PCI_JTAG_DEVID_REG regptr(MSP_SLP_BASE + 0x13c) - -/* - ######################################################################## - # Register content & macro definitions # - ######################################################################## - */ - -/* - *************************************************************************** - * DEV_ID defines * - *************************************************************************** - */ -#define DEV_ID_PCI_DIS (1 << 26) /* Set if PCI disabled */ -#define DEV_ID_PCI_HOST (1 << 20) /* Set if PCI host */ -#define DEV_ID_SINGLE_PC (1 << 19) /* Set if single PC Card */ -#define DEV_ID_FAMILY (0xff << 8) /* family ID code */ -#define POLO_ZEUS_SUB_FAMILY (0x7 << 16) /* sub family for Polo/Zeus */ - -#define MSPFPGA_ID (0x00 << 8) /* you are on your own here */ -#define MSP5000_ID (0x50 << 8) -#define MSP4F00_ID (0x4f << 8) /* FPGA version of MSP4200 */ -#define MSP4E00_ID (0x4f << 8) /* FPGA version of MSP7120 */ -#define MSP4200_ID (0x42 << 8) -#define MSP4000_ID (0x40 << 8) -#define MSP2XXX_ID (0x20 << 8) -#define MSPZEUS_ID (0x10 << 8) - -#define MSP2004_SUB_ID (0x0 << 16) -#define MSP2005_SUB_ID (0x1 << 16) -#define MSP2006_SUB_ID (0x1 << 16) -#define MSP2007_SUB_ID (0x2 << 16) -#define MSP2010_SUB_ID (0x3 << 16) -#define MSP2015_SUB_ID (0x4 << 16) -#define MSP2020_SUB_ID (0x5 << 16) -#define MSP2100_SUB_ID (0x6 << 16) - -/* - *************************************************************************** - * RESET defines * - *************************************************************************** - */ -#define MSP_GR_RST (0x01 << 0) /* Global reset bit */ -#define MSP_MR_RST (0x01 << 1) /* MIPS reset bit */ -#define MSP_PD_RST (0x01 << 2) /* PVC DMA reset bit */ -#define MSP_PP_RST (0x01 << 3) /* PVC reset bit */ -/* reserved */ -#define MSP_EA_RST (0x01 << 6) /* Mac A reset bit */ -#define MSP_EB_RST (0x01 << 7) /* Mac B reset bit */ -#define MSP_SE_RST (0x01 << 8) /* Security Eng reset bit */ -#define MSP_PB_RST (0x01 << 9) /* Per block reset bit */ -#define MSP_EC_RST (0x01 << 10) /* Mac C reset bit */ -#define MSP_TW_RST (0x01 << 11) /* TWI reset bit */ -#define MSP_SPI_RST (0x01 << 12) /* SPI/MPI reset bit */ -#define MSP_U1_RST (0x01 << 13) /* UART1 reset bit */ -#define MSP_U0_RST (0x01 << 14) /* UART0 reset bit */ - -/* - *************************************************************************** - * UART defines * - *************************************************************************** - */ -#define MSP_BASE_BAUD 25000000 -#define MSP_UART_REG_LEN 0x20 - -/* - *************************************************************************** - * ELB defines * - *************************************************************************** - */ -#define PCCARD_32 0x02 /* Set if is PCCARD 32 (Cardbus) */ -#define SINGLE_PCCARD 0x01 /* Set to enable single PC card */ - -/* - *************************************************************************** - * CIC defines * - *************************************************************************** - */ - -/* CIC_EXT_CFG_REG */ -#define EXT_INT_POL(eirq) (1 << (eirq + 8)) -#define EXT_INT_EDGE(eirq) (1 << eirq) - -#define CIC_EXT_SET_TRIGGER_LEVEL(reg, eirq) (reg &= ~EXT_INT_EDGE(eirq)) -#define CIC_EXT_SET_TRIGGER_EDGE(reg, eirq) (reg |= EXT_INT_EDGE(eirq)) -#define CIC_EXT_SET_ACTIVE_HI(reg, eirq) (reg |= EXT_INT_POL(eirq)) -#define CIC_EXT_SET_ACTIVE_LO(reg, eirq) (reg &= ~EXT_INT_POL(eirq)) -#define CIC_EXT_SET_ACTIVE_RISING CIC_EXT_SET_ACTIVE_HI -#define CIC_EXT_SET_ACTIVE_FALLING CIC_EXT_SET_ACTIVE_LO - -#define CIC_EXT_IS_TRIGGER_LEVEL(reg, eirq) \ - ((reg & EXT_INT_EDGE(eirq)) == 0) -#define CIC_EXT_IS_TRIGGER_EDGE(reg, eirq) (reg & EXT_INT_EDGE(eirq)) -#define CIC_EXT_IS_ACTIVE_HI(reg, eirq) (reg & EXT_INT_POL(eirq)) -#define CIC_EXT_IS_ACTIVE_LO(reg, eirq) \ - ((reg & EXT_INT_POL(eirq)) == 0) -#define CIC_EXT_IS_ACTIVE_RISING CIC_EXT_IS_ACTIVE_HI -#define CIC_EXT_IS_ACTIVE_FALLING CIC_EXT_IS_ACTIVE_LO - -/* - *************************************************************************** - * Memory Controller defines * - *************************************************************************** - */ - -/* Indirect memory controller registers */ -#define DDRC_CFG(n) (n) -#define DDRC_DEBUG(n) (0x04 + n) -#define DDRC_CTL(n) (0x40 + n) - -/* Macro to perform DDRC indirect write */ -#define DDRC_INDIRECT_WRITE(reg, mask, value) \ -({ \ - *MEM_SS_ADDR = (((mask) & 0xf) << 8) | ((reg) & 0xff); \ - *MEM_SS_DATA = (value); \ - *MEM_SS_WRITE = 1; \ -}) - -/* - *************************************************************************** - * SPI/MPI Mode * - *************************************************************************** - */ -#define SPI_MPI_RX_BUSY 0x00008000 /* SPI/MPI Receive Busy */ -#define SPI_MPI_FIFO_EMPTY 0x00004000 /* SPI/MPI Fifo Empty */ -#define SPI_MPI_TX_BUSY 0x00002000 /* SPI/MPI Transmit Busy */ -#define SPI_MPI_FIFO_FULL 0x00001000 /* SPI/MPU FIFO full */ - -/* - *************************************************************************** - * SPI/MPI Control Register * - *************************************************************************** - */ -#define SPI_MPI_RX_START 0x00000004 /* Start receive command */ -#define SPI_MPI_FLUSH_Q 0x00000002 /* Flush SPI/MPI Queue */ -#define SPI_MPI_TX_START 0x00000001 /* Start Transmit Command */ - -#endif /* !_ASM_MSP_REGS_H */ diff --git a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_slp_int.h b/arch/mips/include/asm/mach-pmcs-msp71xx/msp_slp_int.h deleted file mode 100644 index 9a763eb5e5f5..000000000000 --- a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_slp_int.h +++ /dev/null @@ -1,129 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Defines for the MSP interrupt controller. - * - * Copyright (C) 1999 MIPS Technologies, Inc. All rights reserved. - * Author: Carsten Langgaard, carstenl@mips.com - * - * ######################################################################## - * - * ######################################################################## - */ - -#ifndef _MSP_SLP_INT_H -#define _MSP_SLP_INT_H - -/* - * The PMC-Sierra SLP interrupts are arranged in a 3 level cascaded - * hierarchical system. The first level are the direct MIPS interrupts - * and are assigned the interrupt range 0-7. The second level is the SLM - * interrupt controller and is assigned the range 8-39. The third level - * comprises the Peripherial block, the PCI block, the PCI MSI block and - * the SLP. The PCI interrupts and the SLP errors are handled by the - * relevant subsystems so the core interrupt code needs only concern - * itself with the Peripheral block. These are assigned interrupts in - * the range 40-71. - */ - -/* - * IRQs directly connected to CPU - */ -#define MSP_MIPS_INTBASE 0 -#define MSP_INT_SW0 0 /* IRQ for swint0, C_SW0 */ -#define MSP_INT_SW1 1 /* IRQ for swint1, C_SW1 */ -#define MSP_INT_MAC0 2 /* IRQ for MAC 0, C_IRQ0 */ -#define MSP_INT_MAC1 3 /* IRQ for MAC 1, C_IRQ1 */ -#define MSP_INT_C_IRQ2 4 /* Wired off, C_IRQ2 */ -#define MSP_INT_VE 5 /* IRQ for Voice Engine, C_IRQ3 */ -#define MSP_INT_SLP 6 /* IRQ for SLM block, C_IRQ4 */ -#define MSP_INT_TIMER 7 /* IRQ for the MIPS timer, C_IRQ5 */ - -/* - * IRQs cascaded on CPU interrupt 4 (CAUSE bit 12, C_IRQ4) - * These defines should be tied to the register definition for the SLM - * interrupt routine. For now, just use hard-coded values. - */ -#define MSP_SLP_INTBASE (MSP_MIPS_INTBASE + 8) -#define MSP_INT_EXT0 (MSP_SLP_INTBASE + 0) - /* External interrupt 0 */ -#define MSP_INT_EXT1 (MSP_SLP_INTBASE + 1) - /* External interrupt 1 */ -#define MSP_INT_EXT2 (MSP_SLP_INTBASE + 2) - /* External interrupt 2 */ -#define MSP_INT_EXT3 (MSP_SLP_INTBASE + 3) - /* External interrupt 3 */ -/* Reserved 4-7 */ - -/* - ************************************************************************* - * DANGER/DANGER/DANGER/DANGER/DANGER/DANGER/DANGER/DANGER/DANGER/DANGER * - * Some MSP produces have this interrupt labelled as Voice and some are * - * SEC mbox ... * - ************************************************************************* - */ -#define MSP_INT_SLP_VE (MSP_SLP_INTBASE + 8) - /* Cascaded IRQ for Voice Engine*/ -#define MSP_INT_SLP_TDM (MSP_SLP_INTBASE + 9) - /* TDM interrupt */ -#define MSP_INT_SLP_MAC0 (MSP_SLP_INTBASE + 10) - /* Cascaded IRQ for MAC 0 */ -#define MSP_INT_SLP_MAC1 (MSP_SLP_INTBASE + 11) - /* Cascaded IRQ for MAC 1 */ -#define MSP_INT_SEC (MSP_SLP_INTBASE + 12) - /* IRQ for security engine */ -#define MSP_INT_PER (MSP_SLP_INTBASE + 13) - /* Peripheral interrupt */ -#define MSP_INT_TIMER0 (MSP_SLP_INTBASE + 14) - /* SLP timer 0 */ -#define MSP_INT_TIMER1 (MSP_SLP_INTBASE + 15) - /* SLP timer 1 */ -#define MSP_INT_TIMER2 (MSP_SLP_INTBASE + 16) - /* SLP timer 2 */ -#define MSP_INT_SLP_TIMER (MSP_SLP_INTBASE + 17) - /* Cascaded MIPS timer */ -#define MSP_INT_BLKCP (MSP_SLP_INTBASE + 18) - /* Block Copy */ -#define MSP_INT_UART0 (MSP_SLP_INTBASE + 19) - /* UART 0 */ -#define MSP_INT_PCI (MSP_SLP_INTBASE + 20) - /* PCI subsystem */ -#define MSP_INT_PCI_DBELL (MSP_SLP_INTBASE + 21) - /* PCI doorbell */ -#define MSP_INT_PCI_MSI (MSP_SLP_INTBASE + 22) - /* PCI Message Signal */ -#define MSP_INT_PCI_BC0 (MSP_SLP_INTBASE + 23) - /* PCI Block Copy 0 */ -#define MSP_INT_PCI_BC1 (MSP_SLP_INTBASE + 24) - /* PCI Block Copy 1 */ -#define MSP_INT_SLP_ERR (MSP_SLP_INTBASE + 25) - /* SLP error condition */ -#define MSP_INT_MAC2 (MSP_SLP_INTBASE + 26) - /* IRQ for MAC2 */ -/* Reserved 26-31 */ - -/* - * IRQs cascaded on SLP PER interrupt (MSP_INT_PER) - */ -#define MSP_PER_INTBASE (MSP_SLP_INTBASE + 32) -/* Reserved 0-1 */ -#define MSP_INT_UART1 (MSP_PER_INTBASE + 2) - /* UART 1 */ -/* Reserved 3-5 */ -#define MSP_INT_2WIRE (MSP_PER_INTBASE + 6) - /* 2-wire */ -#define MSP_INT_TM0 (MSP_PER_INTBASE + 7) - /* Peripheral timer block out 0 */ -#define MSP_INT_TM1 (MSP_PER_INTBASE + 8) - /* Peripheral timer block out 1 */ -/* Reserved 9 */ -#define MSP_INT_SPRX (MSP_PER_INTBASE + 10) - /* SPI RX complete */ -#define MSP_INT_SPTX (MSP_PER_INTBASE + 11) - /* SPI TX complete */ -#define MSP_INT_GPIO (MSP_PER_INTBASE + 12) - /* GPIO */ -#define MSP_INT_PER_ERR (MSP_PER_INTBASE + 13) - /* Peripheral error */ -/* Reserved 14-31 */ - -#endif /* !_MSP_SLP_INT_H */ diff --git a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_usb.h b/arch/mips/include/asm/mach-pmcs-msp71xx/msp_usb.h deleted file mode 100644 index 3cc3edb336b6..000000000000 --- a/arch/mips/include/asm/mach-pmcs-msp71xx/msp_usb.h +++ /dev/null @@ -1,124 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/****************************************************************** - * Copyright (c) 2000-2007 PMC-Sierra INC. - * - * PMC-SIERRA INC. DISCLAIMS ANY LIABILITY OF ANY KIND - * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS - * SOFTWARE. - */ -#ifndef MSP_USB_H_ -#define MSP_USB_H_ - -#define NUM_USB_DEVS 1 - -/* Register spaces for USB host 0 */ -#define MSP_USB0_MAB_START (MSP_USB0_BASE + 0x0) -#define MSP_USB0_MAB_END (MSP_USB0_BASE + 0x17) -#define MSP_USB0_ID_START (MSP_USB0_BASE + 0x40000) -#define MSP_USB0_ID_END (MSP_USB0_BASE + 0x4008f) -#define MSP_USB0_HS_START (MSP_USB0_BASE + 0x40100) -#define MSP_USB0_HS_END (MSP_USB0_BASE + 0x401FF) - -/* Register spaces for USB host 1 */ -#define MSP_USB1_MAB_START (MSP_USB1_BASE + 0x0) -#define MSP_USB1_MAB_END (MSP_USB1_BASE + 0x17) -#define MSP_USB1_ID_START (MSP_USB1_BASE + 0x40000) -#define MSP_USB1_ID_END (MSP_USB1_BASE + 0x4008f) -#define MSP_USB1_HS_START (MSP_USB1_BASE + 0x40100) -#define MSP_USB1_HS_END (MSP_USB1_BASE + 0x401ff) - -/* USB Identification registers */ -struct msp_usbid_regs { - u32 id; /* 0x0: Identification register */ - u32 hwgen; /* 0x4: General HW params */ - u32 hwhost; /* 0x8: Host HW params */ - u32 hwdev; /* 0xc: Device HW params */ - u32 hwtxbuf; /* 0x10: Tx buffer HW params */ - u32 hwrxbuf; /* 0x14: Rx buffer HW params */ - u32 reserved[26]; - u32 timer0_load; /* 0x80: General-purpose timer 0 load*/ - u32 timer0_ctrl; /* 0x84: General-purpose timer 0 control */ - u32 timer1_load; /* 0x88: General-purpose timer 1 load*/ - u32 timer1_ctrl; /* 0x8c: General-purpose timer 1 control */ -}; - -/* MSBus to AMBA registers */ -struct msp_mab_regs { - u32 isr; /* 0x0: Interrupt status */ - u32 imr; /* 0x4: Interrupt mask */ - u32 thcr0; /* 0x8: Transaction header capture 0 */ - u32 thcr1; /* 0xc: Transaction header capture 1 */ - u32 int_stat; /* 0x10: Interrupt status summary */ - u32 phy_cfg; /* 0x14: USB phy config */ -}; - -/* EHCI registers */ -struct msp_usbhs_regs { - u32 hciver; /* 0x0: Version and offset to operational regs */ - u32 hcsparams; /* 0x4: Host control structural parameters */ - u32 hccparams; /* 0x8: Host control capability parameters */ - u32 reserved0[5]; - u32 dciver; /* 0x20: Device interface version */ - u32 dccparams; /* 0x24: Device control capability parameters */ - u32 reserved1[6]; - u32 cmd; /* 0x40: USB command */ - u32 sts; /* 0x44: USB status */ - u32 int_ena; /* 0x48: USB interrupt enable */ - u32 frindex; /* 0x4c: Frame index */ - u32 reserved3; - union { - struct { - u32 flb_addr; /* 0x54: Frame list base address */ - u32 next_async_addr; /* 0x58: next asynchronous addr */ - u32 ttctrl; /* 0x5c: embedded transaction translator - async buffer status */ - u32 burst_size; /* 0x60: Controller burst size */ - u32 tx_fifo_ctrl; /* 0x64: Tx latency FIFO tuning */ - u32 reserved0[4]; - u32 endpt_nak; /* 0x78: Endpoint NAK */ - u32 endpt_nak_ena; /* 0x7c: Endpoint NAK enable */ - u32 cfg_flag; /* 0x80: Config flag */ - u32 port_sc1; /* 0x84: Port status & control 1 */ - u32 reserved1[7]; - u32 otgsc; /* 0xa4: OTG status & control */ - u32 mode; /* 0xa8: USB controller mode */ - } host; - - struct { - u32 dev_addr; /* 0x54: Device address */ - u32 endpt_list_addr; /* 0x58: Endpoint list address */ - u32 reserved0[7]; - u32 endpt_nak; /* 0x74 */ - u32 endpt_nak_ctrl; /* 0x78 */ - u32 cfg_flag; /* 0x80 */ - u32 port_sc1; /* 0x84: Port status & control 1 */ - u32 reserved[7]; - u32 otgsc; /* 0xa4: OTG status & control */ - u32 mode; /* 0xa8: USB controller mode */ - u32 endpt_setup_stat; /* 0xac */ - u32 endpt_prime; /* 0xb0 */ - u32 endpt_flush; /* 0xb4 */ - u32 endpt_stat; /* 0xb8 */ - u32 endpt_complete; /* 0xbc */ - u32 endpt_ctrl0; /* 0xc0 */ - u32 endpt_ctrl1; /* 0xc4 */ - u32 endpt_ctrl2; /* 0xc8 */ - u32 endpt_ctrl3; /* 0xcc */ - } device; - } u; -}; -/* - * Container for the more-generic platform_device. - * This exists mainly as a way to map the non-standard register - * spaces and make them accessible to the USB ISR. - */ -struct mspusb_device { - struct msp_mab_regs __iomem *mab_regs; - struct msp_usbid_regs __iomem *usbid_regs; - struct msp_usbhs_regs __iomem *usbhs_regs; - struct platform_device dev; -}; - -#define to_mspusb_device(x) container_of((x), struct mspusb_device, dev) -#define TO_HOST_ID(x) ((x) & 0x3) -#endif /*MSP_USB_H_*/ diff --git a/arch/mips/include/asm/mach-pmcs-msp71xx/war.h b/arch/mips/include/asm/mach-pmcs-msp71xx/war.h deleted file mode 100644 index 31c546f58bb5..000000000000 --- a/arch/mips/include/asm/mach-pmcs-msp71xx/war.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2002, 2004, 2007 by Ralf Baechle - */ -#ifndef __ASM_MIPS_PMC_SIERRA_WAR_H -#define __ASM_MIPS_PMC_SIERRA_WAR_H - -#define R4600_V1_INDEX_ICACHEOP_WAR 0 -#define R4600_V1_HIT_CACHEOP_WAR 0 -#define R4600_V2_HIT_CACHEOP_WAR 0 -#define BCM1250_M3_WAR 0 -#define SIBYTE_1956_WAR 0 -#define MIPS4K_ICACHE_REFILL_WAR 0 -#define MIPS_CACHE_SYNC_WAR 0 -#define TX49XX_ICACHE_INDEX_INV_WAR 0 -#define ICACHE_REFILLS_WORKAROUND_WAR 0 -#define R10000_LLSC_WAR 0 -#if defined(CONFIG_PMC_MSP7120_EVAL) || defined(CONFIG_PMC_MSP7120_GW) || \ - defined(CONFIG_PMC_MSP7120_FPGA) -#define MIPS34K_MISSED_ITLB_WAR 1 -#else -#define MIPS34K_MISSED_ITLB_WAR 0 -#endif - -#endif /* __ASM_MIPS_PMC_SIERRA_WAR_H */ diff --git a/arch/mips/pci/Makefile b/arch/mips/pci/Makefile index c6142b289086..aeac09090fb4 100644 --- a/arch/mips/pci/Makefile +++ b/arch/mips/pci/Makefile @@ -35,9 +35,6 @@ obj-$(CONFIG_LEMOTE_FULOONG2E) += fixup-fuloong2e.o ops-loongson2.o obj-$(CONFIG_LEMOTE_MACH2F) += fixup-lemote2f.o ops-loongson2.o obj-$(CONFIG_MACH_LOONGSON64) += fixup-loongson3.o ops-loongson3.o obj-$(CONFIG_MIPS_MALTA) += fixup-malta.o pci-malta.o -obj-$(CONFIG_PMC_MSP7120_GW) += fixup-pmcmsp.o ops-pmcmsp.o -obj-$(CONFIG_PMC_MSP7120_EVAL) += fixup-pmcmsp.o ops-pmcmsp.o -obj-$(CONFIG_PMC_MSP7120_FPGA) += fixup-pmcmsp.o ops-pmcmsp.o obj-$(CONFIG_SGI_IP27) += pci-ip27.o obj-$(CONFIG_SGI_IP32) += fixup-ip32.o ops-mace.o pci-ip32.o obj-$(CONFIG_SIBYTE_SB1250) += fixup-sb1250.o pci-sb1250.o diff --git a/arch/mips/pci/fixup-pmcmsp.c b/arch/mips/pci/fixup-pmcmsp.c deleted file mode 100644 index 4ad2ef02087b..000000000000 --- a/arch/mips/pci/fixup-pmcmsp.c +++ /dev/null @@ -1,216 +0,0 @@ -/* - * PMC-Sierra MSP board specific pci fixups. - * - * Copyright 2001 MontaVista Software Inc. - * Copyright 2005-2007 PMC-Sierra, Inc - * - * Author: MontaVista Software, Inc. - * ppopov@mvista.com or source@mvista.com - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#ifdef CONFIG_PCI - -#include -#include -#include -#include - -#include - -#include -#include - -/* PCI interrupt pins */ -#define IRQ4 MSP_INT_EXT4 -#define IRQ5 MSP_INT_EXT5 -#define IRQ6 MSP_INT_EXT6 - -#if defined(CONFIG_PMC_MSP7120_GW) -/* Garibaldi Board IRQ wiring to PCI slots */ -static char irq_tab[][5] = { - /* INTA INTB INTC INTD */ - {0, 0, 0, 0, 0 }, /* (AD[0]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[1]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[2]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[3]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[4]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[5]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[6]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[7]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[8]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[9]): Unused */ - {0, 0, 0, 0, 0 }, /* 0 (AD[10]): Unused */ - {0, 0, 0, 0, 0 }, /* 1 (AD[11]): Unused */ - {0, 0, 0, 0, 0 }, /* 2 (AD[12]): Unused */ - {0, 0, 0, 0, 0 }, /* 3 (AD[13]): Unused */ - {0, 0, 0, 0, 0 }, /* 4 (AD[14]): Unused */ - {0, 0, 0, 0, 0 }, /* 5 (AD[15]): Unused */ - {0, 0, 0, 0, 0 }, /* 6 (AD[16]): Unused */ - {0, 0, 0, 0, 0 }, /* 7 (AD[17]): Unused */ - {0, 0, 0, 0, 0 }, /* 8 (AD[18]): Unused */ - {0, 0, 0, 0, 0 }, /* 9 (AD[19]): Unused */ - {0, 0, 0, 0, 0 }, /* 10 (AD[20]): Unused */ - {0, 0, 0, 0, 0 }, /* 11 (AD[21]): Unused */ - {0, 0, 0, 0, 0 }, /* 12 (AD[22]): Unused */ - {0, 0, 0, 0, 0 }, /* 13 (AD[23]): Unused */ - {0, 0, 0, 0, 0 }, /* 14 (AD[24]): Unused */ - {0, 0, 0, 0, 0 }, /* 15 (AD[25]): Unused */ - {0, 0, 0, 0, 0 }, /* 16 (AD[26]): Unused */ - {0, 0, 0, 0, 0 }, /* 17 (AD[27]): Unused */ - {0, IRQ4, IRQ4, 0, 0 }, /* 18 (AD[28]): slot 0 */ - {0, 0, 0, 0, 0 }, /* 19 (AD[29]): Unused */ - {0, IRQ5, IRQ5, 0, 0 }, /* 20 (AD[30]): slot 1 */ - {0, IRQ6, IRQ6, 0, 0 } /* 21 (AD[31]): slot 2 */ -}; - -#elif defined(CONFIG_PMC_MSP7120_EVAL) - -/* MSP7120 Eval Board IRQ wiring to PCI slots */ -static char irq_tab[][5] = { - /* INTA INTB INTC INTD */ - {0, 0, 0, 0, 0 }, /* (AD[0]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[1]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[2]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[3]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[4]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[5]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[6]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[7]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[8]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[9]): Unused */ - {0, 0, 0, 0, 0 }, /* 0 (AD[10]): Unused */ - {0, 0, 0, 0, 0 }, /* 1 (AD[11]): Unused */ - {0, 0, 0, 0, 0 }, /* 2 (AD[12]): Unused */ - {0, 0, 0, 0, 0 }, /* 3 (AD[13]): Unused */ - {0, 0, 0, 0, 0 }, /* 4 (AD[14]): Unused */ - {0, 0, 0, 0, 0 }, /* 5 (AD[15]): Unused */ - {0, IRQ6, IRQ6, 0, 0 }, /* 6 (AD[16]): slot 3 (mini) */ - {0, IRQ5, IRQ5, 0, 0 }, /* 7 (AD[17]): slot 2 (mini) */ - {0, IRQ4, IRQ4, IRQ4, IRQ4}, /* 8 (AD[18]): slot 0 (PCI) */ - {0, IRQ5, IRQ5, IRQ5, IRQ5}, /* 9 (AD[19]): slot 1 (PCI) */ - {0, 0, 0, 0, 0 }, /* 10 (AD[20]): Unused */ - {0, 0, 0, 0, 0 }, /* 11 (AD[21]): Unused */ - {0, 0, 0, 0, 0 }, /* 12 (AD[22]): Unused */ - {0, 0, 0, 0, 0 }, /* 13 (AD[23]): Unused */ - {0, 0, 0, 0, 0 }, /* 14 (AD[24]): Unused */ - {0, 0, 0, 0, 0 }, /* 15 (AD[25]): Unused */ - {0, 0, 0, 0, 0 }, /* 16 (AD[26]): Unused */ - {0, 0, 0, 0, 0 }, /* 17 (AD[27]): Unused */ - {0, 0, 0, 0, 0 }, /* 18 (AD[28]): Unused */ - {0, 0, 0, 0, 0 }, /* 19 (AD[29]): Unused */ - {0, 0, 0, 0, 0 }, /* 20 (AD[30]): Unused */ - {0, 0, 0, 0, 0 } /* 21 (AD[31]): Unused */ -}; - -#else - -/* Unknown board -- don't assign any IRQs */ -static char irq_tab[][5] = { - /* INTA INTB INTC INTD */ - {0, 0, 0, 0, 0 }, /* (AD[0]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[1]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[2]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[3]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[4]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[5]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[6]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[7]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[8]): Unused */ - {0, 0, 0, 0, 0 }, /* (AD[9]): Unused */ - {0, 0, 0, 0, 0 }, /* 0 (AD[10]): Unused */ - {0, 0, 0, 0, 0 }, /* 1 (AD[11]): Unused */ - {0, 0, 0, 0, 0 }, /* 2 (AD[12]): Unused */ - {0, 0, 0, 0, 0 }, /* 3 (AD[13]): Unused */ - {0, 0, 0, 0, 0 }, /* 4 (AD[14]): Unused */ - {0, 0, 0, 0, 0 }, /* 5 (AD[15]): Unused */ - {0, 0, 0, 0, 0 }, /* 6 (AD[16]): Unused */ - {0, 0, 0, 0, 0 }, /* 7 (AD[17]): Unused */ - {0, 0, 0, 0, 0 }, /* 8 (AD[18]): Unused */ - {0, 0, 0, 0, 0 }, /* 9 (AD[19]): Unused */ - {0, 0, 0, 0, 0 }, /* 10 (AD[20]): Unused */ - {0, 0, 0, 0, 0 }, /* 11 (AD[21]): Unused */ - {0, 0, 0, 0, 0 }, /* 12 (AD[22]): Unused */ - {0, 0, 0, 0, 0 }, /* 13 (AD[23]): Unused */ - {0, 0, 0, 0, 0 }, /* 14 (AD[24]): Unused */ - {0, 0, 0, 0, 0 }, /* 15 (AD[25]): Unused */ - {0, 0, 0, 0, 0 }, /* 16 (AD[26]): Unused */ - {0, 0, 0, 0, 0 }, /* 17 (AD[27]): Unused */ - {0, 0, 0, 0, 0 }, /* 18 (AD[28]): Unused */ - {0, 0, 0, 0, 0 }, /* 19 (AD[29]): Unused */ - {0, 0, 0, 0, 0 }, /* 20 (AD[30]): Unused */ - {0, 0, 0, 0, 0 } /* 21 (AD[31]): Unused */ -}; -#endif - -/***************************************************************************** - * - * FUNCTION: pcibios_plat_dev_init - * _________________________________________________________________________ - * - * DESCRIPTION: Perform platform specific device initialization at - * pci_enable_device() time. - * None are needed for the MSP7120 PCI Controller. - * - * INPUTS: dev - structure describing the PCI device - * - * OUTPUTS: none - * - * RETURNS: PCIBIOS_SUCCESSFUL - * - ****************************************************************************/ -int pcibios_plat_dev_init(struct pci_dev *dev) -{ - return PCIBIOS_SUCCESSFUL; -} - -/***************************************************************************** - * - * FUNCTION: pcibios_map_irq - * _________________________________________________________________________ - * - * DESCRIPTION: Perform board supplied PCI IRQ mapping routine. - * - * INPUTS: dev - unused - * slot - PCI slot. Identified by which bit of the AD[] bus - * drives the IDSEL line. AD[10] is 0, AD[31] is - * slot 21. - * pin - numbered using the scheme of the PCI_INTERRUPT_PIN - * field of the config header. - * - * OUTPUTS: none - * - * RETURNS: IRQ number - * - ****************************************************************************/ -int pcibios_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) -{ -#if !defined(CONFIG_PMC_MSP7120_GW) && !defined(CONFIG_PMC_MSP7120_EVAL) - printk(KERN_WARNING "PCI: unknown board, no PCI IRQs assigned.\n"); -#endif - printk(KERN_WARNING "PCI: irq_tab returned %d for slot=%d pin=%d\n", - irq_tab[slot][pin], slot, pin); - - return irq_tab[slot][pin]; -} - -#endif /* CONFIG_PCI */ diff --git a/arch/mips/pci/ops-pmcmsp.c b/arch/mips/pci/ops-pmcmsp.c deleted file mode 100644 index ad5dd711c575..000000000000 --- a/arch/mips/pci/ops-pmcmsp.c +++ /dev/null @@ -1,944 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * PMC-Sierra MSP board specific pci_ops - * - * Copyright 2001 MontaVista Software Inc. - * Copyright 2005-2007 PMC-Sierra, Inc - * - * Author: Jun Sun, jsun@mvista.com or jsun@junsun.net - * - * Much of the code is derived from the original DDB5074 port by - * Geert Uytterhoeven - */ - -#define PCI_COUNTERS 1 - -#include -#include -#include - -#if defined(CONFIG_PROC_FS) && defined(PCI_COUNTERS) -#include -#include -#endif /* CONFIG_PROC_FS && PCI_COUNTERS */ - -#include -#include - -#include -#if defined(CONFIG_PMC_MSP7120_GW) || defined(CONFIG_PMC_MSP7120_EVAL) -#include -#endif - -#include -#include -#include -#include -#include - -#define PCI_ACCESS_READ 0 -#define PCI_ACCESS_WRITE 1 - -#if defined(CONFIG_PROC_FS) && defined(PCI_COUNTERS) -static char proc_init; -extern struct proc_dir_entry *proc_bus_pci_dir; -unsigned int pci_int_count[32]; - -static void pci_proc_init(void); - -/***************************************************************************** - * - * FUNCTION: show_msp_pci_counts - * _________________________________________________________________________ - * - * DESCRIPTION: Prints the count of how many times each PCI - * interrupt has asserted. Can be invoked by the - * /proc filesystem. - * - * INPUTS: m - synthetic file construction data - * v - iterator - * - * RETURNS: 0 or error - * - ****************************************************************************/ -static int show_msp_pci_counts(struct seq_file *m, void *v) -{ - int i; - unsigned int intcount, total = 0; - - for (i = 0; i < 32; ++i) { - intcount = pci_int_count[i]; - if (intcount != 0) { - seq_printf(m, "[%d] = %u\n", i, intcount); - total += intcount; - } - } - - seq_printf(m, "total = %u\n", total); - return 0; -} - -/***************************************************************************** - * - * FUNCTION: gen_pci_cfg_wr_show - * _________________________________________________________________________ - * - * DESCRIPTION: Generates a configuration write cycle for debug purposes. - * The IDSEL line asserted and location and data written are - * immaterial. Just want to be able to prove that a - * configuration write can be correctly generated on the - * PCI bus. Intent is that this function by invocable from - * the /proc filesystem. - * - * INPUTS: m - synthetic file construction data - * v - iterator - * - * RETURNS: 0 or error - * - ****************************************************************************/ -static int gen_pci_cfg_wr_show(struct seq_file *m, void *v) -{ - unsigned char where = 0; /* Write to static Device/Vendor ID */ - unsigned char bus_num = 0; /* Bus 0 */ - unsigned char dev_fn = 0xF; /* Arbitrary device number */ - u32 wr_data = 0xFF00AA00; /* Arbitrary data */ - struct msp_pci_regs *preg = (void *)PCI_BASE_REG; - unsigned long value; - int intr; - - seq_puts(m, "PMC MSP PCI: Beginning\n"); - - if (proc_init == 0) { - pci_proc_init(); - proc_init = ~0; - } - - seq_puts(m, "PMC MSP PCI: Before Cfg Wr\n"); - - /* - * Generate PCI Configuration Write Cycle - */ - - /* Clear cause register bits */ - preg->if_status = ~(BPCI_IFSTATUS_BC0F | BPCI_IFSTATUS_BC1F); - - /* Setup address that is to appear on PCI bus */ - preg->config_addr = BPCI_CFGADDR_ENABLE | - (bus_num << BPCI_CFGADDR_BUSNUM_SHF) | - (dev_fn << BPCI_CFGADDR_FUNCTNUM_SHF) | - (where & 0xFC); - - value = cpu_to_le32(wr_data); - - /* Launch the PCI configuration write cycle */ - *PCI_CONFIG_SPACE_REG = value; - - /* - * Check if the PCI configuration cycle (rd or wr) succeeded, by - * checking the status bits for errors like master or target abort. - */ - intr = preg->if_status; - - seq_puts(m, "PMC MSP PCI: After Cfg Wr\n"); - return 0; -} - -/***************************************************************************** - * - * FUNCTION: pci_proc_init - * _________________________________________________________________________ - * - * DESCRIPTION: Create entries in the /proc filesystem for debug access. - * - * INPUTS: none - * - * OUTPUTS: none - * - * RETURNS: none - * - ****************************************************************************/ -static void pci_proc_init(void) -{ - proc_create_single("pmc_msp_pci_rd_cnt", 0, NULL, show_msp_pci_counts); - proc_create_single("pmc_msp_pci_cfg_wr", 0, NULL, gen_pci_cfg_wr_show); -} -#endif /* CONFIG_PROC_FS && PCI_COUNTERS */ - -/***************************************************************************** - * - * STRUCT: pci_io_resource - * _________________________________________________________________________ - * - * DESCRIPTION: Defines the address range that pciauto() will use to - * assign to the I/O BARs of PCI devices. - * - * Use the start and end addresses of the MSP7120 PCI Host - * Controller I/O space, in the form that they appear on the - * PCI bus AFTER MSP7120 has performed address translation. - * - * For I/O accesses, MSP7120 ignores OATRAN and maps I/O - * accesses into the bottom 0xFFF region of address space, - * so that is the range to put into the pci_io_resource - * struct. - * - * In MSP4200, the start address was 0x04 instead of the - * expected 0x00. Will just assume there was a good reason - * for this! - * - * NOTES: Linux, by default, will assign I/O space to the lowest - * region of address space. Since MSP7120 and Linux, - * by default, have no offset in between how they map, the - * io_offset element of pci_controller struct should be set - * to zero. - * ELEMENTS: - * name - String used for a meaningful name. - * - * start - Start address of MSP7120's I/O space, as MSP7120 presents - * the address on the PCI bus. - * - * end - End address of MSP7120's I/O space, as MSP7120 presents - * the address on the PCI bus. - * - * flags - Attributes indicating the type of resource. In this case, - * indicate I/O space. - * - ****************************************************************************/ -static struct resource pci_io_resource = { - .name = "pci IO space", - .start = 0x04, - .end = 0x0FFF, - .flags = IORESOURCE_IO /* I/O space */ -}; - -/***************************************************************************** - * - * STRUCT: pci_mem_resource - * _________________________________________________________________________ - * - * DESCRIPTION: Defines the address range that pciauto() will use to - * assign to the memory BARs of PCI devices. - * - * The .start and .end values are dependent upon how address - * translation is performed by the OATRAN regiser. - * - * The values to use for .start and .end are the values - * in the form they appear on the PCI bus AFTER MSP7120 has - * performed OATRAN address translation. - * - * ELEMENTS: - * name - String used for a meaningful name. - * - * start - Start address of MSP7120's memory space, as MSP7120 presents - * the address on the PCI bus. - * - * end - End address of MSP7120's memory space, as MSP7120 presents - * the address on the PCI bus. - * - * flags - Attributes indicating the type of resource. In this case, - * indicate memory space. - * - ****************************************************************************/ -static struct resource pci_mem_resource = { - .name = "pci memory space", - .start = MSP_PCI_SPACE_BASE, - .end = MSP_PCI_SPACE_END, - .flags = IORESOURCE_MEM /* memory space */ -}; - -/***************************************************************************** - * - * FUNCTION: bpci_interrupt - * _________________________________________________________________________ - * - * DESCRIPTION: PCI status interrupt handler. Updates the count of how - * many times each status bit has been set, then clears - * the status bits. If the appropriate macros are defined, - * these counts can be viewed via the /proc filesystem. - * - * INPUTS: irq - unused - * dev_id - unused - * pt_regs - unused - * - * OUTPUTS: none - * - * RETURNS: PCIBIOS_SUCCESSFUL - success - * - ****************************************************************************/ -static irqreturn_t bpci_interrupt(int irq, void *dev_id) -{ - struct msp_pci_regs *preg = (void *)PCI_BASE_REG; - unsigned int stat = preg->if_status; - -#if defined(CONFIG_PROC_FS) && defined(PCI_COUNTERS) - int i; - for (i = 0; i < 32; ++i) { - if ((1 << i) & stat) - ++pci_int_count[i]; - } -#endif /* PROC_FS && PCI_COUNTERS */ - - /* printk("PCI ISR: Status=%08X\n", stat); */ - - /* write to clear all asserted interrupts */ - preg->if_status = stat; - - return IRQ_HANDLED; -} - -/***************************************************************************** - * - * FUNCTION: msp_pcibios_config_access - * _________________________________________________________________________ - * - * DESCRIPTION: Performs a PCI configuration access (rd or wr), then - * checks that the access succeeded by querying MSP7120's - * PCI status bits. - * - * INPUTS: - * access_type - kind of PCI configuration cycle to perform - * (read or write). Legal values are - * PCI_ACCESS_WRITE and PCI_ACCESS_READ. - * - * bus - pointer to the bus number of the device to - * be targeted for the configuration cycle. - * The only element of the pci_bus structure - * used is bus->number. This argument determines - * if the configuration access will be Type 0 or - * Type 1. Since MSP7120 assumes itself to be the - * PCI Host, any non-zero bus->number generates - * a Type 1 access. - * - * devfn - this is an 8-bit field. The lower three bits - * specify the function number of the device to - * be targeted for the configuration cycle, with - * all three-bit combinations being legal. The - * upper five bits specify the device number, - * with legal values being 10 to 31. - * - * where - address within the Configuration Header - * space to access. - * - * data - for write accesses, contains the data to - * write. - * - * OUTPUTS: - * data - for read accesses, contains the value read. - * - * RETURNS: PCIBIOS_SUCCESSFUL - success - * -1 - access failure - * - ****************************************************************************/ -int msp_pcibios_config_access(unsigned char access_type, - struct pci_bus *bus, - unsigned int devfn, - unsigned char where, - u32 *data) -{ - struct msp_pci_regs *preg = (void *)PCI_BASE_REG; - unsigned char bus_num = bus->number; - unsigned char dev_fn = (unsigned char)devfn; - unsigned long intr; - unsigned long value; - static char pciirqflag; - int ret; -#if defined(CONFIG_PMC_MSP7120_GW) || defined(CONFIG_PMC_MSP7120_EVAL) - unsigned int vpe_status; -#endif - -#if defined(CONFIG_PROC_FS) && defined(PCI_COUNTERS) - if (proc_init == 0) { - pci_proc_init(); - proc_init = ~0; - } -#endif /* CONFIG_PROC_FS && PCI_COUNTERS */ - - /* - * Just the first time this function invokes, allocate - * an interrupt line for PCI host status interrupts. The - * allocation assigns an interrupt handler to the interrupt. - */ - if (pciirqflag == 0) { - ret = request_irq(MSP_INT_PCI,/* Hardcoded internal MSP7120 wiring */ - bpci_interrupt, - IRQF_SHARED, - "PMC MSP PCI Host", - preg); - if (ret != 0) - return ret; - pciirqflag = ~0; - } - -#if defined(CONFIG_PMC_MSP7120_GW) || defined(CONFIG_PMC_MSP7120_EVAL) - vpe_status = dvpe(); -#endif - - /* - * Clear PCI cause register bits. - * - * In Polo, the PCI Host had a dedicated DMA called the - * Block Copy (not to be confused with the general purpose Block - * Copy Engine block). There appear to have been special interrupts - * for this Block Copy, called Block Copy 0 Fault (BC0F) and - * Block Copy 1 Fault (BC1F). MSP4200 and MSP7120 don't have this - * dedicated Block Copy block, so these two interrupts are now - * marked reserved. In case the Block Copy is resurrected in a - * future design, maintain the code that treats these two interrupts - * specially. - * - * Write to clear all interrupts in the PCI status register, aside - * from BC0F and BC1F. - */ - preg->if_status = ~(BPCI_IFSTATUS_BC0F | BPCI_IFSTATUS_BC1F); - - /* Setup address that is to appear on PCI bus */ - preg->config_addr = BPCI_CFGADDR_ENABLE | - (bus_num << BPCI_CFGADDR_BUSNUM_SHF) | - (dev_fn << BPCI_CFGADDR_FUNCTNUM_SHF) | - (where & 0xFC); - - /* IF access is a PCI configuration write */ - if (access_type == PCI_ACCESS_WRITE) { - value = cpu_to_le32(*data); - *PCI_CONFIG_SPACE_REG = value; - } else { - /* ELSE access is a PCI configuration read */ - value = le32_to_cpu(*PCI_CONFIG_SPACE_REG); - *data = value; - } - - /* - * Check if the PCI configuration cycle (rd or wr) succeeded, by - * checking the status bits for errors like master or target abort. - */ - intr = preg->if_status; - - /* Clear config access */ - preg->config_addr = 0; - - /* IF error occurred */ - if (intr & ~(BPCI_IFSTATUS_BC0F | BPCI_IFSTATUS_BC1F)) { - /* Clear status bits */ - preg->if_status = ~(BPCI_IFSTATUS_BC0F | BPCI_IFSTATUS_BC1F); - -#if defined(CONFIG_PMC_MSP7120_GW) || defined(CONFIG_PMC_MSP7120_EVAL) - evpe(vpe_status); -#endif - - return -1; - } - -#if defined(CONFIG_PMC_MSP7120_GW) || defined(CONFIG_PMC_MSP7120_EVAL) - evpe(vpe_status); -#endif - - return PCIBIOS_SUCCESSFUL; -} - -/***************************************************************************** - * - * FUNCTION: msp_pcibios_read_config_byte - * _________________________________________________________________________ - * - * DESCRIPTION: Read a byte from PCI configuration address spac - * Since the hardware can't address 8 bit chunks - * directly, read a 32-bit chunk, then mask off extraneous - * bits. - * - * INPUTS bus - structure containing attributes for the PCI bus - * that the read is destined for. - * devfn - device/function combination that the read is - * destined for. - * where - register within the Configuration Header space - * to access. - * - * OUTPUTS val - read data - * - * RETURNS: PCIBIOS_SUCCESSFUL - success - * -1 - read access failure - * - ****************************************************************************/ -static int -msp_pcibios_read_config_byte(struct pci_bus *bus, - unsigned int devfn, - int where, - u32 *val) -{ - u32 data = 0; - - /* - * If the config access did not complete normally (e.g., underwent - * master abort) do the PCI compliant thing, which is to supply an - * all ones value. - */ - if (msp_pcibios_config_access(PCI_ACCESS_READ, bus, devfn, - where, &data)) { - *val = 0xFFFFFFFF; - return -1; - } - - *val = (data >> ((where & 3) << 3)) & 0x0ff; - - return PCIBIOS_SUCCESSFUL; -} - -/***************************************************************************** - * - * FUNCTION: msp_pcibios_read_config_word - * _________________________________________________________________________ - * - * DESCRIPTION: Read a word (16 bits) from PCI configuration address space. - * Since the hardware can't address 16 bit chunks - * directly, read a 32-bit chunk, then mask off extraneous - * bits. - * - * INPUTS bus - structure containing attributes for the PCI bus - * that the read is destined for. - * devfn - device/function combination that the read is - * destined for. - * where - register within the Configuration Header space - * to access. - * - * OUTPUTS val - read data - * - * RETURNS: PCIBIOS_SUCCESSFUL - success - * PCIBIOS_BAD_REGISTER_NUMBER - bad register address - * -1 - read access failure - * - ****************************************************************************/ -static int -msp_pcibios_read_config_word(struct pci_bus *bus, - unsigned int devfn, - int where, - u32 *val) -{ - u32 data = 0; - - /* if (where & 1) */ /* Commented out non-compliant code. - * Should allow word access to configuration - * registers, with only exception being when - * the word access would wrap around into - * the next dword. - */ - if ((where & 3) == 3) { - *val = 0xFFFFFFFF; - return PCIBIOS_BAD_REGISTER_NUMBER; - } - - /* - * If the config access did not complete normally (e.g., underwent - * master abort) do the PCI compliant thing, which is to supply an - * all ones value. - */ - if (msp_pcibios_config_access(PCI_ACCESS_READ, bus, devfn, - where, &data)) { - *val = 0xFFFFFFFF; - return -1; - } - - *val = (data >> ((where & 3) << 3)) & 0x0ffff; - - return PCIBIOS_SUCCESSFUL; -} - -/***************************************************************************** - * - * FUNCTION: msp_pcibios_read_config_dword - * _________________________________________________________________________ - * - * DESCRIPTION: Read a double word (32 bits) from PCI configuration - * address space. - * - * INPUTS bus - structure containing attributes for the PCI bus - * that the read is destined for. - * devfn - device/function combination that the read is - * destined for. - * where - register within the Configuration Header space - * to access. - * - * OUTPUTS val - read data - * - * RETURNS: PCIBIOS_SUCCESSFUL - success - * PCIBIOS_BAD_REGISTER_NUMBER - bad register address - * -1 - read access failure - * - ****************************************************************************/ -static int -msp_pcibios_read_config_dword(struct pci_bus *bus, - unsigned int devfn, - int where, - u32 *val) -{ - u32 data = 0; - - /* Address must be dword aligned. */ - if (where & 3) { - *val = 0xFFFFFFFF; - return PCIBIOS_BAD_REGISTER_NUMBER; - } - - /* - * If the config access did not complete normally (e.g., underwent - * master abort) do the PCI compliant thing, which is to supply an - * all ones value. - */ - if (msp_pcibios_config_access(PCI_ACCESS_READ, bus, devfn, - where, &data)) { - *val = 0xFFFFFFFF; - return -1; - } - - *val = data; - - return PCIBIOS_SUCCESSFUL; -} - -/***************************************************************************** - * - * FUNCTION: msp_pcibios_write_config_byte - * _________________________________________________________________________ - * - * DESCRIPTION: Write a byte to PCI configuration address space. - * Since the hardware can't address 8 bit chunks - * directly, a read-modify-write is performed. - * - * INPUTS bus - structure containing attributes for the PCI bus - * that the write is destined for. - * devfn - device/function combination that the write is - * destined for. - * where - register within the Configuration Header space - * to access. - * val - value to write - * - * OUTPUTS none - * - * RETURNS: PCIBIOS_SUCCESSFUL - success - * -1 - write access failure - * - ****************************************************************************/ -static int -msp_pcibios_write_config_byte(struct pci_bus *bus, - unsigned int devfn, - int where, - u8 val) -{ - u32 data = 0; - - /* read config space */ - if (msp_pcibios_config_access(PCI_ACCESS_READ, bus, devfn, - where, &data)) - return -1; - - /* modify the byte within the dword */ - data = (data & ~(0xff << ((where & 3) << 3))) | - (val << ((where & 3) << 3)); - - /* write back the full dword */ - if (msp_pcibios_config_access(PCI_ACCESS_WRITE, bus, devfn, - where, &data)) - return -1; - - return PCIBIOS_SUCCESSFUL; -} - -/***************************************************************************** - * - * FUNCTION: msp_pcibios_write_config_word - * _________________________________________________________________________ - * - * DESCRIPTION: Write a word (16-bits) to PCI configuration address space. - * Since the hardware can't address 16 bit chunks - * directly, a read-modify-write is performed. - * - * INPUTS bus - structure containing attributes for the PCI bus - * that the write is destined for. - * devfn - device/function combination that the write is - * destined for. - * where - register within the Configuration Header space - * to access. - * val - value to write - * - * OUTPUTS none - * - * RETURNS: PCIBIOS_SUCCESSFUL - success - * PCIBIOS_BAD_REGISTER_NUMBER - bad register address - * -1 - write access failure - * - ****************************************************************************/ -static int -msp_pcibios_write_config_word(struct pci_bus *bus, - unsigned int devfn, - int where, - u16 val) -{ - u32 data = 0; - - /* Fixed non-compliance: if (where & 1) */ - if ((where & 3) == 3) - return PCIBIOS_BAD_REGISTER_NUMBER; - - /* read config space */ - if (msp_pcibios_config_access(PCI_ACCESS_READ, bus, devfn, - where, &data)) - return -1; - - /* modify the word within the dword */ - data = (data & ~(0xffff << ((where & 3) << 3))) | - (val << ((where & 3) << 3)); - - /* write back the full dword */ - if (msp_pcibios_config_access(PCI_ACCESS_WRITE, bus, devfn, - where, &data)) - return -1; - - return PCIBIOS_SUCCESSFUL; -} - -/***************************************************************************** - * - * FUNCTION: msp_pcibios_write_config_dword - * _________________________________________________________________________ - * - * DESCRIPTION: Write a double word (32-bits) to PCI configuration address - * space. - * - * INPUTS bus - structure containing attributes for the PCI bus - * that the write is destined for. - * devfn - device/function combination that the write is - * destined for. - * where - register within the Configuration Header space - * to access. - * val - value to write - * - * OUTPUTS none - * - * RETURNS: PCIBIOS_SUCCESSFUL - success - * PCIBIOS_BAD_REGISTER_NUMBER - bad register address - * -1 - write access failure - * - ****************************************************************************/ -static int -msp_pcibios_write_config_dword(struct pci_bus *bus, - unsigned int devfn, - int where, - u32 val) -{ - /* check that address is dword aligned */ - if (where & 3) - return PCIBIOS_BAD_REGISTER_NUMBER; - - /* perform write */ - if (msp_pcibios_config_access(PCI_ACCESS_WRITE, bus, devfn, - where, &val)) - return -1; - - return PCIBIOS_SUCCESSFUL; -} - -/***************************************************************************** - * - * FUNCTION: msp_pcibios_read_config - * _________________________________________________________________________ - * - * DESCRIPTION: Interface the PCI configuration read request with - * the appropriate function, based on how many bytes - * the read request is. - * - * INPUTS bus - structure containing attributes for the PCI bus - * that the write is destined for. - * devfn - device/function combination that the write is - * destined for. - * where - register within the Configuration Header space - * to access. - * size - in units of bytes, should be 1, 2, or 4. - * - * OUTPUTS val - value read, with any extraneous bytes masked - * to zero. - * - * RETURNS: PCIBIOS_SUCCESSFUL - success - * -1 - failure - * - ****************************************************************************/ -int -msp_pcibios_read_config(struct pci_bus *bus, - unsigned int devfn, - int where, - int size, - u32 *val) -{ - if (size == 1) { - if (msp_pcibios_read_config_byte(bus, devfn, where, val)) { - return -1; - } - } else if (size == 2) { - if (msp_pcibios_read_config_word(bus, devfn, where, val)) { - return -1; - } - } else if (size == 4) { - if (msp_pcibios_read_config_dword(bus, devfn, where, val)) { - return -1; - } - } else { - *val = 0xFFFFFFFF; - return -1; - } - - return PCIBIOS_SUCCESSFUL; -} - -/***************************************************************************** - * - * FUNCTION: msp_pcibios_write_config - * _________________________________________________________________________ - * - * DESCRIPTION: Interface the PCI configuration write request with - * the appropriate function, based on how many bytes - * the read request is. - * - * INPUTS bus - structure containing attributes for the PCI bus - * that the write is destined for. - * devfn - device/function combination that the write is - * destined for. - * where - register within the Configuration Header space - * to access. - * size - in units of bytes, should be 1, 2, or 4. - * val - value to write - * - * OUTPUTS: none - * - * RETURNS: PCIBIOS_SUCCESSFUL - success - * -1 - failure - * - ****************************************************************************/ -int -msp_pcibios_write_config(struct pci_bus *bus, - unsigned int devfn, - int where, - int size, - u32 val) -{ - if (size == 1) { - if (msp_pcibios_write_config_byte(bus, devfn, - where, (u8)(0xFF & val))) { - return -1; - } - } else if (size == 2) { - if (msp_pcibios_write_config_word(bus, devfn, - where, (u16)(0xFFFF & val))) { - return -1; - } - } else if (size == 4) { - if (msp_pcibios_write_config_dword(bus, devfn, where, val)) { - return -1; - } - } else { - return -1; - } - - return PCIBIOS_SUCCESSFUL; -} - -/***************************************************************************** - * - * STRUCTURE: msp_pci_ops - * _________________________________________________________________________ - * - * DESCRIPTION: structure to abstract the hardware specific PCI - * configuration accesses. - * - * ELEMENTS: - * read - function for Linux to generate PCI Configuration reads. - * write - function for Linux to generate PCI Configuration writes. - * - ****************************************************************************/ -struct pci_ops msp_pci_ops = { - .read = msp_pcibios_read_config, - .write = msp_pcibios_write_config -}; - -/***************************************************************************** - * - * STRUCTURE: msp_pci_controller - * _________________________________________________________________________ - * - * Describes the attributes of the MSP7120 PCI Host Controller - * - * ELEMENTS: - * pci_ops - abstracts the hardware specific PCI configuration - * accesses. - * - * mem_resource - address range pciauto() uses to assign to PCI device - * memory BARs. - * - * mem_offset - offset between how MSP7120 outbound PCI memory - * transaction addresses appear on the PCI bus and how Linux - * wants to configure memory BARs of the PCI devices. - * MSP7120 does nothing funky, so just set to zero. - * - * io_resource - address range pciauto() uses to assign to PCI device - * I/O BARs. - * - * io_offset - offset between how MSP7120 outbound PCI I/O - * transaction addresses appear on the PCI bus and how - * Linux defaults to configure I/O BARs of the PCI devices. - * MSP7120 maps outbound I/O accesses into the bottom - * bottom 4K of PCI address space (and ignores OATRAN). - * Since the Linux default is to configure I/O BARs to the - * bottom 4K, no special offset is needed. Just set to zero. - * - ****************************************************************************/ -static struct pci_controller msp_pci_controller = { - .pci_ops = &msp_pci_ops, - .mem_resource = &pci_mem_resource, - .mem_offset = 0, - .io_map_base = MSP_PCI_IOSPACE_BASE, - .io_resource = &pci_io_resource, - .io_offset = 0 -}; - -/***************************************************************************** - * - * FUNCTION: msp_pci_init - * _________________________________________________________________________ - * - * DESCRIPTION: Initialize the PCI Host Controller and register it with - * Linux so Linux can seize control of the PCI bus. - * - ****************************************************************************/ -void __init msp_pci_init(void) -{ - struct msp_pci_regs *preg = (void *)PCI_BASE_REG; - u32 id; - - /* Extract Device ID */ - id = read_reg32(PCI_JTAG_DEVID_REG, 0xFFFF) >> 12; - - /* Check if JTAG ID identifies MSP7120 */ - if (!MSP_HAS_PCI(id)) { - printk(KERN_WARNING "PCI: No PCI; id reads as %x\n", id); - goto no_pci; - } - - /* - * Enable flushing of the PCI-SDRAM queue upon a read - * of the SDRAM's Memory Configuration Register. - */ - *(unsigned long *)QFLUSH_REG_1 = 3; - - /* Configure PCI Host Controller. */ - preg->if_status = ~0; /* Clear cause register bits */ - preg->config_addr = 0; /* Clear config access */ - preg->oatran = MSP_PCI_OATRAN; /* PCI outbound addr translation */ - preg->if_mask = 0xF8BF87C0; /* Enable all PCI status interrupts */ - - /* configure so inb(), outb(), and family are functional */ - set_io_port_base(MSP_PCI_IOSPACE_BASE); - - /* Tell Linux the details of the MSP7120 PCI Host Controller */ - register_pci_controller(&msp_pci_controller); - - return; - -no_pci: - /* Disable PCI channel */ - printk(KERN_WARNING "PCI: no host PCI bus detected\n"); -} diff --git a/arch/mips/pmcs-msp71xx/Kconfig b/arch/mips/pmcs-msp71xx/Kconfig deleted file mode 100644 index b185b7620c97..000000000000 --- a/arch/mips/pmcs-msp71xx/Kconfig +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -choice - prompt "PMC-Sierra MSP SOC type" - depends on PMC_MSP - -config PMC_MSP4200_EVAL - bool "PMC-Sierra MSP4200 Eval Board" - select IRQ_MSP_SLP - select HAVE_PCI - select MIPS_L1_CACHE_SHIFT_4 - -config PMC_MSP4200_GW - bool "PMC-Sierra MSP4200 VoIP Gateway" - select IRQ_MSP_SLP - select HAVE_PCI - -config PMC_MSP7120_EVAL - bool "PMC-Sierra MSP7120 Eval Board" - select SYS_SUPPORTS_MULTITHREADING - select IRQ_MSP_CIC - select HAVE_PCI - -config PMC_MSP7120_GW - bool "PMC-Sierra MSP7120 Residential Gateway" - select SYS_SUPPORTS_MULTITHREADING - select IRQ_MSP_CIC - select HAVE_PCI - select MSP_HAS_USB - select MSP_ETH - -config PMC_MSP7120_FPGA - bool "PMC-Sierra MSP7120 FPGA" - select SYS_SUPPORTS_MULTITHREADING - select IRQ_MSP_CIC - select HAVE_PCI - -endchoice - -config MSP_HAS_USB - bool - depends on PMC_MSP - -config MSP_ETH - bool - select MSP_HAS_MAC - depends on PMC_MSP - -config MSP_HAS_MAC - bool - depends on PMC_MSP diff --git a/arch/mips/pmcs-msp71xx/Makefile b/arch/mips/pmcs-msp71xx/Makefile deleted file mode 100644 index c040bd6ed62d..000000000000 --- a/arch/mips/pmcs-msp71xx/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# Makefile for the PMC-Sierra MSP SOCs -# -obj-y += msp_prom.o msp_setup.o msp_irq.o \ - msp_time.o msp_serial.o msp_elb.o -obj-$(CONFIG_PMC_MSP7120_GW) += msp_hwbutton.o -obj-$(CONFIG_IRQ_MSP_SLP) += msp_irq_slp.o -obj-$(CONFIG_IRQ_MSP_CIC) += msp_irq_cic.o msp_irq_per.o -obj-$(CONFIG_PCI) += msp_pci.o -obj-$(CONFIG_MSP_HAS_MAC) += msp_eth.o -obj-$(CONFIG_MSP_HAS_USB) += msp_usb.o -obj-$(CONFIG_MIPS_MT_SMP) += msp_smp.o diff --git a/arch/mips/pmcs-msp71xx/Platform b/arch/mips/pmcs-msp71xx/Platform deleted file mode 100644 index 7af0734a5007..000000000000 --- a/arch/mips/pmcs-msp71xx/Platform +++ /dev/null @@ -1,7 +0,0 @@ -# -# PMC-Sierra MSP SOCs -# -platform-$(CONFIG_PMC_MSP) += pmcs-msp71xx/ -cflags-$(CONFIG_PMC_MSP) += -I$(srctree)/arch/mips/include/asm/mach-pmcs-msp71xx \ - -mno-branch-likely -load-$(CONFIG_PMC_MSP) += 0xffffffff80100000 diff --git a/arch/mips/pmcs-msp71xx/msp_elb.c b/arch/mips/pmcs-msp71xx/msp_elb.c deleted file mode 100644 index 3e9641007216..000000000000 --- a/arch/mips/pmcs-msp71xx/msp_elb.c +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Sets up the proper Chip Select configuration registers. It is assumed that - * PMON sets up the ADDR and MASK registers properly. - * - * Copyright 2005-2006 PMC-Sierra, Inc. - * Author: Marc St-Jean, Marc_St-Jean@pmc-sierra.com - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include -#include -#include - -static int __init msp_elb_setup(void) -{ -#if defined(CONFIG_PMC_MSP7120_GW) \ - || defined(CONFIG_PMC_MSP7120_EVAL) - /* - * Force all CNFG to be identical and equal to CS0, - * according to OPS doc - */ - *CS1_CNFG_REG = *CS2_CNFG_REG = *CS3_CNFG_REG = *CS0_CNFG_REG; -#endif - return 0; -} - -subsys_initcall(msp_elb_setup); diff --git a/arch/mips/pmcs-msp71xx/msp_eth.c b/arch/mips/pmcs-msp71xx/msp_eth.c deleted file mode 100644 index 15679b427f44..000000000000 --- a/arch/mips/pmcs-msp71xx/msp_eth.c +++ /dev/null @@ -1,111 +0,0 @@ -/* - * The setup file for ethernet related hardware on PMC-Sierra MSP processors. - * - * Copyright 2010 PMC-Sierra, Inc. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - - -#define MSP_ETHERNET_GPIO0 14 -#define MSP_ETHERNET_GPIO1 15 -#define MSP_ETHERNET_GPIO2 16 - -#define MSP_ETH_ID "pmc_mspeth" -#define MSP_ETH_SIZE 0xE0 -static struct resource msp_eth0_resources[] = { - [0] = { - .start = MSP_MAC0_BASE, - .end = MSP_MAC0_BASE + MSP_ETH_SIZE - 1, - .flags = IORESOURCE_MEM, - }, - [1] = { - .start = MSP_INT_MAC0, - .end = MSP_INT_MAC0, - .flags = IORESOURCE_IRQ, - }, -}; - -static struct resource msp_eth1_resources[] = { - [0] = { - .start = MSP_MAC1_BASE, - .end = MSP_MAC1_BASE + MSP_ETH_SIZE - 1, - .flags = IORESOURCE_MEM, - }, - [1] = { - .start = MSP_INT_MAC1, - .end = MSP_INT_MAC1, - .flags = IORESOURCE_IRQ, - }, -}; - - - -static struct platform_device mspeth_device[] = { - [0] = { - .name = MSP_ETH_ID, - .id = 0, - .num_resources = ARRAY_SIZE(msp_eth0_resources), - .resource = msp_eth0_resources, - }, - [1] = { - .name = MSP_ETH_ID, - .id = 1, - .num_resources = ARRAY_SIZE(msp_eth1_resources), - .resource = msp_eth1_resources, - }, - -}; -#define msp_eth_devs mspeth_device - -int __init msp_eth_setup(void) -{ - int i, ret = 0; - - /* Configure the GPIO and take the ethernet PHY out of reset */ - msp_gpio_pin_mode(MSP_GPIO_OUTPUT, MSP_ETHERNET_GPIO0); - msp_gpio_pin_hi(MSP_ETHERNET_GPIO0); - - for (i = 0; i < ARRAY_SIZE(msp_eth_devs); i++) { - ret = platform_device_register(&msp_eth_devs[i]); - printk(KERN_INFO "device: %d, return value = %d\n", i, ret); - if (ret) { - platform_device_unregister(&msp_eth_devs[i]); - break; - } - } - - if (ret) - printk(KERN_WARNING "Could not initialize " - "MSPETH device structures.\n"); - - return ret; -} -subsys_initcall(msp_eth_setup); diff --git a/arch/mips/pmcs-msp71xx/msp_hwbutton.c b/arch/mips/pmcs-msp71xx/msp_hwbutton.c deleted file mode 100644 index bb57ed9ea2bd..000000000000 --- a/arch/mips/pmcs-msp71xx/msp_hwbutton.c +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Sets up interrupt handlers for various hardware switches which are - * connected to interrupt lines. - * - * Copyright 2005-2207 PMC-Sierra, Inc. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include -#include -#include - -#include -#include -#include - -/* For hwbutton_interrupt->initial_state */ -#define HWBUTTON_HI 0x1 -#define HWBUTTON_LO 0x2 - -/* - * This struct describes a hardware button - */ -struct hwbutton_interrupt { - char *name; /* Name of button */ - int irq; /* Actual LINUX IRQ */ - int eirq; /* Extended IRQ number (0-7) */ - int initial_state; /* The "normal" state of the switch */ - void (*handle_hi)(void *); /* Handler: switch input has gone HI */ - void (*handle_lo)(void *); /* Handler: switch input has gone LO */ - void *data; /* Optional data to pass to handler */ -}; - -#ifdef CONFIG_PMC_MSP7120_GW -extern void msp_restart(char *); - -static void softreset_push(void *data) -{ - printk(KERN_WARNING "SOFTRESET switch was pushed\n"); - - /* - * In the future you could move this to the release handler, - * timing the difference between the 'push' and 'release', and only - * doing this ungraceful restart if the button has been down for - * a certain amount of time; otherwise doing a graceful restart. - */ - - msp_restart(NULL); -} - -static void softreset_release(void *data) -{ - printk(KERN_WARNING "SOFTRESET switch was released\n"); - - /* Do nothing */ -} - -static void standby_on(void *data) -{ - printk(KERN_WARNING "STANDBY switch was set to ON (not implemented)\n"); - - /* TODO: Put board in standby mode */ -} - -static void standby_off(void *data) -{ - printk(KERN_WARNING - "STANDBY switch was set to OFF (not implemented)\n"); - - /* TODO: Take out of standby mode */ -} - -static struct hwbutton_interrupt softreset_sw = { - .name = "Softreset button", - .irq = MSP_INT_EXT0, - .eirq = 0, - .initial_state = HWBUTTON_HI, - .handle_hi = softreset_release, - .handle_lo = softreset_push, - .data = NULL, -}; - -static struct hwbutton_interrupt standby_sw = { - .name = "Standby switch", - .irq = MSP_INT_EXT1, - .eirq = 1, - .initial_state = HWBUTTON_HI, - .handle_hi = standby_off, - .handle_lo = standby_on, - .data = NULL, -}; -#endif /* CONFIG_PMC_MSP7120_GW */ - -static irqreturn_t hwbutton_handler(int irq, void *data) -{ - struct hwbutton_interrupt *hirq = data; - unsigned long cic_ext = *CIC_EXT_CFG_REG; - - if (CIC_EXT_IS_ACTIVE_HI(cic_ext, hirq->eirq)) { - /* Interrupt: pin is now HI */ - CIC_EXT_SET_ACTIVE_LO(cic_ext, hirq->eirq); - hirq->handle_hi(hirq->data); - } else { - /* Interrupt: pin is now LO */ - CIC_EXT_SET_ACTIVE_HI(cic_ext, hirq->eirq); - hirq->handle_lo(hirq->data); - } - - /* - * Invert the POLARITY of this level interrupt to ack the interrupt - * Thus next state change will invoke the opposite message - */ - *CIC_EXT_CFG_REG = cic_ext; - - return IRQ_HANDLED; -} - -static int msp_hwbutton_register(struct hwbutton_interrupt *hirq) -{ - unsigned long cic_ext; - - if (hirq->handle_hi == NULL || hirq->handle_lo == NULL) - return -EINVAL; - - cic_ext = *CIC_EXT_CFG_REG; - CIC_EXT_SET_TRIGGER_LEVEL(cic_ext, hirq->eirq); - if (hirq->initial_state == HWBUTTON_HI) - CIC_EXT_SET_ACTIVE_LO(cic_ext, hirq->eirq); - else - CIC_EXT_SET_ACTIVE_HI(cic_ext, hirq->eirq); - *CIC_EXT_CFG_REG = cic_ext; - - return request_irq(hirq->irq, hwbutton_handler, 0, - hirq->name, hirq); -} - -static int __init msp_hwbutton_setup(void) -{ -#ifdef CONFIG_PMC_MSP7120_GW - msp_hwbutton_register(&softreset_sw); - msp_hwbutton_register(&standby_sw); -#endif - return 0; -} - -subsys_initcall(msp_hwbutton_setup); diff --git a/arch/mips/pmcs-msp71xx/msp_irq.c b/arch/mips/pmcs-msp71xx/msp_irq.c deleted file mode 100644 index d525cc931d89..000000000000 --- a/arch/mips/pmcs-msp71xx/msp_irq.c +++ /dev/null @@ -1,155 +0,0 @@ -/* - * IRQ vector handles - * - * Copyright (C) 1995, 1996, 1997, 2003 by Ralf Baechle - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - */ - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -/* SLP bases systems */ -extern void msp_slp_irq_init(void); -extern void msp_slp_irq_dispatch(void); - -/* CIC based systems */ -extern void msp_cic_irq_init(void); -extern void msp_cic_irq_dispatch(void); - -/* VSMP support init */ -extern void msp_vsmp_int_init(void); - -/* vectored interrupt implementation */ - -/* SW0/1 interrupts are used for SMP */ -static inline void mac0_int_dispatch(void) { do_IRQ(MSP_INT_MAC0); } -static inline void mac1_int_dispatch(void) { do_IRQ(MSP_INT_MAC1); } -static inline void mac2_int_dispatch(void) { do_IRQ(MSP_INT_SAR); } -static inline void usb_int_dispatch(void) { do_IRQ(MSP_INT_USB); } -static inline void sec_int_dispatch(void) { do_IRQ(MSP_INT_SEC); } - -/* - * The PMC-Sierra MSP interrupts are arranged in a 3 level cascaded - * hierarchical system. The first level are the direct MIPS interrupts - * and are assigned the interrupt range 0-7. The second level is the SLM - * interrupt controller and is assigned the range 8-39. The third level - * comprises the Peripherial block, the PCI block, the PCI MSI block and - * the SLP. The PCI interrupts and the SLP errors are handled by the - * relevant subsystems so the core interrupt code needs only concern - * itself with the Peripheral block. These are assigned interrupts in - * the range 40-71. - */ - -asmlinkage void plat_irq_dispatch(void) -{ - u32 pending; - - pending = read_c0_status() & read_c0_cause(); - - /* - * jump to the correct interrupt routine - * These are arranged in priority order and the timer - * comes first! - */ - -#ifdef CONFIG_IRQ_MSP_CIC /* break out the CIC stuff for now */ - if (pending & C_IRQ4) /* do the peripherals first, that's the timer */ - msp_cic_irq_dispatch(); - - else if (pending & C_IRQ0) - do_IRQ(MSP_INT_MAC0); - - else if (pending & C_IRQ1) - do_IRQ(MSP_INT_MAC1); - - else if (pending & C_IRQ2) - do_IRQ(MSP_INT_USB); - - else if (pending & C_IRQ3) - do_IRQ(MSP_INT_SAR); - - else if (pending & C_IRQ5) - do_IRQ(MSP_INT_SEC); - -#else - if (pending & C_IRQ5) - do_IRQ(MSP_INT_TIMER); - - else if (pending & C_IRQ0) - do_IRQ(MSP_INT_MAC0); - - else if (pending & C_IRQ1) - do_IRQ(MSP_INT_MAC1); - - else if (pending & C_IRQ3) - do_IRQ(MSP_INT_VE); - - else if (pending & C_IRQ4) - msp_slp_irq_dispatch(); -#endif - - else if (pending & C_SW0) /* do software after hardware */ - do_IRQ(MSP_INT_SW0); - - else if (pending & C_SW1) - do_IRQ(MSP_INT_SW1); -} - -void __init arch_init_irq(void) -{ - /* assume we'll be using vectored interrupt mode except in UP mode*/ -#ifdef CONFIG_MIPS_MT - BUG_ON(!cpu_has_vint); -#endif - /* initialize the 1st-level CPU based interrupt controller */ - mips_cpu_irq_init(); - -#ifdef CONFIG_IRQ_MSP_CIC - msp_cic_irq_init(); -#ifdef CONFIG_MIPS_MT - set_vi_handler(MSP_INT_CIC, msp_cic_irq_dispatch); - set_vi_handler(MSP_INT_MAC0, mac0_int_dispatch); - set_vi_handler(MSP_INT_MAC1, mac1_int_dispatch); - set_vi_handler(MSP_INT_SAR, mac2_int_dispatch); - set_vi_handler(MSP_INT_USB, usb_int_dispatch); - set_vi_handler(MSP_INT_SEC, sec_int_dispatch); -#ifdef CONFIG_MIPS_MT_SMP - msp_vsmp_int_init(); -#endif /* CONFIG_MIPS_MT_SMP */ -#endif /* CONFIG_MIPS_MT */ - /* setup the cascaded interrupts */ - if (request_irq(MSP_INT_CIC, no_action, IRQF_NO_THREAD, - "MSP CIC cascade", NULL)) - pr_err("Failed to register MSP CIC cascade interrupt\n"); - if (request_irq(MSP_INT_PER, no_action, IRQF_NO_THREAD, - "MSP PER cascade", NULL)) - pr_err("Failed to register MSP PER cascade interrupt\n"); - -#else - /* - * Setup the 2nd-level SLP register based interrupt controller. - * VSMP support support is not enabled for SLP. - */ - msp_slp_irq_init(); - - /* setup the cascaded SLP/PER interrupts */ - if (request_irq(MSP_INT_SLP, no_action, IRQF_NO_THREAD, - "MSP CIC cascade", NULL)) - pr_err("Failed to register MSP CIC cascade interrupt\n"); - if (request_irq(MSP_INT_PER, no_action, IRQF_NO_THREAD, - "MSP PER cascade", NULL)) - pr_err("Failed to register MSP PER cascade interrupt\n"); -#endif -} diff --git a/arch/mips/pmcs-msp71xx/msp_irq_cic.c b/arch/mips/pmcs-msp71xx/msp_irq_cic.c deleted file mode 100644 index 0706010cc99f..000000000000 --- a/arch/mips/pmcs-msp71xx/msp_irq_cic.c +++ /dev/null @@ -1,208 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright 2010 PMC-Sierra, Inc, derived from irq_cpu.c - * - * This file define the irq handler for MSP CIC subsystem interrupts. - */ - -#include -#include -#include -#include -#include - -#include - -#include -#include - -/* - * External API - */ -extern void msp_per_irq_init(void); -extern void msp_per_irq_dispatch(void); - - -/* - * Convenience Macro. Should be somewhere generic. - */ -#define get_current_vpe() \ - ((read_c0_tcbind() >> TCBIND_CURVPE_SHIFT) & TCBIND_CURVPE) - -#ifdef CONFIG_SMP - -#define LOCK_VPE(flags, mtflags) \ -do { \ - local_irq_save(flags); \ - mtflags = dmt(); \ -} while (0) - -#define UNLOCK_VPE(flags, mtflags) \ -do { \ - emt(mtflags); \ - local_irq_restore(flags);\ -} while (0) - -#define LOCK_CORE(flags, mtflags) \ -do { \ - local_irq_save(flags); \ - mtflags = dvpe(); \ -} while (0) - -#define UNLOCK_CORE(flags, mtflags) \ -do { \ - evpe(mtflags); \ - local_irq_restore(flags);\ -} while (0) - -#else - -#define LOCK_VPE(flags, mtflags) -#define UNLOCK_VPE(flags, mtflags) -#endif - -/* ensure writes to cic are completed */ -static inline void cic_wmb(void) -{ - const volatile void __iomem *cic_mem = CIC_VPE0_MSK_REG; - volatile u32 dummy_read; - - wmb(); - dummy_read = __raw_readl(cic_mem); - dummy_read++; -} - -static void unmask_cic_irq(struct irq_data *d) -{ - volatile u32 *cic_msk_reg = CIC_VPE0_MSK_REG; - int vpe; -#ifdef CONFIG_SMP - unsigned int mtflags; - unsigned long flags; - - /* - * Make sure we have IRQ affinity. It may have changed while - * we were processing the IRQ. - */ - if (!cpumask_test_cpu(smp_processor_id(), - irq_data_get_affinity_mask(d))) - return; -#endif - - vpe = get_current_vpe(); - LOCK_VPE(flags, mtflags); - cic_msk_reg[vpe] |= (1 << (d->irq - MSP_CIC_INTBASE)); - UNLOCK_VPE(flags, mtflags); - cic_wmb(); -} - -static void mask_cic_irq(struct irq_data *d) -{ - volatile u32 *cic_msk_reg = CIC_VPE0_MSK_REG; - int vpe = get_current_vpe(); -#ifdef CONFIG_SMP - unsigned long flags, mtflags; -#endif - LOCK_VPE(flags, mtflags); - cic_msk_reg[vpe] &= ~(1 << (d->irq - MSP_CIC_INTBASE)); - UNLOCK_VPE(flags, mtflags); - cic_wmb(); -} -static void msp_cic_irq_ack(struct irq_data *d) -{ - mask_cic_irq(d); - /* - * Only really necessary for 18, 16-14 and sometimes 3:0 - * (since these can be edge sensitive) but it doesn't - * hurt for the others - */ - *CIC_STS_REG = (1 << (d->irq - MSP_CIC_INTBASE)); -} - -/* Note: Limiting to VSMP. */ - -#ifdef CONFIG_MIPS_MT_SMP -static int msp_cic_irq_set_affinity(struct irq_data *d, - const struct cpumask *cpumask, bool force) -{ - int cpu; - unsigned long flags; - unsigned int mtflags; - unsigned long imask = (1 << (d->irq - MSP_CIC_INTBASE)); - volatile u32 *cic_mask = (volatile u32 *)CIC_VPE0_MSK_REG; - - /* timer balancing should be disabled in kernel code */ - BUG_ON(d->irq == MSP_INT_VPE0_TIMER || d->irq == MSP_INT_VPE1_TIMER); - - LOCK_CORE(flags, mtflags); - /* enable if any of each VPE's TCs require this IRQ */ - for_each_online_cpu(cpu) { - if (cpumask_test_cpu(cpu, cpumask)) - cic_mask[cpu] |= imask; - else - cic_mask[cpu] &= ~imask; - - } - - UNLOCK_CORE(flags, mtflags); - return 0; - -} -#endif - -static struct irq_chip msp_cic_irq_controller = { - .name = "MSP_CIC", - .irq_mask = mask_cic_irq, - .irq_mask_ack = msp_cic_irq_ack, - .irq_unmask = unmask_cic_irq, - .irq_ack = msp_cic_irq_ack, -#ifdef CONFIG_MIPS_MT_SMP - .irq_set_affinity = msp_cic_irq_set_affinity, -#endif -}; - -void __init msp_cic_irq_init(void) -{ - int i; - /* Mask/clear interrupts. */ - *CIC_VPE0_MSK_REG = 0x00000000; - *CIC_VPE1_MSK_REG = 0x00000000; - *CIC_STS_REG = 0xFFFFFFFF; - /* - * The MSP7120 RG and EVBD boards use IRQ[6:4] for PCI. - * These inputs map to EXT_INT_POL[6:4] inside the CIC. - * They are to be active low, level sensitive. - */ - *CIC_EXT_CFG_REG &= 0xFFFF8F8F; - - /* initialize all the IRQ descriptors */ - for (i = MSP_CIC_INTBASE ; i < MSP_CIC_INTBASE + 32 ; i++) { - irq_set_chip_and_handler(i, &msp_cic_irq_controller, - handle_level_irq); - } - - /* Initialize the PER interrupt sub-system */ - msp_per_irq_init(); -} - -/* CIC masked by CIC vector processing before dispatch called */ -void msp_cic_irq_dispatch(void) -{ - volatile u32 *cic_msk_reg = (volatile u32 *)CIC_VPE0_MSK_REG; - u32 cic_mask; - u32 pending; - int cic_status = *CIC_STS_REG; - cic_mask = cic_msk_reg[get_current_vpe()]; - pending = cic_status & cic_mask; - if (pending & (1 << (MSP_INT_VPE0_TIMER - MSP_CIC_INTBASE))) { - do_IRQ(MSP_INT_VPE0_TIMER); - } else if (pending & (1 << (MSP_INT_VPE1_TIMER - MSP_CIC_INTBASE))) { - do_IRQ(MSP_INT_VPE1_TIMER); - } else if (pending & (1 << (MSP_INT_PER - MSP_CIC_INTBASE))) { - msp_per_irq_dispatch(); - } else if (pending) { - do_IRQ(ffs(pending) + MSP_CIC_INTBASE - 1); - } else{ - spurious_interrupt(); - } -} diff --git a/arch/mips/pmcs-msp71xx/msp_irq_per.c b/arch/mips/pmcs-msp71xx/msp_irq_per.c deleted file mode 100644 index b284412b2923..000000000000 --- a/arch/mips/pmcs-msp71xx/msp_irq_per.c +++ /dev/null @@ -1,127 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright 2010 PMC-Sierra, Inc, derived from irq_cpu.c - * - * This file define the irq handler for MSP PER subsystem interrupts. - */ - -#include -#include -#include -#include -#include - -#include - -#include -#include - - -/* - * Convenience Macro. Should be somewhere generic. - */ -#define get_current_vpe() \ - ((read_c0_tcbind() >> TCBIND_CURVPE_SHIFT) & TCBIND_CURVPE) - -#ifdef CONFIG_SMP -/* - * The PER registers must be protected from concurrent access. - */ - -static DEFINE_SPINLOCK(per_lock); -#endif - -/* ensure writes to per are completed */ - -static inline void per_wmb(void) -{ - const volatile void __iomem *per_mem = PER_INT_MSK_REG; - volatile u32 dummy_read; - - wmb(); - dummy_read = __raw_readl(per_mem); - dummy_read++; -} - -static inline void unmask_per_irq(struct irq_data *d) -{ -#ifdef CONFIG_SMP - unsigned long flags; - spin_lock_irqsave(&per_lock, flags); - *PER_INT_MSK_REG |= (1 << (d->irq - MSP_PER_INTBASE)); - spin_unlock_irqrestore(&per_lock, flags); -#else - *PER_INT_MSK_REG |= (1 << (d->irq - MSP_PER_INTBASE)); -#endif - per_wmb(); -} - -static inline void mask_per_irq(struct irq_data *d) -{ -#ifdef CONFIG_SMP - unsigned long flags; - spin_lock_irqsave(&per_lock, flags); - *PER_INT_MSK_REG &= ~(1 << (d->irq - MSP_PER_INTBASE)); - spin_unlock_irqrestore(&per_lock, flags); -#else - *PER_INT_MSK_REG &= ~(1 << (d->irq - MSP_PER_INTBASE)); -#endif - per_wmb(); -} - -static inline void msp_per_irq_ack(struct irq_data *d) -{ - mask_per_irq(d); - /* - * In the PER interrupt controller, only bits 11 and 10 - * are write-to-clear, (SPI TX complete, SPI RX complete). - * It does nothing for any others. - */ - *PER_INT_STS_REG = (1 << (d->irq - MSP_PER_INTBASE)); -} - -#ifdef CONFIG_SMP -static int msp_per_irq_set_affinity(struct irq_data *d, - const struct cpumask *affinity, bool force) -{ - /* WTF is this doing ????? */ - unmask_per_irq(d); - return 0; -} -#endif - -static struct irq_chip msp_per_irq_controller = { - .name = "MSP_PER", - .irq_enable = unmask_per_irq, - .irq_disable = mask_per_irq, - .irq_ack = msp_per_irq_ack, -#ifdef CONFIG_SMP - .irq_set_affinity = msp_per_irq_set_affinity, -#endif -}; - -void __init msp_per_irq_init(void) -{ - int i; - /* Mask/clear interrupts. */ - *PER_INT_MSK_REG = 0x00000000; - *PER_INT_STS_REG = 0xFFFFFFFF; - /* initialize all the IRQ descriptors */ - for (i = MSP_PER_INTBASE; i < MSP_PER_INTBASE + 32; i++) { - irq_set_chip(i, &msp_per_irq_controller); - } -} - -void msp_per_irq_dispatch(void) -{ - u32 per_mask = *PER_INT_MSK_REG; - u32 per_status = *PER_INT_STS_REG; - u32 pending; - - pending = per_status & per_mask; - if (pending) { - do_IRQ(ffs(pending) + MSP_PER_INTBASE - 1); - } else { - spurious_interrupt(); - } -} diff --git a/arch/mips/pmcs-msp71xx/msp_irq_slp.c b/arch/mips/pmcs-msp71xx/msp_irq_slp.c deleted file mode 100644 index 097a5fd3b06b..000000000000 --- a/arch/mips/pmcs-msp71xx/msp_irq_slp.c +++ /dev/null @@ -1,102 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * This file define the irq handler for MSP SLM subsystem interrupts. - * - * Copyright 2005-2006 PMC-Sierra, Inc, derived from irq_cpu.c - * Author: Andrew Hughes, Andrew_Hughes@pmc-sierra.com - */ - -#include -#include -#include -#include - -#include - -#include -#include - -static inline void unmask_msp_slp_irq(struct irq_data *d) -{ - unsigned int irq = d->irq; - - /* check for PER interrupt range */ - if (irq < MSP_PER_INTBASE) - *SLP_INT_MSK_REG |= (1 << (irq - MSP_SLP_INTBASE)); - else - *PER_INT_MSK_REG |= (1 << (irq - MSP_PER_INTBASE)); -} - -static inline void mask_msp_slp_irq(struct irq_data *d) -{ - unsigned int irq = d->irq; - - /* check for PER interrupt range */ - if (irq < MSP_PER_INTBASE) - *SLP_INT_MSK_REG &= ~(1 << (irq - MSP_SLP_INTBASE)); - else - *PER_INT_MSK_REG &= ~(1 << (irq - MSP_PER_INTBASE)); -} - -/* - * While we ack the interrupt interrupts are disabled and thus we don't need - * to deal with concurrency issues. Same for msp_slp_irq_end. - */ -static inline void ack_msp_slp_irq(struct irq_data *d) -{ - unsigned int irq = d->irq; - - /* check for PER interrupt range */ - if (irq < MSP_PER_INTBASE) - *SLP_INT_STS_REG = (1 << (irq - MSP_SLP_INTBASE)); - else - *PER_INT_STS_REG = (1 << (irq - MSP_PER_INTBASE)); -} - -static struct irq_chip msp_slp_irq_controller = { - .name = "MSP_SLP", - .irq_ack = ack_msp_slp_irq, - .irq_mask = mask_msp_slp_irq, - .irq_unmask = unmask_msp_slp_irq, -}; - -void __init msp_slp_irq_init(void) -{ - int i; - - /* Mask/clear interrupts. */ - *SLP_INT_MSK_REG = 0x00000000; - *PER_INT_MSK_REG = 0x00000000; - *SLP_INT_STS_REG = 0xFFFFFFFF; - *PER_INT_STS_REG = 0xFFFFFFFF; - - /* initialize all the IRQ descriptors */ - for (i = MSP_SLP_INTBASE; i < MSP_PER_INTBASE + 32; i++) - irq_set_chip_and_handler(i, &msp_slp_irq_controller, - handle_level_irq); -} - -void msp_slp_irq_dispatch(void) -{ - u32 pending; - int intbase; - - intbase = MSP_SLP_INTBASE; - pending = *SLP_INT_STS_REG & *SLP_INT_MSK_REG; - - /* check for PER interrupt */ - if (pending == (1 << (MSP_INT_PER - MSP_SLP_INTBASE))) { - intbase = MSP_PER_INTBASE; - pending = *PER_INT_STS_REG & *PER_INT_MSK_REG; - } - - /* check for spurious interrupt */ - if (pending == 0x00000000) { - printk(KERN_ERR "Spurious %s interrupt?\n", - (intbase == MSP_SLP_INTBASE) ? "SLP" : "PER"); - return; - } - - /* dispatch the irq */ - do_IRQ(ffs(pending) + intbase - 1); -} diff --git a/arch/mips/pmcs-msp71xx/msp_pci.c b/arch/mips/pmcs-msp71xx/msp_pci.c deleted file mode 100644 index 428dea23c35c..000000000000 --- a/arch/mips/pmcs-msp71xx/msp_pci.c +++ /dev/null @@ -1,50 +0,0 @@ -/* - * The setup file for PCI related hardware on PMC-Sierra MSP processors. - * - * Copyright 2005-2006 PMC-Sierra, Inc. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include - -#include -#include - -extern void msp_pci_init(void); - -static int __init msp_pci_setup(void) -{ -#if 0 /* Linux 2.6 initialization code to be completed */ - if (getdeviceid() & DEV_ID_SINGLE_PC) { - /* If single card mode */ - slmRegs *sreg = (slmRegs *) SREG_BASE; - - sreg->single_pc_enable = SINGLE_PCCARD; - } -#endif - - msp_pci_init(); - - return 0; -} - -subsys_initcall(msp_pci_setup); diff --git a/arch/mips/pmcs-msp71xx/msp_prom.c b/arch/mips/pmcs-msp71xx/msp_prom.c deleted file mode 100644 index 800a21b8b8b0..000000000000 --- a/arch/mips/pmcs-msp71xx/msp_prom.c +++ /dev/null @@ -1,513 +0,0 @@ -/* - * BRIEF MODULE DESCRIPTION - * PROM library initialisation code, assuming a version of - * pmon is the boot code. - * - * Copyright 2000,2001 MontaVista Software Inc. - * Author: MontaVista Software, Inc. - * ppopov@mvista.com or source@mvista.com - * - * This file was derived from Carsten Langgaard's - * arch/mips/mips-boards/xx files. - * - * Carsten Langgaard, carstenl@mips.com - * Copyright (C) 1999,2000 MIPS Technologies, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -/* global PROM environment variables and pointers */ -int prom_argc; -char **prom_argv, **prom_envp; -int *prom_vec; - -/* debug flag */ -int init_debug = 1; - -/* memory blocks */ -struct prom_pmemblock mdesc[PROM_MAX_PMEMBLOCKS]; - -#define MAX_PROM_MEM 5 -static phys_addr_t prom_mem_base[MAX_PROM_MEM] __initdata; -static phys_addr_t prom_mem_size[MAX_PROM_MEM] __initdata; -static unsigned int nr_prom_mem __initdata; - -/* default feature sets */ -static char msp_default_features[] = -#if defined(CONFIG_PMC_MSP4200_EVAL) \ - || defined(CONFIG_PMC_MSP4200_GW) - "ERER"; -#elif defined(CONFIG_PMC_MSP7120_EVAL) \ - || defined(CONFIG_PMC_MSP7120_GW) - "EMEMSP"; -#elif defined(CONFIG_PMC_MSP7120_FPGA) - "EMEM"; -#endif - -/* conversion functions */ -static inline unsigned char str2hexnum(unsigned char c) -{ - if (c >= '0' && c <= '9') - return c - '0'; - if (c >= 'a' && c <= 'f') - return c - 'a' + 10; - return 0; /* foo */ -} - -int str2eaddr(unsigned char *ea, unsigned char *str) -{ - int index = 0; - unsigned char num = 0; - - while (*str != '\0') { - if ((*str == '.') || (*str == ':')) { - ea[index++] = num; - num = 0; - str++; - } else { - num = num << 4; - num |= str2hexnum(*str++); - } - } - - if (index == 5) { - ea[index++] = num; - return 0; - } else - return -1; -} -EXPORT_SYMBOL(str2eaddr); - -static inline unsigned long str2hex(unsigned char *str) -{ - int value = 0; - - while (*str) { - value = value << 4; - value |= str2hexnum(*str++); - } - - return value; -} - -/* function to query the system information */ -const char *get_system_type(void) -{ -#if defined(CONFIG_PMC_MSP4200_EVAL) - return "PMC-Sierra MSP4200 Eval Board"; -#elif defined(CONFIG_PMC_MSP4200_GW) - return "PMC-Sierra MSP4200 VoIP Gateway"; -#elif defined(CONFIG_PMC_MSP7120_EVAL) - return "PMC-Sierra MSP7120 Eval Board"; -#elif defined(CONFIG_PMC_MSP7120_GW) - return "PMC-Sierra MSP7120 Residential Gateway"; -#elif defined(CONFIG_PMC_MSP7120_FPGA) - return "PMC-Sierra MSP7120 FPGA"; -#else - #error "What is the type of *your* MSP?" -#endif -} - -int get_ethernet_addr(char *ethaddr_name, char *ethernet_addr) -{ - char *ethaddr_str; - - ethaddr_str = prom_getenv(ethaddr_name); - if (!ethaddr_str) { - printk(KERN_WARNING "%s not set in boot prom\n", ethaddr_name); - return -1; - } - - if (str2eaddr(ethernet_addr, ethaddr_str) == -1) { - printk(KERN_WARNING "%s badly formatted-<%s>\n", - ethaddr_name, ethaddr_str); - return -1; - } - - if (init_debug > 1) { - int i; - printk(KERN_DEBUG "get_ethernet_addr: for %s ", ethaddr_name); - for (i = 0; i < 5; i++) - printk(KERN_DEBUG "%02x:", - (unsigned char)*(ethernet_addr+i)); - printk(KERN_DEBUG "%02x\n", *(ethernet_addr+i)); - } - - return 0; -} -EXPORT_SYMBOL(get_ethernet_addr); - -static char *get_features(void) -{ - char *feature = prom_getenv(FEATURES); - - if (feature == NULL) { - /* default features based on MACHINE_TYPE */ - feature = msp_default_features; - } - - return feature; -} - -static char test_feature(char c) -{ - char *feature = get_features(); - - while (*feature) { - if (*feature++ == c) - return *feature; - feature++; - } - - return FEATURE_NOEXIST; -} - -unsigned long get_deviceid(void) -{ - char *deviceid = prom_getenv(DEVICEID); - - if (deviceid == NULL) - return *DEV_ID_REG; - else - return str2hex(deviceid); -} - -char identify_pci(void) -{ - return test_feature(PCI_KEY); -} -EXPORT_SYMBOL(identify_pci); - -char identify_pcimux(void) -{ - return test_feature(PCIMUX_KEY); -} - -char identify_sec(void) -{ - return test_feature(SEC_KEY); -} -EXPORT_SYMBOL(identify_sec); - -char identify_spad(void) -{ - return test_feature(SPAD_KEY); -} -EXPORT_SYMBOL(identify_spad); - -char identify_tdm(void) -{ - return test_feature(TDM_KEY); -} -EXPORT_SYMBOL(identify_tdm); - -char identify_zsp(void) -{ - return test_feature(ZSP_KEY); -} -EXPORT_SYMBOL(identify_zsp); - -static char identify_enetfeature(char key, unsigned long interface_num) -{ - char *feature = get_features(); - - while (*feature) { - if (*feature++ == key && interface_num-- == 0) - return *feature; - feature++; - } - - return FEATURE_NOEXIST; -} - -char identify_enet(unsigned long interface_num) -{ - return identify_enetfeature(ENET_KEY, interface_num); -} -EXPORT_SYMBOL(identify_enet); - -char identify_enetTxD(unsigned long interface_num) -{ - return identify_enetfeature(ENETTXD_KEY, interface_num); -} -EXPORT_SYMBOL(identify_enetTxD); - -unsigned long identify_family(void) -{ - unsigned long deviceid; - - deviceid = get_deviceid(); - - return deviceid & CPU_DEVID_FAMILY; -} -EXPORT_SYMBOL(identify_family); - -unsigned long identify_revision(void) -{ - unsigned long deviceid; - - deviceid = get_deviceid(); - - return deviceid & CPU_DEVID_REVISION; -} -EXPORT_SYMBOL(identify_revision); - -/* PROM environment functions */ -char *prom_getenv(char *env_name) -{ - /* - * Return a pointer to the given environment variable. prom_envp - * points to a null terminated array of pointers to variables. - * Environment variables are stored in the form of "memsize=64" - */ - - char **var = prom_envp; - int i = strlen(env_name); - - while (*var) { - if (strncmp(env_name, *var, i) == 0) { - return *var + strlen(env_name) + 1; - } - var++; - } - - return NULL; -} - -/* PROM commandline functions */ -void __init prom_init_cmdline(void) -{ - char *cp; - int actr; - - actr = 1; /* Always ignore argv[0] */ - - cp = &(arcs_cmdline[0]); - while (actr < prom_argc) { - strcpy(cp, prom_argv[actr]); - cp += strlen(prom_argv[actr]); - *cp++ = ' '; - actr++; - } - if (cp != &(arcs_cmdline[0])) /* get rid of trailing space */ - --cp; - *cp = '\0'; -} - -/* memory allocation functions */ -static int __init prom_memtype_classify(unsigned int type) -{ - switch (type) { - case yamon_free: - return BOOT_MEM_RAM; - case yamon_prom: - return BOOT_MEM_ROM_DATA; - default: - return BOOT_MEM_RESERVED; - } -} - -void __init prom_meminit(void) -{ - struct prom_pmemblock *p; - - p = prom_getmdesc(); - - while (p->size) { - long type; - unsigned long base, size; - - type = prom_memtype_classify(p->type); - base = p->base; - size = p->size; - - add_memory_region(base, size, type); - p++; - - if (type == BOOT_MEM_ROM_DATA) { - if (nr_prom_mem >= MAX_PROM_MEM) { - pr_err("Too many ROM DATA regions"); - continue; - } - prom_mem_base[nr_prom_mem] = base; - prom_mem_size[nr_prom_mem] = size; - nr_prom_mem++; - } - } -} - -void __init prom_free_prom_memory(void) -{ - int argc; - char **argv; - char **envp; - char *ptr; - int len = 0; - int i; - - /* - * preserve environment variables and command line from pmon/bbload - * first preserve the command line - */ - for (argc = 0; argc < prom_argc; argc++) { - len += sizeof(char *); /* length of pointer */ - len += strlen(prom_argv[argc]) + 1; /* length of string */ - } - len += sizeof(char *); /* plus length of null pointer */ - - argv = kmalloc(len, GFP_KERNEL); - ptr = (char *) &argv[prom_argc + 1]; /* strings follow array */ - - for (argc = 0; argc < prom_argc; argc++) { - argv[argc] = ptr; - strcpy(ptr, prom_argv[argc]); - ptr += strlen(prom_argv[argc]) + 1; - } - argv[prom_argc] = NULL; /* end array with null pointer */ - prom_argv = argv; - - /* next preserve the environment variables */ - len = 0; - i = 0; - for (envp = prom_envp; *envp != NULL; envp++) { - i++; /* count number of environment variables */ - len += sizeof(char *); /* length of pointer */ - len += strlen(*envp) + 1; /* length of string */ - } - len += sizeof(char *); /* plus length of null pointer */ - - envp = kmalloc(len, GFP_KERNEL); - ptr = (char *) &envp[i+1]; - - for (argc = 0; argc < i; argc++) { - envp[argc] = ptr; - strcpy(ptr, prom_envp[argc]); - ptr += strlen(prom_envp[argc]) + 1; - } - envp[i] = NULL; /* end array with null pointer */ - prom_envp = envp; - - for (i = 0; i < nr_prom_mem; i++) { - free_init_pages("prom memory", - prom_mem_base[i], prom_mem_base[i] + prom_mem_size[i]); - } -} - -struct prom_pmemblock *__init prom_getmdesc(void) -{ - static char memsz_env[] __initdata = "memsize"; - static char heaptop_env[] __initdata = "heaptop"; - char *str; - unsigned int memsize; - unsigned int heaptop; - int i; - - str = prom_getenv(memsz_env); - if (!str) { - ppfinit("memsize not set in boot prom, " - "set to default (32Mb)\n"); - memsize = 0x02000000; - } else { - memsize = simple_strtol(str, NULL, 0); - - if (memsize == 0) { - /* if memsize is a bad size, use reasonable default */ - memsize = 0x02000000; - } - - /* convert to physical address (removing caching bits, etc) */ - memsize = CPHYSADDR(memsize); - } - - str = prom_getenv(heaptop_env); - if (!str) { - heaptop = CPHYSADDR((u32)&_text); - ppfinit("heaptop not set in boot prom, " - "set to default 0x%08x\n", heaptop); - } else { - heaptop = simple_strtol(str, NULL, 16); - if (heaptop == 0) { - /* heaptop conversion bad, might have 0xValue */ - heaptop = simple_strtol(str, NULL, 0); - - if (heaptop == 0) { - /* heaptop still bad, use reasonable default */ - heaptop = CPHYSADDR((u32)&_text); - } - } - - /* convert to physical address (removing caching bits, etc) */ - heaptop = CPHYSADDR((u32)heaptop); - } - - /* the base region */ - i = 0; - mdesc[i].type = BOOT_MEM_RESERVED; - mdesc[i].base = 0x00000000; - mdesc[i].size = PAGE_ALIGN(0x300 + 0x80); - /* jtag interrupt vector + sizeof vector */ - - /* PMON data */ - if (heaptop > mdesc[i].base + mdesc[i].size) { - i++; /* 1 */ - mdesc[i].type = BOOT_MEM_ROM_DATA; - mdesc[i].base = mdesc[i-1].base + mdesc[i-1].size; - mdesc[i].size = heaptop - mdesc[i].base; - } - - /* end of PMON data to start of kernel -- probably zero .. */ - if (heaptop != CPHYSADDR((u32)_text)) { - i++; /* 2 */ - mdesc[i].type = BOOT_MEM_RAM; - mdesc[i].base = heaptop; - mdesc[i].size = CPHYSADDR((u32)_text) - mdesc[i].base; - } - - /* kernel proper */ - i++; /* 3 */ - mdesc[i].type = BOOT_MEM_RESERVED; - mdesc[i].base = CPHYSADDR((u32)_text); - mdesc[i].size = CPHYSADDR(PAGE_ALIGN((u32)_end)) - mdesc[i].base; - - /* Remainder of RAM -- under memsize */ - i++; /* 5 */ - mdesc[i].type = yamon_free; - mdesc[i].base = mdesc[i-1].base + mdesc[i-1].size; - mdesc[i].size = memsize - mdesc[i].base; - - return &mdesc[0]; -} diff --git a/arch/mips/pmcs-msp71xx/msp_serial.c b/arch/mips/pmcs-msp71xx/msp_serial.c deleted file mode 100644 index 940c684f6921..000000000000 --- a/arch/mips/pmcs-msp71xx/msp_serial.c +++ /dev/null @@ -1,154 +0,0 @@ -/* - * The setup file for serial related hardware on PMC-Sierra MSP processors. - * - * Copyright 2005 PMC-Sierra, Inc. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include -#include - -struct msp_uart_data { - int last_lcr; -}; - -static void msp_serial_out(struct uart_port *p, int offset, int value) -{ - struct msp_uart_data *d = p->private_data; - - if (offset == UART_LCR) - d->last_lcr = value; - - offset <<= p->regshift; - writeb(value, p->membase + offset); -} - -static unsigned int msp_serial_in(struct uart_port *p, int offset) -{ - offset <<= p->regshift; - - return readb(p->membase + offset); -} - -static int msp_serial_handle_irq(struct uart_port *p) -{ - struct msp_uart_data *d = p->private_data; - unsigned int iir = readb(p->membase + (UART_IIR << p->regshift)); - - if (serial8250_handle_irq(p, iir)) { - return 1; - } else if ((iir & UART_IIR_BUSY) == UART_IIR_BUSY) { - /* - * The DesignWare APB UART has an Busy Detect (0x07) interrupt - * meaning an LCR write attempt occurred while the UART was - * busy. The interrupt must be cleared by reading the UART - * status register (USR) and the LCR re-written. - * - * Note: MSP reserves 0x20 bytes of address space for the UART - * and the USR is mapped in a separate block at an offset of - * 0xc0 from the start of the UART. - */ - (void)readb(p->membase + 0xc0); - writeb(d->last_lcr, p->membase + (UART_LCR << p->regshift)); - - return 1; - } - - return 0; -} - -void __init msp_serial_setup(void) -{ - char *s; - char *endp; - struct uart_port up; - unsigned int uartclk; - - memset(&up, 0, sizeof(up)); - - /* Check if clock was specified in environment */ - s = prom_getenv("uartfreqhz"); - if(!(s && *s && (uartclk = simple_strtoul(s, &endp, 10)) && *endp == 0)) - uartclk = MSP_BASE_BAUD; - ppfinit("UART clock set to %d\n", uartclk); - - /* Initialize first serial port */ - up.mapbase = MSP_UART0_BASE; - up.membase = ioremap(up.mapbase, MSP_UART_REG_LEN); - up.irq = MSP_INT_UART0; - up.uartclk = uartclk; - up.regshift = 2; - up.iotype = UPIO_MEM; - up.flags = UPF_BOOT_AUTOCONF | UPF_SKIP_TEST; - up.type = PORT_16550A; - up.line = 0; - up.serial_out = msp_serial_out; - up.serial_in = msp_serial_in; - up.handle_irq = msp_serial_handle_irq; - up.private_data = kzalloc(sizeof(struct msp_uart_data), GFP_KERNEL); - if (!up.private_data) { - pr_err("failed to allocate uart private data\n"); - return; - } - if (early_serial_setup(&up)) { - kfree(up.private_data); - pr_err("Early serial init of port 0 failed\n"); - } - - /* Initialize the second serial port, if one exists */ - switch (mips_machtype) { - case MACH_MSP4200_EVAL: - case MACH_MSP4200_GW: - case MACH_MSP4200_FPGA: - case MACH_MSP7120_EVAL: - case MACH_MSP7120_GW: - case MACH_MSP7120_FPGA: - /* Enable UART1 on MSP4200 and MSP7120 */ - *GPIO_CFG2_REG = 0x00002299; - break; - - default: - return; /* No second serial port, good-bye. */ - } - - up.mapbase = MSP_UART1_BASE; - up.membase = ioremap(up.mapbase, MSP_UART_REG_LEN); - up.irq = MSP_INT_UART1; - up.line = 1; - up.private_data = (void*)UART1_STATUS_REG; - if (early_serial_setup(&up)) { - kfree(up.private_data); - pr_err("Early serial init of port 1 failed\n"); - } -} diff --git a/arch/mips/pmcs-msp71xx/msp_setup.c b/arch/mips/pmcs-msp71xx/msp_setup.c deleted file mode 100644 index e0f20f487d96..000000000000 --- a/arch/mips/pmcs-msp71xx/msp_setup.c +++ /dev/null @@ -1,228 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * The generic setup file for PMC-Sierra MSP processors - * - * Copyright 2005-2007 PMC-Sierra, Inc, - * Author: Jun Sun, jsun@mvista.com or jsun@junsun.net - */ - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#if defined(CONFIG_PMC_MSP7120_GW) -#include -#define MSP_BOARD_RESET_GPIO 9 -#endif - -extern void msp_serial_setup(void); - -#if defined(CONFIG_PMC_MSP7120_EVAL) || \ - defined(CONFIG_PMC_MSP7120_GW) || \ - defined(CONFIG_PMC_MSP7120_FPGA) -/* - * Performs the reset for MSP7120-based boards - */ -void msp7120_reset(void) -{ - void *start, *end, *iptr; - - /* Diasble all interrupts */ - local_irq_disable(); -#ifdef CONFIG_SYS_SUPPORTS_MULTITHREADING - dvpe(); -#endif - - /* Cache the reset code of this function */ - __asm__ __volatile__ ( - " .set push \n" - " .set arch=r4000 \n" - " la %0,startpoint \n" - " la %1,endpoint \n" - " .set pop \n" - : "=r" (start), "=r" (end) - : - ); - - for (iptr = (void *)((unsigned int)start & ~(L1_CACHE_BYTES - 1)); - iptr < end; iptr += L1_CACHE_BYTES) - cache_op(Fill_I, iptr); - - __asm__ __volatile__ ( - "startpoint: \n" - ); - - /* Put the DDRC into self-refresh mode */ - DDRC_INDIRECT_WRITE(DDRC_CTL(10), 0xb, 1 << 16); - - /* - * IMPORTANT! - * DO NOT do anything from here on out that might even - * think about fetching from RAM - i.e., don't call any - * non-inlined functions, and be VERY sure that any inline - * functions you do call do NOT access any sort of RAM - * anywhere! - */ - - /* Wait a bit for the DDRC to settle */ - mdelay(125); - -#if defined(CONFIG_PMC_MSP7120_GW) - /* - * Set GPIO 9 HI, (tied to board reset logic) - * GPIO 9 is the 4th GPIO of register 3 - * - * NOTE: We cannot use the higher-level msp_gpio_mode()/out() - * as GPIO char driver may not be enabled and it would look up - * data inRAM! - */ - set_value_reg32(GPIO_CFG3_REG, 0xf000, 0x8000); - set_reg32(GPIO_DATA3_REG, 8); - - /* - * In case GPIO9 doesn't reset the board (jumper configurable!) - * fallback to device reset below. - */ -#endif - /* Set bit 1 of the MSP7120 reset register */ - *RST_SET_REG = 0x00000001; - - __asm__ __volatile__ ( - "endpoint: \n" - ); -} -#endif - -void msp_restart(char *command) -{ - printk(KERN_WARNING "Now rebooting .......\n"); - -#if defined(CONFIG_PMC_MSP7120_EVAL) || \ - defined(CONFIG_PMC_MSP7120_GW) || \ - defined(CONFIG_PMC_MSP7120_FPGA) - msp7120_reset(); -#else - /* No chip-specific reset code, just jump to the ROM reset vector */ - set_c0_status(ST0_BEV | ST0_ERL); - change_c0_config(CONF_CM_CMASK, CONF_CM_UNCACHED); - __flush_cache_all(); - write_c0_wired(0); - - __asm__ __volatile__("jr\t%0"::"r"(0xbfc00000)); -#endif -} - -void msp_halt(void) -{ - printk(KERN_WARNING "\n** You can safely turn off the power\n"); - while (1) - /* If possible call official function to get CPU WARs */ - if (cpu_wait) - (*cpu_wait)(); - else - __asm__(".set\tmips3\n\t" "wait\n\t" ".set\tmips0"); -} - -void msp_power_off(void) -{ - msp_halt(); -} - -void __init plat_mem_setup(void) -{ - _machine_restart = msp_restart; - _machine_halt = msp_halt; - pm_power_off = msp_power_off; -} - -void __init prom_init(void) -{ - unsigned long family; - unsigned long revision; - - prom_argc = fw_arg0; - prom_argv = (char **)fw_arg1; - prom_envp = (char **)fw_arg2; - - /* - * Someday we can use this with PMON2000 to get a - * platform call prom routines for output etc. without - * having to use grody hacks. For now it's unused. - * - * struct callvectors *cv = (struct callvectors *) fw_arg3; - */ - family = identify_family(); - revision = identify_revision(); - - switch (family) { - case FAMILY_FPGA: - if (FPGA_IS_MSP4200(revision)) { - /* Old-style revision ID */ - mips_machtype = MACH_MSP4200_FPGA; - } else { - mips_machtype = MACH_MSP_OTHER; - } - break; - - case FAMILY_MSP4200: -#if defined(CONFIG_PMC_MSP4200_EVAL) - mips_machtype = MACH_MSP4200_EVAL; -#elif defined(CONFIG_PMC_MSP4200_GW) - mips_machtype = MACH_MSP4200_GW; -#else - mips_machtype = MACH_MSP_OTHER; -#endif - break; - - case FAMILY_MSP4200_FPGA: - mips_machtype = MACH_MSP4200_FPGA; - break; - - case FAMILY_MSP7100: -#if defined(CONFIG_PMC_MSP7120_EVAL) - mips_machtype = MACH_MSP7120_EVAL; -#elif defined(CONFIG_PMC_MSP7120_GW) - mips_machtype = MACH_MSP7120_GW; -#else - mips_machtype = MACH_MSP_OTHER; -#endif - break; - - case FAMILY_MSP7100_FPGA: - mips_machtype = MACH_MSP7120_FPGA; - break; - - default: - /* we don't recognize the machine */ - mips_machtype = MACH_UNKNOWN; - panic("***Bogosity factor five***, exiting"); - break; - } - - prom_init_cmdline(); - - prom_meminit(); - - /* - * Sub-system setup follows. - * Setup functions can either be called here or using the - * subsys_initcall mechanism (i.e. see msp_pci_setup). The - * order in which they are called can be changed by using the - * link order in arch/mips/pmc-sierra/msp71xx/Makefile. - * - * NOTE: Please keep sub-system specific initialization code - * in separate specific files. - */ - msp_serial_setup(); - - register_vsmp_smp_ops(); -} diff --git a/arch/mips/pmcs-msp71xx/msp_smp.c b/arch/mips/pmcs-msp71xx/msp_smp.c deleted file mode 100644 index 00092e2924ec..000000000000 --- a/arch/mips/pmcs-msp71xx/msp_smp.c +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2000, 2001, 2004 MIPS Technologies, Inc. - * Copyright (C) 2001 Ralf Baechle - * Copyright (C) 2010 PMC-Sierra, Inc. - * - * VSMP support for MSP platforms . Derived from malta vsmp support. - */ -#include -#include - -#include - -#ifdef CONFIG_MIPS_MT_SMP -#define MIPS_CPU_IPI_RESCHED_IRQ 0 /* SW int 0 for resched */ -#define MIPS_CPU_IPI_CALL_IRQ 1 /* SW int 1 for call */ - - -static void ipi_resched_dispatch(void) -{ - do_IRQ(MIPS_CPU_IPI_RESCHED_IRQ); -} - -static void ipi_call_dispatch(void) -{ - do_IRQ(MIPS_CPU_IPI_CALL_IRQ); -} - -static irqreturn_t ipi_resched_interrupt(int irq, void *dev_id) -{ - return IRQ_HANDLED; -} - -static irqreturn_t ipi_call_interrupt(int irq, void *dev_id) -{ - generic_smp_call_function_interrupt(); - - return IRQ_HANDLED; -} - -void __init arch_init_ipiirq(int irq, const char *name, irq_handler_t handler) -{ - if (request_irq(irq, handler, IRQF_PERCPU, name, NULL)) - pr_err("Failed to request irq %d (%s)\n", irq, name); - irq_set_handler(irq, handle_percpu_irq); -} - -void __init msp_vsmp_int_init(void) -{ - set_vi_handler(MIPS_CPU_IPI_RESCHED_IRQ, ipi_resched_dispatch); - set_vi_handler(MIPS_CPU_IPI_CALL_IRQ, ipi_call_dispatch); - arch_init_ipiirq(MIPS_CPU_IPI_RESCHED_IRQ, "IPI_resched", - ipi_resched_interrupt); - arch_init_ipiirq(MIPS_CPU_IPI_CALL_IRQ, "IPI_call", ipi_call_interrupt); -} -#endif /* CONFIG_MIPS_MT_SMP */ diff --git a/arch/mips/pmcs-msp71xx/msp_time.c b/arch/mips/pmcs-msp71xx/msp_time.c deleted file mode 100644 index 9c629829f447..000000000000 --- a/arch/mips/pmcs-msp71xx/msp_time.c +++ /dev/null @@ -1,90 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Setting up the clock on MSP SOCs. No RTC typically. - * - * Carsten Langgaard, carstenl@mips.com - * Copyright (C) 1999,2000 MIPS Technologies, Inc. All rights reserved. - * - * ######################################################################## - * - * ######################################################################## - */ - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -#define get_current_vpe() \ - ((read_c0_tcbind() >> TCBIND_CURVPE_SHIFT) & TCBIND_CURVPE) - -static int tim_installed; - -void __init plat_time_init(void) -{ - char *endp, *s; - unsigned long cpu_rate = 0; - - if (cpu_rate == 0) { - s = prom_getenv("clkfreqhz"); - cpu_rate = simple_strtoul(s, &endp, 10); - if (endp != NULL && *endp != 0) { - printk(KERN_ERR - "Clock rate in Hz parse error: %s\n", s); - cpu_rate = 0; - } - } - - if (cpu_rate == 0) { - s = prom_getenv("clkfreq"); - cpu_rate = 1000 * simple_strtoul(s, &endp, 10); - if (endp != NULL && *endp != 0) { - printk(KERN_ERR - "Clock rate in MHz parse error: %s\n", s); - cpu_rate = 0; - } - } - - if (cpu_rate == 0) { -#if defined(CONFIG_PMC_MSP7120_EVAL) \ - || defined(CONFIG_PMC_MSP7120_GW) - cpu_rate = 400000000; -#elif defined(CONFIG_PMC_MSP7120_FPGA) - cpu_rate = 25000000; -#else - cpu_rate = 150000000; -#endif - printk(KERN_ERR - "Failed to determine CPU clock rate, " - "assuming %ld hz ...\n", cpu_rate); - } - - printk(KERN_WARNING "Clock rate set to %ld\n", cpu_rate); - - /* timer frequency is 1/2 clock rate */ - mips_hpt_frequency = cpu_rate/2; -} - -unsigned int get_c0_compare_int(void) -{ - unsigned long flags = IRQF_PERCPU | IRQF_TIMER | IRQF_SHARED; - - /* MIPS_MT modes may want timer for second VPE */ - if ((get_current_vpe()) && !tim_installed) { - if (request_irq(MSP_INT_VPE1_TIMER, c0_compare_interrupt, flags, - "timer", c0_compare_interrupt)) - pr_err("Failed to register timer interrupt\n"); - tim_installed++; - } - - return get_current_vpe() ? MSP_INT_VPE1_TIMER : MSP_INT_VPE0_TIMER; -} diff --git a/arch/mips/pmcs-msp71xx/msp_usb.c b/arch/mips/pmcs-msp71xx/msp_usb.c deleted file mode 100644 index d38ac70b5a2e..000000000000 --- a/arch/mips/pmcs-msp71xx/msp_usb.c +++ /dev/null @@ -1,173 +0,0 @@ -/* - * The setup file for USB related hardware on PMC-Sierra MSP processors. - * - * Copyright 2006 PMC-Sierra, Inc. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 675 Mass Ave, Cambridge, MA 02139, USA. - */ -#if defined(CONFIG_USB_EHCI_HCD) || defined(CONFIG_USB_GADGET) - -#include -#include -#include - -#include - -#include -#include -#include -#include - - -#if defined(CONFIG_USB_EHCI_HCD) -static struct resource msp_usbhost0_resources[] = { - [0] = { /* EHCI-HS operational and capabilities registers */ - .start = MSP_USB0_HS_START, - .end = MSP_USB0_HS_END, - .flags = IORESOURCE_MEM, - }, - [1] = { - .start = MSP_INT_USB, - .end = MSP_INT_USB, - .flags = IORESOURCE_IRQ, - }, - [2] = { /* MSBus-to-AMBA bridge register space */ - .start = MSP_USB0_MAB_START, - .end = MSP_USB0_MAB_END, - .flags = IORESOURCE_MEM, - }, - [3] = { /* Identification and general hardware parameters */ - .start = MSP_USB0_ID_START, - .end = MSP_USB0_ID_END, - .flags = IORESOURCE_MEM, - }, -}; - -static u64 msp_usbhost0_dma_mask = 0xffffffffUL; - -static struct mspusb_device msp_usbhost0_device = { - .dev = { - .name = "pmcmsp-ehci", - .id = 0, - .dev = { - .dma_mask = &msp_usbhost0_dma_mask, - .coherent_dma_mask = 0xffffffffUL, - }, - .num_resources = ARRAY_SIZE(msp_usbhost0_resources), - .resource = msp_usbhost0_resources, - }, -}; -#endif /* CONFIG_USB_EHCI_HCD */ - -#if defined(CONFIG_USB_GADGET) -static struct resource msp_usbdev0_resources[] = { - [0] = { /* EHCI-HS operational and capabilities registers */ - .start = MSP_USB0_HS_START, - .end = MSP_USB0_HS_END, - .flags = IORESOURCE_MEM, - }, - [1] = { - .start = MSP_INT_USB, - .end = MSP_INT_USB, - .flags = IORESOURCE_IRQ, - }, - [2] = { /* MSBus-to-AMBA bridge register space */ - .start = MSP_USB0_MAB_START, - .end = MSP_USB0_MAB_END, - .flags = IORESOURCE_MEM, - }, - [3] = { /* Identification and general hardware parameters */ - .start = MSP_USB0_ID_START, - .end = MSP_USB0_ID_END, - .flags = IORESOURCE_MEM, - }, -}; - -static u64 msp_usbdev_dma_mask = 0xffffffffUL; - -/* This may need to be converted to a mspusb_device, too. */ -static struct mspusb_device msp_usbdev0_device = { - .dev = { - .name = "msp71xx_udc", - .id = 0, - .dev = { - .dma_mask = &msp_usbdev_dma_mask, - .coherent_dma_mask = 0xffffffffUL, - }, - .num_resources = ARRAY_SIZE(msp_usbdev0_resources), - .resource = msp_usbdev0_resources, - }, -}; -#endif /* CONFIG_USB_GADGET */ - -static int __init msp_usb_setup(void) -{ - char *strp; - char envstr[32]; - struct platform_device *msp_devs[NUM_USB_DEVS]; - unsigned int val; - - /* construct environment name usbmode */ - /* set usbmode as pmon environment var */ - /* - * Could this perhaps be integrated into the "features" env var? - * Use the features key "U", and follow with "H" for host-mode, - * "D" for device-mode. If it works for Ethernet, why not USB... - * -- hammtrev, 2007/03/22 - */ - snprintf(&envstr[0], sizeof(envstr), "usbmode"); - - /* set default host mode */ - val = 1; - - /* get environment string */ - strp = prom_getenv(&envstr[0]); - if (strp) { - /* compare string */ - if (!strcmp(strp, "device")) - val = 0; - } - - if (val) { -#if defined(CONFIG_USB_EHCI_HCD) - msp_devs[0] = &msp_usbhost0_device.dev; - ppfinit("platform add USB HOST done %s.\n", msp_devs[0]->name); -#else - ppfinit("%s: echi_hcd not supported\n", __FILE__); -#endif /* CONFIG_USB_EHCI_HCD */ - } else { -#if defined(CONFIG_USB_GADGET) - /* get device mode structure */ - msp_devs[0] = &msp_usbdev0_device.dev; - ppfinit("platform add USB DEVICE done %s.\n" - , msp_devs[0]->name); -#else - ppfinit("%s: usb_gadget not supported\n", __FILE__); -#endif /* CONFIG_USB_GADGET */ - } - /* add device */ - platform_add_devices(msp_devs, ARRAY_SIZE(msp_devs)); - - return 0; -} - -subsys_initcall(msp_usb_setup); -#endif /* CONFIG_USB_EHCI_HCD || CONFIG_USB_GADGET */ From 1ce4530cce233edd99d66a9fcdea20c86e8536c0 Mon Sep 17 00:00:00 2001 From: Thomas Bogendoerfer Date: Mon, 20 Apr 2020 14:37:24 +0200 Subject: [PATCH 0305/1043] MIPS: Remove NEC MARKEINS/EMMA No (active) developer owns this hardware, so let's remove Linux support. Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kbuild.platforms | 1 - arch/mips/Kconfig | 19 -- arch/mips/configs/markeins_defconfig | 185 -------------- arch/mips/emma/Makefile | 7 - arch/mips/emma/Platform | 4 - arch/mips/emma/common/Makefile | 6 - arch/mips/emma/common/prom.c | 56 ----- arch/mips/emma/markeins/Makefile | 6 - arch/mips/emma/markeins/irq.c | 293 ----------------------- arch/mips/emma/markeins/led.c | 44 ---- arch/mips/emma/markeins/platform.c | 199 --------------- arch/mips/emma/markeins/setup.c | 115 --------- arch/mips/include/asm/emma/emma2rh.h | 248 ------------------- arch/mips/include/asm/emma/markeins.h | 28 --- arch/mips/include/asm/mach-emma2rh/irq.h | 15 -- arch/mips/pci/Makefile | 1 - arch/mips/pci/fixup-emma2rh.c | 84 ------- arch/mips/pci/ops-emma2rh.c | 167 ------------- arch/mips/pci/pci-emma2rh.c | 72 ------ 19 files changed, 1550 deletions(-) delete mode 100644 arch/mips/configs/markeins_defconfig delete mode 100644 arch/mips/emma/Makefile delete mode 100644 arch/mips/emma/Platform delete mode 100644 arch/mips/emma/common/Makefile delete mode 100644 arch/mips/emma/common/prom.c delete mode 100644 arch/mips/emma/markeins/Makefile delete mode 100644 arch/mips/emma/markeins/irq.c delete mode 100644 arch/mips/emma/markeins/led.c delete mode 100644 arch/mips/emma/markeins/platform.c delete mode 100644 arch/mips/emma/markeins/setup.c delete mode 100644 arch/mips/include/asm/emma/emma2rh.h delete mode 100644 arch/mips/include/asm/emma/markeins.h delete mode 100644 arch/mips/include/asm/mach-emma2rh/irq.h delete mode 100644 arch/mips/pci/fixup-emma2rh.c delete mode 100644 arch/mips/pci/ops-emma2rh.c delete mode 100644 arch/mips/pci/pci-emma2rh.c diff --git a/arch/mips/Kbuild.platforms b/arch/mips/Kbuild.platforms index 65c119432e06..292b59afb4ba 100644 --- a/arch/mips/Kbuild.platforms +++ b/arch/mips/Kbuild.platforms @@ -11,7 +11,6 @@ platforms += bmips platforms += cavium-octeon platforms += cobalt platforms += dec -platforms += emma platforms += generic platforms += jazz platforms += jz4740 diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index f2565a88e086..ce1aacc2ee9c 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -580,13 +580,6 @@ config MACH_PIC32 Microchip PIC32 is a family of general-purpose 32 bit MIPS core microcontrollers. -config NEC_MARKEINS - bool "NEC EMMA2RH Mark-eins board" - select SOC_EMMA2RH - select HAVE_PCI - help - This enables support for the NEC Electronics Mark-eins boards. - config MACH_VR41XX bool "NEC VR4100 series based machines" select CEVT_R4K @@ -1292,18 +1285,6 @@ config PCI_XTALK_BRIDGE config NO_EXCEPT_FILL bool -config SOC_EMMA2RH - bool - select CEVT_R4K - select CSRC_R4K - select DMA_NONCOHERENT - select IRQ_MIPS_CPU - select SWAP_IO_SPACE - select SYS_HAS_CPU_R5500 - select SYS_SUPPORTS_32BIT_KERNEL - select SYS_SUPPORTS_64BIT_KERNEL - select SYS_SUPPORTS_BIG_ENDIAN - config SOC_PNX833X bool select CEVT_R4K diff --git a/arch/mips/configs/markeins_defconfig b/arch/mips/configs/markeins_defconfig deleted file mode 100644 index 507ad91b21e7..000000000000 --- a/arch/mips/configs/markeins_defconfig +++ /dev/null @@ -1,185 +0,0 @@ -CONFIG_SYSVIPC=y -CONFIG_POSIX_MQUEUE=y -CONFIG_PREEMPT=y -CONFIG_BSD_PROCESS_ACCT=y -CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y -CONFIG_LOG_BUF_SHIFT=14 -CONFIG_EXPERT=y -CONFIG_SLAB=y -CONFIG_NEC_MARKEINS=y -CONFIG_HZ_1000=y -CONFIG_PCI=y -CONFIG_MODULES=y -CONFIG_MODULE_UNLOAD=y -CONFIG_MODULE_FORCE_UNLOAD=y -CONFIG_MODVERSIONS=y -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_UNIX=y -CONFIG_NET_KEY=y -CONFIG_NET_KEY_MIGRATE=y -CONFIG_INET=y -CONFIG_IP_MULTICAST=y -CONFIG_IP_ADVANCED_ROUTER=y -CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IP_ROUTE_MULTIPATH=y -CONFIG_IP_ROUTE_VERBOSE=y -CONFIG_IP_PNP=y -CONFIG_IP_PNP_BOOTP=y -CONFIG_SYN_COOKIES=y -CONFIG_INET_XFRM_MODE_TRANSPORT=m -CONFIG_INET_XFRM_MODE_TUNNEL=m -CONFIG_INET_XFRM_MODE_BEET=m -CONFIG_TCP_MD5SIG=y -CONFIG_IPV6_MIP6=m -CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m -CONFIG_IPV6_MULTIPLE_TABLES=y -CONFIG_IPV6_SUBTREES=y -CONFIG_NETWORK_SECMARK=y -CONFIG_NETFILTER=y -CONFIG_NF_CONNTRACK=m -CONFIG_NF_CONNTRACK_SECMARK=y -CONFIG_NF_CONNTRACK_EVENTS=y -CONFIG_NF_CONNTRACK_AMANDA=m -CONFIG_NF_CONNTRACK_FTP=m -CONFIG_NF_CONNTRACK_H323=m -CONFIG_NF_CONNTRACK_IRC=m -CONFIG_NF_CONNTRACK_PPTP=m -CONFIG_NF_CONNTRACK_SANE=m -CONFIG_NF_CONNTRACK_SIP=m -CONFIG_NF_CONNTRACK_TFTP=m -CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m -CONFIG_NETFILTER_XT_TARGET_CONNMARK=m -CONFIG_NETFILTER_XT_TARGET_CONNSECMARK=m -CONFIG_NETFILTER_XT_TARGET_DSCP=m -CONFIG_NETFILTER_XT_TARGET_MARK=m -CONFIG_NETFILTER_XT_TARGET_NFLOG=m -CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m -CONFIG_NETFILTER_XT_TARGET_SECMARK=m -CONFIG_NETFILTER_XT_TARGET_TCPMSS=m -CONFIG_NETFILTER_XT_MATCH_COMMENT=m -CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m -CONFIG_NETFILTER_XT_MATCH_CONNMARK=m -CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m -CONFIG_NETFILTER_XT_MATCH_DCCP=m -CONFIG_NETFILTER_XT_MATCH_DSCP=m -CONFIG_NETFILTER_XT_MATCH_ESP=m -CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m -CONFIG_NETFILTER_XT_MATCH_HELPER=m -CONFIG_NETFILTER_XT_MATCH_LENGTH=m -CONFIG_NETFILTER_XT_MATCH_LIMIT=m -CONFIG_NETFILTER_XT_MATCH_MAC=m -CONFIG_NETFILTER_XT_MATCH_MARK=m -CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m -CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m -CONFIG_NETFILTER_XT_MATCH_QUOTA=m -CONFIG_NETFILTER_XT_MATCH_REALM=m -CONFIG_NETFILTER_XT_MATCH_STATE=m -CONFIG_NETFILTER_XT_MATCH_STATISTIC=m -CONFIG_NETFILTER_XT_MATCH_STRING=m -CONFIG_NETFILTER_XT_MATCH_TCPMSS=m -CONFIG_IP_NF_IPTABLES=m -CONFIG_IP_NF_MATCH_AH=m -CONFIG_IP_NF_MATCH_ECN=m -CONFIG_IP_NF_MATCH_TTL=m -CONFIG_IP_NF_FILTER=m -CONFIG_IP_NF_TARGET_REJECT=m -CONFIG_IP_NF_MANGLE=m -CONFIG_IP_NF_TARGET_CLUSTERIP=m -CONFIG_IP_NF_TARGET_ECN=m -CONFIG_IP_NF_TARGET_TTL=m -CONFIG_IP_NF_RAW=m -CONFIG_IP_NF_ARPTABLES=m -CONFIG_IP_NF_ARPFILTER=m -CONFIG_IP_NF_ARP_MANGLE=m -CONFIG_IP6_NF_IPTABLES=m -CONFIG_IP6_NF_MATCH_AH=m -CONFIG_IP6_NF_MATCH_EUI64=m -CONFIG_IP6_NF_MATCH_FRAG=m -CONFIG_IP6_NF_MATCH_OPTS=m -CONFIG_IP6_NF_MATCH_HL=m -CONFIG_IP6_NF_MATCH_IPV6HEADER=m -CONFIG_IP6_NF_MATCH_MH=m -CONFIG_IP6_NF_MATCH_RT=m -CONFIG_IP6_NF_TARGET_HL=m -CONFIG_IP6_NF_FILTER=m -CONFIG_IP6_NF_TARGET_REJECT=m -CONFIG_IP6_NF_MANGLE=m -CONFIG_IP6_NF_RAW=m -CONFIG_FW_LOADER=m -CONFIG_MTD=y -CONFIG_MTD_CMDLINE_PARTS=y -CONFIG_MTD_BLOCK=y -CONFIG_MTD_CFI=y -CONFIG_MTD_CFI_AMDSTD=y -CONFIG_MTD_PHYSMAP=y -CONFIG_BLK_DEV_LOOP=m -CONFIG_BLK_DEV_CRYPTOLOOP=m -CONFIG_SCSI=m -# CONFIG_SCSI_PROC_FS is not set -CONFIG_BLK_DEV_SD=m -CONFIG_CHR_DEV_SG=m -CONFIG_SCSI_SCAN_ASYNC=y -CONFIG_SCSI_AIC94XX=m -# CONFIG_AIC94XX_DEBUG is not set -CONFIG_NETDEVICES=y -CONFIG_TUN=m -CONFIG_CHELSIO_T3=m -CONFIG_NATSEMI=y -CONFIG_QLA3XXX=m -CONFIG_NETXEN_NIC=m -CONFIG_PPP=m -CONFIG_PPP_DEFLATE=m -CONFIG_PPP_ASYNC=m -CONFIG_PPP_SYNC_TTY=m -CONFIG_INPUT_EVDEV=m -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set -# CONFIG_SERIO is not set -# CONFIG_VT is not set -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -# CONFIG_HW_RANDOM is not set -CONFIG_I2C=y -CONFIG_I2C_CHARDEV=y -CONFIG_I2C_DEBUG_CORE=y -CONFIG_I2C_DEBUG_BUS=y -# CONFIG_HID is not set -CONFIG_EXT2_FS=y -CONFIG_EXT2_FS_XATTR=y -CONFIG_EXT2_FS_POSIX_ACL=y -CONFIG_EXT2_FS_SECURITY=y -CONFIG_EXT3_FS=m -CONFIG_XFS_FS=m -# CONFIG_DNOTIFY is not set -CONFIG_AUTOFS4_FS=m -CONFIG_MSDOS_FS=y -CONFIG_VFAT_FS=y -CONFIG_NTFS_FS=m -CONFIG_PROC_KCORE=y -CONFIG_TMPFS=y -CONFIG_TMPFS_POSIX_ACL=y -CONFIG_JFFS2_FS=y -CONFIG_JFFS2_COMPRESSION_OPTIONS=y -CONFIG_CRAMFS=y -CONFIG_NFS_FS=y -CONFIG_NFS_V4=y -CONFIG_ROOT_NFS=y -CONFIG_NFSD=m -CONFIG_NFSD_V3=y -CONFIG_NLS_DEFAULT="" -CONFIG_NLS_CODEPAGE_437=m -CONFIG_NLS_ASCII=m -CONFIG_NLS_ISO8859_1=m -CONFIG_NLS_UTF8=m -CONFIG_CRYPTO_ECB=m -CONFIG_CRYPTO_LRW=m -CONFIG_CRYPTO_PCBC=m -CONFIG_CRYPTO_HMAC=y -CONFIG_CRYPTO_XCBC=m -CONFIG_CRYPTO_CAMELLIA=m -CONFIG_CRYPTO_FCRYPT=m -CONFIG_CMDLINE_BOOL=y -CONFIG_CMDLINE="console=ttyS0,115200 mem=192m ip=bootp root=/dev/nfs rw" diff --git a/arch/mips/emma/Makefile b/arch/mips/emma/Makefile deleted file mode 100644 index bc03082064ca..000000000000 --- a/arch/mips/emma/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_SOC_EMMA2RH) += common/ - -# -# NEC EMMA2RH Mark-eins -# -obj-$(CONFIG_NEC_MARKEINS) += markeins/ diff --git a/arch/mips/emma/Platform b/arch/mips/emma/Platform deleted file mode 100644 index 0282f7f99b88..000000000000 --- a/arch/mips/emma/Platform +++ /dev/null @@ -1,4 +0,0 @@ -platform-$(CONFIG_SOC_EMMA2RH) += emma/ -cflags-$(CONFIG_SOC_EMMA2RH) += \ - -I$(srctree)/arch/mips/include/asm/mach-emma2rh -load-$(CONFIG_NEC_MARKEINS) += 0xffffffff88100000 diff --git a/arch/mips/emma/common/Makefile b/arch/mips/emma/common/Makefile deleted file mode 100644 index a754abd1beb9..000000000000 --- a/arch/mips/emma/common/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-or-later -# -# Copyright (C) NEC Electronics Corporation 2005-2006 -# - -obj-$(CONFIG_NEC_MARKEINS) += prom.o diff --git a/arch/mips/emma/common/prom.c b/arch/mips/emma/common/prom.c deleted file mode 100644 index 7c3a6f32beda..000000000000 --- a/arch/mips/emma/common/prom.c +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) NEC Electronics Corporation 2004-2006 - * - * This file is based on the arch/mips/ddb5xxx/common/prom.c - * - * Copyright 2001 MontaVista Software Inc. - */ -#include -#include -#include -#include - -#include -#include -#include - -const char *get_system_type(void) -{ -#ifdef CONFIG_NEC_MARKEINS - return "NEC EMMA2RH Mark-eins"; -#else -#error Unknown NEC board -#endif -} - -/* [jsun@junsun.net] PMON passes arguments in C main() style */ -void __init prom_init(void) -{ - int argc = fw_arg0; - char **arg = (char **)fw_arg1; - int i; - - /* if user passes kernel args, ignore the default one */ - if (argc > 1) - arcs_cmdline[0] = '\0'; - - /* arg[0] is "g", the rest is boot parameters */ - for (i = 1; i < argc; i++) { - if (strlen(arcs_cmdline) + strlen(arg[i]) + 1 - >= sizeof(arcs_cmdline)) - break; - strcat(arcs_cmdline, arg[i]); - strcat(arcs_cmdline, " "); - } - -#ifdef CONFIG_NEC_MARKEINS - add_memory_region(0, EMMA2RH_RAM_SIZE, BOOT_MEM_RAM); -#else -#error Unknown NEC board -#endif -} - -void __init prom_free_prom_memory(void) -{ -} diff --git a/arch/mips/emma/markeins/Makefile b/arch/mips/emma/markeins/Makefile deleted file mode 100644 index 8c8649069504..000000000000 --- a/arch/mips/emma/markeins/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-or-later -# -# Copyright (C) NEC Electronics Corporation 2005-2006 -# - -obj-$(CONFIG_NEC_MARKEINS) += irq.o setup.o led.o platform.o diff --git a/arch/mips/emma/markeins/irq.c b/arch/mips/emma/markeins/irq.c deleted file mode 100644 index 4aebf559be2e..000000000000 --- a/arch/mips/emma/markeins/irq.c +++ /dev/null @@ -1,293 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) NEC Electronics Corporation 2004-2006 - * - * This file is based on the arch/mips/ddb5xxx/ddb5477/irq.c - * - * Copyright 2001 MontaVista Software Inc. - */ -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include - -static void emma2rh_irq_enable(struct irq_data *d) -{ - unsigned int irq = d->irq - EMMA2RH_IRQ_BASE; - u32 reg_value, reg_bitmask, reg_index; - - reg_index = EMMA2RH_BHIF_INT_EN_0 + - (EMMA2RH_BHIF_INT_EN_1 - EMMA2RH_BHIF_INT_EN_0) * (irq / 32); - reg_value = emma2rh_in32(reg_index); - reg_bitmask = 0x1 << (irq % 32); - emma2rh_out32(reg_index, reg_value | reg_bitmask); -} - -static void emma2rh_irq_disable(struct irq_data *d) -{ - unsigned int irq = d->irq - EMMA2RH_IRQ_BASE; - u32 reg_value, reg_bitmask, reg_index; - - reg_index = EMMA2RH_BHIF_INT_EN_0 + - (EMMA2RH_BHIF_INT_EN_1 - EMMA2RH_BHIF_INT_EN_0) * (irq / 32); - reg_value = emma2rh_in32(reg_index); - reg_bitmask = 0x1 << (irq % 32); - emma2rh_out32(reg_index, reg_value & ~reg_bitmask); -} - -struct irq_chip emma2rh_irq_controller = { - .name = "emma2rh_irq", - .irq_mask = emma2rh_irq_disable, - .irq_unmask = emma2rh_irq_enable, -}; - -void emma2rh_irq_init(void) -{ - u32 i; - - for (i = 0; i < NUM_EMMA2RH_IRQ; i++) - irq_set_chip_and_handler_name(EMMA2RH_IRQ_BASE + i, - &emma2rh_irq_controller, - handle_level_irq, "level"); -} - -static void emma2rh_sw_irq_enable(struct irq_data *d) -{ - unsigned int irq = d->irq - EMMA2RH_SW_IRQ_BASE; - u32 reg; - - reg = emma2rh_in32(EMMA2RH_BHIF_SW_INT_EN); - reg |= 1 << irq; - emma2rh_out32(EMMA2RH_BHIF_SW_INT_EN, reg); -} - -static void emma2rh_sw_irq_disable(struct irq_data *d) -{ - unsigned int irq = d->irq - EMMA2RH_SW_IRQ_BASE; - u32 reg; - - reg = emma2rh_in32(EMMA2RH_BHIF_SW_INT_EN); - reg &= ~(1 << irq); - emma2rh_out32(EMMA2RH_BHIF_SW_INT_EN, reg); -} - -struct irq_chip emma2rh_sw_irq_controller = { - .name = "emma2rh_sw_irq", - .irq_mask = emma2rh_sw_irq_disable, - .irq_unmask = emma2rh_sw_irq_enable, -}; - -void emma2rh_sw_irq_init(void) -{ - u32 i; - - for (i = 0; i < NUM_EMMA2RH_IRQ_SW; i++) - irq_set_chip_and_handler_name(EMMA2RH_SW_IRQ_BASE + i, - &emma2rh_sw_irq_controller, - handle_level_irq, "level"); -} - -static void emma2rh_gpio_irq_enable(struct irq_data *d) -{ - unsigned int irq = d->irq - EMMA2RH_GPIO_IRQ_BASE; - u32 reg; - - reg = emma2rh_in32(EMMA2RH_GPIO_INT_MASK); - reg |= 1 << irq; - emma2rh_out32(EMMA2RH_GPIO_INT_MASK, reg); -} - -static void emma2rh_gpio_irq_disable(struct irq_data *d) -{ - unsigned int irq = d->irq - EMMA2RH_GPIO_IRQ_BASE; - u32 reg; - - reg = emma2rh_in32(EMMA2RH_GPIO_INT_MASK); - reg &= ~(1 << irq); - emma2rh_out32(EMMA2RH_GPIO_INT_MASK, reg); -} - -static void emma2rh_gpio_irq_ack(struct irq_data *d) -{ - unsigned int irq = d->irq - EMMA2RH_GPIO_IRQ_BASE; - - emma2rh_out32(EMMA2RH_GPIO_INT_ST, ~(1 << irq)); -} - -static void emma2rh_gpio_irq_mask_ack(struct irq_data *d) -{ - unsigned int irq = d->irq - EMMA2RH_GPIO_IRQ_BASE; - u32 reg; - - emma2rh_out32(EMMA2RH_GPIO_INT_ST, ~(1 << irq)); - - reg = emma2rh_in32(EMMA2RH_GPIO_INT_MASK); - reg &= ~(1 << irq); - emma2rh_out32(EMMA2RH_GPIO_INT_MASK, reg); -} - -struct irq_chip emma2rh_gpio_irq_controller = { - .name = "emma2rh_gpio_irq", - .irq_ack = emma2rh_gpio_irq_ack, - .irq_mask = emma2rh_gpio_irq_disable, - .irq_mask_ack = emma2rh_gpio_irq_mask_ack, - .irq_unmask = emma2rh_gpio_irq_enable, -}; - -void emma2rh_gpio_irq_init(void) -{ - u32 i; - - for (i = 0; i < NUM_EMMA2RH_IRQ_GPIO; i++) - irq_set_chip_and_handler_name(EMMA2RH_GPIO_IRQ_BASE + i, - &emma2rh_gpio_irq_controller, - handle_edge_irq, "edge"); -} - -/* - * the first level int-handler will jump here if it is a emma2rh irq - */ -void emma2rh_irq_dispatch(void) -{ - u32 intStatus; - u32 bitmask; - u32 i; - - intStatus = emma2rh_in32(EMMA2RH_BHIF_INT_ST_0) & - emma2rh_in32(EMMA2RH_BHIF_INT_EN_0); - -#ifdef EMMA2RH_SW_CASCADE - if (intStatus & (1UL << EMMA2RH_SW_CASCADE)) { - u32 swIntStatus; - swIntStatus = emma2rh_in32(EMMA2RH_BHIF_SW_INT) - & emma2rh_in32(EMMA2RH_BHIF_SW_INT_EN); - for (i = 0, bitmask = 1; i < 32; i++, bitmask <<= 1) { - if (swIntStatus & bitmask) { - do_IRQ(EMMA2RH_SW_IRQ_BASE + i); - return; - } - } - } - /* Skip S/W interrupt */ - intStatus &= ~(1UL << EMMA2RH_SW_CASCADE); -#endif - - for (i = 0, bitmask = 1; i < 32; i++, bitmask <<= 1) { - if (intStatus & bitmask) { - do_IRQ(EMMA2RH_IRQ_BASE + i); - return; - } - } - - intStatus = emma2rh_in32(EMMA2RH_BHIF_INT_ST_1) & - emma2rh_in32(EMMA2RH_BHIF_INT_EN_1); - -#ifdef EMMA2RH_GPIO_CASCADE - if (intStatus & (1UL << (EMMA2RH_GPIO_CASCADE % 32))) { - u32 gpioIntStatus; - gpioIntStatus = emma2rh_in32(EMMA2RH_GPIO_INT_ST) - & emma2rh_in32(EMMA2RH_GPIO_INT_MASK); - for (i = 0, bitmask = 1; i < 32; i++, bitmask <<= 1) { - if (gpioIntStatus & bitmask) { - do_IRQ(EMMA2RH_GPIO_IRQ_BASE + i); - return; - } - } - } - /* Skip GPIO interrupt */ - intStatus &= ~(1UL << (EMMA2RH_GPIO_CASCADE % 32)); -#endif - - for (i = 32, bitmask = 1; i < 64; i++, bitmask <<= 1) { - if (intStatus & bitmask) { - do_IRQ(EMMA2RH_IRQ_BASE + i); - return; - } - } - - intStatus = emma2rh_in32(EMMA2RH_BHIF_INT_ST_2) & - emma2rh_in32(EMMA2RH_BHIF_INT_EN_2); - - for (i = 64, bitmask = 1; i < 96; i++, bitmask <<= 1) { - if (intStatus & bitmask) { - do_IRQ(EMMA2RH_IRQ_BASE + i); - return; - } - } -} - -void __init arch_init_irq(void) -{ - u32 reg; - int irq; - - /* by default, interrupts are disabled. */ - emma2rh_out32(EMMA2RH_BHIF_INT_EN_0, 0); - emma2rh_out32(EMMA2RH_BHIF_INT_EN_1, 0); - emma2rh_out32(EMMA2RH_BHIF_INT_EN_2, 0); - emma2rh_out32(EMMA2RH_BHIF_INT1_EN_0, 0); - emma2rh_out32(EMMA2RH_BHIF_INT1_EN_1, 0); - emma2rh_out32(EMMA2RH_BHIF_INT1_EN_2, 0); - emma2rh_out32(EMMA2RH_BHIF_SW_INT_EN, 0); - - clear_c0_status(0xff00); - set_c0_status(0x0400); - -#define GPIO_PCI (0xf<<15) - /* setup GPIO interrupt for PCI interface */ - /* direction input */ - reg = emma2rh_in32(EMMA2RH_GPIO_DIR); - emma2rh_out32(EMMA2RH_GPIO_DIR, reg & ~GPIO_PCI); - /* disable interrupt */ - reg = emma2rh_in32(EMMA2RH_GPIO_INT_MASK); - emma2rh_out32(EMMA2RH_GPIO_INT_MASK, reg & ~GPIO_PCI); - /* level triggerd */ - reg = emma2rh_in32(EMMA2RH_GPIO_INT_MODE); - emma2rh_out32(EMMA2RH_GPIO_INT_MODE, reg | GPIO_PCI); - reg = emma2rh_in32(EMMA2RH_GPIO_INT_CND_A); - emma2rh_out32(EMMA2RH_GPIO_INT_CND_A, reg & (~GPIO_PCI)); - /* interrupt clear */ - emma2rh_out32(EMMA2RH_GPIO_INT_ST, ~GPIO_PCI); - - /* init all controllers */ - emma2rh_irq_init(); - emma2rh_sw_irq_init(); - emma2rh_gpio_irq_init(); - mips_cpu_irq_init(); - - /* setup cascade interrupts */ - irq = EMMA2RH_IRQ_BASE + EMMA2RH_SW_CASCADE; - if (request_irq(irq, no_action, IRQF_NO_THREAD, "cascade", NULL)) - pr_err("Failed to request irq %d (cascade)\n", irq); - irq = EMMA2RH_IRQ_BASE + EMMA2RH_GPIO_CASCADE; - if (request_irq(irq, no_action, IRQF_NO_THREAD, "cascade", NULL)) - pr_err("Failed to request irq %d (cascade)\n", irq); - irq = MIPS_CPU_IRQ_BASE + 2; - if (request_irq(irq, no_action, IRQF_NO_THREAD, "cascade", NULL)) - pr_err("Failed to request irq %d (cascade)\n", irq); -} - -asmlinkage void plat_irq_dispatch(void) -{ - unsigned int pending = read_c0_status() & read_c0_cause() & ST0_IM; - - if (pending & STATUSF_IP7) - do_IRQ(MIPS_CPU_IRQ_BASE + 7); - else if (pending & STATUSF_IP2) - emma2rh_irq_dispatch(); - else if (pending & STATUSF_IP1) - do_IRQ(MIPS_CPU_IRQ_BASE + 1); - else if (pending & STATUSF_IP0) - do_IRQ(MIPS_CPU_IRQ_BASE + 0); - else - spurious_interrupt(); -} diff --git a/arch/mips/emma/markeins/led.c b/arch/mips/emma/markeins/led.c deleted file mode 100644 index d377542c0ec4..000000000000 --- a/arch/mips/emma/markeins/led.c +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) NEC Electronics Corporation 2004-2006 - */ -#include -#include -#include -#include - -const unsigned long clear = 0x20202020; - -#define LED_BASE 0xb1400038 - -void markeins_led_clear(void) -{ - emma2rh_out32(LED_BASE, clear); - emma2rh_out32(LED_BASE + 4, clear); -} - -void markeins_led(const char *str) -{ - int i; - int len = strlen(str); - - markeins_led_clear(); - if (len > 8) - len = 8; - - if (emma2rh_in32(0xb0000800) & (0x1 << 18)) - for (i = 0; i < len; i++) - emma2rh_out8(LED_BASE + i, str[i]); - else - for (i = 0; i < len; i++) - emma2rh_out8(LED_BASE + (i & 4) + (3 - (i & 3)), - str[i]); -} - -void markeins_led_hex(u32 val) -{ - char str[10]; - - sprintf(str, "%08x", val); - markeins_led(str); -} diff --git a/arch/mips/emma/markeins/platform.c b/arch/mips/emma/markeins/platform.c deleted file mode 100644 index 97eeb9e8fb2b..000000000000 --- a/arch/mips/emma/markeins/platform.c +++ /dev/null @@ -1,199 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright(C) MontaVista Software Inc, 2006 - * - * Author: dmitry pervushin - */ -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - - -#define I2C_EMMA2RH "emma2rh-iic" /* must be in sync with IIC driver */ - -static struct resource i2c_emma_resources_0[] = { - { - .name = NULL, - .start = EMMA2RH_IRQ_PIIC0, - .end = EMMA2RH_IRQ_PIIC0, - .flags = IORESOURCE_IRQ - }, { - .name = NULL, - .start = EMMA2RH_PIIC0_BASE, - .end = EMMA2RH_PIIC0_BASE + 0x1000, - .flags = 0 - }, -}; - -struct resource i2c_emma_resources_1[] = { - { - .name = NULL, - .start = EMMA2RH_IRQ_PIIC1, - .end = EMMA2RH_IRQ_PIIC1, - .flags = IORESOURCE_IRQ - }, { - .name = NULL, - .start = EMMA2RH_PIIC1_BASE, - .end = EMMA2RH_PIIC1_BASE + 0x1000, - .flags = 0 - }, -}; - -struct resource i2c_emma_resources_2[] = { - { - .name = NULL, - .start = EMMA2RH_IRQ_PIIC2, - .end = EMMA2RH_IRQ_PIIC2, - .flags = IORESOURCE_IRQ - }, { - .name = NULL, - .start = EMMA2RH_PIIC2_BASE, - .end = EMMA2RH_PIIC2_BASE + 0x1000, - .flags = 0 - }, -}; - -struct platform_device i2c_emma_devices[] = { - [0] = { - .name = I2C_EMMA2RH, - .id = 0, - .resource = i2c_emma_resources_0, - .num_resources = ARRAY_SIZE(i2c_emma_resources_0), - }, - [1] = { - .name = I2C_EMMA2RH, - .id = 1, - .resource = i2c_emma_resources_1, - .num_resources = ARRAY_SIZE(i2c_emma_resources_1), - }, - [2] = { - .name = I2C_EMMA2RH, - .id = 2, - .resource = i2c_emma_resources_2, - .num_resources = ARRAY_SIZE(i2c_emma_resources_2), - }, -}; - -#define EMMA2RH_SERIAL_CLOCK 18544000 -#define EMMA2RH_SERIAL_FLAGS UPF_BOOT_AUTOCONF | UPF_SKIP_TEST - -static struct plat_serial8250_port platform_serial_ports[] = { - [0] = { - .membase= (void __iomem*)KSEG1ADDR(EMMA2RH_PFUR0_BASE + 3), - .mapbase = EMMA2RH_PFUR0_BASE + 3, - .irq = EMMA2RH_IRQ_PFUR0, - .uartclk = EMMA2RH_SERIAL_CLOCK, - .regshift = 4, - .iotype = UPIO_MEM, - .flags = EMMA2RH_SERIAL_FLAGS, - }, [1] = { - .membase = (void __iomem*)KSEG1ADDR(EMMA2RH_PFUR1_BASE + 3), - .mapbase = EMMA2RH_PFUR1_BASE + 3, - .irq = EMMA2RH_IRQ_PFUR1, - .uartclk = EMMA2RH_SERIAL_CLOCK, - .regshift = 4, - .iotype = UPIO_MEM, - .flags = EMMA2RH_SERIAL_FLAGS, - }, [2] = { - .membase = (void __iomem*)KSEG1ADDR(EMMA2RH_PFUR2_BASE + 3), - .mapbase = EMMA2RH_PFUR2_BASE + 3, - .irq = EMMA2RH_IRQ_PFUR2, - .uartclk = EMMA2RH_SERIAL_CLOCK, - .regshift = 4, - .iotype = UPIO_MEM, - .flags = EMMA2RH_SERIAL_FLAGS, - }, [3] = { - .flags = 0, - }, -}; - -static struct platform_device serial_emma = { - .name = "serial8250", - .dev = { - .platform_data = &platform_serial_ports, - }, -}; - -static struct mtd_partition markeins_parts[] = { - [0] = { - .name = "RootFS", - .offset = 0x00000000, - .size = 0x00c00000, - }, - [1] = { - .name = "boot code area", - .offset = MTDPART_OFS_APPEND, - .size = 0x00100000, - }, - [2] = { - .name = "kernel image", - .offset = MTDPART_OFS_APPEND, - .size = 0x00300000, - }, - [3] = { - .name = "RootFS2", - .offset = MTDPART_OFS_APPEND, - .size = 0x00c00000, - }, - [4] = { - .name = "boot code area2", - .offset = MTDPART_OFS_APPEND, - .size = 0x00100000, - }, - [5] = { - .name = "kernel image2", - .offset = MTDPART_OFS_APPEND, - .size = MTDPART_SIZ_FULL, - }, -}; - -static struct physmap_flash_data markeins_flash_data = { - .width = 2, - .nr_parts = ARRAY_SIZE(markeins_parts), - .parts = markeins_parts -}; - -static struct resource markeins_flash_resource = { - .start = 0x1e000000, - .end = 0x02000000, - .flags = IORESOURCE_MEM -}; - -static struct platform_device markeins_flash_device = { - .name = "physmap-flash", - .id = 0, - .dev = { - .platform_data = &markeins_flash_data, - }, - .num_resources = 1, - .resource = &markeins_flash_resource, -}; - -static struct platform_device *devices[] = { - i2c_emma_devices, - i2c_emma_devices + 1, - i2c_emma_devices + 2, - &serial_emma, - &markeins_flash_device, -}; - -static int __init platform_devices_setup(void) -{ - return platform_add_devices(devices, ARRAY_SIZE(devices)); -} - -arch_initcall(platform_devices_setup); diff --git a/arch/mips/emma/markeins/setup.c b/arch/mips/emma/markeins/setup.c deleted file mode 100644 index c8a91c2a63bc..000000000000 --- a/arch/mips/emma/markeins/setup.c +++ /dev/null @@ -1,115 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) NEC Electronics Corporation 2004-2006 - * - * This file is based on the arch/mips/ddb5xxx/ddb5477/setup.c. - * - * Copyright 2001 MontaVista Software Inc. - */ -#include -#include -#include - -#include -#include - -#include - -#define USE_CPU_COUNTER_TIMER /* whether we use cpu counter */ - -extern void markeins_led(const char *); - -static int bus_frequency; - -static void markeins_machine_restart(char *command) -{ - static void (*back_to_prom) (void) = (void (*)(void))0xbfc00000; - - printk("cannot EMMA2RH Mark-eins restart.\n"); - markeins_led("restart."); - back_to_prom(); -} - -static void markeins_machine_halt(void) -{ - printk("EMMA2RH Mark-eins halted.\n"); - markeins_led("halted."); - while (1) ; -} - -static void markeins_machine_power_off(void) -{ - markeins_led("poweroff."); - while (1) ; -} - -static unsigned long __initdata emma2rh_clock[4] = { - 166500000, 187312500, 199800000, 210600000 -}; - -static unsigned int __init detect_bus_frequency(unsigned long rtc_base) -{ - u32 reg; - - /* detect from boot strap */ - reg = emma2rh_in32(EMMA2RH_BHIF_STRAP_0); - reg = (reg >> 4) & 0x3; - - return emma2rh_clock[reg]; -} - -void __init plat_time_init(void) -{ - u32 reg; - if (bus_frequency == 0) - bus_frequency = detect_bus_frequency(0); - - reg = emma2rh_in32(EMMA2RH_BHIF_STRAP_0); - if ((reg & 0x3) == 0) - reg = (reg >> 6) & 0x3; - else { - reg = emma2rh_in32(EMMA2RH_BHIF_MAIN_CTRL); - reg = (reg >> 4) & 0x3; - } - mips_hpt_frequency = (bus_frequency * (4 + reg)) / 4 / 2; -} - -static void markeins_board_init(void); -extern void markeins_irq_setup(void); - -static inline void __init markeins_sio_setup(void) -{ -} - -void __init plat_mem_setup(void) -{ - /* initialize board - we don't trust the loader */ - markeins_board_init(); - - set_io_port_base(KSEG1ADDR(EMMA2RH_PCI_IO_BASE)); - - _machine_restart = markeins_machine_restart; - _machine_halt = markeins_machine_halt; - pm_power_off = markeins_machine_power_off; - - /* setup resource limits */ - ioport_resource.start = EMMA2RH_PCI_IO_BASE; - ioport_resource.end = EMMA2RH_PCI_IO_BASE + EMMA2RH_PCI_IO_SIZE - 1; - iomem_resource.start = EMMA2RH_IO_BASE; - iomem_resource.end = EMMA2RH_ROM_BASE - 1; - - markeins_sio_setup(); -} - -static void __init markeins_board_init(void) -{ - u32 val; - - val = emma2rh_in32(EMMA2RH_PBRD_INT_EN); /* open serial interrupts. */ - emma2rh_out32(EMMA2RH_PBRD_INT_EN, val | 0xaa); - val = emma2rh_in32(EMMA2RH_PBRD_CLKSEL); /* set serial clocks. */ - emma2rh_out32(EMMA2RH_PBRD_CLKSEL, val | 0x5); /* 18MHz */ - emma2rh_out32(EMMA2RH_PCI_CONTROL, 0); - - markeins_led("MVL E2RH"); -} diff --git a/arch/mips/include/asm/emma/emma2rh.h b/arch/mips/include/asm/emma/emma2rh.h deleted file mode 100644 index a25cdb378fe8..000000000000 --- a/arch/mips/include/asm/emma/emma2rh.h +++ /dev/null @@ -1,248 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) NEC Electronics Corporation 2005-2006 - * - * This file based on include/asm-mips/ddb5xxx/ddb5xxx.h - * Copyright 2001 MontaVista Software Inc. - */ -#ifndef __ASM_EMMA_EMMA2RH_H -#define __ASM_EMMA_EMMA2RH_H - -#include - -/* - * EMMA2RH registers - */ -#define REGBASE 0x10000000 - -#define EMMA2RH_BHIF_STRAP_0 (0x000010+REGBASE) -#define EMMA2RH_BHIF_INT_ST_0 (0x000030+REGBASE) -#define EMMA2RH_BHIF_INT_ST_1 (0x000034+REGBASE) -#define EMMA2RH_BHIF_INT_ST_2 (0x000038+REGBASE) -#define EMMA2RH_BHIF_INT_EN_0 (0x000040+REGBASE) -#define EMMA2RH_BHIF_INT_EN_1 (0x000044+REGBASE) -#define EMMA2RH_BHIF_INT_EN_2 (0x000048+REGBASE) -#define EMMA2RH_BHIF_INT1_EN_0 (0x000050+REGBASE) -#define EMMA2RH_BHIF_INT1_EN_1 (0x000054+REGBASE) -#define EMMA2RH_BHIF_INT1_EN_2 (0x000058+REGBASE) -#define EMMA2RH_BHIF_SW_INT (0x000070+REGBASE) -#define EMMA2RH_BHIF_SW_INT_EN (0x000080+REGBASE) -#define EMMA2RH_BHIF_SW_INT_CLR (0x000090+REGBASE) -#define EMMA2RH_BHIF_MAIN_CTRL (0x0000b4+REGBASE) -#define EMMA2RH_BHIF_EXCEPT_VECT_BASE_ADDRESS (0x0000c0+REGBASE) -#define EMMA2RH_GPIO_DIR (0x110d20+REGBASE) -#define EMMA2RH_GPIO_INT_ST (0x110d30+REGBASE) -#define EMMA2RH_GPIO_INT_MASK (0x110d3c+REGBASE) -#define EMMA2RH_GPIO_INT_MODE (0x110d48+REGBASE) -#define EMMA2RH_GPIO_INT_CND_A (0x110d54+REGBASE) -#define EMMA2RH_GPIO_INT_CND_B (0x110d60+REGBASE) -#define EMMA2RH_PBRD_INT_EN (0x100010+REGBASE) -#define EMMA2RH_PBRD_CLKSEL (0x100028+REGBASE) -#define EMMA2RH_PFUR0_BASE (0x101000+REGBASE) -#define EMMA2RH_PFUR1_BASE (0x102000+REGBASE) -#define EMMA2RH_PFUR2_BASE (0x103000+REGBASE) -#define EMMA2RH_PIIC0_BASE (0x107000+REGBASE) -#define EMMA2RH_PIIC1_BASE (0x108000+REGBASE) -#define EMMA2RH_PIIC2_BASE (0x109000+REGBASE) -#define EMMA2RH_PCI_CONTROL (0x200000+REGBASE) -#define EMMA2RH_PCI_ARBIT_CTR (0x200004+REGBASE) -#define EMMA2RH_PCI_IWIN0_CTR (0x200010+REGBASE) -#define EMMA2RH_PCI_IWIN1_CTR (0x200014+REGBASE) -#define EMMA2RH_PCI_INIT_ESWP (0x200018+REGBASE) -#define EMMA2RH_PCI_INT (0x200020+REGBASE) -#define EMMA2RH_PCI_INT_EN (0x200024+REGBASE) -#define EMMA2RH_PCI_TWIN_CTR (0x200030+REGBASE) -#define EMMA2RH_PCI_TWIN_BADR (0x200034+REGBASE) -#define EMMA2RH_PCI_TWIN0_DADR (0x200038+REGBASE) -#define EMMA2RH_PCI_TWIN1_DADR (0x20003c+REGBASE) - -/* - * Memory map (physical address) - * - * Note most of the following address must be properly aligned by the - * corresponding size. For example, if PCI_IO_SIZE is 16MB, then - * PCI_IO_BASE must be aligned along 16MB boundary. - */ - -/* the actual ram size is detected at run-time */ -#define EMMA2RH_RAM_BASE 0x00000000 -#define EMMA2RH_RAM_SIZE 0x10000000 /* less than 256MB */ - -#define EMMA2RH_IO_BASE 0x10000000 -#define EMMA2RH_IO_SIZE 0x01000000 /* 16 MB */ - -#define EMMA2RH_GENERALIO_BASE 0x11000000 -#define EMMA2RH_GENERALIO_SIZE 0x01000000 /* 16 MB */ - -#define EMMA2RH_PCI_IO_BASE 0x12000000 -#define EMMA2RH_PCI_IO_SIZE 0x02000000 /* 32 MB */ - -#define EMMA2RH_PCI_MEM_BASE 0x14000000 -#define EMMA2RH_PCI_MEM_SIZE 0x08000000 /* 128 MB */ - -#define EMMA2RH_ROM_BASE 0x1c000000 -#define EMMA2RH_ROM_SIZE 0x04000000 /* 64 MB */ - -#define EMMA2RH_PCI_CONFIG_BASE EMMA2RH_PCI_IO_BASE -#define EMMA2RH_PCI_CONFIG_SIZE EMMA2RH_PCI_IO_SIZE - -#define NUM_EMMA2RH_IRQ 96 - -#define EMMA2RH_IRQ_BASE (MIPS_CPU_IRQ_BASE + 8) - -/* - * emma2rh irq defs - */ - -#define EMMA2RH_IRQ_INT(n) (EMMA2RH_IRQ_BASE + (n)) - -#define EMMA2RH_IRQ_PFUR0 EMMA2RH_IRQ_INT(49) -#define EMMA2RH_IRQ_PFUR1 EMMA2RH_IRQ_INT(50) -#define EMMA2RH_IRQ_PFUR2 EMMA2RH_IRQ_INT(51) -#define EMMA2RH_IRQ_PIIC0 EMMA2RH_IRQ_INT(56) -#define EMMA2RH_IRQ_PIIC1 EMMA2RH_IRQ_INT(57) -#define EMMA2RH_IRQ_PIIC2 EMMA2RH_IRQ_INT(58) - -/* - * EMMA2RH Register Access - */ - -#define EMMA2RH_BASE (0xa0000000) - -static inline void emma2rh_sync(void) -{ - volatile u32 *p = (volatile u32 *)0xbfc00000; - (void)(*p); -} - -static inline void emma2rh_out32(u32 offset, u32 val) -{ - *(volatile u32 *)(EMMA2RH_BASE | offset) = val; - emma2rh_sync(); -} - -static inline u32 emma2rh_in32(u32 offset) -{ - u32 val = *(volatile u32 *)(EMMA2RH_BASE | offset); - return val; -} - -static inline void emma2rh_out16(u32 offset, u16 val) -{ - *(volatile u16 *)(EMMA2RH_BASE | offset) = val; - emma2rh_sync(); -} - -static inline u16 emma2rh_in16(u32 offset) -{ - u16 val = *(volatile u16 *)(EMMA2RH_BASE | offset); - return val; -} - -static inline void emma2rh_out8(u32 offset, u8 val) -{ - *(volatile u8 *)(EMMA2RH_BASE | offset) = val; - emma2rh_sync(); -} - -static inline u8 emma2rh_in8(u32 offset) -{ - u8 val = *(volatile u8 *)(EMMA2RH_BASE | offset); - return val; -} - -/** - * IIC registers map - **/ - -/*---------------------------------------------------------------------------*/ -/* CNT - Control register (00H R/W) */ -/*---------------------------------------------------------------------------*/ -#define SPT 0x00000001 -#define STT 0x00000002 -#define ACKE 0x00000004 -#define WTIM 0x00000008 -#define SPIE 0x00000010 -#define WREL 0x00000020 -#define LREL 0x00000040 -#define IICE 0x00000080 -#define CNT_RESERVED 0x000000ff /* reserved bit 0 */ - -#define I2C_EMMA_START (IICE | STT) -#define I2C_EMMA_STOP (IICE | SPT) -#define I2C_EMMA_REPSTART I2C_EMMA_START - -/*---------------------------------------------------------------------------*/ -/* STA - Status register (10H Read) */ -/*---------------------------------------------------------------------------*/ -#define MSTS 0x00000080 -#define ALD 0x00000040 -#define EXC 0x00000020 -#define COI 0x00000010 -#define TRC 0x00000008 -#define ACKD 0x00000004 -#define STD 0x00000002 -#define SPD 0x00000001 - -/*---------------------------------------------------------------------------*/ -/* CSEL - Clock select register (20H R/W) */ -/*---------------------------------------------------------------------------*/ -#define FCL 0x00000080 -#define ND50 0x00000040 -#define CLD 0x00000020 -#define DAD 0x00000010 -#define SMC 0x00000008 -#define DFC 0x00000004 -#define CL 0x00000003 -#define CSEL_RESERVED 0x000000ff /* reserved bit 0 */ - -#define FAST397 0x0000008b -#define FAST297 0x0000008a -#define FAST347 0x0000000b -#define FAST260 0x0000000a -#define FAST130 0x00000008 -#define STANDARD108 0x00000083 -#define STANDARD83 0x00000082 -#define STANDARD95 0x00000003 -#define STANDARD73 0x00000002 -#define STANDARD36 0x00000001 -#define STANDARD71 0x00000000 - -/*---------------------------------------------------------------------------*/ -/* SVA - Slave address register (30H R/W) */ -/*---------------------------------------------------------------------------*/ -#define SVA 0x000000fe - -/*---------------------------------------------------------------------------*/ -/* SHR - Shift register (40H R/W) */ -/*---------------------------------------------------------------------------*/ -#define SR 0x000000ff - -/*---------------------------------------------------------------------------*/ -/* INT - Interrupt register (50H R/W) */ -/* INTM - Interrupt mask register (60H R/W) */ -/*---------------------------------------------------------------------------*/ -#define INTE0 0x00000001 - -/*********************************************************************** - * I2C registers - *********************************************************************** - */ -#define I2C_EMMA_CNT 0x00 -#define I2C_EMMA_STA 0x10 -#define I2C_EMMA_CSEL 0x20 -#define I2C_EMMA_SVA 0x30 -#define I2C_EMMA_SHR 0x40 -#define I2C_EMMA_INT 0x50 -#define I2C_EMMA_INTM 0x60 - -/* - * include the board dependent part - */ -#ifdef CONFIG_NEC_MARKEINS -#include -#else -#error "Unknown EMMA2RH board!" -#endif - -#endif /* __ASM_EMMA_EMMA2RH_H */ diff --git a/arch/mips/include/asm/emma/markeins.h b/arch/mips/include/asm/emma/markeins.h deleted file mode 100644 index 2d7e1339d36f..000000000000 --- a/arch/mips/include/asm/emma/markeins.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) NEC Electronics Corporation 2005-2006 - * - * This file based on include/asm-mips/ddb5xxx/ddb5xxx.h - * Copyright 2001 MontaVista Software Inc. - */ - -#ifndef MARKEINS_H -#define MARKEINS_H - -#define NUM_EMMA2RH_IRQ_SW 32 -#define NUM_EMMA2RH_IRQ_GPIO 32 - -#define EMMA2RH_SW_CASCADE (EMMA2RH_IRQ_INT(7) - EMMA2RH_IRQ_INT(0)) -#define EMMA2RH_GPIO_CASCADE (EMMA2RH_IRQ_INT(46) - EMMA2RH_IRQ_INT(0)) - -#define EMMA2RH_SW_IRQ_BASE (EMMA2RH_IRQ_BASE + NUM_EMMA2RH_IRQ) -#define EMMA2RH_GPIO_IRQ_BASE (EMMA2RH_SW_IRQ_BASE + NUM_EMMA2RH_IRQ_SW) - -#define EMMA2RH_SW_IRQ_INT(n) (EMMA2RH_SW_IRQ_BASE + (n)) - -#define MARKEINS_PCI_IRQ_INTA EMMA2RH_GPIO_IRQ_BASE+15 -#define MARKEINS_PCI_IRQ_INTB EMMA2RH_GPIO_IRQ_BASE+16 -#define MARKEINS_PCI_IRQ_INTC EMMA2RH_GPIO_IRQ_BASE+17 -#define MARKEINS_PCI_IRQ_INTD EMMA2RH_GPIO_IRQ_BASE+18 - -#endif /* CONFIG_MARKEINS */ diff --git a/arch/mips/include/asm/mach-emma2rh/irq.h b/arch/mips/include/asm/mach-emma2rh/irq.h deleted file mode 100644 index d32736736bb3..000000000000 --- a/arch/mips/include/asm/mach-emma2rh/irq.h +++ /dev/null @@ -1,15 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2003 by Ralf Baechle - */ -#ifndef __ASM_MACH_EMMA2RH_IRQ_H -#define __ASM_MACH_EMMA2RH_IRQ_H - -#define NR_IRQS 256 - -#include - -#endif /* __ASM_MACH_EMMA2RH_IRQ_H */ diff --git a/arch/mips/pci/Makefile b/arch/mips/pci/Makefile index aeac09090fb4..f238d7b9e0b2 100644 --- a/arch/mips/pci/Makefile +++ b/arch/mips/pci/Makefile @@ -15,7 +15,6 @@ obj-$(CONFIG_PCI_GT64XXX_PCI0) += ops-gt64xxx_pci0.o obj-$(CONFIG_MIPS_MSC) += ops-msc.o obj-$(CONFIG_SOC_TX3927) += ops-tx3927.o obj-$(CONFIG_PCI_VR41XX) += ops-vr41xx.o pci-vr41xx.o -obj-$(CONFIG_NEC_MARKEINS) += ops-emma2rh.o pci-emma2rh.o fixup-emma2rh.o obj-$(CONFIG_PCI_TX4927) += ops-tx4927.o obj-$(CONFIG_BCM47XX) += pci-bcm47xx.o obj-$(CONFIG_BCM63XX) += pci-bcm63xx.o fixup-bcm63xx.o \ diff --git a/arch/mips/pci/fixup-emma2rh.c b/arch/mips/pci/fixup-emma2rh.c deleted file mode 100644 index 2541f9bc12de..000000000000 --- a/arch/mips/pci/fixup-emma2rh.c +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) NEC Electronics Corporation 2004-2006 - * - * This file is based on the arch/mips/ddb5xxx/ddb5477/pci.c - * - * Copyright 2001 MontaVista Software Inc. - */ - -#include -#include -#include -#include - -#include - -#include - -#define EMMA2RH_PCI_HOST_SLOT 0x09 -#define EMMA2RH_USB_SLOT 0x03 -#define PCI_DEVICE_ID_NEC_EMMA2RH 0x014b /* EMMA2RH PCI Host */ - -/* - * we fix up irqs based on the slot number. - * The first entry is at AD:11. - * Fortunately this works because, although we have two pci buses, - * they all have different slot numbers (except for rockhopper slot 20 - * which is handled below). - * - */ - -#define MAX_SLOT_NUM 10 -static unsigned char irq_map[][5] = { - [3] = {0, MARKEINS_PCI_IRQ_INTB, MARKEINS_PCI_IRQ_INTC, - MARKEINS_PCI_IRQ_INTD, 0,}, - [4] = {0, MARKEINS_PCI_IRQ_INTA, 0, 0, 0,}, - [5] = {0, 0, 0, 0, 0,}, - [6] = {0, MARKEINS_PCI_IRQ_INTC, MARKEINS_PCI_IRQ_INTD, - MARKEINS_PCI_IRQ_INTA, MARKEINS_PCI_IRQ_INTB,}, -}; - -static void nec_usb_controller_fixup(struct pci_dev *dev) -{ - if (PCI_SLOT(dev->devfn) == EMMA2RH_USB_SLOT) - /* on board USB controller configuration */ - pci_write_config_dword(dev, 0xe4, 1 << 5); -} - -DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NEC, PCI_DEVICE_ID_NEC_USB, - nec_usb_controller_fixup); - -/* - * Prevent the PCI layer from seeing the resources allocated to this device - * if it is the host bridge by marking it as such. These resources are of - * no consequence to the PCI layer (they are handled elsewhere). - */ -static void emma2rh_pci_host_fixup(struct pci_dev *dev) -{ - int i; - - if (PCI_SLOT(dev->devfn) == EMMA2RH_PCI_HOST_SLOT) { - dev->class &= 0xff; - dev->class |= PCI_CLASS_BRIDGE_HOST << 8; - for (i = 0; i < PCI_NUM_RESOURCES; i++) { - dev->resource[i].start = 0; - dev->resource[i].end = 0; - dev->resource[i].flags = 0; - } - } -} - -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_NEC, PCI_DEVICE_ID_NEC_EMMA2RH, - emma2rh_pci_host_fixup); - -int pcibios_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) -{ - return irq_map[slot][pin]; -} - -/* Do platform specific device initialization at pci_enable_device() time */ -int pcibios_plat_dev_init(struct pci_dev *dev) -{ - return 0; -} diff --git a/arch/mips/pci/ops-emma2rh.c b/arch/mips/pci/ops-emma2rh.c deleted file mode 100644 index 65f47344536c..000000000000 --- a/arch/mips/pci/ops-emma2rh.c +++ /dev/null @@ -1,167 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) NEC Electronics Corporation 2004-2006 - * - * This file is based on the arch/mips/pci/ops-vr41xx.c - * - * Copyright 2001 MontaVista Software Inc. - */ - -#include -#include -#include - -#include - -#include - -#define RTABORT (0x1<<9) -#define RMABORT (0x1<<10) -#define EMMA2RH_PCI_SLOT_NUM 9 /* 0000:09.0 is final PCI device */ - -/* - * access config space - */ - -static int check_args(struct pci_bus *bus, u32 devfn, u32 * bus_num) -{ - /* check if the bus is top-level */ - if (bus->parent != NULL) - *bus_num = bus->number; - else - *bus_num = 0; - - if (*bus_num == 0) { - /* Type 0 */ - if (PCI_SLOT(devfn) >= 10) - return PCIBIOS_DEVICE_NOT_FOUND; - } else { - /* Type 1 */ - if ((*bus_num >= 64) || (PCI_SLOT(devfn) >= 16)) - return PCIBIOS_DEVICE_NOT_FOUND; - } - return 0; -} - -static inline int set_pci_configuration_address(unsigned char bus_num, - unsigned int devfn, int where) -{ - u32 config_win0; - - emma2rh_out32(EMMA2RH_PCI_INT, ~RMABORT); - if (bus_num == 0) - /* - * Type 0 configuration - */ - config_win0 = (1 << (22 + PCI_SLOT(devfn))) | (5 << 9); - else - /* - * Type 1 configuration - */ - config_win0 = (bus_num << 26) | (PCI_SLOT(devfn) << 22) | - (1 << 15) | (5 << 9); - - emma2rh_out32(EMMA2RH_PCI_IWIN0_CTR, config_win0); - - return 0; -} - -static int pci_config_read(struct pci_bus *bus, unsigned int devfn, int where, - int size, uint32_t * val) -{ - u32 bus_num; - u32 base = KSEG1ADDR(EMMA2RH_PCI_CONFIG_BASE); - u32 backup_win0; - u32 data; - - *val = 0xffffffffU; - - if (check_args(bus, devfn, &bus_num) == PCIBIOS_DEVICE_NOT_FOUND) - return PCIBIOS_DEVICE_NOT_FOUND; - - backup_win0 = emma2rh_in32(EMMA2RH_PCI_IWIN0_CTR); - - if (set_pci_configuration_address(bus_num, devfn, where) < 0) - return PCIBIOS_DEVICE_NOT_FOUND; - - data = - *(volatile u32 *)(base + (PCI_FUNC(devfn) << 8) + - (where & 0xfffffffc)); - - switch (size) { - case 1: - *val = (data >> ((where & 3) << 3)) & 0xffU; - break; - case 2: - *val = (data >> ((where & 2) << 3)) & 0xffffU; - break; - case 4: - *val = data; - break; - default: - emma2rh_out32(EMMA2RH_PCI_IWIN0_CTR, backup_win0); - return PCIBIOS_FUNC_NOT_SUPPORTED; - } - - emma2rh_out32(EMMA2RH_PCI_IWIN0_CTR, backup_win0); - - if (emma2rh_in32(EMMA2RH_PCI_INT) & RMABORT) - return PCIBIOS_DEVICE_NOT_FOUND; - - return PCIBIOS_SUCCESSFUL; -} - -static int pci_config_write(struct pci_bus *bus, unsigned int devfn, int where, - int size, u32 val) -{ - u32 bus_num; - u32 base = KSEG1ADDR(EMMA2RH_PCI_CONFIG_BASE); - u32 backup_win0; - u32 data; - int shift; - - if (check_args(bus, devfn, &bus_num) == PCIBIOS_DEVICE_NOT_FOUND) - return PCIBIOS_DEVICE_NOT_FOUND; - - backup_win0 = emma2rh_in32(EMMA2RH_PCI_IWIN0_CTR); - - if (set_pci_configuration_address(bus_num, devfn, where) < 0) - return PCIBIOS_DEVICE_NOT_FOUND; - - /* read modify write */ - data = - *(volatile u32 *)(base + (PCI_FUNC(devfn) << 8) + - (where & 0xfffffffc)); - - switch (size) { - case 1: - shift = (where & 3) << 3; - data &= ~(0xffU << shift); - data |= ((val & 0xffU) << shift); - break; - case 2: - shift = (where & 2) << 3; - data &= ~(0xffffU << shift); - data |= ((val & 0xffffU) << shift); - break; - case 4: - data = val; - break; - default: - emma2rh_out32(EMMA2RH_PCI_IWIN0_CTR, backup_win0); - return PCIBIOS_FUNC_NOT_SUPPORTED; - } - *(volatile u32 *)(base + (PCI_FUNC(devfn) << 8) + - (where & 0xfffffffc)) = data; - - emma2rh_out32(EMMA2RH_PCI_IWIN0_CTR, backup_win0); - if (emma2rh_in32(EMMA2RH_PCI_INT) & RMABORT) - return PCIBIOS_DEVICE_NOT_FOUND; - - return PCIBIOS_SUCCESSFUL; -} - -struct pci_ops emma2rh_pci_ops = { - .read = pci_config_read, - .write = pci_config_write, -}; diff --git a/arch/mips/pci/pci-emma2rh.c b/arch/mips/pci/pci-emma2rh.c deleted file mode 100644 index 156091a3e341..000000000000 --- a/arch/mips/pci/pci-emma2rh.c +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) NEC Electronics Corporation 2004-2006 - * - * This file is based on the arch/mips/ddb5xxx/ddb5477/pci.c - * - * Copyright 2001 MontaVista Software Inc. - */ - -#include -#include -#include -#include - -#include - -#include - -static struct resource pci_io_resource = { - .name = "pci IO space", - .start = EMMA2RH_PCI_IO_BASE, - .end = EMMA2RH_PCI_IO_BASE + EMMA2RH_PCI_IO_SIZE - 1, - .flags = IORESOURCE_IO, -}; - -static struct resource pci_mem_resource = { - .name = "pci memory space", - .start = EMMA2RH_PCI_MEM_BASE, - .end = EMMA2RH_PCI_MEM_BASE + EMMA2RH_PCI_MEM_SIZE - 1, - .flags = IORESOURCE_MEM, -}; - -extern struct pci_ops emma2rh_pci_ops; - -static struct pci_controller emma2rh_pci_controller = { - .pci_ops = &emma2rh_pci_ops, - .mem_resource = &pci_mem_resource, - .io_resource = &pci_io_resource, - .mem_offset = -0x04000000, - .io_offset = 0, -}; - -static void __init emma2rh_pci_init(void) -{ - /* setup PCI interface */ - emma2rh_out32(EMMA2RH_PCI_ARBIT_CTR, 0x70f); - - emma2rh_out32(EMMA2RH_PCI_IWIN0_CTR, 0x80000a18); - emma2rh_out32(EMMA2RH_PCI_CONFIG_BASE + PCI_COMMAND, - PCI_STATUS_DEVSEL_MEDIUM | PCI_STATUS_CAP_LIST | - PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY); - emma2rh_out32(EMMA2RH_PCI_CONFIG_BASE + PCI_BASE_ADDRESS_0, 0x10000000); - emma2rh_out32(EMMA2RH_PCI_CONFIG_BASE + PCI_BASE_ADDRESS_1, 0x00000000); - - emma2rh_out32(EMMA2RH_PCI_IWIN0_CTR, 0x12000000 | 0x218); - emma2rh_out32(EMMA2RH_PCI_IWIN1_CTR, 0x18000000 | 0x600); - emma2rh_out32(EMMA2RH_PCI_INIT_ESWP, 0x00000200); - - emma2rh_out32(EMMA2RH_PCI_TWIN_CTR, 0x00009200); - emma2rh_out32(EMMA2RH_PCI_TWIN_BADR, 0x00000000); - emma2rh_out32(EMMA2RH_PCI_TWIN0_DADR, 0x00000000); - emma2rh_out32(EMMA2RH_PCI_TWIN1_DADR, 0x00000000); -} - -static int __init emma2rh_pci_setup(void) -{ - emma2rh_pci_init(); - register_pci_controller(&emma2rh_pci_controller); - return 0; -} - -arch_initcall(emma2rh_pci_setup); From 615399896ca3787728c56c499c99be79e40ac125 Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:49:46 -0700 Subject: [PATCH 0306/1043] nvme-fc: Sync header to FC-NVME-2 rev 1.08 A couple of minor changes occurred between 1.06 and 1.08: - Addition of NVME_SR_RSP opcode - change of SR_RSP status code 1 to Reserved Signed-off-by: James Smart Reviewed-by: Sagi Grimberg Reviewed-by: Himanshu Madhani Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- include/linux/nvme-fc.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/include/linux/nvme-fc.h b/include/linux/nvme-fc.h index e8c30b39bb27..51fe44e0328b 100644 --- a/include/linux/nvme-fc.h +++ b/include/linux/nvme-fc.h @@ -4,8 +4,8 @@ */ /* - * This file contains definitions relative to FC-NVME-2 r1.06 - * (T11-2019-00210-v001). + * This file contains definitions relative to FC-NVME-2 r1.08 + * (T11-2019-00210-v004). */ #ifndef _NVME_FC_H @@ -81,7 +81,8 @@ struct nvme_fc_ersp_iu { }; -#define FCNVME_NVME_SR_OPCODE 0x01 +#define FCNVME_NVME_SR_OPCODE 0x01 +#define FCNVME_NVME_SR_RSP_OPCODE 0x02 struct nvme_fc_nvme_sr_iu { __u8 fc_id; @@ -94,7 +95,7 @@ struct nvme_fc_nvme_sr_iu { enum { FCNVME_SRSTAT_ACC = 0x0, - FCNVME_SRSTAT_INV_FCID = 0x1, + /* reserved 0x1 */ /* reserved 0x2 */ FCNVME_SRSTAT_LOGICAL_ERR = 0x3, FCNVME_SRSTAT_INV_QUALIF = 0x4, @@ -397,7 +398,7 @@ struct fcnvme_ls_disconnect_conn_rqst { struct fcnvme_ls_rqst_w0 w0; __be32 desc_list_len; struct fcnvme_lsdesc_assoc_id associd; - struct fcnvme_lsdesc_disconn_cmd connectid; + struct fcnvme_lsdesc_conn_id connectid; }; struct fcnvme_ls_disconnect_conn_acc { From 72e6329f86c714785ac195d293cb19dd24507880 Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:49:47 -0700 Subject: [PATCH 0307/1043] nvme-fc and nvmet-fc: revise LLDD api for LS reception and LS request The current LLDD api has: nvme-fc: contains api for transport to do LS requests (and aborts of them). However, there is no interface for reception of LS's and sending responses for them. nvmet-fc: contains api for transport to do reception of LS's and sending of responses for them. However, there is no interface for doing LS requests. Revise the api's so that both nvme-fc and nvmet-fc can send LS's, as well as receiving LS's and sending their responses. Change name of the rcv_ls_req struct to better reflect generic use as a context to used to send an ls rsp. Specifically: nvmefc_tgt_ls_req -> nvmefc_ls_rsp nvmefc_tgt_ls_req.nvmet_fc_private -> nvmefc_ls_rsp.nvme_fc_private Change nvmet_fc_rcv_ls_req() calling sequence to provide handle that can be used by transport in later LS request sequences for an association. nvme-fc nvmet_fc nvme_fcloop: Revise to adapt to changed names in api header. Change calling sequence to nvmet_fc_rcv_ls_req() for hosthandle. Add stubs for new interfaces: host/fc.c: nvme_fc_rcv_ls_req() target/fc.c: nvmet_fc_invalidate_host() lpfc: Revise to adapt code to changed names in api header. Change calling sequence to nvmet_fc_rcv_ls_req() for hosthandle. Signed-off-by: James Smart Reviewed-by: Sagi Grimberg Reviewed-by: Himanshu Madhani Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/fc.c | 35 +++ drivers/nvme/target/fc.c | 77 +++++-- drivers/nvme/target/fcloop.c | 20 +- drivers/scsi/lpfc/lpfc_nvmet.c | 10 +- drivers/scsi/lpfc/lpfc_nvmet.h | 2 +- include/linux/nvme-fc-driver.h | 382 ++++++++++++++++++++++++--------- 6 files changed, 385 insertions(+), 141 deletions(-) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 7dfc4a2ecf1e..8012099fc3ee 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -1464,6 +1464,41 @@ nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl) kfree(lsop); } +/** + * nvme_fc_rcv_ls_req - transport entry point called by an LLDD + * upon the reception of a NVME LS request. + * + * The nvme-fc layer will copy payload to an internal structure for + * processing. As such, upon completion of the routine, the LLDD may + * immediately free/reuse the LS request buffer passed in the call. + * + * If this routine returns error, the LLDD should abort the exchange. + * + * @remoteport: pointer to the (registered) remote port that the LS + * was received from. The remoteport is associated with + * a specific localport. + * @lsrsp: pointer to a nvmefc_ls_rsp response structure to be + * used to reference the exchange corresponding to the LS + * when issuing an ls response. + * @lsreqbuf: pointer to the buffer containing the LS Request + * @lsreqbuf_len: length, in bytes, of the received LS request + */ +int +nvme_fc_rcv_ls_req(struct nvme_fc_remote_port *portptr, + struct nvmefc_ls_rsp *lsrsp, + void *lsreqbuf, u32 lsreqbuf_len) +{ + struct nvme_fc_rport *rport = remoteport_to_rport(portptr); + struct nvme_fc_lport *lport = rport->lport; + + /* validate there's a routine to transmit a response */ + if (!lport->ops->xmt_ls_rsp) + return(-EINVAL); + + return 0; +} +EXPORT_SYMBOL_GPL(nvme_fc_rcv_ls_req); + /* *********************** NVME Ctrl Routines **************************** */ diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index a8ceb7721640..aac7869a70bb 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -28,7 +28,7 @@ struct nvmet_fc_tgtport; struct nvmet_fc_tgt_assoc; struct nvmet_fc_ls_iod { - struct nvmefc_tgt_ls_req *lsreq; + struct nvmefc_ls_rsp *lsrsp; struct nvmefc_tgt_fcp_req *fcpreq; /* only if RS */ struct list_head ls_list; /* tgtport->ls_list */ @@ -1146,6 +1146,42 @@ __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport) spin_unlock_irqrestore(&tgtport->lock, flags); } +/** + * nvmet_fc_invalidate_host - transport entry point called by an LLDD + * to remove references to a hosthandle for LS's. + * + * The nvmet-fc layer ensures that any references to the hosthandle + * on the targetport are forgotten (set to NULL). The LLDD will + * typically call this when a login with a remote host port has been + * lost, thus LS's for the remote host port are no longer possible. + * + * If an LS request is outstanding to the targetport/hosthandle (or + * issued concurrently with the call to invalidate the host), the + * LLDD is responsible for terminating/aborting the LS and completing + * the LS request. It is recommended that these terminations/aborts + * occur after calling to invalidate the host handle to avoid additional + * retries by the nvmet-fc transport. The nvmet-fc transport may + * continue to reference host handle while it cleans up outstanding + * NVME associations. The nvmet-fc transport will call the + * ops->host_release() callback to notify the LLDD that all references + * are complete and the related host handle can be recovered. + * Note: if there are no references, the callback may be called before + * the invalidate host call returns. + * + * @target_port: pointer to the (registered) target port that a prior + * LS was received on and which supplied the transport the + * hosthandle. + * @hosthandle: the handle (pointer) that represents the host port + * that no longer has connectivity and that LS's should + * no longer be directed to. + */ +void +nvmet_fc_invalidate_host(struct nvmet_fc_target_port *target_port, + void *hosthandle) +{ +} +EXPORT_SYMBOL_GPL(nvmet_fc_invalidate_host); + /* * nvmet layer has called to terminate an association */ @@ -1371,7 +1407,7 @@ nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport, dev_err(tgtport->dev, "Create Association LS failed: %s\n", validation_errors[ret]); - iod->lsreq->rsplen = nvmet_fc_format_rjt(acc, + iod->lsrsp->rsplen = nvmet_fc_format_rjt(acc, NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0); @@ -1384,7 +1420,7 @@ nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport, /* format a response */ - iod->lsreq->rsplen = sizeof(*acc); + iod->lsrsp->rsplen = sizeof(*acc); nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, fcnvme_lsdesc_len( @@ -1462,7 +1498,7 @@ nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport, dev_err(tgtport->dev, "Create Connection LS failed: %s\n", validation_errors[ret]); - iod->lsreq->rsplen = nvmet_fc_format_rjt(acc, + iod->lsrsp->rsplen = nvmet_fc_format_rjt(acc, NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, (ret == VERR_NO_ASSOC) ? FCNVME_RJT_RC_INV_ASSOC : @@ -1477,7 +1513,7 @@ nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport, /* format a response */ - iod->lsreq->rsplen = sizeof(*acc); + iod->lsrsp->rsplen = sizeof(*acc); nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)), @@ -1542,7 +1578,7 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, dev_err(tgtport->dev, "Disconnect LS failed: %s\n", validation_errors[ret]); - iod->lsreq->rsplen = nvmet_fc_format_rjt(acc, + iod->lsrsp->rsplen = nvmet_fc_format_rjt(acc, NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, (ret == VERR_NO_ASSOC) ? FCNVME_RJT_RC_INV_ASSOC : @@ -1555,7 +1591,7 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, /* format a response */ - iod->lsreq->rsplen = sizeof(*acc); + iod->lsrsp->rsplen = sizeof(*acc); nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, fcnvme_lsdesc_len( @@ -1577,9 +1613,9 @@ static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req); static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops; static void -nvmet_fc_xmt_ls_rsp_done(struct nvmefc_tgt_ls_req *lsreq) +nvmet_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp) { - struct nvmet_fc_ls_iod *iod = lsreq->nvmet_fc_private; + struct nvmet_fc_ls_iod *iod = lsrsp->nvme_fc_private; struct nvmet_fc_tgtport *tgtport = iod->tgtport; fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma, @@ -1597,9 +1633,9 @@ nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport, fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma, NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE); - ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsreq); + ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsrsp); if (ret) - nvmet_fc_xmt_ls_rsp_done(iod->lsreq); + nvmet_fc_xmt_ls_rsp_done(iod->lsrsp); } /* @@ -1612,12 +1648,12 @@ nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport, struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)iod->rqstbuf; - iod->lsreq->nvmet_fc_private = iod; - iod->lsreq->rspbuf = iod->rspbuf; - iod->lsreq->rspdma = iod->rspdma; - iod->lsreq->done = nvmet_fc_xmt_ls_rsp_done; + iod->lsrsp->nvme_fc_private = iod; + iod->lsrsp->rspbuf = iod->rspbuf; + iod->lsrsp->rspdma = iod->rspdma; + iod->lsrsp->done = nvmet_fc_xmt_ls_rsp_done; /* Be preventative. handlers will later set to valid length */ - iod->lsreq->rsplen = 0; + iod->lsrsp->rsplen = 0; iod->assoc = NULL; @@ -1640,7 +1676,7 @@ nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport, nvmet_fc_ls_disconnect(tgtport, iod); break; default: - iod->lsreq->rsplen = nvmet_fc_format_rjt(iod->rspbuf, + iod->lsrsp->rsplen = nvmet_fc_format_rjt(iod->rspbuf, NVME_FC_MAX_LS_BUFFER_SIZE, w0->ls_cmd, FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0); } @@ -1674,14 +1710,15 @@ nvmet_fc_handle_ls_rqst_work(struct work_struct *work) * * @target_port: pointer to the (registered) target port the LS was * received on. - * @lsreq: pointer to a lsreq request structure to be used to reference + * @lsrsp: pointer to a lsrsp structure to be used to reference * the exchange corresponding to the LS. * @lsreqbuf: pointer to the buffer containing the LS Request * @lsreqbuf_len: length, in bytes, of the received LS request */ int nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port, - struct nvmefc_tgt_ls_req *lsreq, + void *hosthandle, + struct nvmefc_ls_rsp *lsrsp, void *lsreqbuf, u32 lsreqbuf_len) { struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); @@ -1699,7 +1736,7 @@ nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port, return -ENOENT; } - iod->lsreq = lsreq; + iod->lsrsp = lsrsp; iod->fcpreq = NULL; memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len); iod->rqstdatalen = lsreqbuf_len; diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c index f69ce66e2d44..c11805a155e8 100644 --- a/drivers/nvme/target/fcloop.c +++ b/drivers/nvme/target/fcloop.c @@ -228,7 +228,7 @@ struct fcloop_nport { struct fcloop_lsreq { struct nvmefc_ls_req *lsreq; - struct nvmefc_tgt_ls_req tgt_ls_req; + struct nvmefc_ls_rsp ls_rsp; int status; struct list_head ls_list; /* fcloop_rport->ls_list */ }; @@ -267,9 +267,9 @@ struct fcloop_ini_fcpreq { }; static inline struct fcloop_lsreq * -tgt_ls_req_to_lsreq(struct nvmefc_tgt_ls_req *tgt_lsreq) +ls_rsp_to_lsreq(struct nvmefc_ls_rsp *lsrsp) { - return container_of(tgt_lsreq, struct fcloop_lsreq, tgt_ls_req); + return container_of(lsrsp, struct fcloop_lsreq, ls_rsp); } static inline struct fcloop_fcpreq * @@ -344,7 +344,7 @@ fcloop_ls_req(struct nvme_fc_local_port *localport, } tls_req->status = 0; - ret = nvmet_fc_rcv_ls_req(rport->targetport, &tls_req->tgt_ls_req, + ret = nvmet_fc_rcv_ls_req(rport->targetport, NULL, &tls_req->ls_rsp, lsreq->rqstaddr, lsreq->rqstlen); return ret; @@ -352,19 +352,19 @@ fcloop_ls_req(struct nvme_fc_local_port *localport, static int fcloop_xmt_ls_rsp(struct nvmet_fc_target_port *targetport, - struct nvmefc_tgt_ls_req *tgt_lsreq) + struct nvmefc_ls_rsp *lsrsp) { - struct fcloop_lsreq *tls_req = tgt_ls_req_to_lsreq(tgt_lsreq); + struct fcloop_lsreq *tls_req = ls_rsp_to_lsreq(lsrsp); struct nvmefc_ls_req *lsreq = tls_req->lsreq; struct fcloop_tport *tport = targetport->private; struct nvme_fc_remote_port *remoteport = tport->remoteport; struct fcloop_rport *rport; - memcpy(lsreq->rspaddr, tgt_lsreq->rspbuf, - ((lsreq->rsplen < tgt_lsreq->rsplen) ? - lsreq->rsplen : tgt_lsreq->rsplen)); + memcpy(lsreq->rspaddr, lsrsp->rspbuf, + ((lsreq->rsplen < lsrsp->rsplen) ? + lsreq->rsplen : lsrsp->rsplen)); - tgt_lsreq->done(tgt_lsreq); + lsrsp->done(lsrsp); if (remoteport) { rport = remoteport->private; diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index 565419bf8d74..3b25bcb150ea 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -302,7 +302,7 @@ lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, struct lpfc_wcqe_complete *wcqe) { struct lpfc_nvmet_tgtport *tgtp; - struct nvmefc_tgt_ls_req *rsp; + struct nvmefc_ls_rsp *rsp; struct lpfc_nvmet_rcv_ctx *ctxp; uint32_t status, result; @@ -335,7 +335,7 @@ lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, } out: - rsp = &ctxp->ctx.ls_req; + rsp = &ctxp->ctx.ls_rsp; lpfc_nvmeio_data(phba, "NVMET LS CMPL: xri x%x stat x%x result x%x\n", ctxp->oxid, status, result); @@ -828,10 +828,10 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, static int lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport, - struct nvmefc_tgt_ls_req *rsp) + struct nvmefc_ls_rsp *rsp) { struct lpfc_nvmet_rcv_ctx *ctxp = - container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.ls_req); + container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.ls_rsp); struct lpfc_hba *phba = ctxp->phba; struct hbq_dmabuf *nvmebuf = (struct hbq_dmabuf *)ctxp->rqb_buffer; @@ -1999,7 +1999,7 @@ dropit: * lpfc_nvmet_xmt_ls_rsp_cmp should free the allocated ctxp. */ atomic_inc(&tgtp->rcv_ls_req_in); - rc = nvmet_fc_rcv_ls_req(phba->targetport, &ctxp->ctx.ls_req, + rc = nvmet_fc_rcv_ls_req(phba->targetport, NULL, &ctxp->ctx.ls_rsp, payload, size); lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, diff --git a/drivers/scsi/lpfc/lpfc_nvmet.h b/drivers/scsi/lpfc/lpfc_nvmet.h index b80b1639b9a7..f0196f3ef90d 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.h +++ b/drivers/scsi/lpfc/lpfc_nvmet.h @@ -105,7 +105,7 @@ struct lpfc_nvmet_ctx_info { struct lpfc_nvmet_rcv_ctx { union { - struct nvmefc_tgt_ls_req ls_req; + struct nvmefc_ls_rsp ls_rsp; struct nvmefc_tgt_fcp_req fcp_req; } ctx; struct list_head list; diff --git a/include/linux/nvme-fc-driver.h b/include/linux/nvme-fc-driver.h index 10f81629b9ce..41e7795a3ee4 100644 --- a/include/linux/nvme-fc-driver.h +++ b/include/linux/nvme-fc-driver.h @@ -9,6 +9,124 @@ #include +/* + * ********************** FC-NVME LS API ******************** + * + * Data structures used by both FC-NVME hosts and FC-NVME + * targets to perform FC-NVME LS requests or transmit + * responses. + * + * *********************************************************** + */ + +/** + * struct nvmefc_ls_req - Request structure passed from the transport + * to the LLDD to perform a NVME-FC LS request and obtain + * a response. + * Used by nvme-fc transport (host) to send LS's such as + * Create Association, Create Connection and Disconnect + * Association. + * Used by the nvmet-fc transport (controller) to send + * LS's such as Disconnect Association. + * + * Values set by the requestor prior to calling the LLDD ls_req entrypoint: + * @rqstaddr: pointer to request buffer + * @rqstdma: PCI DMA address of request buffer + * @rqstlen: Length, in bytes, of request buffer + * @rspaddr: pointer to response buffer + * @rspdma: PCI DMA address of response buffer + * @rsplen: Length, in bytes, of response buffer + * @timeout: Maximum amount of time, in seconds, to wait for the LS response. + * If timeout exceeded, LLDD to abort LS exchange and complete + * LS request with error status. + * @private: pointer to memory allocated alongside the ls request structure + * that is specifically for the LLDD to use while processing the + * request. The length of the buffer corresponds to the + * lsrqst_priv_sz value specified in the xxx_template supplied + * by the LLDD. + * @done: The callback routine the LLDD is to invoke upon completion of + * the LS request. req argument is the pointer to the original LS + * request structure. Status argument must be 0 upon success, a + * negative errno on failure (example: -ENXIO). + */ +struct nvmefc_ls_req { + void *rqstaddr; + dma_addr_t rqstdma; + u32 rqstlen; + void *rspaddr; + dma_addr_t rspdma; + u32 rsplen; + u32 timeout; + + void *private; + + void (*done)(struct nvmefc_ls_req *req, int status); + +} __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ + + +/** + * struct nvmefc_ls_rsp - Structure passed from the transport to the LLDD + * to request the transmit the NVME-FC LS response to a + * NVME-FC LS request. The structure originates in the LLDD + * and is given to the transport via the xxx_rcv_ls_req() + * transport routine. As such, the structure represents the + * FC exchange context for the NVME-FC LS request that was + * received and which the response is to be sent for. + * Used by the LLDD to pass the nvmet-fc transport (controller) + * received LS's such as Create Association, Create Connection + * and Disconnect Association. + * Used by the LLDD to pass the nvme-fc transport (host) + * received LS's such as Disconnect Association or Disconnect + * Connection. + * + * The structure is allocated by the LLDD whenever a LS Request is received + * from the FC link. The address of the structure is passed to the nvmet-fc + * or nvme-fc layer via the xxx_rcv_ls_req() transport routines. + * + * The address of the structure is to be passed back to the LLDD + * when the response is to be transmit. The LLDD will use the address to + * map back to the LLDD exchange structure which maintains information such + * the remote N_Port that sent the LS as well as any FC exchange context. + * Upon completion of the LS response transmit, the LLDD will pass the + * address of the structure back to the transport LS rsp done() routine, + * allowing the transport release dma resources. Upon completion of + * the done() routine, no further access to the structure will be made by + * the transport and the LLDD can de-allocate the structure. + * + * Field initialization: + * At the time of the xxx_rcv_ls_req() call, there is no content that + * is valid in the structure. + * + * When the structure is used for the LLDD->xmt_ls_rsp() call, the + * transport layer will fully set the fields in order to specify the + * response payload buffer and its length as well as the done routine + * to be called upon completion of the transmit. The transport layer + * will also set a private pointer for its own use in the done routine. + * + * Values set by the transport layer prior to calling the LLDD xmt_ls_rsp + * entrypoint: + * @rspbuf: pointer to the LS response buffer + * @rspdma: PCI DMA address of the LS response buffer + * @rsplen: Length, in bytes, of the LS response buffer + * @done: The callback routine the LLDD is to invoke upon completion of + * transmitting the LS response. req argument is the pointer to + * the original ls request. + * @nvme_fc_private: pointer to an internal transport-specific structure + * used as part of the transport done() processing. The LLDD is + * not to access this pointer. + */ +struct nvmefc_ls_rsp { + void *rspbuf; + dma_addr_t rspdma; + u16 rsplen; + + void (*done)(struct nvmefc_ls_rsp *rsp); + void *nvme_fc_private; /* LLDD is not to access !! */ +}; + + + /* * ********************** LLDD FC-NVME Host API ******************** * @@ -18,7 +136,6 @@ */ - /** * struct nvme_fc_port_info - port-specific ids and FC connection-specific * data element used during NVME Host role @@ -43,49 +160,6 @@ struct nvme_fc_port_info { u32 dev_loss_tmo; }; - -/** - * struct nvmefc_ls_req - Request structure passed from NVME-FC transport - * to LLDD in order to perform a NVME FC-4 LS - * request and obtain a response. - * - * Values set by the NVME-FC layer prior to calling the LLDD ls_req - * entrypoint. - * @rqstaddr: pointer to request buffer - * @rqstdma: PCI DMA address of request buffer - * @rqstlen: Length, in bytes, of request buffer - * @rspaddr: pointer to response buffer - * @rspdma: PCI DMA address of response buffer - * @rsplen: Length, in bytes, of response buffer - * @timeout: Maximum amount of time, in seconds, to wait for the LS response. - * If timeout exceeded, LLDD to abort LS exchange and complete - * LS request with error status. - * @private: pointer to memory allocated alongside the ls request structure - * that is specifically for the LLDD to use while processing the - * request. The length of the buffer corresponds to the - * lsrqst_priv_sz value specified in the nvme_fc_port_template - * supplied by the LLDD. - * @done: The callback routine the LLDD is to invoke upon completion of - * the LS request. req argument is the pointer to the original LS - * request structure. Status argument must be 0 upon success, a - * negative errno on failure (example: -ENXIO). - */ -struct nvmefc_ls_req { - void *rqstaddr; - dma_addr_t rqstdma; - u32 rqstlen; - void *rspaddr; - dma_addr_t rspdma; - u32 rsplen; - u32 timeout; - - void *private; - - void (*done)(struct nvmefc_ls_req *req, int status); - -} __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ - - enum nvmefc_fcp_datadir { NVMEFC_FCP_NODATA, /* payload_length and sg_cnt will be zero */ NVMEFC_FCP_WRITE, @@ -337,6 +411,21 @@ struct nvme_fc_remote_port { * indicating an FC transport Aborted status. * Entrypoint is Mandatory. * + * @xmt_ls_rsp: Called to transmit the response to a FC-NVME FC-4 LS service. + * The nvmefc_ls_rsp structure is the same LLDD-supplied exchange + * structure specified in the nvme_fc_rcv_ls_req() call made when + * the LS request was received. The structure will fully describe + * the buffers for the response payload and the dma address of the + * payload. The LLDD is to transmit the response (or return a + * non-zero errno status), and upon completion of the transmit, call + * the "done" routine specified in the nvmefc_ls_rsp structure + * (argument to done is the address of the nvmefc_ls_rsp structure + * itself). Upon the completion of the done routine, the LLDD shall + * consider the LS handling complete and the nvmefc_ls_rsp structure + * may be freed/released. + * Entrypoint is mandatory if the LLDD calls the nvme_fc_rcv_ls_req() + * entrypoint. + * * @max_hw_queues: indicates the maximum number of hw queues the LLDD * supports for cpu affinitization. * Value is Mandatory. Must be at least 1. @@ -371,7 +460,7 @@ struct nvme_fc_remote_port { * @lsrqst_priv_sz: The LLDD sets this field to the amount of additional * memory that it would like fc nvme layer to allocate on the LLDD's * behalf whenever a ls request structure is allocated. The additional - * memory area solely for the of the LLDD and its location is + * memory area is solely for use by the LLDD and its location is * specified by the ls_request->private pointer. * Value is Mandatory. Allowed to be zero. * @@ -405,6 +494,9 @@ struct nvme_fc_port_template { struct nvme_fc_remote_port *, void *hw_queue_handle, struct nvmefc_fcp_req *); + int (*xmt_ls_rsp)(struct nvme_fc_local_port *localport, + struct nvme_fc_remote_port *rport, + struct nvmefc_ls_rsp *ls_rsp); u32 max_hw_queues; u16 max_sgl_segments; @@ -441,6 +533,34 @@ void nvme_fc_rescan_remoteport(struct nvme_fc_remote_port *remoteport); int nvme_fc_set_remoteport_devloss(struct nvme_fc_remote_port *remoteport, u32 dev_loss_tmo); +/* + * Routine called to pass a NVME-FC LS request, received by the lldd, + * to the nvme-fc transport. + * + * If the return value is zero: the LS was successfully accepted by the + * transport. + * If the return value is non-zero: the transport has not accepted the + * LS. The lldd should ABTS-LS the LS. + * + * Note: if the LLDD receives and ABTS for the LS prior to the transport + * calling the ops->xmt_ls_rsp() routine to transmit a response, the LLDD + * shall mark the LS as aborted, and when the xmt_ls_rsp() is called: the + * response shall not be transmit and the struct nvmefc_ls_rsp() done + * routine shall be called. The LLDD may transmit the ABTS response as + * soon as the LS was marked or can delay until the xmt_ls_rsp() call is + * made. + * Note: if an RCV LS was successfully posted to the transport and the + * remoteport is then unregistered before xmt_ls_rsp() was called for + * the lsrsp structure, the transport will still call xmt_ls_rsp() + * afterward to cleanup the outstanding lsrsp structure. The LLDD should + * noop the transmission of the rsp and call the lsrsp->done() routine + * to allow the lsrsp structure to be released. + */ +int nvme_fc_rcv_ls_req(struct nvme_fc_remote_port *remoteport, + struct nvmefc_ls_rsp *lsrsp, + void *lsreqbuf, u32 lsreqbuf_len); + + /* * *************** LLDD FC-NVME Target/Subsystem API *************** @@ -470,55 +590,6 @@ struct nvmet_fc_port_info { }; -/** - * struct nvmefc_tgt_ls_req - Structure used between LLDD and NVMET-FC - * layer to represent the exchange context for - * a FC-NVME Link Service (LS). - * - * The structure is allocated by the LLDD whenever a LS Request is received - * from the FC link. The address of the structure is passed to the nvmet-fc - * layer via the nvmet_fc_rcv_ls_req() call. The address of the structure - * will be passed back to the LLDD when the response is to be transmit. - * The LLDD is to use the address to map back to the LLDD exchange structure - * which maintains information such as the targetport the LS was received - * on, the remote FC NVME initiator that sent the LS, and any FC exchange - * context. Upon completion of the LS response transmit, the address of the - * structure will be passed back to the LS rsp done() routine, allowing the - * nvmet-fc layer to release dma resources. Upon completion of the done() - * routine, no further access will be made by the nvmet-fc layer and the - * LLDD can de-allocate the structure. - * - * Field initialization: - * At the time of the nvmet_fc_rcv_ls_req() call, there is no content that - * is valid in the structure. - * - * When the structure is used for the LLDD->xmt_ls_rsp() call, the nvmet-fc - * layer will fully set the fields in order to specify the response - * payload buffer and its length as well as the done routine to be called - * upon compeletion of the transmit. The nvmet-fc layer will also set a - * private pointer for its own use in the done routine. - * - * Values set by the NVMET-FC layer prior to calling the LLDD xmt_ls_rsp - * entrypoint. - * @rspbuf: pointer to the LS response buffer - * @rspdma: PCI DMA address of the LS response buffer - * @rsplen: Length, in bytes, of the LS response buffer - * @done: The callback routine the LLDD is to invoke upon completion of - * transmitting the LS response. req argument is the pointer to - * the original ls request. - * @nvmet_fc_private: pointer to an internal NVMET-FC layer structure used - * as part of the NVMET-FC processing. The LLDD is not to access - * this pointer. - */ -struct nvmefc_tgt_ls_req { - void *rspbuf; - dma_addr_t rspdma; - u16 rsplen; - - void (*done)(struct nvmefc_tgt_ls_req *req); - void *nvmet_fc_private; /* LLDD is not to access !! */ -}; - /* Operations that NVME-FC layer may request the LLDD to perform for FCP */ enum { NVMET_FCOP_READDATA = 1, /* xmt data to initiator */ @@ -693,17 +764,19 @@ struct nvmet_fc_target_port { * Entrypoint is Mandatory. * * @xmt_ls_rsp: Called to transmit the response to a FC-NVME FC-4 LS service. - * The nvmefc_tgt_ls_req structure is the same LLDD-supplied exchange + * The nvmefc_ls_rsp structure is the same LLDD-supplied exchange * structure specified in the nvmet_fc_rcv_ls_req() call made when - * the LS request was received. The structure will fully describe + * the LS request was received. The structure will fully describe * the buffers for the response payload and the dma address of the - * payload. The LLDD is to transmit the response (or return a non-zero - * errno status), and upon completion of the transmit, call the - * "done" routine specified in the nvmefc_tgt_ls_req structure - * (argument to done is the ls reqwuest structure itself). - * After calling the done routine, the LLDD shall consider the - * LS handling complete and the nvmefc_tgt_ls_req structure may - * be freed/released. + * payload. The LLDD is to transmit the response (or return a + * non-zero errno status), and upon completion of the transmit, call + * the "done" routine specified in the nvmefc_ls_rsp structure + * (argument to done is the address of the nvmefc_ls_rsp structure + * itself). Upon the completion of the done() routine, the LLDD shall + * consider the LS handling complete and the nvmefc_ls_rsp structure + * may be freed/released. + * The transport will always call the xmt_ls_rsp() routine for any + * LS received. * Entrypoint is Mandatory. * * @fcp_op: Called to perform a data transfer or transmit a response. @@ -798,6 +871,39 @@ struct nvmet_fc_target_port { * should cause the initiator to rescan the discovery controller * on the targetport. * + * @ls_req: Called to issue a FC-NVME FC-4 LS service request. + * The nvme_fc_ls_req structure will fully describe the buffers for + * the request payload and where to place the response payload. + * The targetport that is to issue the LS request is identified by + * the targetport argument. The remote port that is to receive the + * LS request is identified by the hosthandle argument. The nvmet-fc + * transport is only allowed to issue FC-NVME LS's on behalf of an + * association that was created prior by a Create Association LS. + * The hosthandle will originate from the LLDD in the struct + * nvmefc_ls_rsp structure for the Create Association LS that + * was delivered to the transport. The transport will save the + * hosthandle as an attribute of the association. If the LLDD + * loses connectivity with the remote port, it must call the + * nvmet_fc_invalidate_host() routine to remove any references to + * the remote port in the transport. + * The LLDD is to allocate an exchange, issue the LS request, obtain + * the LS response, and call the "done" routine specified in the + * request structure (argument to done is the ls request structure + * itself). + * Entrypoint is Optional - but highly recommended. + * + * @ls_abort: called to request the LLDD to abort the indicated ls request. + * The call may return before the abort has completed. After aborting + * the request, the LLDD must still call the ls request done routine + * indicating an FC transport Aborted status. + * Entrypoint is Mandatory if the ls_req entry point is specified. + * + * @host_release: called to inform the LLDD that the request to invalidate + * the host port indicated by the hosthandle has been fully completed. + * No associations exist with the host port and there will be no + * further references to hosthandle. + * Entrypoint is Mandatory if the lldd calls nvmet_fc_invalidate_host(). + * * @max_hw_queues: indicates the maximum number of hw queues the LLDD * supports for cpu affinitization. * Value is Mandatory. Must be at least 1. @@ -826,11 +932,19 @@ struct nvmet_fc_target_port { * area solely for the of the LLDD and its location is specified by * the targetport->private pointer. * Value is Mandatory. Allowed to be zero. + * + * @lsrqst_priv_sz: The LLDD sets this field to the amount of additional + * memory that it would like nvmet-fc layer to allocate on the LLDD's + * behalf whenever a ls request structure is allocated. The additional + * memory area is solely for use by the LLDD and its location is + * specified by the ls_request->private pointer. + * Value is Mandatory. Allowed to be zero. + * */ struct nvmet_fc_target_template { void (*targetport_delete)(struct nvmet_fc_target_port *tgtport); int (*xmt_ls_rsp)(struct nvmet_fc_target_port *tgtport, - struct nvmefc_tgt_ls_req *tls_req); + struct nvmefc_ls_rsp *ls_rsp); int (*fcp_op)(struct nvmet_fc_target_port *tgtport, struct nvmefc_tgt_fcp_req *fcpreq); void (*fcp_abort)(struct nvmet_fc_target_port *tgtport, @@ -840,6 +954,11 @@ struct nvmet_fc_target_template { void (*defer_rcv)(struct nvmet_fc_target_port *tgtport, struct nvmefc_tgt_fcp_req *fcpreq); void (*discovery_event)(struct nvmet_fc_target_port *tgtport); + int (*ls_req)(struct nvmet_fc_target_port *targetport, + void *hosthandle, struct nvmefc_ls_req *lsreq); + void (*ls_abort)(struct nvmet_fc_target_port *targetport, + void *hosthandle, struct nvmefc_ls_req *lsreq); + void (*host_release)(void *hosthandle); u32 max_hw_queues; u16 max_sgl_segments; @@ -848,7 +967,9 @@ struct nvmet_fc_target_template { u32 target_features; + /* sizes of additional private data for data structures */ u32 target_priv_sz; + u32 lsrqst_priv_sz; }; @@ -859,10 +980,61 @@ int nvmet_fc_register_targetport(struct nvmet_fc_port_info *portinfo, int nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *tgtport); +/* + * Routine called to pass a NVME-FC LS request, received by the lldd, + * to the nvmet-fc transport. + * + * If the return value is zero: the LS was successfully accepted by the + * transport. + * If the return value is non-zero: the transport has not accepted the + * LS. The lldd should ABTS-LS the LS. + * + * Note: if the LLDD receives and ABTS for the LS prior to the transport + * calling the ops->xmt_ls_rsp() routine to transmit a response, the LLDD + * shall mark the LS as aborted, and when the xmt_ls_rsp() is called: the + * response shall not be transmit and the struct nvmefc_ls_rsp() done + * routine shall be called. The LLDD may transmit the ABTS response as + * soon as the LS was marked or can delay until the xmt_ls_rsp() call is + * made. + * Note: if an RCV LS was successfully posted to the transport and the + * targetport is then unregistered before xmt_ls_rsp() was called for + * the lsrsp structure, the transport will still call xmt_ls_rsp() + * afterward to cleanup the outstanding lsrsp structure. The LLDD should + * noop the transmission of the rsp and call the lsrsp->done() routine + * to allow the lsrsp structure to be released. + */ int nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *tgtport, - struct nvmefc_tgt_ls_req *lsreq, + void *hosthandle, + struct nvmefc_ls_rsp *rsp, void *lsreqbuf, u32 lsreqbuf_len); +/* + * Routine called by the LLDD whenever it has a logout or loss of + * connectivity to a NVME-FC host port which there had been active + * NVMe controllers for. The host port is indicated by the + * hosthandle. The hosthandle is given to the nvmet-fc transport + * when a NVME LS was received, typically to create a new association. + * The nvmet-fc transport will cache the hostport value with the + * association for use in LS requests for the association. + * When the LLDD calls this routine, the nvmet-fc transport will + * immediately terminate all associations that were created with + * the hosthandle host port. + * The LLDD, after calling this routine and having control returned, + * must assume the transport may subsequently utilize hosthandle as + * part of sending LS's to terminate the association. The LLDD + * should reject the LS's if they are attempted. + * Once the last association has terminated for the hosthandle host + * port, the nvmet-fc transport will call the ops->host_release() + * callback. As of the callback, the nvmet-fc transport will no + * longer reference hosthandle. + */ +void nvmet_fc_invalidate_host(struct nvmet_fc_target_port *tgtport, + void *hosthandle); + +/* + * If nvmet_fc_rcv_fcp_req returns non-zero, the transport has not accepted + * the FCP cmd. The lldd should ABTS-LS the cmd. + */ int nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *tgtport, struct nvmefc_tgt_fcp_req *fcpreq, void *cmdiubuf, u32 cmdiubuf_len); From ca19bcd086331ec2fa182ad8cd589014beb931be Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:49:48 -0700 Subject: [PATCH 0308/1043] nvme-fc nvmet-fc: refactor for common LS definitions Routines in the target will want to be used in the host as well. Error definitions should now shared as both sides will process requests and responses to requests. Moved common declarations to new fc.h header kept in the host subdirectory. Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Reviewed-by: Himanshu Madhani Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/fc.c | 36 +---------- drivers/nvme/host/fc.h | 133 +++++++++++++++++++++++++++++++++++++++ drivers/nvme/target/fc.c | 115 +++------------------------------ 3 files changed, 143 insertions(+), 141 deletions(-) create mode 100644 drivers/nvme/host/fc.h diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 8012099fc3ee..83f9b2ac7c55 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -14,6 +14,7 @@ #include "fabrics.h" #include #include +#include "fc.h" #include /* *************************** Data Structures/Defines ****************** */ @@ -1140,41 +1141,6 @@ nvme_fc_send_ls_req_async(struct nvme_fc_rport *rport, return __nvme_fc_send_ls_req(rport, lsop, done); } -/* Validation Error indexes into the string table below */ -enum { - VERR_NO_ERROR = 0, - VERR_LSACC = 1, - VERR_LSDESC_RQST = 2, - VERR_LSDESC_RQST_LEN = 3, - VERR_ASSOC_ID = 4, - VERR_ASSOC_ID_LEN = 5, - VERR_CONN_ID = 6, - VERR_CONN_ID_LEN = 7, - VERR_CR_ASSOC = 8, - VERR_CR_ASSOC_ACC_LEN = 9, - VERR_CR_CONN = 10, - VERR_CR_CONN_ACC_LEN = 11, - VERR_DISCONN = 12, - VERR_DISCONN_ACC_LEN = 13, -}; - -static char *validation_errors[] = { - "OK", - "Not LS_ACC", - "Not LSDESC_RQST", - "Bad LSDESC_RQST Length", - "Not Association ID", - "Bad Association ID Length", - "Not Connection ID", - "Bad Connection ID Length", - "Not CR_ASSOC Rqst", - "Bad CR_ASSOC ACC Length", - "Not CR_CONN Rqst", - "Bad CR_CONN ACC Length", - "Not Disconnect Rqst", - "Bad Disconnect ACC Length", -}; - static int nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, u16 qsize, u16 ersp_ratio) diff --git a/drivers/nvme/host/fc.h b/drivers/nvme/host/fc.h new file mode 100644 index 000000000000..d2861cdd58ee --- /dev/null +++ b/drivers/nvme/host/fc.h @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2016, Avago Technologies + */ + +#ifndef _NVME_FC_TRANSPORT_H +#define _NVME_FC_TRANSPORT_H 1 + + +/* + * Common definitions between the nvme_fc (host) transport and + * nvmet_fc (target) transport implementation. + */ + +/* + * ****************** FC-NVME LS HANDLING ****************** + */ + +static inline void +nvme_fc_format_rsp_hdr(void *buf, u8 ls_cmd, __be32 desc_len, u8 rqst_ls_cmd) +{ + struct fcnvme_ls_acc_hdr *acc = buf; + + acc->w0.ls_cmd = ls_cmd; + acc->desc_list_len = desc_len; + acc->rqst.desc_tag = cpu_to_be32(FCNVME_LSDESC_RQST); + acc->rqst.desc_len = + fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)); + acc->rqst.w0.ls_cmd = rqst_ls_cmd; +} + +static inline int +nvme_fc_format_rjt(void *buf, u16 buflen, u8 ls_cmd, + u8 reason, u8 explanation, u8 vendor) +{ + struct fcnvme_ls_rjt *rjt = buf; + + nvme_fc_format_rsp_hdr(buf, FCNVME_LSDESC_RQST, + fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_rjt)), + ls_cmd); + rjt->rjt.desc_tag = cpu_to_be32(FCNVME_LSDESC_RJT); + rjt->rjt.desc_len = fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rjt)); + rjt->rjt.reason_code = reason; + rjt->rjt.reason_explanation = explanation; + rjt->rjt.vendor = vendor; + + return sizeof(struct fcnvme_ls_rjt); +} + +/* Validation Error indexes into the string table below */ +enum { + VERR_NO_ERROR = 0, + VERR_CR_ASSOC_LEN = 1, + VERR_CR_ASSOC_RQST_LEN = 2, + VERR_CR_ASSOC_CMD = 3, + VERR_CR_ASSOC_CMD_LEN = 4, + VERR_ERSP_RATIO = 5, + VERR_ASSOC_ALLOC_FAIL = 6, + VERR_QUEUE_ALLOC_FAIL = 7, + VERR_CR_CONN_LEN = 8, + VERR_CR_CONN_RQST_LEN = 9, + VERR_ASSOC_ID = 10, + VERR_ASSOC_ID_LEN = 11, + VERR_NO_ASSOC = 12, + VERR_CONN_ID = 13, + VERR_CONN_ID_LEN = 14, + VERR_INVAL_CONN = 15, + VERR_CR_CONN_CMD = 16, + VERR_CR_CONN_CMD_LEN = 17, + VERR_DISCONN_LEN = 18, + VERR_DISCONN_RQST_LEN = 19, + VERR_DISCONN_CMD = 20, + VERR_DISCONN_CMD_LEN = 21, + VERR_DISCONN_SCOPE = 22, + VERR_RS_LEN = 23, + VERR_RS_RQST_LEN = 24, + VERR_RS_CMD = 25, + VERR_RS_CMD_LEN = 26, + VERR_RS_RCTL = 27, + VERR_RS_RO = 28, + VERR_LSACC = 29, + VERR_LSDESC_RQST = 30, + VERR_LSDESC_RQST_LEN = 31, + VERR_CR_ASSOC = 32, + VERR_CR_ASSOC_ACC_LEN = 33, + VERR_CR_CONN = 34, + VERR_CR_CONN_ACC_LEN = 35, + VERR_DISCONN = 36, + VERR_DISCONN_ACC_LEN = 37, +}; + +static char *validation_errors[] = { + "OK", + "Bad CR_ASSOC Length", + "Bad CR_ASSOC Rqst Length", + "Not CR_ASSOC Cmd", + "Bad CR_ASSOC Cmd Length", + "Bad Ersp Ratio", + "Association Allocation Failed", + "Queue Allocation Failed", + "Bad CR_CONN Length", + "Bad CR_CONN Rqst Length", + "Not Association ID", + "Bad Association ID Length", + "No Association", + "Not Connection ID", + "Bad Connection ID Length", + "Invalid Connection ID", + "Not CR_CONN Cmd", + "Bad CR_CONN Cmd Length", + "Bad DISCONN Length", + "Bad DISCONN Rqst Length", + "Not DISCONN Cmd", + "Bad DISCONN Cmd Length", + "Bad Disconnect Scope", + "Bad RS Length", + "Bad RS Rqst Length", + "Not RS Cmd", + "Bad RS Cmd Length", + "Bad RS R_CTL", + "Bad RS Relative Offset", + "Not LS_ACC", + "Not LSDESC_RQST", + "Bad LSDESC_RQST Length", + "Not CR_ASSOC Rqst", + "Bad CR_ASSOC ACC Length", + "Not CR_CONN Rqst", + "Bad CR_CONN ACC Length", + "Not Disconnect Rqst", + "Bad Disconnect ACC Length", +}; + +#endif /* _NVME_FC_TRANSPORT_H */ diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index aac7869a70bb..1f3118a3b0a3 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -14,6 +14,7 @@ #include "nvmet.h" #include #include +#include "../host/fc.h" /* *************************** Data Structures/Defines ****************** */ @@ -1257,102 +1258,6 @@ EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport); /* *********************** FC-NVME LS Handling **************************** */ -static void -nvmet_fc_format_rsp_hdr(void *buf, u8 ls_cmd, __be32 desc_len, u8 rqst_ls_cmd) -{ - struct fcnvme_ls_acc_hdr *acc = buf; - - acc->w0.ls_cmd = ls_cmd; - acc->desc_list_len = desc_len; - acc->rqst.desc_tag = cpu_to_be32(FCNVME_LSDESC_RQST); - acc->rqst.desc_len = - fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)); - acc->rqst.w0.ls_cmd = rqst_ls_cmd; -} - -static int -nvmet_fc_format_rjt(void *buf, u16 buflen, u8 ls_cmd, - u8 reason, u8 explanation, u8 vendor) -{ - struct fcnvme_ls_rjt *rjt = buf; - - nvmet_fc_format_rsp_hdr(buf, FCNVME_LSDESC_RQST, - fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_rjt)), - ls_cmd); - rjt->rjt.desc_tag = cpu_to_be32(FCNVME_LSDESC_RJT); - rjt->rjt.desc_len = fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rjt)); - rjt->rjt.reason_code = reason; - rjt->rjt.reason_explanation = explanation; - rjt->rjt.vendor = vendor; - - return sizeof(struct fcnvme_ls_rjt); -} - -/* Validation Error indexes into the string table below */ -enum { - VERR_NO_ERROR = 0, - VERR_CR_ASSOC_LEN = 1, - VERR_CR_ASSOC_RQST_LEN = 2, - VERR_CR_ASSOC_CMD = 3, - VERR_CR_ASSOC_CMD_LEN = 4, - VERR_ERSP_RATIO = 5, - VERR_ASSOC_ALLOC_FAIL = 6, - VERR_QUEUE_ALLOC_FAIL = 7, - VERR_CR_CONN_LEN = 8, - VERR_CR_CONN_RQST_LEN = 9, - VERR_ASSOC_ID = 10, - VERR_ASSOC_ID_LEN = 11, - VERR_NO_ASSOC = 12, - VERR_CONN_ID = 13, - VERR_CONN_ID_LEN = 14, - VERR_NO_CONN = 15, - VERR_CR_CONN_CMD = 16, - VERR_CR_CONN_CMD_LEN = 17, - VERR_DISCONN_LEN = 18, - VERR_DISCONN_RQST_LEN = 19, - VERR_DISCONN_CMD = 20, - VERR_DISCONN_CMD_LEN = 21, - VERR_DISCONN_SCOPE = 22, - VERR_RS_LEN = 23, - VERR_RS_RQST_LEN = 24, - VERR_RS_CMD = 25, - VERR_RS_CMD_LEN = 26, - VERR_RS_RCTL = 27, - VERR_RS_RO = 28, -}; - -static char *validation_errors[] = { - "OK", - "Bad CR_ASSOC Length", - "Bad CR_ASSOC Rqst Length", - "Not CR_ASSOC Cmd", - "Bad CR_ASSOC Cmd Length", - "Bad Ersp Ratio", - "Association Allocation Failed", - "Queue Allocation Failed", - "Bad CR_CONN Length", - "Bad CR_CONN Rqst Length", - "Not Association ID", - "Bad Association ID Length", - "No Association", - "Not Connection ID", - "Bad Connection ID Length", - "No Connection", - "Not CR_CONN Cmd", - "Bad CR_CONN Cmd Length", - "Bad DISCONN Length", - "Bad DISCONN Rqst Length", - "Not DISCONN Cmd", - "Bad DISCONN Cmd Length", - "Bad Disconnect Scope", - "Bad RS Length", - "Bad RS Rqst Length", - "Not RS Cmd", - "Bad RS Cmd Length", - "Bad RS R_CTL", - "Bad RS Relative Offset", -}; - static void nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod) @@ -1407,7 +1312,7 @@ nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport, dev_err(tgtport->dev, "Create Association LS failed: %s\n", validation_errors[ret]); - iod->lsrsp->rsplen = nvmet_fc_format_rjt(acc, + iod->lsrsp->rsplen = nvme_fc_format_rjt(acc, NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0); @@ -1422,7 +1327,7 @@ nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport, iod->lsrsp->rsplen = sizeof(*acc); - nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, + nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, fcnvme_lsdesc_len( sizeof(struct fcnvme_ls_cr_assoc_acc)), FCNVME_LS_CREATE_ASSOCIATION); @@ -1498,7 +1403,7 @@ nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport, dev_err(tgtport->dev, "Create Connection LS failed: %s\n", validation_errors[ret]); - iod->lsrsp->rsplen = nvmet_fc_format_rjt(acc, + iod->lsrsp->rsplen = nvme_fc_format_rjt(acc, NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, (ret == VERR_NO_ASSOC) ? FCNVME_RJT_RC_INV_ASSOC : @@ -1515,7 +1420,7 @@ nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport, iod->lsrsp->rsplen = sizeof(*acc); - nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, + nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)), FCNVME_LS_CREATE_CONNECTION); acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID); @@ -1578,13 +1483,11 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, dev_err(tgtport->dev, "Disconnect LS failed: %s\n", validation_errors[ret]); - iod->lsrsp->rsplen = nvmet_fc_format_rjt(acc, + iod->lsrsp->rsplen = nvme_fc_format_rjt(acc, NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, (ret == VERR_NO_ASSOC) ? FCNVME_RJT_RC_INV_ASSOC : - (ret == VERR_NO_CONN) ? - FCNVME_RJT_RC_INV_CONN : - FCNVME_RJT_RC_LOGIC, + FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0); return; } @@ -1593,7 +1496,7 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, iod->lsrsp->rsplen = sizeof(*acc); - nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, + nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, fcnvme_lsdesc_len( sizeof(struct fcnvme_ls_disconnect_assoc_acc)), FCNVME_LS_DISCONNECT_ASSOC); @@ -1676,7 +1579,7 @@ nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport, nvmet_fc_ls_disconnect(tgtport, iod); break; default: - iod->lsrsp->rsplen = nvmet_fc_format_rjt(iod->rspbuf, + iod->lsrsp->rsplen = nvme_fc_format_rjt(iod->rspbuf, NVME_FC_MAX_LS_BUFFER_SIZE, w0->ls_cmd, FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0); } From 3b8281b02bdc6fc7bed6f20af6fd7933a86b94e2 Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:49:49 -0700 Subject: [PATCH 0309/1043] nvmet-fc: Better size LS buffers Current code uses NVME_FC_MAX_LS_BUFFER_SIZE (2KB) when allocating buffers for LS requests and responses. This is considerable overkill for what is actually defined. Rework code to have unions for all possible requests and responses and size based on the unions. Remove NVME_FC_MAX_LS_BUFFER_SIZE. Signed-off-by: James Smart Reviewed-by: Sagi Grimberg Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/fc.h | 15 ++++++++++++ drivers/nvme/target/fc.c | 53 +++++++++++++++++----------------------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/drivers/nvme/host/fc.h b/drivers/nvme/host/fc.h index d2861cdd58ee..08fa88381d45 100644 --- a/drivers/nvme/host/fc.h +++ b/drivers/nvme/host/fc.h @@ -16,6 +16,21 @@ * ****************** FC-NVME LS HANDLING ****************** */ +union nvmefc_ls_requests { + struct fcnvme_ls_cr_assoc_rqst rq_cr_assoc; + struct fcnvme_ls_cr_conn_rqst rq_cr_conn; + struct fcnvme_ls_disconnect_assoc_rqst rq_dis_assoc; + struct fcnvme_ls_disconnect_conn_rqst rq_dis_conn; +} __aligned(128); /* alignment for other things alloc'd with */ + +union nvmefc_ls_responses { + struct fcnvme_ls_rjt rsp_rjt; + struct fcnvme_ls_cr_assoc_acc rsp_cr_assoc; + struct fcnvme_ls_cr_conn_acc rsp_cr_conn; + struct fcnvme_ls_disconnect_assoc_acc rsp_dis_assoc; + struct fcnvme_ls_disconnect_conn_acc rsp_dis_conn; +} __aligned(128); /* alignment for other things alloc'd with */ + static inline void nvme_fc_format_rsp_hdr(void *buf, u8 ls_cmd, __be32 desc_len, u8 rqst_ls_cmd) { diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index 1f3118a3b0a3..66de6bd8f4fd 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -22,9 +22,6 @@ #define NVMET_LS_CTX_COUNT 256 -/* for this implementation, assume small single frame rqst/rsp */ -#define NVME_FC_MAX_LS_BUFFER_SIZE 2048 - struct nvmet_fc_tgtport; struct nvmet_fc_tgt_assoc; @@ -37,8 +34,8 @@ struct nvmet_fc_ls_iod { struct nvmet_fc_tgtport *tgtport; struct nvmet_fc_tgt_assoc *assoc; - u8 *rqstbuf; - u8 *rspbuf; + union nvmefc_ls_requests *rqstbuf; + union nvmefc_ls_responses *rspbuf; u16 rqstdatalen; dma_addr_t rspdma; @@ -340,15 +337,16 @@ nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport) iod->tgtport = tgtport; list_add_tail(&iod->ls_list, &tgtport->ls_list); - iod->rqstbuf = kcalloc(2, NVME_FC_MAX_LS_BUFFER_SIZE, - GFP_KERNEL); + iod->rqstbuf = kzalloc(sizeof(union nvmefc_ls_requests) + + sizeof(union nvmefc_ls_responses), + GFP_KERNEL); if (!iod->rqstbuf) goto out_fail; - iod->rspbuf = iod->rqstbuf + NVME_FC_MAX_LS_BUFFER_SIZE; + iod->rspbuf = (union nvmefc_ls_responses *)&iod->rqstbuf[1]; iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf, - NVME_FC_MAX_LS_BUFFER_SIZE, + sizeof(*iod->rspbuf), DMA_TO_DEVICE); if (fc_dma_mapping_error(tgtport->dev, iod->rspdma)) goto out_fail; @@ -361,7 +359,7 @@ out_fail: list_del(&iod->ls_list); for (iod--, i--; i >= 0; iod--, i--) { fc_dma_unmap_single(tgtport->dev, iod->rspdma, - NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE); + sizeof(*iod->rspbuf), DMA_TO_DEVICE); kfree(iod->rqstbuf); list_del(&iod->ls_list); } @@ -379,7 +377,7 @@ nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport) for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) { fc_dma_unmap_single(tgtport->dev, - iod->rspdma, NVME_FC_MAX_LS_BUFFER_SIZE, + iod->rspdma, sizeof(*iod->rspbuf), DMA_TO_DEVICE); kfree(iod->rqstbuf); list_del(&iod->ls_list); @@ -1262,10 +1260,8 @@ static void nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod) { - struct fcnvme_ls_cr_assoc_rqst *rqst = - (struct fcnvme_ls_cr_assoc_rqst *)iod->rqstbuf; - struct fcnvme_ls_cr_assoc_acc *acc = - (struct fcnvme_ls_cr_assoc_acc *)iod->rspbuf; + struct fcnvme_ls_cr_assoc_rqst *rqst = &iod->rqstbuf->rq_cr_assoc; + struct fcnvme_ls_cr_assoc_acc *acc = &iod->rspbuf->rsp_cr_assoc; struct nvmet_fc_tgt_queue *queue; int ret = 0; @@ -1313,7 +1309,7 @@ nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport, "Create Association LS failed: %s\n", validation_errors[ret]); iod->lsrsp->rsplen = nvme_fc_format_rjt(acc, - NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, + sizeof(*acc), rqst->w0.ls_cmd, FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0); return; @@ -1348,10 +1344,8 @@ static void nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod) { - struct fcnvme_ls_cr_conn_rqst *rqst = - (struct fcnvme_ls_cr_conn_rqst *)iod->rqstbuf; - struct fcnvme_ls_cr_conn_acc *acc = - (struct fcnvme_ls_cr_conn_acc *)iod->rspbuf; + struct fcnvme_ls_cr_conn_rqst *rqst = &iod->rqstbuf->rq_cr_conn; + struct fcnvme_ls_cr_conn_acc *acc = &iod->rspbuf->rsp_cr_conn; struct nvmet_fc_tgt_queue *queue; int ret = 0; @@ -1404,7 +1398,7 @@ nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport, "Create Connection LS failed: %s\n", validation_errors[ret]); iod->lsrsp->rsplen = nvme_fc_format_rjt(acc, - NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, + sizeof(*acc), rqst->w0.ls_cmd, (ret == VERR_NO_ASSOC) ? FCNVME_RJT_RC_INV_ASSOC : FCNVME_RJT_RC_LOGIC, @@ -1437,9 +1431,9 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod) { struct fcnvme_ls_disconnect_assoc_rqst *rqst = - (struct fcnvme_ls_disconnect_assoc_rqst *)iod->rqstbuf; + &iod->rqstbuf->rq_dis_assoc; struct fcnvme_ls_disconnect_assoc_acc *acc = - (struct fcnvme_ls_disconnect_assoc_acc *)iod->rspbuf; + &iod->rspbuf->rsp_dis_assoc; struct nvmet_fc_tgt_assoc *assoc; int ret = 0; @@ -1484,7 +1478,7 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, "Disconnect LS failed: %s\n", validation_errors[ret]); iod->lsrsp->rsplen = nvme_fc_format_rjt(acc, - NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, + sizeof(*acc), rqst->w0.ls_cmd, (ret == VERR_NO_ASSOC) ? FCNVME_RJT_RC_INV_ASSOC : FCNVME_RJT_RC_LOGIC, @@ -1522,7 +1516,7 @@ nvmet_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp) struct nvmet_fc_tgtport *tgtport = iod->tgtport; fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma, - NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE); + sizeof(*iod->rspbuf), DMA_TO_DEVICE); nvmet_fc_free_ls_iod(tgtport, iod); nvmet_fc_tgtport_put(tgtport); } @@ -1534,7 +1528,7 @@ nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport, int ret; fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma, - NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE); + sizeof(*iod->rspbuf), DMA_TO_DEVICE); ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsrsp); if (ret) @@ -1548,8 +1542,7 @@ static void nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod) { - struct fcnvme_ls_rqst_w0 *w0 = - (struct fcnvme_ls_rqst_w0 *)iod->rqstbuf; + struct fcnvme_ls_rqst_w0 *w0 = &iod->rqstbuf->rq_cr_assoc.w0; iod->lsrsp->nvme_fc_private = iod; iod->lsrsp->rspbuf = iod->rspbuf; @@ -1580,7 +1573,7 @@ nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport, break; default: iod->lsrsp->rsplen = nvme_fc_format_rjt(iod->rspbuf, - NVME_FC_MAX_LS_BUFFER_SIZE, w0->ls_cmd, + sizeof(*iod->rspbuf), w0->ls_cmd, FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0); } @@ -1627,7 +1620,7 @@ nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port, struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); struct nvmet_fc_ls_iod *iod; - if (lsreqbuf_len > NVME_FC_MAX_LS_BUFFER_SIZE) + if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) return -E2BIG; if (!nvmet_fc_tgtport_get(tgtport)) From f56bf76f79f3dc15f17433dda1b567d34f18e699 Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:49:50 -0700 Subject: [PATCH 0310/1043] nvme-fc: Ensure private pointers are NULL if no data Ensure that when allocations are done, and the lldd options indicate no private data is needed, that private pointers will be set to NULL (catches driver error that forgot to set private data size). Slightly reorg the allocations so that private data follows allocations for LS request/response buffers. Ensures better alignments for the buffers as well as the private pointer. Signed-off-by: James Smart Reviewed-by: Sagi Grimberg Reviewed-by: Hannes Reinecke Reviewed-by: Himanshu Madhani Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/fc.c | 81 +++++++++++++++++++++++++--------------- drivers/nvme/target/fc.c | 5 ++- 2 files changed, 54 insertions(+), 32 deletions(-) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 83f9b2ac7c55..bf80b941d739 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -395,7 +395,10 @@ nvme_fc_register_localport(struct nvme_fc_port_info *pinfo, newrec->ops = template; newrec->dev = dev; ida_init(&newrec->endp_cnt); - newrec->localport.private = &newrec[1]; + if (template->local_priv_sz) + newrec->localport.private = &newrec[1]; + else + newrec->localport.private = NULL; newrec->localport.node_name = pinfo->node_name; newrec->localport.port_name = pinfo->port_name; newrec->localport.port_role = pinfo->port_role; @@ -704,7 +707,10 @@ nvme_fc_register_remoteport(struct nvme_fc_local_port *localport, newrec->remoteport.localport = &lport->localport; newrec->dev = lport->dev; newrec->lport = lport; - newrec->remoteport.private = &newrec[1]; + if (lport->ops->remote_priv_sz) + newrec->remoteport.private = &newrec[1]; + else + newrec->remoteport.private = NULL; newrec->remoteport.port_role = pinfo->port_role; newrec->remoteport.node_name = pinfo->node_name; newrec->remoteport.port_name = pinfo->port_name; @@ -1152,18 +1158,23 @@ nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl, int ret, fcret = 0; lsop = kzalloc((sizeof(*lsop) + - ctrl->lport->ops->lsrqst_priv_sz + - sizeof(*assoc_rqst) + sizeof(*assoc_acc)), GFP_KERNEL); + sizeof(*assoc_rqst) + sizeof(*assoc_acc) + + ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL); if (!lsop) { + dev_info(ctrl->ctrl.device, + "NVME-FC{%d}: send Create Association failed: ENOMEM\n", + ctrl->cnum); ret = -ENOMEM; goto out_no_memory; } - lsreq = &lsop->ls_req; - lsreq->private = (void *)&lsop[1]; - assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *) - (lsreq->private + ctrl->lport->ops->lsrqst_priv_sz); + assoc_rqst = (struct fcnvme_ls_cr_assoc_rqst *)&lsop[1]; assoc_acc = (struct fcnvme_ls_cr_assoc_acc *)&assoc_rqst[1]; + lsreq = &lsop->ls_req; + if (ctrl->lport->ops->lsrqst_priv_sz) + lsreq->private = &assoc_acc[1]; + else + lsreq->private = NULL; assoc_rqst->w0.ls_cmd = FCNVME_LS_CREATE_ASSOCIATION; assoc_rqst->desc_list_len = @@ -1261,18 +1272,23 @@ nvme_fc_connect_queue(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, int ret, fcret = 0; lsop = kzalloc((sizeof(*lsop) + - ctrl->lport->ops->lsrqst_priv_sz + - sizeof(*conn_rqst) + sizeof(*conn_acc)), GFP_KERNEL); + sizeof(*conn_rqst) + sizeof(*conn_acc) + + ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL); if (!lsop) { + dev_info(ctrl->ctrl.device, + "NVME-FC{%d}: send Create Connection failed: ENOMEM\n", + ctrl->cnum); ret = -ENOMEM; goto out_no_memory; } - lsreq = &lsop->ls_req; - lsreq->private = (void *)&lsop[1]; - conn_rqst = (struct fcnvme_ls_cr_conn_rqst *) - (lsreq->private + ctrl->lport->ops->lsrqst_priv_sz); + conn_rqst = (struct fcnvme_ls_cr_conn_rqst *)&lsop[1]; conn_acc = (struct fcnvme_ls_cr_conn_acc *)&conn_rqst[1]; + lsreq = &lsop->ls_req; + if (ctrl->lport->ops->lsrqst_priv_sz) + lsreq->private = (void *)&conn_acc[1]; + else + lsreq->private = NULL; conn_rqst->w0.ls_cmd = FCNVME_LS_CREATE_CONNECTION; conn_rqst->desc_list_len = cpu_to_be32( @@ -1386,19 +1402,23 @@ nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl) int ret; lsop = kzalloc((sizeof(*lsop) + - ctrl->lport->ops->lsrqst_priv_sz + - sizeof(*discon_rqst) + sizeof(*discon_acc)), - GFP_KERNEL); - if (!lsop) - /* couldn't sent it... too bad */ + sizeof(*discon_rqst) + sizeof(*discon_acc) + + ctrl->lport->ops->lsrqst_priv_sz), GFP_KERNEL); + if (!lsop) { + dev_info(ctrl->ctrl.device, + "NVME-FC{%d}: send Disconnect Association " + "failed: ENOMEM\n", + ctrl->cnum); return; + } - lsreq = &lsop->ls_req; - - lsreq->private = (void *)&lsop[1]; - discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *) - (lsreq->private + ctrl->lport->ops->lsrqst_priv_sz); + discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1]; discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1]; + lsreq = &lsop->ls_req; + if (ctrl->lport->ops->lsrqst_priv_sz) + lsreq->private = (void *)&discon_acc[1]; + else + lsreq->private = NULL; discon_rqst->w0.ls_cmd = FCNVME_LS_DISCONNECT_ASSOC; discon_rqst->desc_list_len = cpu_to_be32( @@ -1784,15 +1804,17 @@ nvme_fc_init_aen_ops(struct nvme_fc_ctrl *ctrl) struct nvme_fc_fcp_op *aen_op; struct nvme_fc_cmd_iu *cmdiu; struct nvme_command *sqe; - void *private; + void *private = NULL; int i, ret; aen_op = ctrl->aen_ops; for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) { - private = kzalloc(ctrl->lport->ops->fcprqst_priv_sz, + if (ctrl->lport->ops->fcprqst_priv_sz) { + private = kzalloc(ctrl->lport->ops->fcprqst_priv_sz, GFP_KERNEL); - if (!private) - return -ENOMEM; + if (!private) + return -ENOMEM; + } cmdiu = &aen_op->cmd_iu; sqe = &cmdiu->sqe; @@ -1823,9 +1845,6 @@ nvme_fc_term_aen_ops(struct nvme_fc_ctrl *ctrl) aen_op = ctrl->aen_ops; for (i = 0; i < NVME_NR_AEN_COMMANDS; i++, aen_op++) { - if (!aen_op->fcp_req.private) - continue; - __nvme_fc_exit_request(ctrl, aen_op); kfree(aen_op->fcp_req.private); diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index 66de6bd8f4fd..66a60a218994 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -1047,7 +1047,10 @@ nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo, newrec->fc_target_port.node_name = pinfo->node_name; newrec->fc_target_port.port_name = pinfo->port_name; - newrec->fc_target_port.private = &newrec[1]; + if (template->target_priv_sz) + newrec->fc_target_port.private = &newrec[1]; + else + newrec->fc_target_port.private = NULL; newrec->fc_target_port.port_id = pinfo->port_id; newrec->fc_target_port.port_num = idx; INIT_LIST_HEAD(&newrec->tgt_list); From eb4ee8f125157926cf36a3c275b04825f1bf8cfa Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:49:51 -0700 Subject: [PATCH 0311/1043] nvme-fc: convert assoc_active flag to bit op Convert the assoc_active boolean flag to a bitop on the flags field. The bit ops will provide atomicity. To make this change, the flags field was converted to a long type, which also affects the FCCTRL_TERMIO flag. Both FCCTRL_TERMIO and now ASSOC_ACTIVE flags are set/cleared by bit operations. Signed-off-by: James Smart Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/fc.c | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index bf80b941d739..0ac246603063 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -127,9 +127,9 @@ struct nvme_fc_rport { unsigned long dev_loss_end; } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ -enum nvme_fcctrl_flags { - FCCTRL_TERMIO = (1 << 0), -}; +/* fc_ctrl flags values - specified as bit positions */ +#define ASSOC_ACTIVE 0 +#define FCCTRL_TERMIO 1 struct nvme_fc_ctrl { spinlock_t lock; @@ -140,7 +140,6 @@ struct nvme_fc_ctrl { u32 cnum; bool ioq_live; - bool assoc_active; atomic_t err_work_active; u64 association_id; @@ -153,7 +152,7 @@ struct nvme_fc_ctrl { struct work_struct err_work; struct kref ref; - u32 flags; + unsigned long flags; u32 iocnt; wait_queue_head_t ioabort_wait; @@ -1521,7 +1520,7 @@ __nvme_fc_abort_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op) opstate = atomic_xchg(&op->state, FCPOP_STATE_ABORTED); if (opstate != FCPOP_STATE_ACTIVE) atomic_set(&op->state, opstate); - else if (ctrl->flags & FCCTRL_TERMIO) + else if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) ctrl->iocnt++; spin_unlock_irqrestore(&ctrl->lock, flags); @@ -1558,7 +1557,7 @@ __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl, if (opstate == FCPOP_STATE_ABORTED) { spin_lock_irqsave(&ctrl->lock, flags); - if (ctrl->flags & FCCTRL_TERMIO) { + if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) { if (!--ctrl->iocnt) wake_up(&ctrl->ioabort_wait); } @@ -2386,16 +2385,9 @@ nvme_fc_submit_async_event(struct nvme_ctrl *arg) { struct nvme_fc_ctrl *ctrl = to_fc_ctrl(arg); struct nvme_fc_fcp_op *aen_op; - unsigned long flags; - bool terminating = false; blk_status_t ret; - spin_lock_irqsave(&ctrl->lock, flags); - if (ctrl->flags & FCCTRL_TERMIO) - terminating = true; - spin_unlock_irqrestore(&ctrl->lock, flags); - - if (terminating) + if (test_bit(FCCTRL_TERMIO, &ctrl->flags)) return; aen_op = &ctrl->aen_ops[0]; @@ -2604,10 +2596,9 @@ nvme_fc_ctlr_active_on_rport(struct nvme_fc_ctrl *ctrl) struct nvme_fc_rport *rport = ctrl->rport; u32 cnt; - if (ctrl->assoc_active) + if (test_and_set_bit(ASSOC_ACTIVE, &ctrl->flags)) return 1; - ctrl->assoc_active = true; cnt = atomic_inc_return(&rport->act_ctrl_cnt); if (cnt == 1) nvme_fc_rport_active_on_lport(rport); @@ -2622,7 +2613,7 @@ nvme_fc_ctlr_inactive_on_rport(struct nvme_fc_ctrl *ctrl) struct nvme_fc_lport *lport = rport->lport; u32 cnt; - /* ctrl->assoc_active=false will be set independently */ + /* clearing of ctrl->flags ASSOC_ACTIVE bit is in association delete */ cnt = atomic_dec_return(&rport->act_ctrl_cnt); if (cnt == 0) { @@ -2764,7 +2755,7 @@ out_delete_hw_queue: __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0); out_free_queue: nvme_fc_free_queue(&ctrl->queues[0]); - ctrl->assoc_active = false; + clear_bit(ASSOC_ACTIVE, &ctrl->flags); nvme_fc_ctlr_inactive_on_rport(ctrl); return ret; @@ -2781,12 +2772,11 @@ nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl) { unsigned long flags; - if (!ctrl->assoc_active) + if (!test_and_clear_bit(ASSOC_ACTIVE, &ctrl->flags)) return; - ctrl->assoc_active = false; spin_lock_irqsave(&ctrl->lock, flags); - ctrl->flags |= FCCTRL_TERMIO; + set_bit(FCCTRL_TERMIO, &ctrl->flags); ctrl->iocnt = 0; spin_unlock_irqrestore(&ctrl->lock, flags); @@ -2837,7 +2827,7 @@ nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl) /* wait for all io that had to be aborted */ spin_lock_irq(&ctrl->lock); wait_event_lock_irq(ctrl->ioabort_wait, ctrl->iocnt == 0, ctrl->lock); - ctrl->flags &= ~FCCTRL_TERMIO; + clear_bit(FCCTRL_TERMIO, &ctrl->flags); spin_unlock_irq(&ctrl->lock); nvme_fc_term_aen_ops(ctrl); @@ -3109,7 +3099,6 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, ctrl->dev = lport->dev; ctrl->cnum = idx; ctrl->ioq_live = false; - ctrl->assoc_active = false; atomic_set(&ctrl->err_work_active, 0); init_waitqueue_head(&ctrl->ioabort_wait); From fd5a5f2213048b012bc7e19e832e9ae0ec1a2c4a Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:49:52 -0700 Subject: [PATCH 0312/1043] nvme-fc: Update header and host for common definitions for LS handling Given that both host and target now generate and receive LS's create a single table definition for LS names. Each tranport half will have a local version of the table. As Create Association LS is issued by both sides, and received by both sides, create common routines to format the LS and to validate the LS. Convert the host side transport to use the new common Create Association LS formatting routine. Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Reviewed-by: Himanshu Madhani Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/fc.c | 25 ++----------- drivers/nvme/host/fc.h | 79 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 23 deletions(-) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 0ac246603063..c069ab056202 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -1419,29 +1419,8 @@ nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl) else lsreq->private = NULL; - discon_rqst->w0.ls_cmd = FCNVME_LS_DISCONNECT_ASSOC; - discon_rqst->desc_list_len = cpu_to_be32( - sizeof(struct fcnvme_lsdesc_assoc_id) + - sizeof(struct fcnvme_lsdesc_disconn_cmd)); - - discon_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID); - discon_rqst->associd.desc_len = - fcnvme_lsdesc_len( - sizeof(struct fcnvme_lsdesc_assoc_id)); - - discon_rqst->associd.association_id = cpu_to_be64(ctrl->association_id); - - discon_rqst->discon_cmd.desc_tag = cpu_to_be32( - FCNVME_LSDESC_DISCONN_CMD); - discon_rqst->discon_cmd.desc_len = - fcnvme_lsdesc_len( - sizeof(struct fcnvme_lsdesc_disconn_cmd)); - - lsreq->rqstaddr = discon_rqst; - lsreq->rqstlen = sizeof(*discon_rqst); - lsreq->rspaddr = discon_acc; - lsreq->rsplen = sizeof(*discon_acc); - lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC; + nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc, + ctrl->association_id); ret = nvme_fc_send_ls_req_async(ctrl->rport, lsop, nvme_fc_disconnect_assoc_done); diff --git a/drivers/nvme/host/fc.h b/drivers/nvme/host/fc.h index 08fa88381d45..05ce566f2caf 100644 --- a/drivers/nvme/host/fc.h +++ b/drivers/nvme/host/fc.h @@ -17,6 +17,7 @@ */ union nvmefc_ls_requests { + struct fcnvme_ls_rqst_w0 w0; struct fcnvme_ls_cr_assoc_rqst rq_cr_assoc; struct fcnvme_ls_cr_conn_rqst rq_cr_conn; struct fcnvme_ls_disconnect_assoc_rqst rq_dis_assoc; @@ -145,4 +146,82 @@ static char *validation_errors[] = { "Bad Disconnect ACC Length", }; +#define NVME_FC_LAST_LS_CMD_VALUE FCNVME_LS_DISCONNECT_CONN + +static char *nvmefc_ls_names[] = { + "Reserved (0)", + "RJT (1)", + "ACC (2)", + "Create Association", + "Create Connection", + "Disconnect Association", + "Disconnect Connection", +}; + +static inline void +nvmefc_fmt_lsreq_discon_assoc(struct nvmefc_ls_req *lsreq, + struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst, + struct fcnvme_ls_disconnect_assoc_acc *discon_acc, + u64 association_id) +{ + lsreq->rqstaddr = discon_rqst; + lsreq->rqstlen = sizeof(*discon_rqst); + lsreq->rspaddr = discon_acc; + lsreq->rsplen = sizeof(*discon_acc); + lsreq->timeout = NVME_FC_LS_TIMEOUT_SEC; + + discon_rqst->w0.ls_cmd = FCNVME_LS_DISCONNECT_ASSOC; + discon_rqst->desc_list_len = cpu_to_be32( + sizeof(struct fcnvme_lsdesc_assoc_id) + + sizeof(struct fcnvme_lsdesc_disconn_cmd)); + + discon_rqst->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID); + discon_rqst->associd.desc_len = + fcnvme_lsdesc_len( + sizeof(struct fcnvme_lsdesc_assoc_id)); + + discon_rqst->associd.association_id = cpu_to_be64(association_id); + + discon_rqst->discon_cmd.desc_tag = cpu_to_be32( + FCNVME_LSDESC_DISCONN_CMD); + discon_rqst->discon_cmd.desc_len = + fcnvme_lsdesc_len( + sizeof(struct fcnvme_lsdesc_disconn_cmd)); +} + +static inline int +nvmefc_vldt_lsreq_discon_assoc(u32 rqstlen, + struct fcnvme_ls_disconnect_assoc_rqst *rqst) +{ + int ret = 0; + + if (rqstlen < sizeof(struct fcnvme_ls_disconnect_assoc_rqst)) + ret = VERR_DISCONN_LEN; + else if (rqst->desc_list_len != + fcnvme_lsdesc_len( + sizeof(struct fcnvme_ls_disconnect_assoc_rqst))) + ret = VERR_DISCONN_RQST_LEN; + else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID)) + ret = VERR_ASSOC_ID; + else if (rqst->associd.desc_len != + fcnvme_lsdesc_len( + sizeof(struct fcnvme_lsdesc_assoc_id))) + ret = VERR_ASSOC_ID_LEN; + else if (rqst->discon_cmd.desc_tag != + cpu_to_be32(FCNVME_LSDESC_DISCONN_CMD)) + ret = VERR_DISCONN_CMD; + else if (rqst->discon_cmd.desc_len != + fcnvme_lsdesc_len( + sizeof(struct fcnvme_lsdesc_disconn_cmd))) + ret = VERR_DISCONN_CMD_LEN; + /* + * As the standard changed on the LS, check if old format and scope + * something other than Association (e.g. 0). + */ + else if (rqst->discon_cmd.rsvd8[0]) + ret = VERR_DISCONN_SCOPE; + + return ret; +} + #endif /* _NVME_FC_TRANSPORT_H */ From ec3b0e3cc393dee1ad3f4bd1026f2c1f8b1c1ffb Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:49:53 -0700 Subject: [PATCH 0313/1043] nvmet-fc: Update target for common definitions for LS handling Given that both host and target now generate and receive LS's create a single table definition for LS names. Each tranport half will have a local version of the table. Convert the target side transport to use the new common Create Association LS validation routine. Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Reviewed-by: Himanshu Madhani Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/fc.c | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index 66a60a218994..5739df7edc59 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -1442,32 +1442,8 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, memset(acc, 0, sizeof(*acc)); - if (iod->rqstdatalen < sizeof(struct fcnvme_ls_disconnect_assoc_rqst)) - ret = VERR_DISCONN_LEN; - else if (rqst->desc_list_len != - fcnvme_lsdesc_len( - sizeof(struct fcnvme_ls_disconnect_assoc_rqst))) - ret = VERR_DISCONN_RQST_LEN; - else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID)) - ret = VERR_ASSOC_ID; - else if (rqst->associd.desc_len != - fcnvme_lsdesc_len( - sizeof(struct fcnvme_lsdesc_assoc_id))) - ret = VERR_ASSOC_ID_LEN; - else if (rqst->discon_cmd.desc_tag != - cpu_to_be32(FCNVME_LSDESC_DISCONN_CMD)) - ret = VERR_DISCONN_CMD; - else if (rqst->discon_cmd.desc_len != - fcnvme_lsdesc_len( - sizeof(struct fcnvme_lsdesc_disconn_cmd))) - ret = VERR_DISCONN_CMD_LEN; - /* - * As the standard changed on the LS, check if old format and scope - * something other than Association (e.g. 0). - */ - else if (rqst->discon_cmd.rsvd8[0]) - ret = VERR_DISCONN_SCOPE; - else { + ret = nvmefc_vldt_lsreq_discon_assoc(iod->rqstdatalen, rqst); + if (!ret) { /* match an active association */ assoc = nvmet_fc_find_target_assoc(tgtport, be64_to_cpu(rqst->associd.association_id)); From 14fd1e98afafc0027a6a86ea1962e31dceafb400 Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:49:54 -0700 Subject: [PATCH 0314/1043] nvme-fc: Add Disconnect Association Rcv support The nvme-fc host transport did not support the reception of a FC-NVME LS. Reception is necessary to implement full compliance with FC-NVME-2. Populate the LS receive handler, and specifically the handling of a Disconnect Association LS. The response to the LS, if it matched a controller, must be sent after the aborts for any I/O on any connection have been sent. Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/fc.c | 363 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 359 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index c069ab056202..1921d2195541 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -62,6 +62,17 @@ struct nvmefc_ls_req_op { bool req_queued; }; +struct nvmefc_ls_rcv_op { + struct nvme_fc_rport *rport; + struct nvmefc_ls_rsp *lsrsp; + union nvmefc_ls_requests *rqstbuf; + union nvmefc_ls_responses *rspbuf; + u16 rqstdatalen; + bool handled; + dma_addr_t rspdma; + struct list_head lsrcv_list; /* rport->ls_rcv_list */ +} __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ + enum nvme_fcpop_state { FCPOP_STATE_UNINIT = 0, FCPOP_STATE_IDLE = 1, @@ -118,6 +129,7 @@ struct nvme_fc_rport { struct list_head endp_list; /* for lport->endp_list */ struct list_head ctrl_list; struct list_head ls_req_list; + struct list_head ls_rcv_list; struct list_head disc_list; struct device *dev; /* physical device for dma */ struct nvme_fc_lport *lport; @@ -125,6 +137,7 @@ struct nvme_fc_rport { struct kref ref; atomic_t act_ctrl_cnt; unsigned long dev_loss_end; + struct work_struct lsrcv_work; } __aligned(sizeof(u64)); /* alignment for other things alloc'd with */ /* fc_ctrl flags values - specified as bit positions */ @@ -142,6 +155,7 @@ struct nvme_fc_ctrl { bool ioq_live; atomic_t err_work_active; u64 association_id; + struct nvmefc_ls_rcv_op *rcv_disconn; struct list_head ctrl_list; /* rport->ctrl_list */ @@ -219,6 +233,9 @@ static struct device *fc_udev_device; static void __nvme_fc_delete_hw_queue(struct nvme_fc_ctrl *, struct nvme_fc_queue *, unsigned int); +static void nvme_fc_handle_ls_rqst_work(struct work_struct *work); + + static void nvme_fc_free_lport(struct kref *ref) { @@ -704,6 +721,7 @@ nvme_fc_register_remoteport(struct nvme_fc_local_port *localport, atomic_set(&newrec->act_ctrl_cnt, 0); spin_lock_init(&newrec->lock); newrec->remoteport.localport = &lport->localport; + INIT_LIST_HEAD(&newrec->ls_rcv_list); newrec->dev = lport->dev; newrec->lport = lport; if (lport->ops->remote_priv_sz) @@ -717,6 +735,7 @@ nvme_fc_register_remoteport(struct nvme_fc_local_port *localport, newrec->remoteport.port_state = FC_OBJSTATE_ONLINE; newrec->remoteport.port_num = idx; __nvme_fc_set_dev_loss_tmo(newrec, pinfo); + INIT_WORK(&newrec->lsrcv_work, nvme_fc_handle_ls_rqst_work); spin_lock_irqsave(&nvme_fc_lock, flags); list_add_tail(&newrec->endp_list, &lport->endp_list); @@ -1006,6 +1025,7 @@ fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *); static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *); +static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg); static void __nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop) @@ -1154,6 +1174,7 @@ nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl, struct nvmefc_ls_req *lsreq; struct fcnvme_ls_cr_assoc_rqst *assoc_rqst; struct fcnvme_ls_cr_assoc_acc *assoc_acc; + unsigned long flags; int ret, fcret = 0; lsop = kzalloc((sizeof(*lsop) + @@ -1243,11 +1264,13 @@ nvme_fc_connect_admin_queue(struct nvme_fc_ctrl *ctrl, "q %d Create Association LS failed: %s\n", queue->qnum, validation_errors[fcret]); } else { + spin_lock_irqsave(&ctrl->lock, flags); ctrl->association_id = be64_to_cpu(assoc_acc->associd.association_id); queue->connection_id = be64_to_cpu(assoc_acc->connectid.connection_id); set_bit(NVME_FC_Q_CONNECTED, &queue->flags); + spin_unlock_irqrestore(&ctrl->lock, flags); } out_free_buffer: @@ -1428,6 +1451,247 @@ nvme_fc_xmt_disconnect_assoc(struct nvme_fc_ctrl *ctrl) kfree(lsop); } +static void +nvme_fc_xmt_ls_rsp_done(struct nvmefc_ls_rsp *lsrsp) +{ + struct nvmefc_ls_rcv_op *lsop = lsrsp->nvme_fc_private; + struct nvme_fc_rport *rport = lsop->rport; + struct nvme_fc_lport *lport = rport->lport; + unsigned long flags; + + spin_lock_irqsave(&rport->lock, flags); + list_del(&lsop->lsrcv_list); + spin_unlock_irqrestore(&rport->lock, flags); + + fc_dma_sync_single_for_cpu(lport->dev, lsop->rspdma, + sizeof(*lsop->rspbuf), DMA_TO_DEVICE); + fc_dma_unmap_single(lport->dev, lsop->rspdma, + sizeof(*lsop->rspbuf), DMA_TO_DEVICE); + + kfree(lsop); + + nvme_fc_rport_put(rport); +} + +static void +nvme_fc_xmt_ls_rsp(struct nvmefc_ls_rcv_op *lsop) +{ + struct nvme_fc_rport *rport = lsop->rport; + struct nvme_fc_lport *lport = rport->lport; + struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0; + int ret; + + fc_dma_sync_single_for_device(lport->dev, lsop->rspdma, + sizeof(*lsop->rspbuf), DMA_TO_DEVICE); + + ret = lport->ops->xmt_ls_rsp(&lport->localport, &rport->remoteport, + lsop->lsrsp); + if (ret) { + dev_warn(lport->dev, + "LLDD rejected LS RSP xmt: LS %d status %d\n", + w0->ls_cmd, ret); + nvme_fc_xmt_ls_rsp_done(lsop->lsrsp); + return; + } +} + +static struct nvme_fc_ctrl * +nvme_fc_match_disconn_ls(struct nvme_fc_rport *rport, + struct nvmefc_ls_rcv_op *lsop) +{ + struct fcnvme_ls_disconnect_assoc_rqst *rqst = + &lsop->rqstbuf->rq_dis_assoc; + struct nvme_fc_ctrl *ctrl, *ret = NULL; + struct nvmefc_ls_rcv_op *oldls = NULL; + u64 association_id = be64_to_cpu(rqst->associd.association_id); + unsigned long flags; + + spin_lock_irqsave(&rport->lock, flags); + + list_for_each_entry(ctrl, &rport->ctrl_list, ctrl_list) { + if (!nvme_fc_ctrl_get(ctrl)) + continue; + spin_lock(&ctrl->lock); + if (association_id == ctrl->association_id) { + oldls = ctrl->rcv_disconn; + ctrl->rcv_disconn = lsop; + ret = ctrl; + } + spin_unlock(&ctrl->lock); + if (ret) + /* leave the ctrl get reference */ + break; + nvme_fc_ctrl_put(ctrl); + } + + spin_unlock_irqrestore(&rport->lock, flags); + + /* transmit a response for anything that was pending */ + if (oldls) { + dev_info(rport->lport->dev, + "NVME-FC{%d}: Multiple Disconnect Association " + "LS's received\n", ctrl->cnum); + /* overwrite good response with bogus failure */ + oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf, + sizeof(*oldls->rspbuf), + rqst->w0.ls_cmd, + FCNVME_RJT_RC_UNAB, + FCNVME_RJT_EXP_NONE, 0); + nvme_fc_xmt_ls_rsp(oldls); + } + + return ret; +} + +/* + * returns true to mean LS handled and ls_rsp can be sent + * returns false to defer ls_rsp xmt (will be done as part of + * association termination) + */ +static bool +nvme_fc_ls_disconnect_assoc(struct nvmefc_ls_rcv_op *lsop) +{ + struct nvme_fc_rport *rport = lsop->rport; + struct fcnvme_ls_disconnect_assoc_rqst *rqst = + &lsop->rqstbuf->rq_dis_assoc; + struct fcnvme_ls_disconnect_assoc_acc *acc = + &lsop->rspbuf->rsp_dis_assoc; + struct nvme_fc_ctrl *ctrl = NULL; + int ret = 0; + + memset(acc, 0, sizeof(*acc)); + + ret = nvmefc_vldt_lsreq_discon_assoc(lsop->rqstdatalen, rqst); + if (!ret) { + /* match an active association */ + ctrl = nvme_fc_match_disconn_ls(rport, lsop); + if (!ctrl) + ret = VERR_NO_ASSOC; + } + + if (ret) { + dev_info(rport->lport->dev, + "Disconnect LS failed: %s\n", + validation_errors[ret]); + lsop->lsrsp->rsplen = nvme_fc_format_rjt(acc, + sizeof(*acc), rqst->w0.ls_cmd, + (ret == VERR_NO_ASSOC) ? + FCNVME_RJT_RC_INV_ASSOC : + FCNVME_RJT_RC_LOGIC, + FCNVME_RJT_EXP_NONE, 0); + return true; + } + + /* format an ACCept response */ + + lsop->lsrsp->rsplen = sizeof(*acc); + + nvme_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, + fcnvme_lsdesc_len( + sizeof(struct fcnvme_ls_disconnect_assoc_acc)), + FCNVME_LS_DISCONNECT_ASSOC); + + /* + * the transmit of the response will occur after the exchanges + * for the association have been ABTS'd by + * nvme_fc_delete_association(). + */ + + /* fail the association */ + nvme_fc_error_recovery(ctrl, "Disconnect Association LS received"); + + /* release the reference taken by nvme_fc_match_disconn_ls() */ + nvme_fc_ctrl_put(ctrl); + + return false; +} + +/* + * Actual Processing routine for received FC-NVME LS Requests from the LLD + * returns true if a response should be sent afterward, false if rsp will + * be sent asynchronously. + */ +static bool +nvme_fc_handle_ls_rqst(struct nvmefc_ls_rcv_op *lsop) +{ + struct fcnvme_ls_rqst_w0 *w0 = &lsop->rqstbuf->w0; + bool ret = true; + + lsop->lsrsp->nvme_fc_private = lsop; + lsop->lsrsp->rspbuf = lsop->rspbuf; + lsop->lsrsp->rspdma = lsop->rspdma; + lsop->lsrsp->done = nvme_fc_xmt_ls_rsp_done; + /* Be preventative. handlers will later set to valid length */ + lsop->lsrsp->rsplen = 0; + + /* + * handlers: + * parse request input, execute the request, and format the + * LS response + */ + switch (w0->ls_cmd) { + case FCNVME_LS_DISCONNECT_ASSOC: + ret = nvme_fc_ls_disconnect_assoc(lsop); + break; + case FCNVME_LS_DISCONNECT_CONN: + lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf, + sizeof(*lsop->rspbuf), w0->ls_cmd, + FCNVME_RJT_RC_UNSUP, FCNVME_RJT_EXP_NONE, 0); + break; + case FCNVME_LS_CREATE_ASSOCIATION: + case FCNVME_LS_CREATE_CONNECTION: + lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf, + sizeof(*lsop->rspbuf), w0->ls_cmd, + FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0); + break; + default: + lsop->lsrsp->rsplen = nvme_fc_format_rjt(lsop->rspbuf, + sizeof(*lsop->rspbuf), w0->ls_cmd, + FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0); + break; + } + + return(ret); +} + +static void +nvme_fc_handle_ls_rqst_work(struct work_struct *work) +{ + struct nvme_fc_rport *rport = + container_of(work, struct nvme_fc_rport, lsrcv_work); + struct fcnvme_ls_rqst_w0 *w0; + struct nvmefc_ls_rcv_op *lsop; + unsigned long flags; + bool sendrsp; + +restart: + sendrsp = true; + spin_lock_irqsave(&rport->lock, flags); + list_for_each_entry(lsop, &rport->ls_rcv_list, lsrcv_list) { + if (lsop->handled) + continue; + + lsop->handled = true; + if (rport->remoteport.port_state == FC_OBJSTATE_ONLINE) { + spin_unlock_irqrestore(&rport->lock, flags); + sendrsp = nvme_fc_handle_ls_rqst(lsop); + } else { + spin_unlock_irqrestore(&rport->lock, flags); + w0 = &lsop->rqstbuf->w0; + lsop->lsrsp->rsplen = nvme_fc_format_rjt( + lsop->rspbuf, + sizeof(*lsop->rspbuf), + w0->ls_cmd, + FCNVME_RJT_RC_UNAB, + FCNVME_RJT_EXP_NONE, 0); + } + if (sendrsp) + nvme_fc_xmt_ls_rsp(lsop); + goto restart; + } + spin_unlock_irqrestore(&rport->lock, flags); +} + /** * nvme_fc_rcv_ls_req - transport entry point called by an LLDD * upon the reception of a NVME LS request. @@ -1454,20 +1718,92 @@ nvme_fc_rcv_ls_req(struct nvme_fc_remote_port *portptr, { struct nvme_fc_rport *rport = remoteport_to_rport(portptr); struct nvme_fc_lport *lport = rport->lport; + struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf; + struct nvmefc_ls_rcv_op *lsop; + unsigned long flags; + int ret; + + nvme_fc_rport_get(rport); /* validate there's a routine to transmit a response */ - if (!lport->ops->xmt_ls_rsp) - return(-EINVAL); + if (!lport->ops->xmt_ls_rsp) { + dev_info(lport->dev, + "RCV %s LS failed: no LLDD xmt_ls_rsp\n", + (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? + nvmefc_ls_names[w0->ls_cmd] : ""); + ret = -EINVAL; + goto out_put; + } + + if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) { + dev_info(lport->dev, + "RCV %s LS failed: payload too large\n", + (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? + nvmefc_ls_names[w0->ls_cmd] : ""); + ret = -E2BIG; + goto out_put; + } + + lsop = kzalloc(sizeof(*lsop) + + sizeof(union nvmefc_ls_requests) + + sizeof(union nvmefc_ls_responses), + GFP_KERNEL); + if (!lsop) { + dev_info(lport->dev, + "RCV %s LS failed: No memory\n", + (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? + nvmefc_ls_names[w0->ls_cmd] : ""); + ret = -ENOMEM; + goto out_put; + } + lsop->rqstbuf = (union nvmefc_ls_requests *)&lsop[1]; + lsop->rspbuf = (union nvmefc_ls_responses *)&lsop->rqstbuf[1]; + + lsop->rspdma = fc_dma_map_single(lport->dev, lsop->rspbuf, + sizeof(*lsop->rspbuf), + DMA_TO_DEVICE); + if (fc_dma_mapping_error(lport->dev, lsop->rspdma)) { + dev_info(lport->dev, + "RCV %s LS failed: DMA mapping failure\n", + (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? + nvmefc_ls_names[w0->ls_cmd] : ""); + ret = -EFAULT; + goto out_free; + } + + lsop->rport = rport; + lsop->lsrsp = lsrsp; + + memcpy(lsop->rqstbuf, lsreqbuf, lsreqbuf_len); + lsop->rqstdatalen = lsreqbuf_len; + + spin_lock_irqsave(&rport->lock, flags); + if (rport->remoteport.port_state != FC_OBJSTATE_ONLINE) { + spin_unlock_irqrestore(&rport->lock, flags); + ret = -ENOTCONN; + goto out_unmap; + } + list_add_tail(&lsop->lsrcv_list, &rport->ls_rcv_list); + spin_unlock_irqrestore(&rport->lock, flags); + + schedule_work(&rport->lsrcv_work); return 0; + +out_unmap: + fc_dma_unmap_single(lport->dev, lsop->rspdma, + sizeof(*lsop->rspbuf), DMA_TO_DEVICE); +out_free: + kfree(lsop); +out_put: + nvme_fc_rport_put(rport); + return ret; } EXPORT_SYMBOL_GPL(nvme_fc_rcv_ls_req); /* *********************** NVME Ctrl Routines **************************** */ -static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg); - static void __nvme_fc_exit_request(struct nvme_fc_ctrl *ctrl, struct nvme_fc_fcp_op *op) @@ -2612,6 +2948,8 @@ static int nvme_fc_create_association(struct nvme_fc_ctrl *ctrl) { struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; + struct nvmefc_ls_rcv_op *disls = NULL; + unsigned long flags; int ret; bool changed; @@ -2729,7 +3067,13 @@ out_term_aen_ops: out_disconnect_admin_queue: /* send a Disconnect(association) LS to fc-nvme target */ nvme_fc_xmt_disconnect_assoc(ctrl); + spin_lock_irqsave(&ctrl->lock, flags); ctrl->association_id = 0; + disls = ctrl->rcv_disconn; + ctrl->rcv_disconn = NULL; + spin_unlock_irqrestore(&ctrl->lock, flags); + if (disls) + nvme_fc_xmt_ls_rsp(disls); out_delete_hw_queue: __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[0], 0); out_free_queue: @@ -2749,6 +3093,7 @@ out_free_queue: static void nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl) { + struct nvmefc_ls_rcv_op *disls = NULL; unsigned long flags; if (!test_and_clear_bit(ASSOC_ACTIVE, &ctrl->flags)) @@ -2820,7 +3165,17 @@ nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl) if (ctrl->association_id) nvme_fc_xmt_disconnect_assoc(ctrl); + spin_lock_irqsave(&ctrl->lock, flags); ctrl->association_id = 0; + disls = ctrl->rcv_disconn; + ctrl->rcv_disconn = NULL; + spin_unlock_irqrestore(&ctrl->lock, flags); + if (disls) + /* + * if a Disconnect Request was waiting for a response, send + * now that all ABTS's have been issued (and are complete). + */ + nvme_fc_xmt_ls_rsp(disls); if (ctrl->ctrl.tagset) { nvme_fc_delete_hw_io_queues(ctrl); From a5c2b4f633cf06df62d24b0ef11f824e8da646a5 Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:49:55 -0700 Subject: [PATCH 0315/1043] nvmet-fc: add LS failure messages Add LS reception failure messages Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/fc.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index 5739df7edc59..a91c443c9098 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -1598,15 +1598,31 @@ nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port, { struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); struct nvmet_fc_ls_iod *iod; + struct fcnvme_ls_rqst_w0 *w0 = (struct fcnvme_ls_rqst_w0 *)lsreqbuf; - if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) + if (lsreqbuf_len > sizeof(union nvmefc_ls_requests)) { + dev_info(tgtport->dev, + "RCV %s LS failed: payload too large (%d)\n", + (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? + nvmefc_ls_names[w0->ls_cmd] : "", + lsreqbuf_len); return -E2BIG; + } - if (!nvmet_fc_tgtport_get(tgtport)) + if (!nvmet_fc_tgtport_get(tgtport)) { + dev_info(tgtport->dev, + "RCV %s LS failed: target deleting\n", + (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? + nvmefc_ls_names[w0->ls_cmd] : ""); return -ESHUTDOWN; + } iod = nvmet_fc_alloc_ls_iod(tgtport); if (!iod) { + dev_info(tgtport->dev, + "RCV %s LS failed: context allocation failed\n", + (w0->ls_cmd <= NVME_FC_LAST_LS_CMD_VALUE) ? + nvmefc_ls_names[w0->ls_cmd] : ""); nvmet_fc_tgtport_put(tgtport); return -ENOENT; } From 0dfb992e0ec2e7b9e5ccf92d2261aaa6b5cc57a8 Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:49:56 -0700 Subject: [PATCH 0316/1043] nvmet-fc: perform small cleanups on unneeded checks While code reviewing saw a couple of items that can be cleaned up: - In nvmet_fc_delete_target_queue(), the routine unlocks, then checks and relocks. Reorganize to avoid the unlock/relock. - In nvmet_fc_delete_target_queue(), there's a check on the disconnect state that is unnecessary as the routine validates the state before starting any action. Signed-off-by: James Smart Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/fc.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index a91c443c9098..01488fc35d46 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -677,7 +677,7 @@ nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue) struct nvmet_fc_fcp_iod *fod = queue->fod; struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr; unsigned long flags; - int i, writedataactive; + int i; bool disconnect; disconnect = atomic_xchg(&queue->connected, 0); @@ -688,20 +688,18 @@ nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue) if (fod->active) { spin_lock(&fod->flock); fod->abort = true; - writedataactive = fod->writedataactive; - spin_unlock(&fod->flock); /* * only call lldd abort routine if waiting for * writedata. other outstanding ops should finish * on their own. */ - if (writedataactive) { - spin_lock(&fod->flock); + if (fod->writedataactive) { fod->aborted = true; spin_unlock(&fod->flock); tgtport->ops->fcp_abort( &tgtport->fc_target_port, fod->fcpreq); - } + } else + spin_unlock(&fod->flock); } } @@ -741,8 +739,7 @@ nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue) flush_workqueue(queue->work_q); - if (disconnect) - nvmet_sq_destroy(&queue->nvme_sq); + nvmet_sq_destroy(&queue->nvme_sq); nvmet_fc_tgt_q_put(queue); } From 58ab8ff9dca2142ba16ea02f7db9ba06eebbc37c Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:49:57 -0700 Subject: [PATCH 0317/1043] nvmet-fc: track hostport handle for associations In preparation for sending LS requests for an association that terminates, save and track the hosthandle that is part of the LS's that are received to create associations. Support consists of: - Create a hostport structure that will be 1:1 mapped to a host port handle. The hostport structure is specific to a targetport. - Whenever an association is created, create a host port for the hosthandle the Create Association LS was received from. There will be only 1 hostport structure created, with all associations that have the same hosthandle sharing the hostport structure. - When the association is terminated, the hostport reference will be removed. After the last association for the host port is removed, the hostport will be deleted. - Add support for the new nvmet_fc_invalidate_host() interface. In the past, the LLDD didn't notify loss of connectivity to host ports - the LLD would simply reject new requests and wait for the kato timeout to kill the association. Now, when host port connectivity is lost, the LLDD can notify the transport. The transport will initiate the termination of all associations for that host port. When the last association has been terminated and the hosthandle will no longer be referenced, the new host_release callback will be made to the lldd. - For compatibility with prior behavior which didn't report the hosthandle: the LLDD must set hosthandle to NULL. In these cases, not LS request will be made, and no host_release callbacks will be made either. Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/fc.c | 177 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 170 insertions(+), 7 deletions(-) diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index 01488fc35d46..3ccf27c328b2 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -33,6 +33,7 @@ struct nvmet_fc_ls_iod { struct nvmet_fc_tgtport *tgtport; struct nvmet_fc_tgt_assoc *assoc; + void *hosthandle; union nvmefc_ls_requests *rqstbuf; union nvmefc_ls_responses *rspbuf; @@ -81,7 +82,6 @@ struct nvmet_fc_fcp_iod { }; struct nvmet_fc_tgtport { - struct nvmet_fc_target_port fc_target_port; struct list_head tgt_list; /* nvmet_fc_target_list */ @@ -93,6 +93,7 @@ struct nvmet_fc_tgtport { struct list_head ls_list; struct list_head ls_busylist; struct list_head assoc_list; + struct list_head host_list; struct ida assoc_cnt; struct nvmet_fc_port_entry *pe; struct kref ref; @@ -134,14 +135,24 @@ struct nvmet_fc_tgt_queue { struct nvmet_fc_fcp_iod fod[]; /* array of fcp_iods */ } __aligned(sizeof(unsigned long long)); +struct nvmet_fc_hostport { + struct nvmet_fc_tgtport *tgtport; + void *hosthandle; + struct list_head host_list; + struct kref ref; + u8 invalid; +}; + struct nvmet_fc_tgt_assoc { u64 association_id; u32 a_id; struct nvmet_fc_tgtport *tgtport; + struct nvmet_fc_hostport *hostport; struct list_head a_list; struct nvmet_fc_tgt_queue *queues[NVMET_NR_QUEUES + 1]; struct kref ref; struct work_struct del_work; + atomic_t del_work_active; }; @@ -773,6 +784,102 @@ nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport, return NULL; } +static void +nvmet_fc_hostport_free(struct kref *ref) +{ + struct nvmet_fc_hostport *hostport = + container_of(ref, struct nvmet_fc_hostport, ref); + struct nvmet_fc_tgtport *tgtport = hostport->tgtport; + unsigned long flags; + + spin_lock_irqsave(&tgtport->lock, flags); + list_del(&hostport->host_list); + spin_unlock_irqrestore(&tgtport->lock, flags); + if (tgtport->ops->host_release && hostport->invalid) + tgtport->ops->host_release(hostport->hosthandle); + kfree(hostport); + nvmet_fc_tgtport_put(tgtport); +} + +static void +nvmet_fc_hostport_put(struct nvmet_fc_hostport *hostport) +{ + kref_put(&hostport->ref, nvmet_fc_hostport_free); +} + +static int +nvmet_fc_hostport_get(struct nvmet_fc_hostport *hostport) +{ + return kref_get_unless_zero(&hostport->ref); +} + +static void +nvmet_fc_free_hostport(struct nvmet_fc_hostport *hostport) +{ + /* if LLDD not implemented, leave as NULL */ + if (!hostport->hosthandle) + return; + + nvmet_fc_hostport_put(hostport); +} + +static struct nvmet_fc_hostport * +nvmet_fc_alloc_hostport(struct nvmet_fc_tgtport *tgtport, void *hosthandle) +{ + struct nvmet_fc_hostport *newhost, *host, *match = NULL; + unsigned long flags; + + /* if LLDD not implemented, leave as NULL */ + if (!hosthandle) + return NULL; + + /* take reference for what will be the newly allocated hostport */ + if (!nvmet_fc_tgtport_get(tgtport)) + return ERR_PTR(-EINVAL); + + newhost = kzalloc(sizeof(*newhost), GFP_KERNEL); + if (!newhost) { + spin_lock_irqsave(&tgtport->lock, flags); + list_for_each_entry(host, &tgtport->host_list, host_list) { + if (host->hosthandle == hosthandle && !host->invalid) { + if (nvmet_fc_hostport_get(host)) { + match = host; + break; + } + } + } + spin_unlock_irqrestore(&tgtport->lock, flags); + /* no allocation - release reference */ + nvmet_fc_tgtport_put(tgtport); + return (match) ? match : ERR_PTR(-ENOMEM); + } + + newhost->tgtport = tgtport; + newhost->hosthandle = hosthandle; + INIT_LIST_HEAD(&newhost->host_list); + kref_init(&newhost->ref); + + spin_lock_irqsave(&tgtport->lock, flags); + list_for_each_entry(host, &tgtport->host_list, host_list) { + if (host->hosthandle == hosthandle && !host->invalid) { + if (nvmet_fc_hostport_get(host)) { + match = host; + break; + } + } + } + if (match) { + kfree(newhost); + newhost = NULL; + /* releasing allocation - release reference */ + nvmet_fc_tgtport_put(tgtport); + } else + list_add_tail(&newhost->host_list, &tgtport->host_list); + spin_unlock_irqrestore(&tgtport->lock, flags); + + return (match) ? match : newhost; +} + static void nvmet_fc_delete_assoc(struct work_struct *work) { @@ -780,11 +887,12 @@ nvmet_fc_delete_assoc(struct work_struct *work) container_of(work, struct nvmet_fc_tgt_assoc, del_work); nvmet_fc_delete_target_assoc(assoc); + atomic_set(&assoc->del_work_active, 0); nvmet_fc_tgt_a_put(assoc); } static struct nvmet_fc_tgt_assoc * -nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport) +nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle) { struct nvmet_fc_tgt_assoc *assoc, *tmpassoc; unsigned long flags; @@ -801,13 +909,18 @@ nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport) goto out_free_assoc; if (!nvmet_fc_tgtport_get(tgtport)) - goto out_ida_put; + goto out_ida; + + assoc->hostport = nvmet_fc_alloc_hostport(tgtport, hosthandle); + if (IS_ERR(assoc->hostport)) + goto out_put; assoc->tgtport = tgtport; assoc->a_id = idx; INIT_LIST_HEAD(&assoc->a_list); kref_init(&assoc->ref); INIT_WORK(&assoc->del_work, nvmet_fc_delete_assoc); + atomic_set(&assoc->del_work_active, 0); while (needrandom) { get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID); @@ -829,7 +942,9 @@ nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport) return assoc; -out_ida_put: +out_put: + nvmet_fc_tgtport_put(tgtport); +out_ida: ida_simple_remove(&tgtport->assoc_cnt, idx); out_free_assoc: kfree(assoc); @@ -844,6 +959,7 @@ nvmet_fc_target_assoc_free(struct kref *ref) struct nvmet_fc_tgtport *tgtport = assoc->tgtport; unsigned long flags; + nvmet_fc_free_hostport(assoc->hostport); spin_lock_irqsave(&tgtport->lock, flags); list_del(&assoc->a_list); spin_unlock_irqrestore(&tgtport->lock, flags); @@ -1057,6 +1173,7 @@ nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo, INIT_LIST_HEAD(&newrec->ls_list); INIT_LIST_HEAD(&newrec->ls_busylist); INIT_LIST_HEAD(&newrec->assoc_list); + INIT_LIST_HEAD(&newrec->host_list); kref_init(&newrec->ref); ida_init(&newrec->assoc_cnt); newrec->max_sg_cnt = template->max_sgl_segments; @@ -1133,14 +1250,21 @@ __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport) { struct nvmet_fc_tgt_assoc *assoc, *next; unsigned long flags; + int ret; spin_lock_irqsave(&tgtport->lock, flags); list_for_each_entry_safe(assoc, next, &tgtport->assoc_list, a_list) { if (!nvmet_fc_tgt_a_get(assoc)) continue; - if (!schedule_work(&assoc->del_work)) + ret = atomic_cmpxchg(&assoc->del_work_active, 0, 1); + if (ret == 0) { + if (!schedule_work(&assoc->del_work)) + nvmet_fc_tgt_a_put(assoc); + } else { + /* already deleting - release local reference */ nvmet_fc_tgt_a_put(assoc); + } } spin_unlock_irqrestore(&tgtport->lock, flags); } @@ -1178,6 +1302,36 @@ void nvmet_fc_invalidate_host(struct nvmet_fc_target_port *target_port, void *hosthandle) { + struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); + struct nvmet_fc_tgt_assoc *assoc, *next; + unsigned long flags; + bool noassoc = true; + int ret; + + spin_lock_irqsave(&tgtport->lock, flags); + list_for_each_entry_safe(assoc, next, + &tgtport->assoc_list, a_list) { + if (!assoc->hostport || + assoc->hostport->hosthandle != hosthandle) + continue; + if (!nvmet_fc_tgt_a_get(assoc)) + continue; + assoc->hostport->invalid = 1; + noassoc = false; + ret = atomic_cmpxchg(&assoc->del_work_active, 0, 1); + if (ret == 0) { + if (!schedule_work(&assoc->del_work)) + nvmet_fc_tgt_a_put(assoc); + } else { + /* already deleting - release local reference */ + nvmet_fc_tgt_a_put(assoc); + } + } + spin_unlock_irqrestore(&tgtport->lock, flags); + + /* if there's nothing to wait for - call the callback */ + if (noassoc && tgtport->ops->host_release) + tgtport->ops->host_release(hosthandle); } EXPORT_SYMBOL_GPL(nvmet_fc_invalidate_host); @@ -1192,6 +1346,7 @@ nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl) struct nvmet_fc_tgt_queue *queue; unsigned long flags; bool found_ctrl = false; + int ret; /* this is a bit ugly, but don't want to make locks layered */ spin_lock_irqsave(&nvmet_fc_tgtlock, flags); @@ -1215,8 +1370,14 @@ nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl) nvmet_fc_tgtport_put(tgtport); if (found_ctrl) { - if (!schedule_work(&assoc->del_work)) + ret = atomic_cmpxchg(&assoc->del_work_active, 0, 1); + if (ret == 0) { + if (!schedule_work(&assoc->del_work)) + nvmet_fc_tgt_a_put(assoc); + } else { + /* already deleting - release local reference */ nvmet_fc_tgt_a_put(assoc); + } return; } @@ -1293,7 +1454,8 @@ nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport, else { /* new association w/ admin queue */ - iod->assoc = nvmet_fc_alloc_target_assoc(tgtport); + iod->assoc = nvmet_fc_alloc_target_assoc( + tgtport, iod->hosthandle); if (!iod->assoc) ret = VERR_ASSOC_ALLOC_FAIL; else { @@ -1628,6 +1790,7 @@ nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port, iod->fcpreq = NULL; memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len); iod->rqstdatalen = lsreqbuf_len; + iod->hosthandle = hosthandle; schedule_work(&iod->work); From 349c694ee71ce0dfe4b9ccfac76ef5c1efb476cf Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:49:58 -0700 Subject: [PATCH 0318/1043] nvmet-fc: rename ls_list to ls_rcv_list In preparation to add ls request support, rename the current ls_list, which is RCV LS request only, to ls_rcv_list. Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/fc.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index 3ccf27c328b2..6a5af99f19ba 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -29,7 +29,7 @@ struct nvmet_fc_ls_iod { struct nvmefc_ls_rsp *lsrsp; struct nvmefc_tgt_fcp_req *fcpreq; /* only if RS */ - struct list_head ls_list; /* tgtport->ls_list */ + struct list_head ls_rcv_list; /* tgtport->ls_rcv_list */ struct nvmet_fc_tgtport *tgtport; struct nvmet_fc_tgt_assoc *assoc; @@ -90,7 +90,7 @@ struct nvmet_fc_tgtport { struct nvmet_fc_ls_iod *iod; spinlock_t lock; - struct list_head ls_list; + struct list_head ls_rcv_list; struct list_head ls_busylist; struct list_head assoc_list; struct list_head host_list; @@ -346,7 +346,7 @@ nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport) for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) { INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work); iod->tgtport = tgtport; - list_add_tail(&iod->ls_list, &tgtport->ls_list); + list_add_tail(&iod->ls_rcv_list, &tgtport->ls_rcv_list); iod->rqstbuf = kzalloc(sizeof(union nvmefc_ls_requests) + sizeof(union nvmefc_ls_responses), @@ -367,12 +367,12 @@ nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport) out_fail: kfree(iod->rqstbuf); - list_del(&iod->ls_list); + list_del(&iod->ls_rcv_list); for (iod--, i--; i >= 0; iod--, i--) { fc_dma_unmap_single(tgtport->dev, iod->rspdma, sizeof(*iod->rspbuf), DMA_TO_DEVICE); kfree(iod->rqstbuf); - list_del(&iod->ls_list); + list_del(&iod->ls_rcv_list); } kfree(iod); @@ -391,7 +391,7 @@ nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport) iod->rspdma, sizeof(*iod->rspbuf), DMA_TO_DEVICE); kfree(iod->rqstbuf); - list_del(&iod->ls_list); + list_del(&iod->ls_rcv_list); } kfree(tgtport->iod); } @@ -403,10 +403,10 @@ nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport) unsigned long flags; spin_lock_irqsave(&tgtport->lock, flags); - iod = list_first_entry_or_null(&tgtport->ls_list, - struct nvmet_fc_ls_iod, ls_list); + iod = list_first_entry_or_null(&tgtport->ls_rcv_list, + struct nvmet_fc_ls_iod, ls_rcv_list); if (iod) - list_move_tail(&iod->ls_list, &tgtport->ls_busylist); + list_move_tail(&iod->ls_rcv_list, &tgtport->ls_busylist); spin_unlock_irqrestore(&tgtport->lock, flags); return iod; } @@ -419,7 +419,7 @@ nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport, unsigned long flags; spin_lock_irqsave(&tgtport->lock, flags); - list_move(&iod->ls_list, &tgtport->ls_list); + list_move(&iod->ls_rcv_list, &tgtport->ls_rcv_list); spin_unlock_irqrestore(&tgtport->lock, flags); } @@ -1170,7 +1170,7 @@ nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo, newrec->dev = dev; newrec->ops = template; spin_lock_init(&newrec->lock); - INIT_LIST_HEAD(&newrec->ls_list); + INIT_LIST_HEAD(&newrec->ls_rcv_list); INIT_LIST_HEAD(&newrec->ls_busylist); INIT_LIST_HEAD(&newrec->assoc_list); INIT_LIST_HEAD(&newrec->host_list); From 47bf3241064498878ffed10a69131be9154201eb Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:49:59 -0700 Subject: [PATCH 0319/1043] nvmet-fc: Add Disconnect Association Xmt support As part of FC-NVME-2 (and ammendment on FC-NVME), the target is to send a Disconnect LS after an association is terminated and any exchanges for the association have been ABTS'd. The target is also not to send the receipt to any Disconnect Association LS, received to initiate the association termination or received while the association is terminating, until the Disconnect LS has been transmit. Add support for sending Disconnect Association LS after all I/O's complete (which is after ABTS'd certainly). Utilizes the new LLDD api to send ls requests. There is no need to track the Disconnect LS response or to retry after timeout. All spec requirements will have been met by waiting for i/o completion to initiate the transmission. Add support for tracking the reception of Disconnect Association and defering the response transmission until after the Disconnect Association LS has been transmit. Signed-off-by: James Smart Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/fc.c | 296 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 286 insertions(+), 10 deletions(-) diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index 6a5af99f19ba..02d9751bb7ee 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -25,7 +25,7 @@ struct nvmet_fc_tgtport; struct nvmet_fc_tgt_assoc; -struct nvmet_fc_ls_iod { +struct nvmet_fc_ls_iod { /* for an LS RQST RCV */ struct nvmefc_ls_rsp *lsrsp; struct nvmefc_tgt_fcp_req *fcpreq; /* only if RS */ @@ -45,6 +45,18 @@ struct nvmet_fc_ls_iod { struct work_struct work; } __aligned(sizeof(unsigned long long)); +struct nvmet_fc_ls_req_op { /* for an LS RQST XMT */ + struct nvmefc_ls_req ls_req; + + struct nvmet_fc_tgtport *tgtport; + void *hosthandle; + + int ls_error; + struct list_head lsreq_list; /* tgtport->ls_req_list */ + bool req_queued; +}; + + /* desired maximum for a single sequence - if sg list allows it */ #define NVMET_FC_MAX_SEQ_LENGTH (256 * 1024) @@ -91,6 +103,7 @@ struct nvmet_fc_tgtport { struct nvmet_fc_ls_iod *iod; spinlock_t lock; struct list_head ls_rcv_list; + struct list_head ls_req_list; struct list_head ls_busylist; struct list_head assoc_list; struct list_head host_list; @@ -146,8 +159,10 @@ struct nvmet_fc_hostport { struct nvmet_fc_tgt_assoc { u64 association_id; u32 a_id; + atomic_t terminating; struct nvmet_fc_tgtport *tgtport; struct nvmet_fc_hostport *hostport; + struct nvmet_fc_ls_iod *rcv_disconn; struct list_head a_list; struct nvmet_fc_tgt_queue *queues[NVMET_NR_QUEUES + 1]; struct kref ref; @@ -236,6 +251,8 @@ static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport); static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_fcp_iod *fod); static void nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc); +static void nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport, + struct nvmet_fc_ls_iod *iod); /* *********************** FC-NVME DMA Handling **************************** */ @@ -327,6 +344,188 @@ fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, } +/* ********************** FC-NVME LS XMT Handling ************************* */ + + +static void +__nvmet_fc_finish_ls_req(struct nvmet_fc_ls_req_op *lsop) +{ + struct nvmet_fc_tgtport *tgtport = lsop->tgtport; + struct nvmefc_ls_req *lsreq = &lsop->ls_req; + unsigned long flags; + + spin_lock_irqsave(&tgtport->lock, flags); + + if (!lsop->req_queued) { + spin_unlock_irqrestore(&tgtport->lock, flags); + return; + } + + list_del(&lsop->lsreq_list); + + lsop->req_queued = false; + + spin_unlock_irqrestore(&tgtport->lock, flags); + + fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma, + (lsreq->rqstlen + lsreq->rsplen), + DMA_BIDIRECTIONAL); + + nvmet_fc_tgtport_put(tgtport); +} + +static int +__nvmet_fc_send_ls_req(struct nvmet_fc_tgtport *tgtport, + struct nvmet_fc_ls_req_op *lsop, + void (*done)(struct nvmefc_ls_req *req, int status)) +{ + struct nvmefc_ls_req *lsreq = &lsop->ls_req; + unsigned long flags; + int ret = 0; + + if (!tgtport->ops->ls_req) + return -EOPNOTSUPP; + + if (!nvmet_fc_tgtport_get(tgtport)) + return -ESHUTDOWN; + + lsreq->done = done; + lsop->req_queued = false; + INIT_LIST_HEAD(&lsop->lsreq_list); + + lsreq->rqstdma = fc_dma_map_single(tgtport->dev, lsreq->rqstaddr, + lsreq->rqstlen + lsreq->rsplen, + DMA_BIDIRECTIONAL); + if (fc_dma_mapping_error(tgtport->dev, lsreq->rqstdma)) { + ret = -EFAULT; + goto out_puttgtport; + } + lsreq->rspdma = lsreq->rqstdma + lsreq->rqstlen; + + spin_lock_irqsave(&tgtport->lock, flags); + + list_add_tail(&lsop->lsreq_list, &tgtport->ls_req_list); + + lsop->req_queued = true; + + spin_unlock_irqrestore(&tgtport->lock, flags); + + ret = tgtport->ops->ls_req(&tgtport->fc_target_port, lsop->hosthandle, + lsreq); + if (ret) + goto out_unlink; + + return 0; + +out_unlink: + lsop->ls_error = ret; + spin_lock_irqsave(&tgtport->lock, flags); + lsop->req_queued = false; + list_del(&lsop->lsreq_list); + spin_unlock_irqrestore(&tgtport->lock, flags); + fc_dma_unmap_single(tgtport->dev, lsreq->rqstdma, + (lsreq->rqstlen + lsreq->rsplen), + DMA_BIDIRECTIONAL); +out_puttgtport: + nvmet_fc_tgtport_put(tgtport); + + return ret; +} + +static int +nvmet_fc_send_ls_req_async(struct nvmet_fc_tgtport *tgtport, + struct nvmet_fc_ls_req_op *lsop, + void (*done)(struct nvmefc_ls_req *req, int status)) +{ + /* don't wait for completion */ + + return __nvmet_fc_send_ls_req(tgtport, lsop, done); +} + +static void +nvmet_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status) +{ + struct nvmet_fc_ls_req_op *lsop = + container_of(lsreq, struct nvmet_fc_ls_req_op, ls_req); + + __nvmet_fc_finish_ls_req(lsop); + + /* fc-nvme target doesn't care about success or failure of cmd */ + + kfree(lsop); +} + +/* + * This routine sends a FC-NVME LS to disconnect (aka terminate) + * the FC-NVME Association. Terminating the association also + * terminates the FC-NVME connections (per queue, both admin and io + * queues) that are part of the association. E.g. things are torn + * down, and the related FC-NVME Association ID and Connection IDs + * become invalid. + * + * The behavior of the fc-nvme target is such that it's + * understanding of the association and connections will implicitly + * be torn down. The action is implicit as it may be due to a loss of + * connectivity with the fc-nvme host, so the target may never get a + * response even if it tried. As such, the action of this routine + * is to asynchronously send the LS, ignore any results of the LS, and + * continue on with terminating the association. If the fc-nvme host + * is present and receives the LS, it too can tear down. + */ +static void +nvmet_fc_xmt_disconnect_assoc(struct nvmet_fc_tgt_assoc *assoc) +{ + struct nvmet_fc_tgtport *tgtport = assoc->tgtport; + struct fcnvme_ls_disconnect_assoc_rqst *discon_rqst; + struct fcnvme_ls_disconnect_assoc_acc *discon_acc; + struct nvmet_fc_ls_req_op *lsop; + struct nvmefc_ls_req *lsreq; + int ret; + + /* + * If ls_req is NULL or no hosthandle, it's an older lldd and no + * message is normal. Otherwise, send unless the hostport has + * already been invalidated by the lldd. + */ + if (!tgtport->ops->ls_req || !assoc->hostport || + assoc->hostport->invalid) + return; + + lsop = kzalloc((sizeof(*lsop) + + sizeof(*discon_rqst) + sizeof(*discon_acc) + + tgtport->ops->lsrqst_priv_sz), GFP_KERNEL); + if (!lsop) { + dev_info(tgtport->dev, + "{%d:%d} send Disconnect Association failed: ENOMEM\n", + tgtport->fc_target_port.port_num, assoc->a_id); + return; + } + + discon_rqst = (struct fcnvme_ls_disconnect_assoc_rqst *)&lsop[1]; + discon_acc = (struct fcnvme_ls_disconnect_assoc_acc *)&discon_rqst[1]; + lsreq = &lsop->ls_req; + if (tgtport->ops->lsrqst_priv_sz) + lsreq->private = (void *)&discon_acc[1]; + else + lsreq->private = NULL; + + lsop->tgtport = tgtport; + lsop->hosthandle = assoc->hostport->hosthandle; + + nvmefc_fmt_lsreq_discon_assoc(lsreq, discon_rqst, discon_acc, + assoc->association_id); + + ret = nvmet_fc_send_ls_req_async(tgtport, lsop, + nvmet_fc_disconnect_assoc_done); + if (ret) { + dev_info(tgtport->dev, + "{%d:%d} XMT Disconnect Association failed: %d\n", + tgtport->fc_target_port.port_num, assoc->a_id, ret); + kfree(lsop); + } +} + + /* *********************** FC-NVME Port Management ************************ */ @@ -693,6 +892,10 @@ nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue) disconnect = atomic_xchg(&queue->connected, 0); + /* if not connected, nothing to do */ + if (!disconnect) + return; + spin_lock_irqsave(&queue->qlock, flags); /* abort outstanding io's */ for (i = 0; i < queue->sqsize; fod++, i++) { @@ -921,6 +1124,7 @@ nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle) kref_init(&assoc->ref); INIT_WORK(&assoc->del_work, nvmet_fc_delete_assoc); atomic_set(&assoc->del_work_active, 0); + atomic_set(&assoc->terminating, 0); while (needrandom) { get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID); @@ -957,13 +1161,24 @@ nvmet_fc_target_assoc_free(struct kref *ref) struct nvmet_fc_tgt_assoc *assoc = container_of(ref, struct nvmet_fc_tgt_assoc, ref); struct nvmet_fc_tgtport *tgtport = assoc->tgtport; + struct nvmet_fc_ls_iod *oldls; unsigned long flags; + /* Send Disconnect now that all i/o has completed */ + nvmet_fc_xmt_disconnect_assoc(assoc); + nvmet_fc_free_hostport(assoc->hostport); spin_lock_irqsave(&tgtport->lock, flags); list_del(&assoc->a_list); + oldls = assoc->rcv_disconn; spin_unlock_irqrestore(&tgtport->lock, flags); + /* if pending Rcv Disconnect Association LS, send rsp now */ + if (oldls) + nvmet_fc_xmt_ls_rsp(tgtport, oldls); ida_simple_remove(&tgtport->assoc_cnt, assoc->a_id); + dev_info(tgtport->dev, + "{%d:%d} Association freed\n", + tgtport->fc_target_port.port_num, assoc->a_id); kfree(assoc); nvmet_fc_tgtport_put(tgtport); } @@ -986,7 +1201,13 @@ nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc) struct nvmet_fc_tgtport *tgtport = assoc->tgtport; struct nvmet_fc_tgt_queue *queue; unsigned long flags; - int i; + int i, terminating; + + terminating = atomic_xchg(&assoc->terminating, 1); + + /* if already terminating, do nothing */ + if (terminating) + return; spin_lock_irqsave(&tgtport->lock, flags); for (i = NVMET_NR_QUEUES; i >= 0; i--) { @@ -1002,6 +1223,10 @@ nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc) } spin_unlock_irqrestore(&tgtport->lock, flags); + dev_info(tgtport->dev, + "{%d:%d} Association deleted\n", + tgtport->fc_target_port.port_num, assoc->a_id); + nvmet_fc_tgt_a_put(assoc); } @@ -1171,6 +1396,7 @@ nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo, newrec->ops = template; spin_lock_init(&newrec->lock); INIT_LIST_HEAD(&newrec->ls_rcv_list); + INIT_LIST_HEAD(&newrec->ls_req_list); INIT_LIST_HEAD(&newrec->ls_busylist); INIT_LIST_HEAD(&newrec->assoc_list); INIT_LIST_HEAD(&newrec->host_list); @@ -1407,6 +1633,13 @@ nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port) /* terminate any outstanding associations */ __nvmet_fc_free_assocs(tgtport); + /* + * should terminate LS's as well. However, LS's will be generated + * at the tail end of association termination, so they likely don't + * exist yet. And even if they did, it's worthwhile to just let + * them finish and targetport ref counting will clean things up. + */ + nvmet_fc_tgtport_put(tgtport); return 0; @@ -1414,7 +1647,7 @@ nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port) EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport); -/* *********************** FC-NVME LS Handling **************************** */ +/* ********************** FC-NVME LS RCV Handling ************************* */ static void @@ -1481,6 +1714,10 @@ nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport, atomic_set(&queue->connected, 1); queue->sqhd = 0; /* best place to init value */ + dev_info(tgtport->dev, + "{%d:%d} Association created\n", + tgtport->fc_target_port.port_num, iod->assoc->a_id); + /* format a response */ iod->lsrsp->rsplen = sizeof(*acc); @@ -1588,7 +1825,11 @@ nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport, be16_to_cpu(rqst->connect_cmd.qid))); } -static void +/* + * Returns true if the LS response is to be transmit + * Returns false if the LS response is to be delayed + */ +static int nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod) { @@ -1597,13 +1838,15 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, struct fcnvme_ls_disconnect_assoc_acc *acc = &iod->rspbuf->rsp_dis_assoc; struct nvmet_fc_tgt_assoc *assoc; + struct nvmet_fc_ls_iod *oldls = NULL; + unsigned long flags; int ret = 0; memset(acc, 0, sizeof(*acc)); ret = nvmefc_vldt_lsreq_discon_assoc(iod->rqstdatalen, rqst); if (!ret) { - /* match an active association */ + /* match an active association - takes an assoc ref if !NULL */ assoc = nvmet_fc_find_target_assoc(tgtport, be64_to_cpu(rqst->associd.association_id)); iod->assoc = assoc; @@ -1621,7 +1864,7 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, FCNVME_RJT_RC_INV_ASSOC : FCNVME_RJT_RC_LOGIC, FCNVME_RJT_EXP_NONE, 0); - return; + return true; } /* format a response */ @@ -1634,9 +1877,40 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, FCNVME_LS_DISCONNECT_ASSOC); /* release get taken in nvmet_fc_find_target_assoc */ - nvmet_fc_tgt_a_put(iod->assoc); + nvmet_fc_tgt_a_put(assoc); - nvmet_fc_delete_target_assoc(iod->assoc); + /* + * The rules for LS response says the response cannot + * go back until ABTS's have been sent for all outstanding + * I/O and a Disconnect Association LS has been sent. + * So... save off the Disconnect LS to send the response + * later. If there was a prior LS already saved, replace + * it with the newer one and send a can't perform reject + * on the older one. + */ + spin_lock_irqsave(&tgtport->lock, flags); + oldls = assoc->rcv_disconn; + assoc->rcv_disconn = iod; + spin_unlock_irqrestore(&tgtport->lock, flags); + + nvmet_fc_delete_target_assoc(assoc); + + if (oldls) { + dev_info(tgtport->dev, + "{%d:%d} Multiple Disconnect Association LS's " + "received\n", + tgtport->fc_target_port.port_num, assoc->a_id); + /* overwrite good response with bogus failure */ + oldls->lsrsp->rsplen = nvme_fc_format_rjt(oldls->rspbuf, + sizeof(*iod->rspbuf), + /* ok to use rqst, LS is same */ + rqst->w0.ls_cmd, + FCNVME_RJT_RC_UNAB, + FCNVME_RJT_EXP_NONE, 0); + nvmet_fc_xmt_ls_rsp(tgtport, oldls); + } + + return false; } @@ -1681,6 +1955,7 @@ nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport, struct nvmet_fc_ls_iod *iod) { struct fcnvme_ls_rqst_w0 *w0 = &iod->rqstbuf->rq_cr_assoc.w0; + bool sendrsp = true; iod->lsrsp->nvme_fc_private = iod; iod->lsrsp->rspbuf = iod->rspbuf; @@ -1707,7 +1982,7 @@ nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport, break; case FCNVME_LS_DISCONNECT_ASSOC: /* Terminate a Queue/Connection or the Association */ - nvmet_fc_ls_disconnect(tgtport, iod); + sendrsp = nvmet_fc_ls_disconnect(tgtport, iod); break; default: iod->lsrsp->rsplen = nvme_fc_format_rjt(iod->rspbuf, @@ -1715,7 +1990,8 @@ nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport, FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0); } - nvmet_fc_xmt_ls_rsp(tgtport, iod); + if (sendrsp) + nvmet_fc_xmt_ls_rsp(tgtport, iod); } /* From ea39765843faf5f4426ffda000b0ca02217a1eeb Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:50:00 -0700 Subject: [PATCH 0320/1043] nvme-fcloop: refactor to enable target to host LS Currently nvmefc-loop only sends LS's from host to target. Slightly rework data structures and routine names to reflect this path. Allows a straight-forward conversion to be used by ls's from target to host. Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/fcloop.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c index c11805a155e8..fac7dbe572db 100644 --- a/drivers/nvme/target/fcloop.c +++ b/drivers/nvme/target/fcloop.c @@ -226,9 +226,15 @@ struct fcloop_nport { u32 port_id; }; +enum { + H2T = 0, + T2H = 1, +}; + struct fcloop_lsreq { struct nvmefc_ls_req *lsreq; struct nvmefc_ls_rsp ls_rsp; + int lsdir; /* H2T or T2H */ int status; struct list_head ls_list; /* fcloop_rport->ls_list */ }; @@ -323,7 +329,7 @@ fcloop_rport_lsrqst_work(struct work_struct *work) } static int -fcloop_ls_req(struct nvme_fc_local_port *localport, +fcloop_h2t_ls_req(struct nvme_fc_local_port *localport, struct nvme_fc_remote_port *remoteport, struct nvmefc_ls_req *lsreq) { @@ -331,6 +337,7 @@ fcloop_ls_req(struct nvme_fc_local_port *localport, struct fcloop_rport *rport = remoteport->private; int ret = 0; + tls_req->lsdir = H2T; tls_req->lsreq = lsreq; INIT_LIST_HEAD(&tls_req->ls_list); @@ -351,7 +358,7 @@ fcloop_ls_req(struct nvme_fc_local_port *localport, } static int -fcloop_xmt_ls_rsp(struct nvmet_fc_target_port *targetport, +fcloop_h2t_xmt_ls_rsp(struct nvmet_fc_target_port *targetport, struct nvmefc_ls_rsp *lsrsp) { struct fcloop_lsreq *tls_req = ls_rsp_to_lsreq(lsrsp); @@ -762,7 +769,7 @@ fcloop_fcp_req_release(struct nvmet_fc_target_port *tgtport, } static void -fcloop_ls_abort(struct nvme_fc_local_port *localport, +fcloop_h2t_ls_abort(struct nvme_fc_local_port *localport, struct nvme_fc_remote_port *remoteport, struct nvmefc_ls_req *lsreq) { @@ -879,9 +886,9 @@ static struct nvme_fc_port_template fctemplate = { .remoteport_delete = fcloop_remoteport_delete, .create_queue = fcloop_create_queue, .delete_queue = fcloop_delete_queue, - .ls_req = fcloop_ls_req, + .ls_req = fcloop_h2t_ls_req, .fcp_io = fcloop_fcp_req, - .ls_abort = fcloop_ls_abort, + .ls_abort = fcloop_h2t_ls_abort, .fcp_abort = fcloop_fcp_abort, .max_hw_queues = FCLOOP_HW_QUEUES, .max_sgl_segments = FCLOOP_SGL_SEGS, @@ -896,7 +903,7 @@ static struct nvme_fc_port_template fctemplate = { static struct nvmet_fc_target_template tgttemplate = { .targetport_delete = fcloop_targetport_delete, - .xmt_ls_rsp = fcloop_xmt_ls_rsp, + .xmt_ls_rsp = fcloop_h2t_xmt_ls_rsp, .fcp_op = fcloop_fcp_op, .fcp_abort = fcloop_tgt_fcp_abort, .fcp_req_release = fcloop_fcp_req_release, From 437c0b824dbd05dbdab772ed1e0f69ffec76119d Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:50:01 -0700 Subject: [PATCH 0321/1043] nvme-fcloop: add target to host LS request support Add support for performing LS requests from target to host. Include sending request from targetport, reception into host, host sending ls rsp. Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/fcloop.c | 130 +++++++++++++++++++++++++++++++---- 1 file changed, 118 insertions(+), 12 deletions(-) diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c index fac7dbe572db..2ff1d1334a03 100644 --- a/drivers/nvme/target/fcloop.c +++ b/drivers/nvme/target/fcloop.c @@ -208,10 +208,13 @@ struct fcloop_rport { }; struct fcloop_tport { - struct nvmet_fc_target_port *targetport; - struct nvme_fc_remote_port *remoteport; - struct fcloop_nport *nport; - struct fcloop_lport *lport; + struct nvmet_fc_target_port *targetport; + struct nvme_fc_remote_port *remoteport; + struct fcloop_nport *nport; + struct fcloop_lport *lport; + spinlock_t lock; + struct list_head ls_list; + struct work_struct ls_work; }; struct fcloop_nport { @@ -226,11 +229,6 @@ struct fcloop_nport { u32 port_id; }; -enum { - H2T = 0, - T2H = 1, -}; - struct fcloop_lsreq { struct nvmefc_ls_req *lsreq; struct nvmefc_ls_rsp ls_rsp; @@ -337,7 +335,6 @@ fcloop_h2t_ls_req(struct nvme_fc_local_port *localport, struct fcloop_rport *rport = remoteport->private; int ret = 0; - tls_req->lsdir = H2T; tls_req->lsreq = lsreq; INIT_LIST_HEAD(&tls_req->ls_list); @@ -351,8 +348,9 @@ fcloop_h2t_ls_req(struct nvme_fc_local_port *localport, } tls_req->status = 0; - ret = nvmet_fc_rcv_ls_req(rport->targetport, NULL, &tls_req->ls_rsp, - lsreq->rqstaddr, lsreq->rqstlen); + ret = nvmet_fc_rcv_ls_req(rport->targetport, rport, + &tls_req->ls_rsp, + lsreq->rqstaddr, lsreq->rqstlen); return ret; } @@ -384,6 +382,99 @@ fcloop_h2t_xmt_ls_rsp(struct nvmet_fc_target_port *targetport, return 0; } +static void +fcloop_tport_lsrqst_work(struct work_struct *work) +{ + struct fcloop_tport *tport = + container_of(work, struct fcloop_tport, ls_work); + struct fcloop_lsreq *tls_req; + + spin_lock(&tport->lock); + for (;;) { + tls_req = list_first_entry_or_null(&tport->ls_list, + struct fcloop_lsreq, ls_list); + if (!tls_req) + break; + + list_del(&tls_req->ls_list); + spin_unlock(&tport->lock); + + tls_req->lsreq->done(tls_req->lsreq, tls_req->status); + /* + * callee may free memory containing tls_req. + * do not reference lsreq after this. + */ + + spin_lock(&tport->lock); + } + spin_unlock(&tport->lock); +} + +static int +fcloop_t2h_ls_req(struct nvmet_fc_target_port *targetport, void *hosthandle, + struct nvmefc_ls_req *lsreq) +{ + struct fcloop_lsreq *tls_req = lsreq->private; + struct fcloop_tport *tport = targetport->private; + int ret = 0; + + /* + * hosthandle should be the dst.rport value. + * hosthandle ignored as fcloop currently is + * 1:1 tgtport vs remoteport + */ + tls_req->lsreq = lsreq; + INIT_LIST_HEAD(&tls_req->ls_list); + + if (!tport->remoteport) { + tls_req->status = -ECONNREFUSED; + spin_lock(&tport->lock); + list_add_tail(&tport->ls_list, &tls_req->ls_list); + spin_unlock(&tport->lock); + schedule_work(&tport->ls_work); + return ret; + } + + tls_req->status = 0; + ret = nvme_fc_rcv_ls_req(tport->remoteport, &tls_req->ls_rsp, + lsreq->rqstaddr, lsreq->rqstlen); + + return ret; +} + +static int +fcloop_t2h_xmt_ls_rsp(struct nvme_fc_local_port *localport, + struct nvme_fc_remote_port *remoteport, + struct nvmefc_ls_rsp *lsrsp) +{ + struct fcloop_lsreq *tls_req = ls_rsp_to_lsreq(lsrsp); + struct nvmefc_ls_req *lsreq = tls_req->lsreq; + struct fcloop_rport *rport = remoteport->private; + struct nvmet_fc_target_port *targetport = rport->targetport; + struct fcloop_tport *tport; + + memcpy(lsreq->rspaddr, lsrsp->rspbuf, + ((lsreq->rsplen < lsrsp->rsplen) ? + lsreq->rsplen : lsrsp->rsplen)); + lsrsp->done(lsrsp); + + if (targetport) { + tport = targetport->private; + spin_lock(&tport->lock); + list_add_tail(&tport->ls_list, &tls_req->ls_list); + spin_unlock(&tport->lock); + schedule_work(&tport->ls_work); + } + + return 0; +} + +static void +fcloop_t2h_host_release(void *hosthandle) +{ + /* host handle ignored for now */ +} + /* * Simulate reception of RSCN and converting it to a initiator transport * call to rescan a remote port. @@ -775,6 +866,12 @@ fcloop_h2t_ls_abort(struct nvme_fc_local_port *localport, { } +static void +fcloop_t2h_ls_abort(struct nvmet_fc_target_port *targetport, + void *hosthandle, struct nvmefc_ls_req *lsreq) +{ +} + static void fcloop_fcp_abort(struct nvme_fc_local_port *localport, struct nvme_fc_remote_port *remoteport, @@ -874,6 +971,7 @@ fcloop_targetport_delete(struct nvmet_fc_target_port *targetport) { struct fcloop_tport *tport = targetport->private; + flush_work(&tport->ls_work); fcloop_nport_put(tport->nport); } @@ -890,6 +988,7 @@ static struct nvme_fc_port_template fctemplate = { .fcp_io = fcloop_fcp_req, .ls_abort = fcloop_h2t_ls_abort, .fcp_abort = fcloop_fcp_abort, + .xmt_ls_rsp = fcloop_t2h_xmt_ls_rsp, .max_hw_queues = FCLOOP_HW_QUEUES, .max_sgl_segments = FCLOOP_SGL_SEGS, .max_dif_sgl_segments = FCLOOP_SGL_SEGS, @@ -908,6 +1007,9 @@ static struct nvmet_fc_target_template tgttemplate = { .fcp_abort = fcloop_tgt_fcp_abort, .fcp_req_release = fcloop_fcp_req_release, .discovery_event = fcloop_tgt_discovery_evt, + .ls_req = fcloop_t2h_ls_req, + .ls_abort = fcloop_t2h_ls_abort, + .host_release = fcloop_t2h_host_release, .max_hw_queues = FCLOOP_HW_QUEUES, .max_sgl_segments = FCLOOP_SGL_SEGS, .max_dif_sgl_segments = FCLOOP_SGL_SEGS, @@ -916,6 +1018,7 @@ static struct nvmet_fc_target_template tgttemplate = { .target_features = 0, /* sizes of additional private data for data structures */ .target_priv_sz = sizeof(struct fcloop_tport), + .lsrqst_priv_sz = sizeof(struct fcloop_lsreq), }; static ssize_t @@ -1265,6 +1368,9 @@ fcloop_create_target_port(struct device *dev, struct device_attribute *attr, tport->nport = nport; tport->lport = nport->lport; nport->tport = tport; + spin_lock_init(&tport->lock); + INIT_WORK(&tport->ls_work, fcloop_tport_lsrqst_work); + INIT_LIST_HEAD(&tport->ls_list); return count; } From 2a1160a03ac477b95d596bc4a0955ee3d7d0f3c9 Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:50:02 -0700 Subject: [PATCH 0322/1043] lpfc: Refactor lpfc nvme headers A lot of files in lpfc include nvme headers, building up relationships that require a file to change for its headers when there is no other change necessary. It would be better to localize the nvme headers. There is also no need for separate nvme (initiator) and nvmet (tgt) header files. Refactor the inclusion of nvme headers so that all nvme items are included by lpfc_nvme.h Merge lpfc_nvmet.h into lpfc_nvme.h so that there is a single header used by both the nvme and nvmet sides. This prepares for structure sharing between the two roles. Prep to add shared function prototypes for upcoming shared routines. Signed-off-by: Paul Ely Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/scsi/lpfc/lpfc_attr.c | 3 - drivers/scsi/lpfc/lpfc_ct.c | 1 - drivers/scsi/lpfc/lpfc_debugfs.c | 3 - drivers/scsi/lpfc/lpfc_hbadisc.c | 2 - drivers/scsi/lpfc/lpfc_init.c | 3 - drivers/scsi/lpfc/lpfc_mem.c | 4 - drivers/scsi/lpfc/lpfc_nportdisc.c | 2 - drivers/scsi/lpfc/lpfc_nvme.c | 3 - drivers/scsi/lpfc/lpfc_nvme.h | 146 ++++++++++++++++++++++++++ drivers/scsi/lpfc/lpfc_nvmet.c | 5 - drivers/scsi/lpfc/lpfc_nvmet.h | 158 ----------------------------- drivers/scsi/lpfc/lpfc_sli.c | 3 - 12 files changed, 146 insertions(+), 187 deletions(-) delete mode 100644 drivers/scsi/lpfc/lpfc_nvmet.h diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c index 1354c141d614..f089867674cb 100644 --- a/drivers/scsi/lpfc/lpfc_attr.c +++ b/drivers/scsi/lpfc/lpfc_attr.c @@ -37,8 +37,6 @@ #include #include -#include - #include "lpfc_hw4.h" #include "lpfc_hw.h" #include "lpfc_sli.h" @@ -48,7 +46,6 @@ #include "lpfc.h" #include "lpfc_scsi.h" #include "lpfc_nvme.h" -#include "lpfc_nvmet.h" #include "lpfc_logmsg.h" #include "lpfc_version.h" #include "lpfc_compat.h" diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c index 2aa578d20f8c..196f6ae9952e 100644 --- a/drivers/scsi/lpfc/lpfc_ct.c +++ b/drivers/scsi/lpfc/lpfc_ct.c @@ -44,7 +44,6 @@ #include "lpfc_disc.h" #include "lpfc.h" #include "lpfc_scsi.h" -#include "lpfc_nvme.h" #include "lpfc_logmsg.h" #include "lpfc_crtn.h" #include "lpfc_version.h" diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c index 8a6e02aa553f..8e78e49e3f9f 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.c +++ b/drivers/scsi/lpfc/lpfc_debugfs.c @@ -39,8 +39,6 @@ #include #include -#include - #include "lpfc_hw4.h" #include "lpfc_hw.h" #include "lpfc_sli.h" @@ -50,7 +48,6 @@ #include "lpfc.h" #include "lpfc_scsi.h" #include "lpfc_nvme.h" -#include "lpfc_nvmet.h" #include "lpfc_logmsg.h" #include "lpfc_crtn.h" #include "lpfc_vport.h" diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c index 789eecbf32eb..8dec7b7c06d1 100644 --- a/drivers/scsi/lpfc/lpfc_hbadisc.c +++ b/drivers/scsi/lpfc/lpfc_hbadisc.c @@ -36,8 +36,6 @@ #include #include -#include - #include "lpfc_hw4.h" #include "lpfc_hw.h" #include "lpfc_nl.h" diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 4104bdcdbb6f..a29566010d94 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -50,8 +50,6 @@ #include #include -#include - #include "lpfc_hw4.h" #include "lpfc_hw.h" #include "lpfc_sli.h" @@ -61,7 +59,6 @@ #include "lpfc.h" #include "lpfc_scsi.h" #include "lpfc_nvme.h" -#include "lpfc_nvmet.h" #include "lpfc_logmsg.h" #include "lpfc_crtn.h" #include "lpfc_vport.h" diff --git a/drivers/scsi/lpfc/lpfc_mem.c b/drivers/scsi/lpfc/lpfc_mem.c index 7082279e4c01..726f6619230f 100644 --- a/drivers/scsi/lpfc/lpfc_mem.c +++ b/drivers/scsi/lpfc/lpfc_mem.c @@ -31,8 +31,6 @@ #include #include -#include - #include "lpfc_hw4.h" #include "lpfc_hw.h" #include "lpfc_sli.h" @@ -41,8 +39,6 @@ #include "lpfc_disc.h" #include "lpfc.h" #include "lpfc_scsi.h" -#include "lpfc_nvme.h" -#include "lpfc_nvmet.h" #include "lpfc_crtn.h" #include "lpfc_logmsg.h" diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c index a024e5a3918f..81f4ba1c24b4 100644 --- a/drivers/scsi/lpfc/lpfc_nportdisc.c +++ b/drivers/scsi/lpfc/lpfc_nportdisc.c @@ -32,8 +32,6 @@ #include #include -#include - #include "lpfc_hw4.h" #include "lpfc_hw.h" #include "lpfc_sli.h" diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index a45936e08031..6045000477ac 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -36,9 +36,6 @@ #include #include -#include -#include -#include #include "lpfc_version.h" #include "lpfc_hw4.h" #include "lpfc_hw.h" diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h index 593c48ff634e..e59dc1f5c18a 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.h +++ b/drivers/scsi/lpfc/lpfc_nvme.h @@ -21,6 +21,10 @@ * included with this package. * ********************************************************************/ +#include +#include +#include + #define LPFC_NVME_DEFAULT_SEGS (64 + 1) /* 256K IOs */ #define LPFC_NVME_ERSP_LEN 0x20 @@ -74,3 +78,145 @@ struct lpfc_nvme_rport { struct lpfc_nvme_fcpreq_priv { struct lpfc_io_buf *nvme_buf; }; + + +#define LPFC_NVMET_DEFAULT_SEGS (64 + 1) /* 256K IOs */ +#define LPFC_NVMET_RQE_MIN_POST 128 +#define LPFC_NVMET_RQE_DEF_POST 512 +#define LPFC_NVMET_RQE_DEF_COUNT 2048 +#define LPFC_NVMET_SUCCESS_LEN 12 + +#define LPFC_NVMET_MRQ_AUTO 0 +#define LPFC_NVMET_MRQ_MAX 16 + +#define LPFC_NVMET_WAIT_TMO (5 * MSEC_PER_SEC) + +/* Used for NVME Target */ +struct lpfc_nvmet_tgtport { + struct lpfc_hba *phba; + struct completion *tport_unreg_cmp; + + /* Stats counters - lpfc_nvmet_unsol_ls_buffer */ + atomic_t rcv_ls_req_in; + atomic_t rcv_ls_req_out; + atomic_t rcv_ls_req_drop; + atomic_t xmt_ls_abort; + atomic_t xmt_ls_abort_cmpl; + + /* Stats counters - lpfc_nvmet_xmt_ls_rsp */ + atomic_t xmt_ls_rsp; + atomic_t xmt_ls_drop; + + /* Stats counters - lpfc_nvmet_xmt_ls_rsp_cmp */ + atomic_t xmt_ls_rsp_error; + atomic_t xmt_ls_rsp_aborted; + atomic_t xmt_ls_rsp_xb_set; + atomic_t xmt_ls_rsp_cmpl; + + /* Stats counters - lpfc_nvmet_unsol_fcp_buffer */ + atomic_t rcv_fcp_cmd_in; + atomic_t rcv_fcp_cmd_out; + atomic_t rcv_fcp_cmd_drop; + atomic_t rcv_fcp_cmd_defer; + atomic_t xmt_fcp_release; + + /* Stats counters - lpfc_nvmet_xmt_fcp_op */ + atomic_t xmt_fcp_drop; + atomic_t xmt_fcp_read_rsp; + atomic_t xmt_fcp_read; + atomic_t xmt_fcp_write; + atomic_t xmt_fcp_rsp; + + /* Stats counters - lpfc_nvmet_xmt_fcp_op_cmp */ + atomic_t xmt_fcp_rsp_xb_set; + atomic_t xmt_fcp_rsp_cmpl; + atomic_t xmt_fcp_rsp_error; + atomic_t xmt_fcp_rsp_aborted; + atomic_t xmt_fcp_rsp_drop; + + /* Stats counters - lpfc_nvmet_xmt_fcp_abort */ + atomic_t xmt_fcp_xri_abort_cqe; + atomic_t xmt_fcp_abort; + atomic_t xmt_fcp_abort_cmpl; + atomic_t xmt_abort_sol; + atomic_t xmt_abort_unsol; + atomic_t xmt_abort_rsp; + atomic_t xmt_abort_rsp_error; + + /* Stats counters - defer IO */ + atomic_t defer_ctx; + atomic_t defer_fod; + atomic_t defer_wqfull; +}; + +struct lpfc_nvmet_ctx_info { + struct list_head nvmet_ctx_list; + spinlock_t nvmet_ctx_list_lock; /* lock per CPU */ + struct lpfc_nvmet_ctx_info *nvmet_ctx_next_cpu; + struct lpfc_nvmet_ctx_info *nvmet_ctx_start_cpu; + uint16_t nvmet_ctx_list_cnt; + char pad[16]; /* pad to a cache-line */ +}; + +/* This retrieves the context info associated with the specified cpu / mrq */ +#define lpfc_get_ctx_list(phba, cpu, mrq) \ + (phba->sli4_hba.nvmet_ctx_info + ((cpu * phba->cfg_nvmet_mrq) + mrq)) + +struct lpfc_nvmet_rcv_ctx { + union { + struct nvmefc_ls_rsp ls_rsp; + struct nvmefc_tgt_fcp_req fcp_req; + } ctx; + struct list_head list; + struct lpfc_hba *phba; + struct lpfc_iocbq *wqeq; + struct lpfc_iocbq *abort_wqeq; + spinlock_t ctxlock; /* protect flag access */ + uint32_t sid; + uint32_t offset; + uint16_t oxid; + uint16_t size; + uint16_t entry_cnt; + uint16_t cpu; + uint16_t idx; + uint16_t state; + /* States */ +#define LPFC_NVMET_STE_LS_RCV 1 +#define LPFC_NVMET_STE_LS_ABORT 2 +#define LPFC_NVMET_STE_LS_RSP 3 +#define LPFC_NVMET_STE_RCV 4 +#define LPFC_NVMET_STE_DATA 5 +#define LPFC_NVMET_STE_ABORT 6 +#define LPFC_NVMET_STE_DONE 7 +#define LPFC_NVMET_STE_FREE 0xff + uint16_t flag; +#define LPFC_NVMET_IO_INP 0x1 /* IO is in progress on exchange */ +#define LPFC_NVMET_ABORT_OP 0x2 /* Abort WQE issued on exchange */ +#define LPFC_NVMET_XBUSY 0x4 /* XB bit set on IO cmpl */ +#define LPFC_NVMET_CTX_RLS 0x8 /* ctx free requested */ +#define LPFC_NVMET_ABTS_RCV 0x10 /* ABTS received on exchange */ +#define LPFC_NVMET_CTX_REUSE_WQ 0x20 /* ctx reused via WQ */ +#define LPFC_NVMET_DEFER_WQFULL 0x40 /* Waiting on a free WQE */ +#define LPFC_NVMET_TNOTIFY 0x80 /* notify transport of abts */ + struct rqb_dmabuf *rqb_buffer; + struct lpfc_nvmet_ctxbuf *ctxbuf; + struct lpfc_sli4_hdw_queue *hdwq; + +#ifdef CONFIG_SCSI_LPFC_DEBUG_FS + uint64_t ts_isr_cmd; + uint64_t ts_cmd_nvme; + uint64_t ts_nvme_data; + uint64_t ts_data_wqput; + uint64_t ts_isr_data; + uint64_t ts_data_nvme; + uint64_t ts_nvme_status; + uint64_t ts_status_wqput; + uint64_t ts_isr_status; + uint64_t ts_status_nvme; +#endif +}; + + +/* routines found in lpfc_nvme.c */ + +/* routines found in lpfc_nvmet.c */ diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index 3b25bcb150ea..9576bc30ec85 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -36,10 +36,6 @@ #include #include -#include -#include -#include - #include "lpfc_version.h" #include "lpfc_hw4.h" #include "lpfc_hw.h" @@ -50,7 +46,6 @@ #include "lpfc.h" #include "lpfc_scsi.h" #include "lpfc_nvme.h" -#include "lpfc_nvmet.h" #include "lpfc_logmsg.h" #include "lpfc_crtn.h" #include "lpfc_vport.h" diff --git a/drivers/scsi/lpfc/lpfc_nvmet.h b/drivers/scsi/lpfc/lpfc_nvmet.h deleted file mode 100644 index f0196f3ef90d..000000000000 --- a/drivers/scsi/lpfc/lpfc_nvmet.h +++ /dev/null @@ -1,158 +0,0 @@ -/******************************************************************* - * This file is part of the Emulex Linux Device Driver for * - * Fibre Channel Host Bus Adapters. * - * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * - * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * - * Copyright (C) 2004-2016 Emulex. All rights reserved. * - * EMULEX and SLI are trademarks of Emulex. * - * www.broadcom.com * - * Portions Copyright (C) 2004-2005 Christoph Hellwig * - * * - * This program is free software; you can redistribute it and/or * - * modify it under the terms of version 2 of the GNU General * - * Public License as published by the Free Software Foundation. * - * This program is distributed in the hope that it will be useful. * - * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * - * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * - * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * - * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * - * TO BE LEGALLY INVALID. See the GNU General Public License for * - * more details, a copy of which can be found in the file COPYING * - * included with this package. * - ********************************************************************/ - -#define LPFC_NVMET_DEFAULT_SEGS (64 + 1) /* 256K IOs */ -#define LPFC_NVMET_RQE_MIN_POST 128 -#define LPFC_NVMET_RQE_DEF_POST 512 -#define LPFC_NVMET_RQE_DEF_COUNT 2048 -#define LPFC_NVMET_SUCCESS_LEN 12 - -#define LPFC_NVMET_MRQ_AUTO 0 -#define LPFC_NVMET_MRQ_MAX 16 - -#define LPFC_NVMET_WAIT_TMO (5 * MSEC_PER_SEC) - -/* Used for NVME Target */ -struct lpfc_nvmet_tgtport { - struct lpfc_hba *phba; - struct completion *tport_unreg_cmp; - - /* Stats counters - lpfc_nvmet_unsol_ls_buffer */ - atomic_t rcv_ls_req_in; - atomic_t rcv_ls_req_out; - atomic_t rcv_ls_req_drop; - atomic_t xmt_ls_abort; - atomic_t xmt_ls_abort_cmpl; - - /* Stats counters - lpfc_nvmet_xmt_ls_rsp */ - atomic_t xmt_ls_rsp; - atomic_t xmt_ls_drop; - - /* Stats counters - lpfc_nvmet_xmt_ls_rsp_cmp */ - atomic_t xmt_ls_rsp_error; - atomic_t xmt_ls_rsp_aborted; - atomic_t xmt_ls_rsp_xb_set; - atomic_t xmt_ls_rsp_cmpl; - - /* Stats counters - lpfc_nvmet_unsol_fcp_buffer */ - atomic_t rcv_fcp_cmd_in; - atomic_t rcv_fcp_cmd_out; - atomic_t rcv_fcp_cmd_drop; - atomic_t rcv_fcp_cmd_defer; - atomic_t xmt_fcp_release; - - /* Stats counters - lpfc_nvmet_xmt_fcp_op */ - atomic_t xmt_fcp_drop; - atomic_t xmt_fcp_read_rsp; - atomic_t xmt_fcp_read; - atomic_t xmt_fcp_write; - atomic_t xmt_fcp_rsp; - - /* Stats counters - lpfc_nvmet_xmt_fcp_op_cmp */ - atomic_t xmt_fcp_rsp_xb_set; - atomic_t xmt_fcp_rsp_cmpl; - atomic_t xmt_fcp_rsp_error; - atomic_t xmt_fcp_rsp_aborted; - atomic_t xmt_fcp_rsp_drop; - - /* Stats counters - lpfc_nvmet_xmt_fcp_abort */ - atomic_t xmt_fcp_xri_abort_cqe; - atomic_t xmt_fcp_abort; - atomic_t xmt_fcp_abort_cmpl; - atomic_t xmt_abort_sol; - atomic_t xmt_abort_unsol; - atomic_t xmt_abort_rsp; - atomic_t xmt_abort_rsp_error; - - /* Stats counters - defer IO */ - atomic_t defer_ctx; - atomic_t defer_fod; - atomic_t defer_wqfull; -}; - -struct lpfc_nvmet_ctx_info { - struct list_head nvmet_ctx_list; - spinlock_t nvmet_ctx_list_lock; /* lock per CPU */ - struct lpfc_nvmet_ctx_info *nvmet_ctx_next_cpu; - struct lpfc_nvmet_ctx_info *nvmet_ctx_start_cpu; - uint16_t nvmet_ctx_list_cnt; - char pad[16]; /* pad to a cache-line */ -}; - -/* This retrieves the context info associated with the specified cpu / mrq */ -#define lpfc_get_ctx_list(phba, cpu, mrq) \ - (phba->sli4_hba.nvmet_ctx_info + ((cpu * phba->cfg_nvmet_mrq) + mrq)) - -struct lpfc_nvmet_rcv_ctx { - union { - struct nvmefc_ls_rsp ls_rsp; - struct nvmefc_tgt_fcp_req fcp_req; - } ctx; - struct list_head list; - struct lpfc_hba *phba; - struct lpfc_iocbq *wqeq; - struct lpfc_iocbq *abort_wqeq; - spinlock_t ctxlock; /* protect flag access */ - uint32_t sid; - uint32_t offset; - uint16_t oxid; - uint16_t size; - uint16_t entry_cnt; - uint16_t cpu; - uint16_t idx; - uint16_t state; - /* States */ -#define LPFC_NVMET_STE_LS_RCV 1 -#define LPFC_NVMET_STE_LS_ABORT 2 -#define LPFC_NVMET_STE_LS_RSP 3 -#define LPFC_NVMET_STE_RCV 4 -#define LPFC_NVMET_STE_DATA 5 -#define LPFC_NVMET_STE_ABORT 6 -#define LPFC_NVMET_STE_DONE 7 -#define LPFC_NVMET_STE_FREE 0xff - uint16_t flag; -#define LPFC_NVMET_IO_INP 0x1 /* IO is in progress on exchange */ -#define LPFC_NVMET_ABORT_OP 0x2 /* Abort WQE issued on exchange */ -#define LPFC_NVMET_XBUSY 0x4 /* XB bit set on IO cmpl */ -#define LPFC_NVMET_CTX_RLS 0x8 /* ctx free requested */ -#define LPFC_NVMET_ABTS_RCV 0x10 /* ABTS received on exchange */ -#define LPFC_NVMET_CTX_REUSE_WQ 0x20 /* ctx reused via WQ */ -#define LPFC_NVMET_DEFER_WQFULL 0x40 /* Waiting on a free WQE */ -#define LPFC_NVMET_TNOTIFY 0x80 /* notify transport of abts */ - struct rqb_dmabuf *rqb_buffer; - struct lpfc_nvmet_ctxbuf *ctxbuf; - struct lpfc_sli4_hdw_queue *hdwq; - -#ifdef CONFIG_SCSI_LPFC_DEBUG_FS - uint64_t ts_isr_cmd; - uint64_t ts_cmd_nvme; - uint64_t ts_nvme_data; - uint64_t ts_data_wqput; - uint64_t ts_isr_data; - uint64_t ts_data_nvme; - uint64_t ts_nvme_status; - uint64_t ts_status_wqput; - uint64_t ts_isr_status; - uint64_t ts_status_nvme; -#endif -}; diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index b6fb665e6ec4..d57918dec15c 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -39,8 +39,6 @@ #include #endif -#include - #include "lpfc_hw4.h" #include "lpfc_hw.h" #include "lpfc_sli.h" @@ -50,7 +48,6 @@ #include "lpfc.h" #include "lpfc_scsi.h" #include "lpfc_nvme.h" -#include "lpfc_nvmet.h" #include "lpfc_crtn.h" #include "lpfc_logmsg.h" #include "lpfc_compat.h" From 7cacae2ad04762803ad93bdf08dc482106817ec7 Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:50:03 -0700 Subject: [PATCH 0323/1043] lpfc: Refactor nvmet_rcv_ctx to create lpfc_async_xchg_ctx To support FC-NVME-2 support (actually FC-NVME (rev 1) with Ammendment 1), both the nvme (host) and nvmet (controller/target) sides will need to be able to receive LS requests. Currently, this support is in the nvmet side only. To prepare for both sides supporting LS receive, rename lpfc_nvmet_rcv_ctx to lpfc_async_xchg_ctx and commonize the definition. Signed-off-by: Paul Ely Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/scsi/lpfc/lpfc.h | 2 +- drivers/scsi/lpfc/lpfc_crtn.h | 1 - drivers/scsi/lpfc/lpfc_debugfs.c | 2 +- drivers/scsi/lpfc/lpfc_init.c | 2 +- drivers/scsi/lpfc/lpfc_nvme.h | 7 +- drivers/scsi/lpfc/lpfc_nvmet.c | 109 ++++++++++++++++--------------- drivers/scsi/lpfc/lpfc_sli.c | 2 +- 7 files changed, 63 insertions(+), 62 deletions(-) diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h index 8e2a356911a9..62e96d4fdcc6 100644 --- a/drivers/scsi/lpfc/lpfc.h +++ b/drivers/scsi/lpfc/lpfc.h @@ -143,7 +143,7 @@ struct lpfc_dmabuf { struct lpfc_nvmet_ctxbuf { struct list_head list; - struct lpfc_nvmet_rcv_ctx *context; + struct lpfc_async_xchg_ctx *context; struct lpfc_iocbq *iocbq; struct lpfc_sglq *sglq; struct work_struct defer_work; diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h index 76dc8d9493d2..635df4a0188c 100644 --- a/drivers/scsi/lpfc/lpfc_crtn.h +++ b/drivers/scsi/lpfc/lpfc_crtn.h @@ -24,7 +24,6 @@ typedef int (*node_filter)(struct lpfc_nodelist *, void *); struct fc_rport; struct fc_frame_header; -struct lpfc_nvmet_rcv_ctx; void lpfc_down_link(struct lpfc_hba *, LPFC_MBOXQ_t *); void lpfc_sli_read_link_ste(struct lpfc_hba *); void lpfc_dump_mem(struct lpfc_hba *, LPFC_MBOXQ_t *, uint16_t, uint16_t); diff --git a/drivers/scsi/lpfc/lpfc_debugfs.c b/drivers/scsi/lpfc/lpfc_debugfs.c index 8e78e49e3f9f..4e94148b9352 100644 --- a/drivers/scsi/lpfc/lpfc_debugfs.c +++ b/drivers/scsi/lpfc/lpfc_debugfs.c @@ -1032,7 +1032,7 @@ lpfc_debugfs_nvmestat_data(struct lpfc_vport *vport, char *buf, int size) { struct lpfc_hba *phba = vport->phba; struct lpfc_nvmet_tgtport *tgtp; - struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp; + struct lpfc_async_xchg_ctx *ctxp, *next_ctxp; struct nvme_fc_local_port *localport; struct lpfc_fc4_ctrl_stat *cstat; struct lpfc_nvme_lport *lport; diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index a29566010d94..0e0b40710ea9 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -1029,7 +1029,7 @@ static int lpfc_hba_down_post_s4(struct lpfc_hba *phba) { struct lpfc_io_buf *psb, *psb_next; - struct lpfc_nvmet_rcv_ctx *ctxp, *ctxp_next; + struct lpfc_async_xchg_ctx *ctxp, *ctxp_next; struct lpfc_sli4_hdw_queue *qp; LIST_HEAD(aborts); LIST_HEAD(nvme_aborts); diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h index e59dc1f5c18a..be7d262f8cf3 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.h +++ b/drivers/scsi/lpfc/lpfc_nvme.h @@ -162,13 +162,14 @@ struct lpfc_nvmet_ctx_info { #define lpfc_get_ctx_list(phba, cpu, mrq) \ (phba->sli4_hba.nvmet_ctx_info + ((cpu * phba->cfg_nvmet_mrq) + mrq)) -struct lpfc_nvmet_rcv_ctx { +struct lpfc_async_xchg_ctx { union { - struct nvmefc_ls_rsp ls_rsp; struct nvmefc_tgt_fcp_req fcp_req; - } ctx; + } hdlrctx; struct list_head list; struct lpfc_hba *phba; + struct nvmefc_ls_req *ls_req; + struct nvmefc_ls_rsp ls_rsp; struct lpfc_iocbq *wqeq; struct lpfc_iocbq *abort_wqeq; spinlock_t ctxlock; /* protect flag access */ diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index 9576bc30ec85..03e8010564af 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -52,22 +52,22 @@ #include "lpfc_debugfs.h" static struct lpfc_iocbq *lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *, - struct lpfc_nvmet_rcv_ctx *, + struct lpfc_async_xchg_ctx *, dma_addr_t rspbuf, uint16_t rspsize); static struct lpfc_iocbq *lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *, - struct lpfc_nvmet_rcv_ctx *); + struct lpfc_async_xchg_ctx *); static int lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *, - struct lpfc_nvmet_rcv_ctx *, + struct lpfc_async_xchg_ctx *, uint32_t, uint16_t); static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *, - struct lpfc_nvmet_rcv_ctx *, + struct lpfc_async_xchg_ctx *, uint32_t, uint16_t); static int lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *, - struct lpfc_nvmet_rcv_ctx *, + struct lpfc_async_xchg_ctx *, uint32_t, uint16_t); static void lpfc_nvmet_wqfull_flush(struct lpfc_hba *, struct lpfc_queue *, - struct lpfc_nvmet_rcv_ctx *); + struct lpfc_async_xchg_ctx *); static void lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *); static void lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf); @@ -216,10 +216,10 @@ lpfc_nvmet_cmd_template(void) } #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) -static struct lpfc_nvmet_rcv_ctx * +static struct lpfc_async_xchg_ctx * lpfc_nvmet_get_ctx_for_xri(struct lpfc_hba *phba, u16 xri) { - struct lpfc_nvmet_rcv_ctx *ctxp; + struct lpfc_async_xchg_ctx *ctxp; unsigned long iflag; bool found = false; @@ -238,10 +238,10 @@ lpfc_nvmet_get_ctx_for_xri(struct lpfc_hba *phba, u16 xri) return NULL; } -static struct lpfc_nvmet_rcv_ctx * +static struct lpfc_async_xchg_ctx * lpfc_nvmet_get_ctx_for_oxid(struct lpfc_hba *phba, u16 oxid, u32 sid) { - struct lpfc_nvmet_rcv_ctx *ctxp; + struct lpfc_async_xchg_ctx *ctxp; unsigned long iflag; bool found = false; @@ -262,7 +262,8 @@ lpfc_nvmet_get_ctx_for_oxid(struct lpfc_hba *phba, u16 oxid, u32 sid) #endif static void -lpfc_nvmet_defer_release(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp) +lpfc_nvmet_defer_release(struct lpfc_hba *phba, + struct lpfc_async_xchg_ctx *ctxp) { lockdep_assert_held(&ctxp->ctxlock); @@ -298,7 +299,7 @@ lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, { struct lpfc_nvmet_tgtport *tgtp; struct nvmefc_ls_rsp *rsp; - struct lpfc_nvmet_rcv_ctx *ctxp; + struct lpfc_async_xchg_ctx *ctxp; uint32_t status, result; status = bf_get(lpfc_wcqe_c_status, wcqe); @@ -330,7 +331,7 @@ lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, } out: - rsp = &ctxp->ctx.ls_rsp; + rsp = &ctxp->ls_rsp; lpfc_nvmeio_data(phba, "NVMET LS CMPL: xri x%x stat x%x result x%x\n", ctxp->oxid, status, result); @@ -364,7 +365,7 @@ void lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) { #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) - struct lpfc_nvmet_rcv_ctx *ctxp = ctx_buf->context; + struct lpfc_async_xchg_ctx *ctxp = ctx_buf->context; struct lpfc_nvmet_tgtport *tgtp; struct fc_frame_header *fc_hdr; struct rqb_dmabuf *nvmebuf; @@ -416,7 +417,7 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) size = nvmebuf->bytes_recv; sid = sli4_sid_from_fc_hdr(fc_hdr); - ctxp = (struct lpfc_nvmet_rcv_ctx *)ctx_buf->context; + ctxp = (struct lpfc_async_xchg_ctx *)ctx_buf->context; ctxp->wqeq = NULL; ctxp->offset = 0; ctxp->phba = phba; @@ -490,7 +491,7 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) #ifdef CONFIG_SCSI_LPFC_DEBUG_FS static void lpfc_nvmet_ktime(struct lpfc_hba *phba, - struct lpfc_nvmet_rcv_ctx *ctxp) + struct lpfc_async_xchg_ctx *ctxp) { uint64_t seg1, seg2, seg3, seg4, seg5; uint64_t seg6, seg7, seg8, seg9, seg10; @@ -699,7 +700,7 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, { struct lpfc_nvmet_tgtport *tgtp; struct nvmefc_tgt_fcp_req *rsp; - struct lpfc_nvmet_rcv_ctx *ctxp; + struct lpfc_async_xchg_ctx *ctxp; uint32_t status, result, op, start_clean, logerr; #ifdef CONFIG_SCSI_LPFC_DEBUG_FS int id; @@ -708,7 +709,7 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, ctxp = cmdwqe->context2; ctxp->flag &= ~LPFC_NVMET_IO_INP; - rsp = &ctxp->ctx.fcp_req; + rsp = &ctxp->hdlrctx.fcp_req; op = rsp->op; status = bf_get(lpfc_wcqe_c_status, wcqe); @@ -825,8 +826,8 @@ static int lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport, struct nvmefc_ls_rsp *rsp) { - struct lpfc_nvmet_rcv_ctx *ctxp = - container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.ls_rsp); + struct lpfc_async_xchg_ctx *ctxp = + container_of(rsp, struct lpfc_async_xchg_ctx, ls_rsp); struct lpfc_hba *phba = ctxp->phba; struct hbq_dmabuf *nvmebuf = (struct hbq_dmabuf *)ctxp->rqb_buffer; @@ -916,8 +917,8 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport, struct nvmefc_tgt_fcp_req *rsp) { struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private; - struct lpfc_nvmet_rcv_ctx *ctxp = - container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req); + struct lpfc_async_xchg_ctx *ctxp = + container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req); struct lpfc_hba *phba = ctxp->phba; struct lpfc_queue *wq; struct lpfc_iocbq *nvmewqeq; @@ -1051,8 +1052,8 @@ lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport, struct nvmefc_tgt_fcp_req *req) { struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private; - struct lpfc_nvmet_rcv_ctx *ctxp = - container_of(req, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req); + struct lpfc_async_xchg_ctx *ctxp = + container_of(req, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req); struct lpfc_hba *phba = ctxp->phba; struct lpfc_queue *wq; unsigned long flags; @@ -1113,8 +1114,8 @@ lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport, struct nvmefc_tgt_fcp_req *rsp) { struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private; - struct lpfc_nvmet_rcv_ctx *ctxp = - container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req); + struct lpfc_async_xchg_ctx *ctxp = + container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req); struct lpfc_hba *phba = ctxp->phba; unsigned long flags; bool aborting = false; @@ -1156,8 +1157,8 @@ lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport, struct nvmefc_tgt_fcp_req *rsp) { struct lpfc_nvmet_tgtport *tgtp; - struct lpfc_nvmet_rcv_ctx *ctxp = - container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req); + struct lpfc_async_xchg_ctx *ctxp = + container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req); struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer; struct lpfc_hba *phba = ctxp->phba; unsigned long iflag; @@ -1563,7 +1564,7 @@ lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri); - struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp; + struct lpfc_async_xchg_ctx *ctxp, *next_ctxp; struct lpfc_nvmet_tgtport *tgtp; struct nvmefc_tgt_fcp_req *req = NULL; struct lpfc_nodelist *ndlp; @@ -1649,7 +1650,7 @@ lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n", xri, raw_smp_processor_id(), 0); - req = &ctxp->ctx.fcp_req; + req = &ctxp->hdlrctx.fcp_req; if (req) nvmet_fc_rcv_fcp_abort(phba->targetport, req); } @@ -1662,7 +1663,7 @@ lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, { #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) struct lpfc_hba *phba = vport->phba; - struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp; + struct lpfc_async_xchg_ctx *ctxp, *next_ctxp; struct nvmefc_tgt_fcp_req *rsp; uint32_t sid; uint16_t oxid, xri; @@ -1695,7 +1696,7 @@ lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, "6319 NVMET Rcv ABTS:acc xri x%x\n", xri); - rsp = &ctxp->ctx.fcp_req; + rsp = &ctxp->hdlrctx.fcp_req; nvmet_fc_rcv_fcp_abort(phba->targetport, rsp); /* Respond with BA_ACC accordingly */ @@ -1769,7 +1770,7 @@ lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, if (ctxp->flag & LPFC_NVMET_TNOTIFY) { /* Notify the transport */ nvmet_fc_rcv_fcp_abort(phba->targetport, - &ctxp->ctx.fcp_req); + &ctxp->hdlrctx.fcp_req); } else { cancel_work_sync(&ctxp->ctxbuf->defer_work); spin_lock_irqsave(&ctxp->ctxlock, iflag); @@ -1797,7 +1798,7 @@ lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, static void lpfc_nvmet_wqfull_flush(struct lpfc_hba *phba, struct lpfc_queue *wq, - struct lpfc_nvmet_rcv_ctx *ctxp) + struct lpfc_async_xchg_ctx *ctxp) { struct lpfc_sli_ring *pring; struct lpfc_iocbq *nvmewqeq; @@ -1848,7 +1849,7 @@ lpfc_nvmet_wqfull_process(struct lpfc_hba *phba, #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) struct lpfc_sli_ring *pring; struct lpfc_iocbq *nvmewqeq; - struct lpfc_nvmet_rcv_ctx *ctxp; + struct lpfc_async_xchg_ctx *ctxp; unsigned long iflags; int rc; @@ -1862,7 +1863,7 @@ lpfc_nvmet_wqfull_process(struct lpfc_hba *phba, list_remove_head(&wq->wqfull_list, nvmewqeq, struct lpfc_iocbq, list); spin_unlock_irqrestore(&pring->ring_lock, iflags); - ctxp = (struct lpfc_nvmet_rcv_ctx *)nvmewqeq->context2; + ctxp = (struct lpfc_async_xchg_ctx *)nvmewqeq->context2; rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq); spin_lock_irqsave(&pring->ring_lock, iflags); if (rc == -EBUSY) { @@ -1874,7 +1875,7 @@ lpfc_nvmet_wqfull_process(struct lpfc_hba *phba, if (rc == WQE_SUCCESS) { #ifdef CONFIG_SCSI_LPFC_DEBUG_FS if (ctxp->ts_cmd_nvme) { - if (ctxp->ctx.fcp_req.op == NVMET_FCOP_RSP) + if (ctxp->hdlrctx.fcp_req.op == NVMET_FCOP_RSP) ctxp->ts_status_wqput = ktime_get_ns(); else ctxp->ts_data_wqput = ktime_get_ns(); @@ -1940,7 +1941,7 @@ lpfc_nvmet_unsol_ls_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) struct lpfc_nvmet_tgtport *tgtp; struct fc_frame_header *fc_hdr; - struct lpfc_nvmet_rcv_ctx *ctxp; + struct lpfc_async_xchg_ctx *ctxp; uint32_t *payload; uint32_t size, oxid, sid, rc; @@ -1963,7 +1964,7 @@ lpfc_nvmet_unsol_ls_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, size = bf_get(lpfc_rcqe_length, &nvmebuf->cq_event.cqe.rcqe_cmpl); sid = sli4_sid_from_fc_hdr(fc_hdr); - ctxp = kzalloc(sizeof(struct lpfc_nvmet_rcv_ctx), GFP_ATOMIC); + ctxp = kzalloc(sizeof(struct lpfc_async_xchg_ctx), GFP_ATOMIC); if (ctxp == NULL) { atomic_inc(&tgtp->rcv_ls_req_drop); lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, @@ -1994,7 +1995,7 @@ dropit: * lpfc_nvmet_xmt_ls_rsp_cmp should free the allocated ctxp. */ atomic_inc(&tgtp->rcv_ls_req_in); - rc = nvmet_fc_rcv_ls_req(phba->targetport, NULL, &ctxp->ctx.ls_rsp, + rc = nvmet_fc_rcv_ls_req(phba->targetport, NULL, &ctxp->ls_rsp, payload, size); lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, @@ -2028,7 +2029,7 @@ static void lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf) { #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) - struct lpfc_nvmet_rcv_ctx *ctxp = ctx_buf->context; + struct lpfc_async_xchg_ctx *ctxp = ctx_buf->context; struct lpfc_hba *phba = ctxp->phba; struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer; struct lpfc_nvmet_tgtport *tgtp; @@ -2072,7 +2073,7 @@ lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf) * A buffer has already been reposted for this IO, so just free * the nvmebuf. */ - rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req, + rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->hdlrctx.fcp_req, payload, ctxp->size); /* Process FCP command */ if (rc == 0) { @@ -2219,7 +2220,7 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, uint64_t isr_timestamp, uint8_t cqflag) { - struct lpfc_nvmet_rcv_ctx *ctxp; + struct lpfc_async_xchg_ctx *ctxp; struct lpfc_nvmet_tgtport *tgtp; struct fc_frame_header *fc_hdr; struct lpfc_nvmet_ctxbuf *ctx_buf; @@ -2301,7 +2302,7 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, sid = sli4_sid_from_fc_hdr(fc_hdr); - ctxp = (struct lpfc_nvmet_rcv_ctx *)ctx_buf->context; + ctxp = (struct lpfc_async_xchg_ctx *)ctx_buf->context; spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag); list_add_tail(&ctxp->list, &phba->sli4_hba.t_active_ctx_list); spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag); @@ -2457,7 +2458,7 @@ lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba, **/ static struct lpfc_iocbq * lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *phba, - struct lpfc_nvmet_rcv_ctx *ctxp, + struct lpfc_async_xchg_ctx *ctxp, dma_addr_t rspbuf, uint16_t rspsize) { struct lpfc_nodelist *ndlp; @@ -2579,9 +2580,9 @@ nvme_wqe_free_wqeq_exit: static struct lpfc_iocbq * lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba, - struct lpfc_nvmet_rcv_ctx *ctxp) + struct lpfc_async_xchg_ctx *ctxp) { - struct nvmefc_tgt_fcp_req *rsp = &ctxp->ctx.fcp_req; + struct nvmefc_tgt_fcp_req *rsp = &ctxp->hdlrctx.fcp_req; struct lpfc_nvmet_tgtport *tgtp; struct sli4_sge *sgl; struct lpfc_nodelist *ndlp; @@ -2926,7 +2927,7 @@ static void lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, struct lpfc_wcqe_complete *wcqe) { - struct lpfc_nvmet_rcv_ctx *ctxp; + struct lpfc_async_xchg_ctx *ctxp; struct lpfc_nvmet_tgtport *tgtp; uint32_t result; unsigned long flags; @@ -2995,7 +2996,7 @@ static void lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, struct lpfc_wcqe_complete *wcqe) { - struct lpfc_nvmet_rcv_ctx *ctxp; + struct lpfc_async_xchg_ctx *ctxp; struct lpfc_nvmet_tgtport *tgtp; unsigned long flags; uint32_t result; @@ -3076,7 +3077,7 @@ static void lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, struct lpfc_wcqe_complete *wcqe) { - struct lpfc_nvmet_rcv_ctx *ctxp; + struct lpfc_async_xchg_ctx *ctxp; struct lpfc_nvmet_tgtport *tgtp; uint32_t result; @@ -3117,7 +3118,7 @@ lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, static int lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba, - struct lpfc_nvmet_rcv_ctx *ctxp, + struct lpfc_async_xchg_ctx *ctxp, uint32_t sid, uint16_t xri) { struct lpfc_nvmet_tgtport *tgtp; @@ -3212,7 +3213,7 @@ lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba, static int lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, - struct lpfc_nvmet_rcv_ctx *ctxp, + struct lpfc_async_xchg_ctx *ctxp, uint32_t sid, uint16_t xri) { struct lpfc_nvmet_tgtport *tgtp; @@ -3338,7 +3339,7 @@ lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba, - struct lpfc_nvmet_rcv_ctx *ctxp, + struct lpfc_async_xchg_ctx *ctxp, uint32_t sid, uint16_t xri) { struct lpfc_nvmet_tgtport *tgtp; @@ -3403,7 +3404,7 @@ aerr: static int lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *phba, - struct lpfc_nvmet_rcv_ctx *ctxp, + struct lpfc_async_xchg_ctx *ctxp, uint32_t sid, uint16_t xri) { struct lpfc_nvmet_tgtport *tgtp; diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index d57918dec15c..d4d810cb8944 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -19888,7 +19888,7 @@ lpfc_sli4_issue_wqe(struct lpfc_hba *phba, struct lpfc_sli4_hdw_queue *qp, struct lpfc_iocbq *pwqe) { union lpfc_wqe128 *wqe = &pwqe->wqe; - struct lpfc_nvmet_rcv_ctx *ctxp; + struct lpfc_async_xchg_ctx *ctxp; struct lpfc_queue *wq; struct lpfc_sglq *sglq; struct lpfc_sli_ring *pring; From 7b7f551b0403e0f740c6af2553b46ba2d3531c80 Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:50:04 -0700 Subject: [PATCH 0324/1043] lpfc: Commonize lpfc_async_xchg_ctx state and flag definitions The last step of commonization is to remove the 'T' suffix from state and flag field definitions. This is minor, but removes the mental association that it solely applies to nvmet use. Signed-off-by: Paul Ely Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/scsi/lpfc/lpfc_init.c | 2 +- drivers/scsi/lpfc/lpfc_nvme.h | 37 ++++---- drivers/scsi/lpfc/lpfc_nvmet.c | 158 ++++++++++++++++----------------- 3 files changed, 100 insertions(+), 97 deletions(-) diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c index 0e0b40710ea9..ea99483345f2 100644 --- a/drivers/scsi/lpfc/lpfc_init.c +++ b/drivers/scsi/lpfc/lpfc_init.c @@ -1096,7 +1096,7 @@ lpfc_hba_down_post_s4(struct lpfc_hba *phba) &nvmet_aborts); spin_unlock_irq(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_for_each_entry_safe(ctxp, ctxp_next, &nvmet_aborts, list) { - ctxp->flag &= ~(LPFC_NVMET_XBUSY | LPFC_NVMET_ABORT_OP); + ctxp->flag &= ~(LPFC_NVME_XBUSY | LPFC_NVME_ABORT_OP); lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); } } diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h index be7d262f8cf3..220b51af43da 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.h +++ b/drivers/scsi/lpfc/lpfc_nvme.h @@ -162,6 +162,26 @@ struct lpfc_nvmet_ctx_info { #define lpfc_get_ctx_list(phba, cpu, mrq) \ (phba->sli4_hba.nvmet_ctx_info + ((cpu * phba->cfg_nvmet_mrq) + mrq)) +/* Values for state field of struct lpfc_async_xchg_ctx */ +#define LPFC_NVME_STE_LS_RCV 1 +#define LPFC_NVME_STE_LS_ABORT 2 +#define LPFC_NVME_STE_LS_RSP 3 +#define LPFC_NVME_STE_RCV 4 +#define LPFC_NVME_STE_DATA 5 +#define LPFC_NVME_STE_ABORT 6 +#define LPFC_NVME_STE_DONE 7 +#define LPFC_NVME_STE_FREE 0xff + +/* Values for flag field of struct lpfc_async_xchg_ctx */ +#define LPFC_NVME_IO_INP 0x1 /* IO is in progress on exchange */ +#define LPFC_NVME_ABORT_OP 0x2 /* Abort WQE issued on exchange */ +#define LPFC_NVME_XBUSY 0x4 /* XB bit set on IO cmpl */ +#define LPFC_NVME_CTX_RLS 0x8 /* ctx free requested */ +#define LPFC_NVME_ABTS_RCV 0x10 /* ABTS received on exchange */ +#define LPFC_NVME_CTX_REUSE_WQ 0x20 /* ctx reused via WQ */ +#define LPFC_NVME_DEFER_WQFULL 0x40 /* Waiting on a free WQE */ +#define LPFC_NVME_TNOTIFY 0x80 /* notify transport of abts */ + struct lpfc_async_xchg_ctx { union { struct nvmefc_tgt_fcp_req fcp_req; @@ -181,24 +201,7 @@ struct lpfc_async_xchg_ctx { uint16_t cpu; uint16_t idx; uint16_t state; - /* States */ -#define LPFC_NVMET_STE_LS_RCV 1 -#define LPFC_NVMET_STE_LS_ABORT 2 -#define LPFC_NVMET_STE_LS_RSP 3 -#define LPFC_NVMET_STE_RCV 4 -#define LPFC_NVMET_STE_DATA 5 -#define LPFC_NVMET_STE_ABORT 6 -#define LPFC_NVMET_STE_DONE 7 -#define LPFC_NVMET_STE_FREE 0xff uint16_t flag; -#define LPFC_NVMET_IO_INP 0x1 /* IO is in progress on exchange */ -#define LPFC_NVMET_ABORT_OP 0x2 /* Abort WQE issued on exchange */ -#define LPFC_NVMET_XBUSY 0x4 /* XB bit set on IO cmpl */ -#define LPFC_NVMET_CTX_RLS 0x8 /* ctx free requested */ -#define LPFC_NVMET_ABTS_RCV 0x10 /* ABTS received on exchange */ -#define LPFC_NVMET_CTX_REUSE_WQ 0x20 /* ctx reused via WQ */ -#define LPFC_NVMET_DEFER_WQFULL 0x40 /* Waiting on a free WQE */ -#define LPFC_NVMET_TNOTIFY 0x80 /* notify transport of abts */ struct rqb_dmabuf *rqb_buffer; struct lpfc_nvmet_ctxbuf *ctxbuf; struct lpfc_sli4_hdw_queue *hdwq; diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index 03e8010564af..e0d5be4617ac 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -271,10 +271,10 @@ lpfc_nvmet_defer_release(struct lpfc_hba *phba, "6313 NVMET Defer ctx release oxid x%x flg x%x\n", ctxp->oxid, ctxp->flag); - if (ctxp->flag & LPFC_NVMET_CTX_RLS) + if (ctxp->flag & LPFC_NVME_CTX_RLS) return; - ctxp->flag |= LPFC_NVMET_CTX_RLS; + ctxp->flag |= LPFC_NVME_CTX_RLS; spin_lock(&phba->sli4_hba.t_active_list_lock); list_del(&ctxp->list); spin_unlock(&phba->sli4_hba.t_active_list_lock); @@ -306,7 +306,7 @@ lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, result = wcqe->parameter; ctxp = cmdwqe->context2; - if (ctxp->state != LPFC_NVMET_STE_LS_RSP || ctxp->entry_cnt != 2) { + if (ctxp->state != LPFC_NVME_STE_LS_RSP || ctxp->entry_cnt != 2) { lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, "6410 NVMET LS cmpl state mismatch IO x%x: " "%d %d\n", @@ -374,7 +374,7 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) int cpu; unsigned long iflag; - if (ctxp->state == LPFC_NVMET_STE_FREE) { + if (ctxp->state == LPFC_NVME_STE_FREE) { lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, "6411 NVMET free, already free IO x%x: %d %d\n", ctxp->oxid, ctxp->state, ctxp->entry_cnt); @@ -386,8 +386,8 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) /* check if freed in another path whilst acquiring lock */ if (nvmebuf) { ctxp->rqb_buffer = NULL; - if (ctxp->flag & LPFC_NVMET_CTX_REUSE_WQ) { - ctxp->flag &= ~LPFC_NVMET_CTX_REUSE_WQ; + if (ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) { + ctxp->flag &= ~LPFC_NVME_CTX_REUSE_WQ; spin_unlock_irqrestore(&ctxp->ctxlock, iflag); nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf); @@ -400,7 +400,7 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) spin_unlock_irqrestore(&ctxp->ctxlock, iflag); } } - ctxp->state = LPFC_NVMET_STE_FREE; + ctxp->state = LPFC_NVME_STE_FREE; spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag); if (phba->sli4_hba.nvmet_io_wait_cnt) { @@ -424,7 +424,7 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) ctxp->size = size; ctxp->oxid = oxid; ctxp->sid = sid; - ctxp->state = LPFC_NVMET_STE_RCV; + ctxp->state = LPFC_NVME_STE_RCV; ctxp->entry_cnt = 1; ctxp->flag = 0; ctxp->ctxbuf = ctx_buf; @@ -449,7 +449,7 @@ lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf) /* Indicate that a replacement buffer has been posted */ spin_lock_irqsave(&ctxp->ctxlock, iflag); - ctxp->flag |= LPFC_NVMET_CTX_REUSE_WQ; + ctxp->flag |= LPFC_NVME_CTX_REUSE_WQ; spin_unlock_irqrestore(&ctxp->ctxlock, iflag); if (!queue_work(phba->wq, &ctx_buf->defer_work)) { @@ -707,7 +707,7 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, #endif ctxp = cmdwqe->context2; - ctxp->flag &= ~LPFC_NVMET_IO_INP; + ctxp->flag &= ~LPFC_NVME_IO_INP; rsp = &ctxp->hdlrctx.fcp_req; op = rsp->op; @@ -736,13 +736,13 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, /* pick up SLI4 exhange busy condition */ if (bf_get(lpfc_wcqe_c_xb, wcqe)) { - ctxp->flag |= LPFC_NVMET_XBUSY; + ctxp->flag |= LPFC_NVME_XBUSY; logerr |= LOG_NVME_ABTS; if (tgtp) atomic_inc(&tgtp->xmt_fcp_rsp_xb_set); } else { - ctxp->flag &= ~LPFC_NVMET_XBUSY; + ctxp->flag &= ~LPFC_NVME_XBUSY; } lpfc_printf_log(phba, KERN_INFO, logerr, @@ -764,7 +764,7 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, if ((op == NVMET_FCOP_READDATA_RSP) || (op == NVMET_FCOP_RSP)) { /* Sanity check */ - ctxp->state = LPFC_NVMET_STE_DONE; + ctxp->state = LPFC_NVME_STE_DONE; ctxp->entry_cnt++; #ifdef CONFIG_SCSI_LPFC_DEBUG_FS @@ -846,14 +846,14 @@ lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport, lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, "6023 NVMET LS rsp oxid x%x\n", ctxp->oxid); - if ((ctxp->state != LPFC_NVMET_STE_LS_RCV) || + if ((ctxp->state != LPFC_NVME_STE_LS_RCV) || (ctxp->entry_cnt != 1)) { lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, "6412 NVMET LS rsp state mismatch " "oxid x%x: %d %d\n", ctxp->oxid, ctxp->state, ctxp->entry_cnt); } - ctxp->state = LPFC_NVMET_STE_LS_RSP; + ctxp->state = LPFC_NVME_STE_LS_RSP; ctxp->entry_cnt++; nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, ctxp, rsp->rspdma, @@ -964,8 +964,8 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport, #endif /* Sanity check */ - if ((ctxp->flag & LPFC_NVMET_ABTS_RCV) || - (ctxp->state == LPFC_NVMET_STE_ABORT)) { + if ((ctxp->flag & LPFC_NVME_ABTS_RCV) || + (ctxp->state == LPFC_NVME_STE_ABORT)) { atomic_inc(&lpfc_nvmep->xmt_fcp_drop); lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, "6102 IO oxid x%x aborted\n", @@ -993,7 +993,7 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport, lpfc_nvmeio_data(phba, "NVMET FCP CMND: xri x%x op x%x len x%x\n", ctxp->oxid, rsp->op, rsp->rsplen); - ctxp->flag |= LPFC_NVMET_IO_INP; + ctxp->flag |= LPFC_NVME_IO_INP; rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq); if (rc == WQE_SUCCESS) { #ifdef CONFIG_SCSI_LPFC_DEBUG_FS @@ -1012,7 +1012,7 @@ lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport, * WQ was full, so queue nvmewqeq to be sent after * WQE release CQE */ - ctxp->flag |= LPFC_NVMET_DEFER_WQFULL; + ctxp->flag |= LPFC_NVME_DEFER_WQFULL; wq = ctxp->hdwq->io_wq; pring = wq->pring; spin_lock_irqsave(&pring->ring_lock, iflags); @@ -1081,13 +1081,13 @@ lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport, /* Since iaab/iaar are NOT set, we need to check * if the firmware is in process of aborting IO */ - if (ctxp->flag & (LPFC_NVMET_XBUSY | LPFC_NVMET_ABORT_OP)) { + if (ctxp->flag & (LPFC_NVME_XBUSY | LPFC_NVME_ABORT_OP)) { spin_unlock_irqrestore(&ctxp->ctxlock, flags); return; } - ctxp->flag |= LPFC_NVMET_ABORT_OP; + ctxp->flag |= LPFC_NVME_ABORT_OP; - if (ctxp->flag & LPFC_NVMET_DEFER_WQFULL) { + if (ctxp->flag & LPFC_NVME_DEFER_WQFULL) { spin_unlock_irqrestore(&ctxp->ctxlock, flags); lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); @@ -1097,11 +1097,11 @@ lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport, } spin_unlock_irqrestore(&ctxp->ctxlock, flags); - /* An state of LPFC_NVMET_STE_RCV means we have just received + /* A state of LPFC_NVME_STE_RCV means we have just received * the NVME command and have not started processing it. * (by issuing any IO WQEs on this exchange yet) */ - if (ctxp->state == LPFC_NVMET_STE_RCV) + if (ctxp->state == LPFC_NVME_STE_RCV) lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); else @@ -1121,19 +1121,19 @@ lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport, bool aborting = false; spin_lock_irqsave(&ctxp->ctxlock, flags); - if (ctxp->flag & LPFC_NVMET_XBUSY) + if (ctxp->flag & LPFC_NVME_XBUSY) lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, "6027 NVMET release with XBUSY flag x%x" " oxid x%x\n", ctxp->flag, ctxp->oxid); - else if (ctxp->state != LPFC_NVMET_STE_DONE && - ctxp->state != LPFC_NVMET_STE_ABORT) + else if (ctxp->state != LPFC_NVME_STE_DONE && + ctxp->state != LPFC_NVME_STE_ABORT) lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, "6413 NVMET release bad state %d %d oxid x%x\n", ctxp->state, ctxp->entry_cnt, ctxp->oxid); - if ((ctxp->flag & LPFC_NVMET_ABORT_OP) || - (ctxp->flag & LPFC_NVMET_XBUSY)) { + if ((ctxp->flag & LPFC_NVME_ABORT_OP) || + (ctxp->flag & LPFC_NVME_XBUSY)) { aborting = true; /* let the abort path do the real release */ lpfc_nvmet_defer_release(phba, ctxp); @@ -1144,7 +1144,7 @@ lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport, ctxp->state, aborting); atomic_inc(&lpfc_nvmep->xmt_fcp_release); - ctxp->flag &= ~LPFC_NVMET_TNOTIFY; + ctxp->flag &= ~LPFC_NVME_TNOTIFY; if (aborting) return; @@ -1364,7 +1364,7 @@ lpfc_nvmet_setup_io_context(struct lpfc_hba *phba) return -ENOMEM; } ctx_buf->context->ctxbuf = ctx_buf; - ctx_buf->context->state = LPFC_NVMET_STE_FREE; + ctx_buf->context->state = LPFC_NVME_STE_FREE; ctx_buf->iocbq = lpfc_sli_get_iocbq(phba); if (!ctx_buf->iocbq) { @@ -1595,12 +1595,12 @@ lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, /* Check if we already received a free context call * and we have completed processing an abort situation. */ - if (ctxp->flag & LPFC_NVMET_CTX_RLS && - !(ctxp->flag & LPFC_NVMET_ABORT_OP)) { + if (ctxp->flag & LPFC_NVME_CTX_RLS && + !(ctxp->flag & LPFC_NVME_ABORT_OP)) { list_del_init(&ctxp->list); released = true; } - ctxp->flag &= ~LPFC_NVMET_XBUSY; + ctxp->flag &= ~LPFC_NVME_XBUSY; spin_unlock(&ctxp->ctxlock); spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); @@ -1642,8 +1642,8 @@ lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba, rxid); spin_lock_irqsave(&ctxp->ctxlock, iflag); - ctxp->flag |= LPFC_NVMET_ABTS_RCV; - ctxp->state = LPFC_NVMET_STE_ABORT; + ctxp->flag |= LPFC_NVME_ABTS_RCV; + ctxp->state = LPFC_NVME_STE_ABORT; spin_unlock_irqrestore(&ctxp->ctxlock, iflag); lpfc_nvmeio_data(phba, @@ -1686,7 +1686,7 @@ lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, spin_unlock_irqrestore(&phba->hbalock, iflag); spin_lock_irqsave(&ctxp->ctxlock, iflag); - ctxp->flag |= LPFC_NVMET_ABTS_RCV; + ctxp->flag |= LPFC_NVME_ABTS_RCV; spin_unlock_irqrestore(&ctxp->ctxlock, iflag); lpfc_nvmeio_data(phba, @@ -1755,7 +1755,7 @@ lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, xri = ctxp->ctxbuf->sglq->sli4_xritag; spin_lock_irqsave(&ctxp->ctxlock, iflag); - ctxp->flag |= (LPFC_NVMET_ABTS_RCV | LPFC_NVMET_ABORT_OP); + ctxp->flag |= (LPFC_NVME_ABTS_RCV | LPFC_NVME_ABORT_OP); spin_unlock_irqrestore(&ctxp->ctxlock, iflag); lpfc_nvmeio_data(phba, @@ -1767,7 +1767,7 @@ lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport, "flag x%x state x%x\n", ctxp->oxid, xri, ctxp->flag, ctxp->state); - if (ctxp->flag & LPFC_NVMET_TNOTIFY) { + if (ctxp->flag & LPFC_NVME_TNOTIFY) { /* Notify the transport */ nvmet_fc_rcv_fcp_abort(phba->targetport, &ctxp->hdlrctx.fcp_req); @@ -1982,7 +1982,7 @@ dropit: ctxp->oxid = oxid; ctxp->sid = sid; ctxp->wqeq = NULL; - ctxp->state = LPFC_NVMET_STE_LS_RCV; + ctxp->state = LPFC_NVME_STE_LS_RCV; ctxp->entry_cnt = 1; ctxp->rqb_buffer = (void *)nvmebuf; ctxp->hdwq = &phba->sli4_hba.hdwq[0]; @@ -2050,7 +2050,7 @@ lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf) return; } - if (ctxp->flag & LPFC_NVMET_ABTS_RCV) { + if (ctxp->flag & LPFC_NVME_ABTS_RCV) { lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, "6324 IO oxid x%x aborted\n", ctxp->oxid); @@ -2059,7 +2059,7 @@ lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf) payload = (uint32_t *)(nvmebuf->dbuf.virt); tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; - ctxp->flag |= LPFC_NVMET_TNOTIFY; + ctxp->flag |= LPFC_NVME_TNOTIFY; #ifdef CONFIG_SCSI_LPFC_DEBUG_FS if (ctxp->ts_isr_cmd) ctxp->ts_cmd_nvme = ktime_get_ns(); @@ -2079,7 +2079,7 @@ lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf) if (rc == 0) { atomic_inc(&tgtp->rcv_fcp_cmd_out); spin_lock_irqsave(&ctxp->ctxlock, iflags); - if ((ctxp->flag & LPFC_NVMET_CTX_REUSE_WQ) || + if ((ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) || (nvmebuf != ctxp->rqb_buffer)) { spin_unlock_irqrestore(&ctxp->ctxlock, iflags); return; @@ -2098,7 +2098,7 @@ lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf) atomic_inc(&tgtp->rcv_fcp_cmd_out); atomic_inc(&tgtp->defer_fod); spin_lock_irqsave(&ctxp->ctxlock, iflags); - if (ctxp->flag & LPFC_NVMET_CTX_REUSE_WQ) { + if (ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) { spin_unlock_irqrestore(&ctxp->ctxlock, iflags); return; } @@ -2113,7 +2113,7 @@ lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf) phba->sli4_hba.nvmet_mrq_data[qno], 1, qno); return; } - ctxp->flag &= ~LPFC_NVMET_TNOTIFY; + ctxp->flag &= ~LPFC_NVME_TNOTIFY; atomic_inc(&tgtp->rcv_fcp_cmd_drop); lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, "2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n", @@ -2306,7 +2306,7 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag); list_add_tail(&ctxp->list, &phba->sli4_hba.t_active_ctx_list); spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag); - if (ctxp->state != LPFC_NVMET_STE_FREE) { + if (ctxp->state != LPFC_NVME_STE_FREE) { lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, "6414 NVMET Context corrupt %d %d oxid x%x\n", ctxp->state, ctxp->entry_cnt, ctxp->oxid); @@ -2318,7 +2318,7 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, ctxp->oxid = oxid; ctxp->sid = sid; ctxp->idx = idx; - ctxp->state = LPFC_NVMET_STE_RCV; + ctxp->state = LPFC_NVME_STE_RCV; ctxp->entry_cnt = 1; ctxp->flag = 0; ctxp->ctxbuf = ctx_buf; @@ -2643,9 +2643,9 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba, } /* Sanity check */ - if (((ctxp->state == LPFC_NVMET_STE_RCV) && + if (((ctxp->state == LPFC_NVME_STE_RCV) && (ctxp->entry_cnt == 1)) || - (ctxp->state == LPFC_NVMET_STE_DATA)) { + (ctxp->state == LPFC_NVME_STE_DATA)) { wqe = &nvmewqe->wqe; } else { lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, @@ -2908,7 +2908,7 @@ lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba, sgl++; ctxp->offset += cnt; } - ctxp->state = LPFC_NVMET_STE_DATA; + ctxp->state = LPFC_NVME_STE_DATA; ctxp->entry_cnt++; return nvmewqe; } @@ -2937,23 +2937,23 @@ lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, result = wcqe->parameter; tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; - if (ctxp->flag & LPFC_NVMET_ABORT_OP) + if (ctxp->flag & LPFC_NVME_ABORT_OP) atomic_inc(&tgtp->xmt_fcp_abort_cmpl); spin_lock_irqsave(&ctxp->ctxlock, flags); - ctxp->state = LPFC_NVMET_STE_DONE; + ctxp->state = LPFC_NVME_STE_DONE; /* Check if we already received a free context call * and we have completed processing an abort situation. */ - if ((ctxp->flag & LPFC_NVMET_CTX_RLS) && - !(ctxp->flag & LPFC_NVMET_XBUSY)) { + if ((ctxp->flag & LPFC_NVME_CTX_RLS) && + !(ctxp->flag & LPFC_NVME_XBUSY)) { spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_del_init(&ctxp->list); spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); released = true; } - ctxp->flag &= ~LPFC_NVMET_ABORT_OP; + ctxp->flag &= ~LPFC_NVME_ABORT_OP; spin_unlock_irqrestore(&ctxp->ctxlock, flags); atomic_inc(&tgtp->xmt_abort_rsp); @@ -2977,7 +2977,7 @@ lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, lpfc_sli_release_iocbq(phba, cmdwqe); /* Since iaab/iaar are NOT set, there is no work left. - * For LPFC_NVMET_XBUSY, lpfc_sli4_nvmet_xri_aborted + * For LPFC_NVME_XBUSY, lpfc_sli4_nvmet_xri_aborted * should have been called already. */ } @@ -3016,11 +3016,11 @@ lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; spin_lock_irqsave(&ctxp->ctxlock, flags); - if (ctxp->flag & LPFC_NVMET_ABORT_OP) + if (ctxp->flag & LPFC_NVME_ABORT_OP) atomic_inc(&tgtp->xmt_fcp_abort_cmpl); /* Sanity check */ - if (ctxp->state != LPFC_NVMET_STE_ABORT) { + if (ctxp->state != LPFC_NVME_STE_ABORT) { lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, "6112 ABTS Wrong state:%d oxid x%x\n", ctxp->state, ctxp->oxid); @@ -3029,15 +3029,15 @@ lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, /* Check if we already received a free context call * and we have completed processing an abort situation. */ - ctxp->state = LPFC_NVMET_STE_DONE; - if ((ctxp->flag & LPFC_NVMET_CTX_RLS) && - !(ctxp->flag & LPFC_NVMET_XBUSY)) { + ctxp->state = LPFC_NVME_STE_DONE; + if ((ctxp->flag & LPFC_NVME_CTX_RLS) && + !(ctxp->flag & LPFC_NVME_XBUSY)) { spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_del_init(&ctxp->list); spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); released = true; } - ctxp->flag &= ~LPFC_NVMET_ABORT_OP; + ctxp->flag &= ~LPFC_NVME_ABORT_OP; spin_unlock_irqrestore(&ctxp->ctxlock, flags); atomic_inc(&tgtp->xmt_abort_rsp); @@ -3058,7 +3058,7 @@ lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf); /* Since iaab/iaar are NOT set, there is no work left. - * For LPFC_NVMET_XBUSY, lpfc_sli4_nvmet_xri_aborted + * For LPFC_NVME_XBUSY, lpfc_sli4_nvmet_xri_aborted * should have been called already. */ } @@ -3103,7 +3103,7 @@ lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, return; } - if (ctxp->state != LPFC_NVMET_STE_LS_ABORT) { + if (ctxp->state != LPFC_NVME_STE_LS_ABORT) { lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, "6416 NVMET LS abort cmpl state mismatch: " "oxid x%x: %d %d\n", @@ -3240,7 +3240,7 @@ lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, /* No failure to an ABTS request. */ spin_lock_irqsave(&ctxp->ctxlock, flags); - ctxp->flag &= ~LPFC_NVMET_ABORT_OP; + ctxp->flag &= ~LPFC_NVME_ABORT_OP; spin_unlock_irqrestore(&ctxp->ctxlock, flags); return 0; } @@ -3254,13 +3254,13 @@ lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, "6161 ABORT failed: No wqeqs: " "xri: x%x\n", ctxp->oxid); /* No failure to an ABTS request. */ - ctxp->flag &= ~LPFC_NVMET_ABORT_OP; + ctxp->flag &= ~LPFC_NVME_ABORT_OP; spin_unlock_irqrestore(&ctxp->ctxlock, flags); return 0; } abts_wqeq = ctxp->abort_wqeq; - ctxp->state = LPFC_NVMET_STE_ABORT; - opt = (ctxp->flag & LPFC_NVMET_ABTS_RCV) ? INHIBIT_ABORT : 0; + ctxp->state = LPFC_NVME_STE_ABORT; + opt = (ctxp->flag & LPFC_NVME_ABTS_RCV) ? INHIBIT_ABORT : 0; spin_unlock_irqrestore(&ctxp->ctxlock, flags); /* Announce entry to new IO submit field. */ @@ -3283,7 +3283,7 @@ lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, phba->hba_flag, ctxp->oxid); lpfc_sli_release_iocbq(phba, abts_wqeq); spin_lock_irqsave(&ctxp->ctxlock, flags); - ctxp->flag &= ~LPFC_NVMET_ABORT_OP; + ctxp->flag &= ~LPFC_NVME_ABORT_OP; spin_unlock_irqrestore(&ctxp->ctxlock, flags); return 0; } @@ -3298,7 +3298,7 @@ lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, ctxp->oxid); lpfc_sli_release_iocbq(phba, abts_wqeq); spin_lock_irqsave(&ctxp->ctxlock, flags); - ctxp->flag &= ~LPFC_NVMET_ABORT_OP; + ctxp->flag &= ~LPFC_NVME_ABORT_OP; spin_unlock_irqrestore(&ctxp->ctxlock, flags); return 0; } @@ -3327,7 +3327,7 @@ lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba, atomic_inc(&tgtp->xmt_abort_rsp_error); spin_lock_irqsave(&ctxp->ctxlock, flags); - ctxp->flag &= ~LPFC_NVMET_ABORT_OP; + ctxp->flag &= ~LPFC_NVME_ABORT_OP; spin_unlock_irqrestore(&ctxp->ctxlock, flags); lpfc_sli_release_iocbq(phba, abts_wqeq); lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, @@ -3354,14 +3354,14 @@ lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba, ctxp->wqeq->hba_wqidx = 0; } - if (ctxp->state == LPFC_NVMET_STE_FREE) { + if (ctxp->state == LPFC_NVME_STE_FREE) { lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, "6417 NVMET ABORT ctx freed %d %d oxid x%x\n", ctxp->state, ctxp->entry_cnt, ctxp->oxid); rc = WQE_BUSY; goto aerr; } - ctxp->state = LPFC_NVMET_STE_ABORT; + ctxp->state = LPFC_NVME_STE_ABORT; ctxp->entry_cnt++; rc = lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri); if (rc == 0) @@ -3383,13 +3383,13 @@ lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba, aerr: spin_lock_irqsave(&ctxp->ctxlock, flags); - if (ctxp->flag & LPFC_NVMET_CTX_RLS) { + if (ctxp->flag & LPFC_NVME_CTX_RLS) { spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock); list_del_init(&ctxp->list); spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); released = true; } - ctxp->flag &= ~(LPFC_NVMET_ABORT_OP | LPFC_NVMET_CTX_RLS); + ctxp->flag &= ~(LPFC_NVME_ABORT_OP | LPFC_NVME_CTX_RLS); spin_unlock_irqrestore(&ctxp->ctxlock, flags); atomic_inc(&tgtp->xmt_abort_rsp_error); @@ -3412,16 +3412,16 @@ lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *phba, unsigned long flags; int rc; - if ((ctxp->state == LPFC_NVMET_STE_LS_RCV && ctxp->entry_cnt == 1) || - (ctxp->state == LPFC_NVMET_STE_LS_RSP && ctxp->entry_cnt == 2)) { - ctxp->state = LPFC_NVMET_STE_LS_ABORT; + if ((ctxp->state == LPFC_NVME_STE_LS_RCV && ctxp->entry_cnt == 1) || + (ctxp->state == LPFC_NVME_STE_LS_RSP && ctxp->entry_cnt == 2)) { + ctxp->state = LPFC_NVME_STE_LS_ABORT; ctxp->entry_cnt++; } else { lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, "6418 NVMET LS abort state mismatch " "IO x%x: %d %d\n", ctxp->oxid, ctxp->state, ctxp->entry_cnt); - ctxp->state = LPFC_NVMET_STE_LS_ABORT; + ctxp->state = LPFC_NVME_STE_LS_ABORT; } tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; From 3a8070c567aaaa6038b52113ce01527992604c40 Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:50:05 -0700 Subject: [PATCH 0325/1043] lpfc: Refactor NVME LS receive handling In preparation for supporting both intiator mode and target mode receiving NVME LS's, commonize the existing NVME LS request receive handling found in the base driver and in the nvmet side. Using the original lpfc_nvmet_unsol_ls_event() and lpfc_nvme_unsol_ls_buffer() routines as a templates, commonize the reception of an NVME LS request. The common routine will validate the LS request, that it was received from a logged-in node, and allocate a lpfc_async_xchg_ctx that is used to manage the LS request. The role of the port is then inspected to determine which handler is to receive the LS - nvme or nvmet. As such, the nvmet handler is tied back in. A handler is created in nvme and is stubbed out. Signed-off-by: Paul Ely Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/scsi/lpfc/lpfc_crtn.h | 6 +- drivers/scsi/lpfc/lpfc_nvme.c | 19 ++++ drivers/scsi/lpfc/lpfc_nvme.h | 5 + drivers/scsi/lpfc/lpfc_nvmet.c | 163 ++++++++------------------------- drivers/scsi/lpfc/lpfc_sli.c | 121 +++++++++++++++++++++++- 5 files changed, 184 insertions(+), 130 deletions(-) diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h index 635df4a0188c..07b4f0fd3fe2 100644 --- a/drivers/scsi/lpfc/lpfc_crtn.h +++ b/drivers/scsi/lpfc/lpfc_crtn.h @@ -563,8 +563,10 @@ void lpfc_nvme_update_localport(struct lpfc_vport *vport); int lpfc_nvmet_create_targetport(struct lpfc_hba *phba); int lpfc_nvmet_update_targetport(struct lpfc_hba *phba); void lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba); -void lpfc_nvmet_unsol_ls_event(struct lpfc_hba *phba, - struct lpfc_sli_ring *pring, struct lpfc_iocbq *piocb); +int lpfc_nvme_handle_lsreq(struct lpfc_hba *phba, + struct lpfc_async_xchg_ctx *axchg); +int lpfc_nvmet_handle_lsreq(struct lpfc_hba *phba, + struct lpfc_async_xchg_ctx *axchg); void lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba, uint32_t idx, struct rqb_dmabuf *nvmebuf, uint64_t isr_ts, uint8_t cqflag); diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index 6045000477ac..c2113f6c76c1 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -393,6 +393,25 @@ lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport) return; } +/** + * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request + * @phba: pointer to lpfc hba data structure. + * @axchg: pointer to exchange context for the NVME LS request + * + * This routine is used for processing an asychronously received NVME LS + * request. Any remaining validation is done and the LS is then forwarded + * to the nvme-fc transport via nvme_fc_rcv_ls_req(). + * + * Returns 0 if LS was handled and delivered to the transport + * Returns 1 if LS failed to be handled and should be dropped + */ +int +lpfc_nvme_handle_lsreq(struct lpfc_hba *phba, + struct lpfc_async_xchg_ctx *axchg) +{ + return 1; +} + static void lpfc_nvme_cmpl_gen_req(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, struct lpfc_wcqe_complete *wcqe) diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h index 220b51af43da..10e8d868608e 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.h +++ b/drivers/scsi/lpfc/lpfc_nvme.h @@ -188,6 +188,7 @@ struct lpfc_async_xchg_ctx { } hdlrctx; struct list_head list; struct lpfc_hba *phba; + struct lpfc_nodelist *ndlp; struct nvmefc_ls_req *ls_req; struct nvmefc_ls_rsp ls_rsp; struct lpfc_iocbq *wqeq; @@ -202,6 +203,7 @@ struct lpfc_async_xchg_ctx { uint16_t idx; uint16_t state; uint16_t flag; + void *payload; struct rqb_dmabuf *rqb_buffer; struct lpfc_nvmet_ctxbuf *ctxbuf; struct lpfc_sli4_hdw_queue *hdwq; @@ -224,3 +226,6 @@ struct lpfc_async_xchg_ctx { /* routines found in lpfc_nvme.c */ /* routines found in lpfc_nvmet.c */ +int lpfc_nvme_unsol_ls_issue_abort(struct lpfc_hba *phba, + struct lpfc_async_xchg_ctx *ctxp, uint32_t sid, + uint16_t xri); diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index e0d5be4617ac..a95c8ef6373a 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -63,9 +63,6 @@ static int lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *, static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *, struct lpfc_async_xchg_ctx *, uint32_t, uint16_t); -static int lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *, - struct lpfc_async_xchg_ctx *, - uint32_t, uint16_t); static void lpfc_nvmet_wqfull_flush(struct lpfc_hba *, struct lpfc_queue *, struct lpfc_async_xchg_ctx *); static void lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *); @@ -865,7 +862,7 @@ lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport, ctxp->oxid); lpfc_in_buf_free(phba, &nvmebuf->dbuf); atomic_inc(&nvmep->xmt_ls_abort); - lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, + lpfc_nvme_unsol_ls_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); return -ENOMEM; } @@ -908,7 +905,7 @@ lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport, lpfc_in_buf_free(phba, &nvmebuf->dbuf); atomic_inc(&nvmep->xmt_ls_abort); - lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); + lpfc_nvme_unsol_ls_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); return -ENXIO; } @@ -1922,107 +1919,49 @@ lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba) } /** - * lpfc_nvmet_unsol_ls_buffer - Process an unsolicited event data buffer + * lpfc_nvmet_handle_lsreq - Process an NVME LS request * @phba: pointer to lpfc hba data structure. - * @pring: pointer to a SLI ring. - * @nvmebuf: pointer to lpfc nvme command HBQ data structure. + * @axchg: pointer to exchange context for the NVME LS request * - * This routine is used for processing the WQE associated with a unsolicited - * event. It first determines whether there is an existing ndlp that matches - * the DID from the unsolicited WQE. If not, it will create a new one with - * the DID from the unsolicited WQE. The ELS command from the unsolicited - * WQE is then used to invoke the proper routine and to set up proper state - * of the discovery state machine. - **/ -static void -lpfc_nvmet_unsol_ls_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, - struct hbq_dmabuf *nvmebuf) + * This routine is used for processing an asychronously received NVME LS + * request. Any remaining validation is done and the LS is then forwarded + * to the nvmet-fc transport via nvmet_fc_rcv_ls_req(). + * + * The calling sequence should be: nvmet_fc_rcv_ls_req() -> (processing) + * -> lpfc_nvmet_xmt_ls_rsp/cmp -> req->done. + * lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg. + * + * Returns 0 if LS was handled and delivered to the transport + * Returns 1 if LS failed to be handled and should be dropped + */ +int +lpfc_nvmet_handle_lsreq(struct lpfc_hba *phba, + struct lpfc_async_xchg_ctx *axchg) { #if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) - struct lpfc_nvmet_tgtport *tgtp; - struct fc_frame_header *fc_hdr; - struct lpfc_async_xchg_ctx *ctxp; - uint32_t *payload; - uint32_t size, oxid, sid, rc; + struct lpfc_nvmet_tgtport *tgtp = phba->targetport->private; + uint32_t *payload = axchg->payload; + int rc; - - if (!nvmebuf || !phba->targetport) { - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "6154 LS Drop IO\n"); - oxid = 0; - size = 0; - sid = 0; - ctxp = NULL; - goto dropit; - } - - fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt); - oxid = be16_to_cpu(fc_hdr->fh_ox_id); - - tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; - payload = (uint32_t *)(nvmebuf->dbuf.virt); - size = bf_get(lpfc_rcqe_length, &nvmebuf->cq_event.cqe.rcqe_cmpl); - sid = sli4_sid_from_fc_hdr(fc_hdr); - - ctxp = kzalloc(sizeof(struct lpfc_async_xchg_ctx), GFP_ATOMIC); - if (ctxp == NULL) { - atomic_inc(&tgtp->rcv_ls_req_drop); - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "6155 LS Drop IO x%x: Alloc\n", - oxid); -dropit: - lpfc_nvmeio_data(phba, "NVMET LS DROP: " - "xri x%x sz %d from %06x\n", - oxid, size, sid); - lpfc_in_buf_free(phba, &nvmebuf->dbuf); - return; - } - ctxp->phba = phba; - ctxp->size = size; - ctxp->oxid = oxid; - ctxp->sid = sid; - ctxp->wqeq = NULL; - ctxp->state = LPFC_NVME_STE_LS_RCV; - ctxp->entry_cnt = 1; - ctxp->rqb_buffer = (void *)nvmebuf; - ctxp->hdwq = &phba->sli4_hba.hdwq[0]; - - lpfc_nvmeio_data(phba, "NVMET LS RCV: xri x%x sz %d from %06x\n", - oxid, size, sid); - /* - * The calling sequence should be: - * nvmet_fc_rcv_ls_req -> lpfc_nvmet_xmt_ls_rsp/cmp ->_req->done - * lpfc_nvmet_xmt_ls_rsp_cmp should free the allocated ctxp. - */ atomic_inc(&tgtp->rcv_ls_req_in); - rc = nvmet_fc_rcv_ls_req(phba->targetport, NULL, &ctxp->ls_rsp, - payload, size); + + rc = nvmet_fc_rcv_ls_req(phba->targetport, NULL, &axchg->ls_rsp, + axchg->payload, axchg->size); lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, "6037 NVMET Unsol rcv: sz %d rc %d: %08x %08x %08x " - "%08x %08x %08x\n", size, rc, + "%08x %08x %08x\n", axchg->size, rc, *payload, *(payload+1), *(payload+2), *(payload+3), *(payload+4), *(payload+5)); - if (rc == 0) { + if (!rc) { atomic_inc(&tgtp->rcv_ls_req_out); - return; + return 0; } - lpfc_nvmeio_data(phba, "NVMET LS DROP: xri x%x sz %d from %06x\n", - oxid, size, sid); - atomic_inc(&tgtp->rcv_ls_req_drop); - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "6156 LS Drop IO x%x: nvmet_fc_rcv_ls_req %d\n", - ctxp->oxid, rc); - - /* We assume a rcv'ed cmd ALWAYs fits into 1 buffer */ - lpfc_in_buf_free(phba, &nvmebuf->dbuf); - - atomic_inc(&tgtp->xmt_ls_abort); - lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, sid, oxid); #endif + return 1; } static void @@ -2364,40 +2303,6 @@ lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba, } } -/** - * lpfc_nvmet_unsol_ls_event - Process an unsolicited event from an nvme nport - * @phba: pointer to lpfc hba data structure. - * @pring: pointer to a SLI ring. - * @nvmebuf: pointer to received nvme data structure. - * - * This routine is used to process an unsolicited event received from a SLI - * (Service Level Interface) ring. The actual processing of the data buffer - * associated with the unsolicited event is done by invoking the routine - * lpfc_nvmet_unsol_ls_buffer() after properly set up the buffer from the - * SLI RQ on which the unsolicited event was received. - **/ -void -lpfc_nvmet_unsol_ls_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, - struct lpfc_iocbq *piocb) -{ - struct lpfc_dmabuf *d_buf; - struct hbq_dmabuf *nvmebuf; - - d_buf = piocb->context2; - nvmebuf = container_of(d_buf, struct hbq_dmabuf, dbuf); - - if (!nvmebuf) { - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "3015 LS Drop IO\n"); - return; - } - if (phba->nvmet_support == 0) { - lpfc_in_buf_free(phba, &nvmebuf->dbuf); - return; - } - lpfc_nvmet_unsol_ls_buffer(phba, pring, nvmebuf); -} - /** * lpfc_nvmet_unsol_fcp_event - Process an unsolicited event from an nvme nport * @phba: pointer to lpfc hba data structure. @@ -3402,8 +3307,16 @@ aerr: return 1; } -static int -lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *phba, +/** + * lpfc_nvme_unsol_ls_issue_abort - issue ABTS on an exchange received + * via async frame receive where the frame is not handled. + * @phba: pointer to adapter structure + * @ctxp: pointer to the asynchronously received received sequence + * @sid: address of the remote port to send the ABTS to + * @xri: oxid value to for the ABTS (other side's exchange id). + **/ +int +lpfc_nvme_unsol_ls_issue_abort(struct lpfc_hba *phba, struct lpfc_async_xchg_ctx *ctxp, uint32_t sid, uint16_t xri) { diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index d4d810cb8944..1aaf40081e21 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -2792,6 +2792,121 @@ lpfc_sli_get_buff(struct lpfc_hba *phba, return &hbq_entry->dbuf; } +/** + * lpfc_nvme_unsol_ls_handler - Process an unsolicited event data buffer + * containing a NVME LS request. + * @phba: pointer to lpfc hba data structure. + * @piocb: pointer to the iocbq struct representing the sequence starting + * frame. + * + * This routine initially validates the NVME LS, validates there is a login + * with the port that sent the LS, and then calls the appropriate nvme host + * or target LS request handler. + **/ +static void +lpfc_nvme_unsol_ls_handler(struct lpfc_hba *phba, struct lpfc_iocbq *piocb) +{ + struct lpfc_nodelist *ndlp; + struct lpfc_dmabuf *d_buf; + struct hbq_dmabuf *nvmebuf; + struct fc_frame_header *fc_hdr; + struct lpfc_async_xchg_ctx *axchg = NULL; + char *failwhy = NULL; + uint32_t oxid, sid, did, fctl, size; + int ret; + + d_buf = piocb->context2; + + nvmebuf = container_of(d_buf, struct hbq_dmabuf, dbuf); + fc_hdr = nvmebuf->hbuf.virt; + oxid = be16_to_cpu(fc_hdr->fh_ox_id); + sid = sli4_sid_from_fc_hdr(fc_hdr); + did = sli4_did_from_fc_hdr(fc_hdr); + fctl = (fc_hdr->fh_f_ctl[0] << 16 | + fc_hdr->fh_f_ctl[1] << 8 | + fc_hdr->fh_f_ctl[2]); + size = bf_get(lpfc_rcqe_length, &nvmebuf->cq_event.cqe.rcqe_cmpl); + + lpfc_nvmeio_data(phba, "NVME LS RCV: xri x%x sz %d from %06x\n", + oxid, size, sid); + + if (phba->pport->load_flag & FC_UNLOADING) { + failwhy = "Driver Unloading"; + } else if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) { + failwhy = "NVME FC4 Disabled"; + } else if (!phba->nvmet_support && !phba->pport->localport) { + failwhy = "No Localport"; + } else if (phba->nvmet_support && !phba->targetport) { + failwhy = "No Targetport"; + } else if (unlikely(fc_hdr->fh_r_ctl != FC_RCTL_ELS4_REQ)) { + failwhy = "Bad NVME LS R_CTL"; + } else if (unlikely((fctl & 0x00FF0000) != + (FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT))) { + failwhy = "Bad NVME LS F_CTL"; + } else { + axchg = kzalloc(sizeof(*axchg), GFP_ATOMIC); + if (!axchg) + failwhy = "No CTX memory"; + } + + if (unlikely(failwhy)) { + lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC | LOG_NVME_IOERR, + "6154 Drop NVME LS: SID %06X OXID x%X: %s\n", + sid, oxid, failwhy); + goto out_fail; + } + + /* validate the source of the LS is logged in */ + ndlp = lpfc_findnode_did(phba->pport, sid); + if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || + ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && + (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { + lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC, + "6216 NVME Unsol rcv: No ndlp: " + "NPort_ID x%x oxid x%x\n", + sid, oxid); + goto out_fail; + } + + axchg->phba = phba; + axchg->ndlp = ndlp; + axchg->size = size; + axchg->oxid = oxid; + axchg->sid = sid; + axchg->wqeq = NULL; + axchg->state = LPFC_NVME_STE_LS_RCV; + axchg->entry_cnt = 1; + axchg->rqb_buffer = (void *)nvmebuf; + axchg->hdwq = &phba->sli4_hba.hdwq[0]; + axchg->payload = nvmebuf->dbuf.virt; + INIT_LIST_HEAD(&axchg->list); + + if (phba->nvmet_support) + ret = lpfc_nvmet_handle_lsreq(phba, axchg); + else + ret = lpfc_nvme_handle_lsreq(phba, axchg); + + /* if zero, LS was successfully handled. If non-zero, LS not handled */ + if (!ret) + return; + + lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC | LOG_NVME_IOERR, + "6155 Drop NVME LS from DID %06X: SID %06X OXID x%X " + "NVMe%s handler failed %d\n", + did, sid, oxid, + (phba->nvmet_support) ? "T" : "I", ret); + +out_fail: + kfree(axchg); + + /* recycle receive buffer */ + lpfc_in_buf_free(phba, &nvmebuf->dbuf); + + /* If start of new exchange, abort it */ + if (fctl & FC_FC_FIRST_SEQ && !(fctl & FC_FC_EX_CTX)) + lpfc_nvme_unsol_ls_issue_abort(phba, axchg, sid, oxid); +} + /** * lpfc_complete_unsol_iocb - Complete an unsolicited sequence * @phba: Pointer to HBA context object. @@ -2813,7 +2928,7 @@ lpfc_complete_unsol_iocb(struct lpfc_hba *phba, struct lpfc_sli_ring *pring, switch (fch_type) { case FC_TYPE_NVME: - lpfc_nvmet_unsol_ls_event(phba, pring, saveq); + lpfc_nvme_unsol_ls_handler(phba, saveq); return 1; default: break; @@ -13978,8 +14093,8 @@ lpfc_sli4_nvmet_handle_rcqe(struct lpfc_hba *phba, struct lpfc_queue *cq, /* Just some basic sanity checks on FCP Command frame */ fctl = (fc_hdr->fh_f_ctl[0] << 16 | - fc_hdr->fh_f_ctl[1] << 8 | - fc_hdr->fh_f_ctl[2]); + fc_hdr->fh_f_ctl[1] << 8 | + fc_hdr->fh_f_ctl[2]); if (((fctl & (FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT)) != (FC_FC_FIRST_SEQ | FC_FC_END_SEQ | FC_FC_SEQ_INIT)) || From 6514b25d3fba0610cd6c42aa36e34937bed0e4d8 Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:50:06 -0700 Subject: [PATCH 0326/1043] lpfc: Refactor Send LS Request support Currently, the ability to send an NVME LS request is limited to the nvme (host) side of the driver. In preparation of both the nvme and nvmet sides support Send LS Request, rework the existing send ls_req and ls_req completion routines such that there is common code that can be used by both sides. Signed-off-by: Paul Ely Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/scsi/lpfc/lpfc_nvme.c | 319 ++++++++++++++++++++-------------- drivers/scsi/lpfc/lpfc_nvme.h | 13 ++ 2 files changed, 199 insertions(+), 133 deletions(-) diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index c2113f6c76c1..1b7651c24f06 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -412,18 +412,74 @@ lpfc_nvme_handle_lsreq(struct lpfc_hba *phba, return 1; } +/** + * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME + * LS request. + * @phba: Pointer to HBA context object + * @vport: The local port that issued the LS + * @cmdwqe: Pointer to driver command WQE object. + * @wcqe: Pointer to driver response CQE object. + * + * This function is the generic completion handler for NVME LS requests. + * The function updates any states and statistics, calls the transport + * ls_req done() routine, then tears down the command and buffers used + * for the LS request. + **/ +void +__lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport, + struct lpfc_iocbq *cmdwqe, + struct lpfc_wcqe_complete *wcqe) +{ + struct nvmefc_ls_req *pnvme_lsreq; + struct lpfc_dmabuf *buf_ptr; + struct lpfc_nodelist *ndlp; + uint32_t status; + + pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2; + ndlp = (struct lpfc_nodelist *)cmdwqe->context1; + status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; + + lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, + "6047 NVMEx LS REQ %px cmpl DID %x Xri: %x " + "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px " + "ndlp:x%px\n", + pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, + cmdwqe->sli4_xritag, status, + (wcqe->parameter & 0xffff), + cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp); + + lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n", + cmdwqe->sli4_xritag, status, wcqe->parameter); + + if (cmdwqe->context3) { + buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3; + lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys); + kfree(buf_ptr); + cmdwqe->context3 = NULL; + } + if (pnvme_lsreq->done) + pnvme_lsreq->done(pnvme_lsreq, status); + else + lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, + "6046 NVMEx cmpl without done call back? " + "Data %px DID %x Xri: %x status %x\n", + pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, + cmdwqe->sli4_xritag, status); + if (ndlp) { + lpfc_nlp_put(ndlp); + cmdwqe->context1 = NULL; + } + lpfc_sli_release_iocbq(phba, cmdwqe); +} + static void -lpfc_nvme_cmpl_gen_req(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, +lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, struct lpfc_wcqe_complete *wcqe) { struct lpfc_vport *vport = cmdwqe->vport; struct lpfc_nvme_lport *lport; uint32_t status; - struct nvmefc_ls_req *pnvme_lsreq; - struct lpfc_dmabuf *buf_ptr; - struct lpfc_nodelist *ndlp; - pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2; status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; if (vport->localport) { @@ -438,38 +494,7 @@ lpfc_nvme_cmpl_gen_req(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, } } - ndlp = (struct lpfc_nodelist *)cmdwqe->context1; - lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, - "6047 nvme cmpl Enter " - "Data %px DID %x Xri: %x status %x reason x%x " - "cmd:x%px lsreg:x%px bmp:x%px ndlp:x%px\n", - pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, - cmdwqe->sli4_xritag, status, - (wcqe->parameter & 0xffff), - cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp); - - lpfc_nvmeio_data(phba, "NVME LS CMPL: xri x%x stat x%x parm x%x\n", - cmdwqe->sli4_xritag, status, wcqe->parameter); - - if (cmdwqe->context3) { - buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3; - lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys); - kfree(buf_ptr); - cmdwqe->context3 = NULL; - } - if (pnvme_lsreq->done) - pnvme_lsreq->done(pnvme_lsreq, status); - else - lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, - "6046 nvme cmpl without done call back? " - "Data %px DID %x Xri: %x status %x\n", - pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, - cmdwqe->sli4_xritag, status); - if (ndlp) { - lpfc_nlp_put(ndlp); - cmdwqe->context1 = NULL; - } - lpfc_sli_release_iocbq(phba, cmdwqe); + __lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe); } static int @@ -573,13 +598,6 @@ lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp, /* Issue GEN REQ WQE for NPORT */ - lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS, - "6050 Issue GEN REQ WQE to NPORT x%x " - "Data: x%x x%x wq:x%px lsreq:x%px bmp:x%px " - "xmit:%d 1st:%d\n", - ndlp->nlp_DID, genwqe->iotag, - vport->port_state, - genwqe, pnvme_lsreq, bmp, xmit_len, first_len); genwqe->wqe_cmpl = cmpl; genwqe->iocb_cmpl = NULL; genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT; @@ -591,105 +609,108 @@ lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp, rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe); if (rc) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS, + lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC | LOG_ELS, "6045 Issue GEN REQ WQE to NPORT x%x " - "Data: x%x x%x\n", + "Data: x%x x%x rc x%x\n", ndlp->nlp_DID, genwqe->iotag, - vport->port_state); + vport->port_state, rc); lpfc_sli_release_iocbq(phba, genwqe); return 1; } + + lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS, + "6050 Issue GEN REQ WQE to NPORT x%x " + "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px " + "bmp:x%px xmit:%d 1st:%d\n", + ndlp->nlp_DID, genwqe->sli4_xritag, + vport->port_state, + genwqe, pnvme_lsreq, bmp, xmit_len, first_len); return 0; } + /** - * lpfc_nvme_ls_req - Issue an Link Service request - * @lpfc_pnvme: Pointer to the driver's nvme instance data - * @lpfc_nvme_lport: Pointer to the driver's local port data - * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq + * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request + * @vport: The local port issuing the LS + * @ndlp: The remote port to send the LS to + * @pnvme_lsreq: Pointer to LS request structure from the transport * - * Driver registers this routine to handle any link service request - * from the nvme_fc transport to a remote nvme-aware port. + * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST + * WQE to perform the LS operation. * * Return value : * 0 - Success - * TODO: What are the failure codes. + * non-zero: various error codes, in form of -Exxx **/ -static int -lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport, - struct nvme_fc_remote_port *pnvme_rport, - struct nvmefc_ls_req *pnvme_lsreq) +int +__lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, + struct nvmefc_ls_req *pnvme_lsreq, + void (*gen_req_cmp)(struct lpfc_hba *phba, + struct lpfc_iocbq *cmdwqe, + struct lpfc_wcqe_complete *wcqe)) { - int ret = 0; - struct lpfc_nvme_lport *lport; - struct lpfc_nvme_rport *rport; - struct lpfc_vport *vport; - struct lpfc_nodelist *ndlp; - struct ulp_bde64 *bpl; struct lpfc_dmabuf *bmp; + struct ulp_bde64 *bpl; + int ret; uint16_t ntype, nstate; - /* there are two dma buf in the request, actually there is one and - * the second one is just the start address + cmd size. - * Before calling lpfc_nvme_gen_req these buffers need to be wrapped - * in a lpfc_dmabuf struct. When freeing we just free the wrapper - * because the nvem layer owns the data bufs. - * We do not have to break these packets open, we don't care what is in - * them. And we do not have to look at the resonse data, we only care - * that we got a response. All of the caring is going to happen in the - * nvme-fc layer. - */ - - lport = (struct lpfc_nvme_lport *)pnvme_lport->private; - rport = (struct lpfc_nvme_rport *)pnvme_rport->private; - if (unlikely(!lport) || unlikely(!rport)) - return -EINVAL; - - vport = lport->vport; - - if (vport->load_flag & FC_UNLOADING) - return -ENODEV; - - /* Need the ndlp. It is stored in the driver's rport. */ - ndlp = rport->ndlp; if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, - "6051 Remoteport x%px, rport has invalid ndlp. " - "Failing LS Req\n", pnvme_rport); + lpfc_printf_vlog(vport, KERN_ERR, + LOG_NVME_DISC | LOG_NODE | LOG_NVME_IOERR, + "6051 NVMEx LS REQ: Bad NDLP x%px, Failing " + "LS Req\n", + ndlp); return -ENODEV; } - /* The remote node has to be a mapped nvme target or an - * unmapped nvme initiator or it's an error. - */ ntype = ndlp->nlp_type; nstate = ndlp->nlp_state; if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) || (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, - "6088 DID x%06x not ready for " - "IO. State x%x, Type x%x\n", - pnvme_rport->port_id, - ndlp->nlp_state, ndlp->nlp_type); + lpfc_printf_vlog(vport, KERN_ERR, + LOG_NVME_DISC | LOG_NODE | LOG_NVME_IOERR, + "6088 NVMEx LS REQ: Fail DID x%06x not " + "ready for IO. Type x%x, State x%x\n", + ndlp->nlp_DID, ntype, nstate); return -ENODEV; } - bmp = kmalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL); + + /* + * there are two dma buf in the request, actually there is one and + * the second one is just the start address + cmd size. + * Before calling lpfc_nvme_gen_req these buffers need to be wrapped + * in a lpfc_dmabuf struct. When freeing we just free the wrapper + * because the nvem layer owns the data bufs. + * We do not have to break these packets open, we don't care what is + * in them. And we do not have to look at the resonse data, we only + * care that we got a response. All of the caring is going to happen + * in the nvme-fc layer. + */ + + bmp = kmalloc(sizeof(*bmp), GFP_KERNEL); if (!bmp) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, - "6044 Could not find node for DID %x\n", - pnvme_rport->port_id); - return 2; + lpfc_printf_vlog(vport, KERN_ERR, + LOG_NVME_DISC | LOG_NVME_IOERR, + "6044 NVMEx LS REQ: Could not alloc LS buf " + "for DID %x\n", + ndlp->nlp_DID); + return -ENOMEM; } - INIT_LIST_HEAD(&bmp->list); + bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys)); if (!bmp->virt) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, - "6042 Could not find node for DID %x\n", - pnvme_rport->port_id); + lpfc_printf_vlog(vport, KERN_ERR, + LOG_NVME_DISC | LOG_NVME_IOERR, + "6042 NVMEx LS REQ: Could not alloc mbuf " + "for DID %x\n", + ndlp->nlp_DID); kfree(bmp); - return 3; + return -ENOMEM; } + + INIT_LIST_HEAD(&bmp->list); + bpl = (struct ulp_bde64 *)bmp->virt; bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma)); bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma)); @@ -704,37 +725,69 @@ lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport, bpl->tus.f.bdeSize = pnvme_lsreq->rsplen; bpl->tus.w = le32_to_cpu(bpl->tus.w); - /* Expand print to include key fields. */ lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, - "6149 Issue LS Req to DID 0x%06x lport x%px, " - "rport x%px lsreq x%px rqstlen:%d rsplen:%d " - "%pad %pad\n", - ndlp->nlp_DID, pnvme_lport, pnvme_rport, - pnvme_lsreq, pnvme_lsreq->rqstlen, - pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, - &pnvme_lsreq->rspdma); + "6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, " + "rqstlen:%d rsplen:%d %pad %pad\n", + ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen, + pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, + &pnvme_lsreq->rspdma); + + ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr, + pnvme_lsreq, gen_req_cmp, ndlp, 2, + LPFC_NVME_LS_TIMEOUT, 0); + if (ret != WQE_SUCCESS) { + lpfc_printf_vlog(vport, KERN_ERR, + LOG_NVME_DISC | LOG_NVME_IOERR, + "6052 NVMEx REQ: EXIT. issue ls wqe failed " + "lsreq x%px Status %x DID %x\n", + pnvme_lsreq, ret, ndlp->nlp_DID); + lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys); + kfree(bmp); + return -EIO; + } + + return 0; +} + +/** + * lpfc_nvme_ls_req - Issue an NVME Link Service request + * @lpfc_nvme_lport: Transport localport that LS is to be issued from. + * @lpfc_nvme_rport: Transport remoteport that LS is to be sent to. + * @pnvme_lsreq - the transport nvme_ls_req structure for the LS + * + * Driver registers this routine to handle any link service request + * from the nvme_fc transport to a remote nvme-aware port. + * + * Return value : + * 0 - Success + * non-zero: various error codes, in form of -Exxx + **/ +static int +lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport, + struct nvme_fc_remote_port *pnvme_rport, + struct nvmefc_ls_req *pnvme_lsreq) +{ + struct lpfc_nvme_lport *lport; + struct lpfc_nvme_rport *rport; + struct lpfc_vport *vport; + int ret; + + lport = (struct lpfc_nvme_lport *)pnvme_lport->private; + rport = (struct lpfc_nvme_rport *)pnvme_rport->private; + if (unlikely(!lport) || unlikely(!rport)) + return -EINVAL; + + vport = lport->vport; + if (vport->load_flag & FC_UNLOADING) + return -ENODEV; atomic_inc(&lport->fc4NvmeLsRequests); - /* Hardcode the wait to 30 seconds. Connections are failing otherwise. - * This code allows it all to work. - */ - ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr, - pnvme_lsreq, lpfc_nvme_cmpl_gen_req, - ndlp, 2, 30, 0); - if (ret != WQE_SUCCESS) { + ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq, + lpfc_nvme_ls_req_cmp); + if (ret) atomic_inc(&lport->xmt_ls_err); - lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, - "6052 EXIT. issue ls wqe failed lport x%px, " - "rport x%px lsreq x%px Status %x DID %x\n", - pnvme_lport, pnvme_rport, pnvme_lsreq, - ret, ndlp->nlp_DID); - lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys); - kfree(bmp); - return ret; - } - /* Stub in routine and return 0 for now. */ return ret; } diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h index 10e8d868608e..6f8d44aa47b2 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.h +++ b/drivers/scsi/lpfc/lpfc_nvme.h @@ -79,6 +79,12 @@ struct lpfc_nvme_fcpreq_priv { struct lpfc_io_buf *nvme_buf; }; +/* + * set NVME LS request timeouts to 30s. It is larger than the 2*R_A_TOV + * set by the spec, which appears to have issues with some devices. + */ +#define LPFC_NVME_LS_TIMEOUT 30 + #define LPFC_NVMET_DEFAULT_SEGS (64 + 1) /* 256K IOs */ #define LPFC_NVMET_RQE_MIN_POST 128 @@ -224,6 +230,13 @@ struct lpfc_async_xchg_ctx { /* routines found in lpfc_nvme.c */ +int __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, + struct nvmefc_ls_req *pnvme_lsreq, + void (*gen_req_cmp)(struct lpfc_hba *phba, + struct lpfc_iocbq *cmdwqe, + struct lpfc_wcqe_complete *wcqe)); +void __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport, + struct lpfc_iocbq *cmdwqe, struct lpfc_wcqe_complete *wcqe); /* routines found in lpfc_nvmet.c */ int lpfc_nvme_unsol_ls_issue_abort(struct lpfc_hba *phba, From e96a22b0b7c252295180c12128af380282e3b8c5 Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:50:07 -0700 Subject: [PATCH 0327/1043] lpfc: Refactor Send LS Abort support Send LS Abort support is needed when Send LS Request is supported. Currently, the ability to abort an NVME LS request is limited to the nvme (host) side of the driver. In preparation of both the nvme and nvmet sides supporting Send LS Abort, rework the existing ls_req abort routines such that there is common code that can be used by both sides. While refactoring it was seen the logic in the abort routine was incorrect. It attempted to abort all NVME LS's on the indicated port. As such, the routine was reworked to abort only the NVME LS request that was specified. Signed-off-by: Paul Ely Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/scsi/lpfc/lpfc_nvme.c | 129 ++++++++++++++++++++-------------- drivers/scsi/lpfc/lpfc_nvme.h | 2 + 2 files changed, 79 insertions(+), 52 deletions(-) diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index 1b7651c24f06..406f71b327a1 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -792,17 +792,82 @@ lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport, } /** - * lpfc_nvme_ls_abort - Issue an Link Service request - * @lpfc_pnvme: Pointer to the driver's nvme instance data - * @lpfc_nvme_lport: Pointer to the driver's local port data - * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq + * __lpfc_nvme_ls_abort - Generic service routine to abort a prior + * NVME LS request + * @vport: The local port that issued the LS + * @ndlp: The remote port the LS was sent to + * @pnvme_lsreq: Pointer to LS request structure from the transport * - * Driver registers this routine to handle any link service request - * from the nvme_fc transport to a remote nvme-aware port. + * The driver validates the ndlp, looks for the LS, and aborts the + * LS if found. * - * Return value : - * 0 - Success - * TODO: What are the failure codes. + * Returns: + * 0 : if LS found and aborted + * non-zero: various error conditions in form -Exxx + **/ +int +__lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, + struct nvmefc_ls_req *pnvme_lsreq) +{ + struct lpfc_hba *phba = vport->phba; + struct lpfc_sli_ring *pring; + struct lpfc_iocbq *wqe, *next_wqe; + bool foundit = false; + + if (!ndlp) { + lpfc_printf_log(phba, KERN_ERR, + LOG_NVME_DISC | LOG_NODE | + LOG_NVME_IOERR | LOG_NVME_ABTS, + "6049 NVMEx LS REQ Abort: Bad NDLP x%px DID " + "x%06x, Failing LS Req\n", + ndlp, ndlp ? ndlp->nlp_DID : 0); + return -EINVAL; + } + + lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, + "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq " + "x%p rqstlen:%d rsplen:%d %pad %pad\n", + pnvme_lsreq, pnvme_lsreq->rqstlen, + pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, + &pnvme_lsreq->rspdma); + + /* + * Lock the ELS ring txcmplq and look for the wqe that matches + * this ELS. If found, issue an abort on the wqe. + */ + pring = phba->sli4_hba.nvmels_wq->pring; + spin_lock_irq(&phba->hbalock); + spin_lock(&pring->ring_lock); + list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) { + if (wqe->context2 == pnvme_lsreq) { + wqe->iocb_flag |= LPFC_DRIVER_ABORTED; + foundit = true; + break; + } + } + spin_unlock(&pring->ring_lock); + + if (foundit) + lpfc_sli_issue_abort_iotag(phba, pring, wqe); + spin_unlock_irq(&phba->hbalock); + + if (foundit) + return 0; + + lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, + "6213 NVMEx LS REQ Abort: Unable to locate req x%p\n", + pnvme_lsreq); + return 1; +} + +/** + * lpfc_nvme_ls_abort - Abort a prior NVME LS request + * @lpfc_nvme_lport: Transport localport that LS is to be issued from. + * @lpfc_nvme_rport: Transport remoteport that LS is to be sent to. + * @pnvme_lsreq - the transport nvme_ls_req structure for the LS + * + * Driver registers this routine to abort a NVME LS request that is + * in progress (from the transports perspective). **/ static void lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, @@ -813,9 +878,7 @@ lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, struct lpfc_vport *vport; struct lpfc_hba *phba; struct lpfc_nodelist *ndlp; - LIST_HEAD(abort_list); - struct lpfc_sli_ring *pring; - struct lpfc_iocbq *wqe, *next_wqe; + int ret; lport = (struct lpfc_nvme_lport *)pnvme_lport->private; if (unlikely(!lport)) @@ -827,48 +890,10 @@ lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, return; ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); - if (!ndlp) { - lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, - "6049 Could not find node for DID %x\n", - pnvme_rport->port_id); - return; - } - /* Expand print to include key fields. */ - lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, - "6040 ENTER. lport x%px, rport x%px lsreq x%px rqstlen:%d " - "rsplen:%d %pad %pad\n", - pnvme_lport, pnvme_rport, - pnvme_lsreq, pnvme_lsreq->rqstlen, - pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, - &pnvme_lsreq->rspdma); - - /* - * Lock the ELS ring txcmplq and build a local list of all ELS IOs - * that need an ABTS. The IOs need to stay on the txcmplq so that - * the abort operation completes them successfully. - */ - pring = phba->sli4_hba.nvmels_wq->pring; - spin_lock_irq(&phba->hbalock); - spin_lock(&pring->ring_lock); - list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) { - /* Add to abort_list on on NDLP match. */ - if (lpfc_check_sli_ndlp(phba, pring, wqe, ndlp)) { - wqe->iocb_flag |= LPFC_DRIVER_ABORTED; - list_add_tail(&wqe->dlist, &abort_list); - } - } - spin_unlock(&pring->ring_lock); - spin_unlock_irq(&phba->hbalock); - - /* Abort the targeted IOs and remove them from the abort list. */ - list_for_each_entry_safe(wqe, next_wqe, &abort_list, dlist) { + ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq); + if (!ret) atomic_inc(&lport->xmt_ls_abort); - spin_lock_irq(&phba->hbalock); - list_del_init(&wqe->dlist); - lpfc_sli_issue_abort_iotag(phba, pring, wqe); - spin_unlock_irq(&phba->hbalock); - } } /* Fix up the existing sgls for NVME IO. */ diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h index 6f8d44aa47b2..a9ad81d0c182 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.h +++ b/drivers/scsi/lpfc/lpfc_nvme.h @@ -237,6 +237,8 @@ int __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, struct lpfc_wcqe_complete *wcqe)); void __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport, struct lpfc_iocbq *cmdwqe, struct lpfc_wcqe_complete *wcqe); +int __lpfc_nvme_ls_abort(struct lpfc_vport *vport, + struct lpfc_nodelist *ndlp, struct nvmefc_ls_req *pnvme_lsreq); /* routines found in lpfc_nvmet.c */ int lpfc_nvme_unsol_ls_issue_abort(struct lpfc_hba *phba, From fe1bedec5b9ce741fd6d4ebd6ede56e2c429467b Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:50:08 -0700 Subject: [PATCH 0328/1043] lpfc: Refactor Send LS Response support Currently, the ability to send an NVME LS response is limited to the nvmet (controller/target) side of the driver. In preparation of both the nvme and nvmet sides supporting Send LS Response, rework the existing send ls_rsp and ls_rsp completion routines such that there is common code that can be used by both sides. Signed-off-by: Paul Ely Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/scsi/lpfc/lpfc_nvme.h | 7 + drivers/scsi/lpfc/lpfc_nvmet.c | 255 +++++++++++++++++++++++---------- 2 files changed, 184 insertions(+), 78 deletions(-) diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h index a9ad81d0c182..f28a6789e252 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.h +++ b/drivers/scsi/lpfc/lpfc_nvme.h @@ -244,3 +244,10 @@ int __lpfc_nvme_ls_abort(struct lpfc_vport *vport, int lpfc_nvme_unsol_ls_issue_abort(struct lpfc_hba *phba, struct lpfc_async_xchg_ctx *ctxp, uint32_t sid, uint16_t xri); +int __lpfc_nvme_xmt_ls_rsp(struct lpfc_async_xchg_ctx *axchg, + struct nvmefc_ls_rsp *ls_rsp, + void (*xmt_ls_rsp_cmp)(struct lpfc_hba *phba, + struct lpfc_iocbq *cmdwqe, + struct lpfc_wcqe_complete *wcqe)); +void __lpfc_nvme_xmt_ls_rsp_cmp(struct lpfc_hba *phba, + struct lpfc_iocbq *cmdwqe, struct lpfc_wcqe_complete *wcqe); diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index a95c8ef6373a..6a1365df6c65 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -280,6 +280,53 @@ lpfc_nvmet_defer_release(struct lpfc_hba *phba, spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock); } +/** + * __lpfc_nvme_xmt_ls_rsp_cmp - Generic completion handler for the + * transmission of an NVME LS response. + * @phba: Pointer to HBA context object. + * @cmdwqe: Pointer to driver command WQE object. + * @wcqe: Pointer to driver response CQE object. + * + * The function is called from SLI ring event handler with no + * lock held. The function frees memory resources used for the command + * used to send the NVME LS RSP. + **/ +void +__lpfc_nvme_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, + struct lpfc_wcqe_complete *wcqe) +{ + struct lpfc_async_xchg_ctx *axchg = cmdwqe->context2; + struct nvmefc_ls_rsp *ls_rsp = &axchg->ls_rsp; + uint32_t status, result; + + status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; + result = wcqe->parameter; + + if (axchg->state != LPFC_NVME_STE_LS_RSP || axchg->entry_cnt != 2) { + lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC | LOG_NVME_IOERR, + "6410 NVMEx LS cmpl state mismatch IO x%x: " + "%d %d\n", + axchg->oxid, axchg->state, axchg->entry_cnt); + } + + lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x result x%x\n", + axchg->oxid, status, result); + + lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, + "6038 NVMEx LS rsp cmpl: %d %d oxid x%x\n", + status, result, axchg->oxid); + + lpfc_nlp_put(cmdwqe->context1); + cmdwqe->context2 = NULL; + cmdwqe->context3 = NULL; + lpfc_sli_release_iocbq(phba, cmdwqe); + ls_rsp->done(ls_rsp); + lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, + "6200 NVMEx LS rsp cmpl done status %d oxid x%x\n", + status, axchg->oxid); + kfree(axchg); +} + /** * lpfc_nvmet_xmt_ls_rsp_cmp - Completion handler for LS Response * @phba: Pointer to HBA context object. @@ -288,33 +335,23 @@ lpfc_nvmet_defer_release(struct lpfc_hba *phba, * * The function is called from SLI ring event handler with no * lock held. This function is the completion handler for NVME LS commands - * The function frees memory resources used for the NVME commands. + * The function updates any states and statistics, then calls the + * generic completion handler to free resources. **/ static void lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, struct lpfc_wcqe_complete *wcqe) { struct lpfc_nvmet_tgtport *tgtp; - struct nvmefc_ls_rsp *rsp; - struct lpfc_async_xchg_ctx *ctxp; uint32_t status, result; - status = bf_get(lpfc_wcqe_c_status, wcqe); - result = wcqe->parameter; - ctxp = cmdwqe->context2; - - if (ctxp->state != LPFC_NVME_STE_LS_RSP || ctxp->entry_cnt != 2) { - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "6410 NVMET LS cmpl state mismatch IO x%x: " - "%d %d\n", - ctxp->oxid, ctxp->state, ctxp->entry_cnt); - } - if (!phba->targetport) - goto out; + goto finish; + + status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; + result = wcqe->parameter; tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; - if (tgtp) { if (status) { atomic_inc(&tgtp->xmt_ls_rsp_error); @@ -327,22 +364,8 @@ lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, } } -out: - rsp = &ctxp->ls_rsp; - - lpfc_nvmeio_data(phba, "NVMET LS CMPL: xri x%x stat x%x result x%x\n", - ctxp->oxid, status, result); - - lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, - "6038 NVMET LS rsp cmpl: %d %d oxid x%x\n", - status, result, ctxp->oxid); - - lpfc_nlp_put(cmdwqe->context1); - cmdwqe->context2 = NULL; - cmdwqe->context3 = NULL; - lpfc_sli_release_iocbq(phba, cmdwqe); - rsp->done(rsp); - kfree(ctxp); +finish: + __lpfc_nvme_xmt_ls_rsp_cmp(phba, cmdwqe, wcqe); } /** @@ -819,52 +842,61 @@ lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, #endif } -static int -lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport, - struct nvmefc_ls_rsp *rsp) +/** + * __lpfc_nvme_xmt_ls_rsp - Generic service routine to issue transmit + * an NVME LS rsp for a prior NVME LS request that was received. + * @axchg: pointer to exchange context for the NVME LS request the response + * is for. + * @ls_rsp: pointer to the transport LS RSP that is to be sent + * @xmt_ls_rsp_cmp: completion routine to call upon RSP transmit done + * + * This routine is used to format and send a WQE to transmit a NVME LS + * Response. The response is for a prior NVME LS request that was + * received and posted to the transport. + * + * Returns: + * 0 : if response successfully transmit + * non-zero : if response failed to transmit, of the form -Exxx. + **/ +int +__lpfc_nvme_xmt_ls_rsp(struct lpfc_async_xchg_ctx *axchg, + struct nvmefc_ls_rsp *ls_rsp, + void (*xmt_ls_rsp_cmp)(struct lpfc_hba *phba, + struct lpfc_iocbq *cmdwqe, + struct lpfc_wcqe_complete *wcqe)) { - struct lpfc_async_xchg_ctx *ctxp = - container_of(rsp, struct lpfc_async_xchg_ctx, ls_rsp); - struct lpfc_hba *phba = ctxp->phba; - struct hbq_dmabuf *nvmebuf = - (struct hbq_dmabuf *)ctxp->rqb_buffer; + struct lpfc_hba *phba = axchg->phba; + struct hbq_dmabuf *nvmebuf = (struct hbq_dmabuf *)axchg->rqb_buffer; struct lpfc_iocbq *nvmewqeq; - struct lpfc_nvmet_tgtport *nvmep = tgtport->private; struct lpfc_dmabuf dmabuf; struct ulp_bde64 bpl; int rc; - if (phba->pport->load_flag & FC_UNLOADING) - return -ENODEV; - if (phba->pport->load_flag & FC_UNLOADING) return -ENODEV; lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, - "6023 NVMET LS rsp oxid x%x\n", ctxp->oxid); + "6023 NVMEx LS rsp oxid x%x\n", axchg->oxid); - if ((ctxp->state != LPFC_NVME_STE_LS_RCV) || - (ctxp->entry_cnt != 1)) { - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "6412 NVMET LS rsp state mismatch " + if (axchg->state != LPFC_NVME_STE_LS_RCV || axchg->entry_cnt != 1) { + lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC | LOG_NVME_IOERR, + "6412 NVMEx LS rsp state mismatch " "oxid x%x: %d %d\n", - ctxp->oxid, ctxp->state, ctxp->entry_cnt); + axchg->oxid, axchg->state, axchg->entry_cnt); + return -EALREADY; } - ctxp->state = LPFC_NVME_STE_LS_RSP; - ctxp->entry_cnt++; + axchg->state = LPFC_NVME_STE_LS_RSP; + axchg->entry_cnt++; - nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, ctxp, rsp->rspdma, - rsp->rsplen); + nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, axchg, ls_rsp->rspdma, + ls_rsp->rsplen); if (nvmewqeq == NULL) { - atomic_inc(&nvmep->xmt_ls_drop); - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "6150 LS Drop IO x%x: Prep\n", - ctxp->oxid); - lpfc_in_buf_free(phba, &nvmebuf->dbuf); - atomic_inc(&nvmep->xmt_ls_abort); - lpfc_nvme_unsol_ls_issue_abort(phba, ctxp, - ctxp->sid, ctxp->oxid); - return -ENOMEM; + lpfc_printf_log(phba, KERN_ERR, + LOG_NVME_DISC | LOG_NVME_IOERR | LOG_NVME_ABTS, + "6150 NVMEx LS Drop Rsp x%x: Prep\n", + axchg->oxid); + rc = -ENOMEM; + goto out_free_buf; } /* Save numBdes for bpl2sgl */ @@ -874,39 +906,106 @@ lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport, dmabuf.virt = &bpl; bpl.addrLow = nvmewqeq->wqe.xmit_sequence.bde.addrLow; bpl.addrHigh = nvmewqeq->wqe.xmit_sequence.bde.addrHigh; - bpl.tus.f.bdeSize = rsp->rsplen; + bpl.tus.f.bdeSize = ls_rsp->rsplen; bpl.tus.f.bdeFlags = 0; bpl.tus.w = le32_to_cpu(bpl.tus.w); + /* + * Note: although we're using stack space for the dmabuf, the + * call to lpfc_sli4_issue_wqe is synchronous, so it will not + * be referenced after it returns back to this routine. + */ - nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_rsp_cmp; + nvmewqeq->wqe_cmpl = xmt_ls_rsp_cmp; nvmewqeq->iocb_cmpl = NULL; - nvmewqeq->context2 = ctxp; + nvmewqeq->context2 = axchg; - lpfc_nvmeio_data(phba, "NVMET LS RESP: xri x%x wqidx x%x len x%x\n", - ctxp->oxid, nvmewqeq->hba_wqidx, rsp->rsplen); + lpfc_nvmeio_data(phba, "NVMEx LS RSP: xri x%x wqidx x%x len x%x\n", + axchg->oxid, nvmewqeq->hba_wqidx, ls_rsp->rsplen); + + rc = lpfc_sli4_issue_wqe(phba, axchg->hdwq, nvmewqeq); + + /* clear to be sure there's no reference */ + nvmewqeq->context3 = NULL; - rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq); if (rc == WQE_SUCCESS) { /* * Okay to repost buffer here, but wait till cmpl * before freeing ctxp and iocbq. */ lpfc_in_buf_free(phba, &nvmebuf->dbuf); - atomic_inc(&nvmep->xmt_ls_rsp); return 0; } - /* Give back resources */ - atomic_inc(&nvmep->xmt_ls_drop); - lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, - "6151 LS Drop IO x%x: Issue %d\n", - ctxp->oxid, rc); + + lpfc_printf_log(phba, KERN_ERR, + LOG_NVME_DISC | LOG_NVME_IOERR | LOG_NVME_ABTS, + "6151 NVMEx LS RSP x%x: failed to transmit %d\n", + axchg->oxid, rc); + + rc = -ENXIO; lpfc_nlp_put(nvmewqeq->context1); +out_free_buf: + /* Give back resources */ lpfc_in_buf_free(phba, &nvmebuf->dbuf); - atomic_inc(&nvmep->xmt_ls_abort); - lpfc_nvme_unsol_ls_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid); - return -ENXIO; + + /* + * As transport doesn't track completions of responses, if the rsp + * fails to send, the transport will effectively ignore the rsp + * and consider the LS done. However, the driver has an active + * exchange open for the LS - so be sure to abort the exchange + * if the response isn't sent. + */ + lpfc_nvme_unsol_ls_issue_abort(phba, axchg, axchg->sid, axchg->oxid); + return rc; +} + +/** + * lpfc_nvmet_xmt_ls_rsp - Transmit NVME LS response + * @tgtport: pointer to target port that NVME LS is to be transmit from. + * @ls_rsp: pointer to the transport LS RSP that is to be sent + * + * Driver registers this routine to transmit responses for received NVME + * LS requests. + * + * This routine is used to format and send a WQE to transmit a NVME LS + * Response. The ls_rsp is used to reverse-map the LS to the original + * NVME LS request sequence, which provides addressing information for + * the remote port the LS to be sent to, as well as the exchange id + * that is the LS is bound to. + * + * Returns: + * 0 : if response successfully transmit + * non-zero : if response failed to transmit, of the form -Exxx. + **/ +static int +lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport, + struct nvmefc_ls_rsp *ls_rsp) +{ + struct lpfc_async_xchg_ctx *axchg = + container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp); + struct lpfc_nvmet_tgtport *nvmep = tgtport->private; + int rc; + + if (axchg->phba->pport->load_flag & FC_UNLOADING) + return -ENODEV; + + rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, lpfc_nvmet_xmt_ls_rsp_cmp); + + if (rc) { + atomic_inc(&nvmep->xmt_ls_drop); + /* + * unless the failure is due to having already sent + * the response, an abort will be generated for the + * exchange if the rsp can't be sent. + */ + if (rc != -EALREADY) + atomic_inc(&nvmep->xmt_ls_abort); + return rc; + } + + atomic_inc(&nvmep->xmt_ls_rsp); + return 0; } static int From 9aa09e98b288649544c74d1a7b88223f36e4bffd Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:50:09 -0700 Subject: [PATCH 0329/1043] lpfc: nvme: Add Receive LS Request and Send LS Response support to nvme Now that common helpers exist, add the ability to receive NVME LS requests to the driver. New requests will be delivered to the transport by nvme_fc_rcv_ls_req(). In order to complete the LS, add support for Send LS Response and send LS response completion handling to the driver. Signed-off-by: Paul Ely Signed-off-by: James Smart Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/scsi/lpfc/lpfc_nvme.c | 71 +++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index 406f71b327a1..21bbccf0dc31 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -402,6 +402,10 @@ lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport) * request. Any remaining validation is done and the LS is then forwarded * to the nvme-fc transport via nvme_fc_rcv_ls_req(). * + * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing) + * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done. + * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg. + * * Returns 0 if LS was handled and delivered to the transport * Returns 1 if LS failed to be handled and should be dropped */ @@ -409,6 +413,40 @@ int lpfc_nvme_handle_lsreq(struct lpfc_hba *phba, struct lpfc_async_xchg_ctx *axchg) { +#if (IS_ENABLED(CONFIG_NVME_FC)) + struct lpfc_vport *vport; + struct lpfc_nvme_rport *lpfc_rport; + struct nvme_fc_remote_port *remoteport; + struct lpfc_nvme_lport *lport; + uint32_t *payload = axchg->payload; + int rc; + + vport = axchg->ndlp->vport; + lpfc_rport = axchg->ndlp->nrport; + if (!lpfc_rport) + return -EINVAL; + + remoteport = lpfc_rport->remoteport; + if (!vport->localport) + return -EINVAL; + + lport = vport->localport->private; + if (!lport) + return -EINVAL; + + rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload, + axchg->size); + + lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, + "6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x " + "%08x %08x %08x\n", + axchg->size, rc, + *payload, *(payload+1), *(payload+2), + *(payload+3), *(payload+4), *(payload+5)); + + if (!rc) + return 0; +#endif return 1; } @@ -860,6 +898,37 @@ __lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, return 1; } +static int +lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport, + struct nvme_fc_remote_port *remoteport, + struct nvmefc_ls_rsp *ls_rsp) +{ + struct lpfc_async_xchg_ctx *axchg = + container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp); + struct lpfc_nvme_lport *lport; + int rc; + + if (axchg->phba->pport->load_flag & FC_UNLOADING) + return -ENODEV; + + lport = (struct lpfc_nvme_lport *)localport->private; + + rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp); + + if (rc) { + /* + * unless the failure is due to having already sent + * the response, an abort will be generated for the + * exchange if the rsp can't be sent. + */ + if (rc != -EALREADY) + atomic_inc(&lport->xmt_ls_abort); + return rc; + } + + return 0; +} + /** * lpfc_nvme_ls_abort - Abort a prior NVME LS request * @lpfc_nvme_lport: Transport localport that LS is to be issued from. @@ -2005,6 +2074,7 @@ static struct nvme_fc_port_template lpfc_nvme_template = { .fcp_io = lpfc_nvme_fcp_io_submit, .ls_abort = lpfc_nvme_ls_abort, .fcp_abort = lpfc_nvme_fcp_abort, + .xmt_ls_rsp = lpfc_nvme_xmt_ls_rsp, .max_hw_queues = 1, .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS, @@ -2200,6 +2270,7 @@ lpfc_nvme_create_localport(struct lpfc_vport *vport) atomic_set(&lport->cmpl_fcp_err, 0); atomic_set(&lport->cmpl_ls_xb, 0); atomic_set(&lport->cmpl_ls_err, 0); + atomic_set(&lport->fc4NvmeLsRequests, 0); atomic_set(&lport->fc4NvmeLsCmpls, 0); } From 4c2805aab519a39e8adf281afcef40174d48fd3f Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:50:10 -0700 Subject: [PATCH 0330/1043] lpfc: nvmet: Add support for NVME LS request hosthandle As the nvmet layer does not have the concept of a remoteport object, which can be used to identify the entity on the other end of the fabric that is to receive an LS, the hosthandle was introduced. The driver passes the hosthandle, a value representative of the remote port, with a ls request receive. The LS request will create the association. The transport will remember the hosthandle for the association, and if there is a need to initiate a LS request to the remote port for the association, the hosthandle will be used. When the driver loses connectivity with the remote port, it needs to notify the transport that the hosthandle is no longer valid, allowing the transport to terminate associations related to the hosthandle. This patch adds support to the driver for the hosthandle. The driver will use the ndlp pointer of the remote port for the hosthandle in calls to nvmet_fc_rcv_ls_req(). The discovery engine is updated to invalidate the hosthandle whenever connectivity with the remote port is lost. Signed-off-by: Paul Ely Signed-off-by: James Smart Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/scsi/lpfc/lpfc_crtn.h | 2 ++ drivers/scsi/lpfc/lpfc_hbadisc.c | 6 ++++ drivers/scsi/lpfc/lpfc_nportdisc.c | 11 +++++++ drivers/scsi/lpfc/lpfc_nvme.h | 3 ++ drivers/scsi/lpfc/lpfc_nvmet.c | 53 +++++++++++++++++++++++++++++- 5 files changed, 74 insertions(+), 1 deletion(-) diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h index 07b4f0fd3fe2..9ee6b930a655 100644 --- a/drivers/scsi/lpfc/lpfc_crtn.h +++ b/drivers/scsi/lpfc/lpfc_crtn.h @@ -571,6 +571,8 @@ void lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba, uint32_t idx, struct rqb_dmabuf *nvmebuf, uint64_t isr_ts, uint8_t cqflag); void lpfc_nvme_mod_param_dep(struct lpfc_hba *phba); +void lpfc_nvmet_invalidate_host(struct lpfc_hba *phba, + struct lpfc_nodelist *ndlp); void lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, struct lpfc_wcqe_complete *abts_cmpl); diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c index 8dec7b7c06d1..f5952f8cd4b5 100644 --- a/drivers/scsi/lpfc/lpfc_hbadisc.c +++ b/drivers/scsi/lpfc/lpfc_hbadisc.c @@ -823,6 +823,12 @@ lpfc_cleanup_rpis(struct lpfc_vport *vport, int remove) if ((phba->sli_rev < LPFC_SLI_REV4) && (!remove && ndlp->nlp_type & NLP_FABRIC)) continue; + + /* Notify transport of connectivity loss to trigger cleanup. */ + if (phba->nvmet_support && + ndlp->nlp_state == NLP_STE_UNMAPPED_NODE) + lpfc_nvmet_invalidate_host(phba, ndlp); + lpfc_disc_state_machine(vport, ndlp, NULL, remove ? NLP_EVT_DEVICE_RM diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c index 81f4ba1c24b4..d8501bd959e7 100644 --- a/drivers/scsi/lpfc/lpfc_nportdisc.c +++ b/drivers/scsi/lpfc/lpfc_nportdisc.c @@ -489,6 +489,11 @@ lpfc_rcv_plogi(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, (unsigned long long) wwn_to_u64(sp->portName.u.wwn)); + /* Notify transport of connectivity loss to trigger cleanup. */ + if (phba->nvmet_support && + ndlp->nlp_state == NLP_STE_UNMAPPED_NODE) + lpfc_nvmet_invalidate_host(phba, ndlp); + ndlp->nlp_prev_state = ndlp->nlp_state; /* rport needs to be unregistered first */ lpfc_nlp_set_state(vport, ndlp, NLP_STE_NPR_NODE); @@ -839,6 +844,12 @@ lpfc_rcv_logo(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, lpfc_els_rsp_acc(vport, ELS_CMD_PRLO, cmdiocb, ndlp, NULL); else lpfc_els_rsp_acc(vport, ELS_CMD_ACC, cmdiocb, ndlp, NULL); + + /* Notify transport of connectivity loss to trigger cleanup. */ + if (phba->nvmet_support && + ndlp->nlp_state == NLP_STE_UNMAPPED_NODE) + lpfc_nvmet_invalidate_host(phba, ndlp); + if (ndlp->nlp_DID == Fabric_DID) { if (vport->port_state <= LPFC_FDISC) goto out; diff --git a/drivers/scsi/lpfc/lpfc_nvme.h b/drivers/scsi/lpfc/lpfc_nvme.h index f28a6789e252..4a4c3f780e1f 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.h +++ b/drivers/scsi/lpfc/lpfc_nvme.h @@ -98,9 +98,12 @@ struct lpfc_nvme_fcpreq_priv { #define LPFC_NVMET_WAIT_TMO (5 * MSEC_PER_SEC) /* Used for NVME Target */ +#define LPFC_NVMET_INV_HOST_ACTIVE 1 + struct lpfc_nvmet_tgtport { struct lpfc_hba *phba; struct completion *tport_unreg_cmp; + atomic_t state; /* tracks nvmet hosthandle invalidation */ /* Stats counters - lpfc_nvmet_unsol_ls_buffer */ atomic_t rcv_ls_req_in; diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index 6a1365df6c65..5584eaafbfb2 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -1282,6 +1282,24 @@ lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport, spin_unlock_irqrestore(&ctxp->ctxlock, iflag); } +static void +lpfc_nvmet_host_release(void *hosthandle) +{ + struct lpfc_nodelist *ndlp = hosthandle; + struct lpfc_hba *phba = NULL; + struct lpfc_nvmet_tgtport *tgtp; + + phba = ndlp->phba; + if (!phba->targetport || !phba->targetport->private) + return; + + lpfc_printf_log(phba, KERN_ERR, LOG_NVME, + "6202 NVMET XPT releasing hosthandle x%px\n", + hosthandle); + tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; + atomic_set(&tgtp->state, 0); +} + static void lpfc_nvmet_discovery_event(struct nvmet_fc_target_port *tgtport) { @@ -1306,6 +1324,7 @@ static struct nvmet_fc_target_template lpfc_tgttemplate = { .fcp_req_release = lpfc_nvmet_xmt_fcp_release, .defer_rcv = lpfc_nvmet_defer_rcv, .discovery_event = lpfc_nvmet_discovery_event, + .host_release = lpfc_nvmet_host_release, .max_hw_queues = 1, .max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS, @@ -2044,7 +2063,12 @@ lpfc_nvmet_handle_lsreq(struct lpfc_hba *phba, atomic_inc(&tgtp->rcv_ls_req_in); - rc = nvmet_fc_rcv_ls_req(phba->targetport, NULL, &axchg->ls_rsp, + /* + * Driver passes the ndlp as the hosthandle argument allowing + * the transport to generate LS requests for any associateions + * that are created. + */ + rc = nvmet_fc_rcv_ls_req(phba->targetport, axchg->ndlp, &axchg->ls_rsp, axchg->payload, axchg->size); lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, @@ -3476,3 +3500,30 @@ out: "6056 Failed to Issue ABTS. Status x%x\n", rc); return 0; } + +/** + * lpfc_nvmet_invalidate_host + * + * @phba - pointer to the driver instance bound to an adapter port. + * @ndlp - pointer to an lpfc_nodelist type + * + * This routine upcalls the nvmet transport to invalidate an NVME + * host to which this target instance had active connections. + */ +void +lpfc_nvmet_invalidate_host(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) +{ + struct lpfc_nvmet_tgtport *tgtp; + + lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_NVME_ABTS, + "6203 Invalidating hosthandle x%px\n", + ndlp); + + tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; + atomic_set(&tgtp->state, LPFC_NVMET_INV_HOST_ACTIVE); + +#if (IS_ENABLED(CONFIG_NVME_TARGET_FC)) + /* Need to get the nvmet_fc_target_port pointer here.*/ + nvmet_fc_invalidate_host(phba->targetport, ndlp); +#endif +} From 54840bed372c7779f23ece8514853fa83887b02e Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 31 Mar 2020 09:50:11 -0700 Subject: [PATCH 0331/1043] lpfc: nvmet: Add Send LS Request and Abort LS Request support Now that common helpers exist, add the ability to Send an NVME LS Request and to Abort an outstanding LS Request to the nvmet side of the driver. Signed-off-by: Paul Ely Signed-off-by: James Smart Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/scsi/lpfc/lpfc_nvmet.c | 95 ++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index 5584eaafbfb2..1c6bbbba70b5 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -1282,6 +1282,98 @@ lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport, spin_unlock_irqrestore(&ctxp->ctxlock, iflag); } +/** + * lpfc_nvmet_ls_req_cmp - completion handler for a nvme ls request + * @phba: Pointer to HBA context object + * @cmdwqe: Pointer to driver command WQE object. + * @wcqe: Pointer to driver response CQE object. + * + * This function is the completion handler for NVME LS requests. + * The function updates any states and statistics, then calls the + * generic completion handler to finish completion of the request. + **/ +static void +lpfc_nvmet_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, + struct lpfc_wcqe_complete *wcqe) +{ + __lpfc_nvme_ls_req_cmp(phba, cmdwqe->vport, cmdwqe, wcqe); +} + +/** + * lpfc_nvmet_ls_req - Issue an Link Service request + * @targetport - pointer to target instance registered with nvmet transport. + * @hosthandle - hosthandle set by the driver in a prior ls_rqst_rcv. + * Driver sets this value to the ndlp pointer. + * @pnvme_lsreq - the transport nvme_ls_req structure for the LS + * + * Driver registers this routine to handle any link service request + * from the nvme_fc transport to a remote nvme-aware port. + * + * Return value : + * 0 - Success + * non-zero: various error codes, in form of -Exxx + **/ +static int +lpfc_nvmet_ls_req(struct nvmet_fc_target_port *targetport, + void *hosthandle, + struct nvmefc_ls_req *pnvme_lsreq) +{ + struct lpfc_nvmet_tgtport *lpfc_nvmet = targetport->private; + struct lpfc_hba *phba; + struct lpfc_nodelist *ndlp; + int ret; + u32 hstate; + + if (!lpfc_nvmet) + return -EINVAL; + + phba = lpfc_nvmet->phba; + if (phba->pport->load_flag & FC_UNLOADING) + return -EINVAL; + + hstate = atomic_read(&lpfc_nvmet->state); + if (hstate == LPFC_NVMET_INV_HOST_ACTIVE) + return -EACCES; + + ndlp = (struct lpfc_nodelist *)hosthandle; + + ret = __lpfc_nvme_ls_req(phba->pport, ndlp, pnvme_lsreq, + lpfc_nvmet_ls_req_cmp); + + return ret; +} + +/** + * lpfc_nvmet_ls_abort - Abort a prior NVME LS request + * @targetport: Transport targetport, that LS was issued from. + * @hosthandle - hosthandle set by the driver in a prior ls_rqst_rcv. + * Driver sets this value to the ndlp pointer. + * @pnvme_lsreq - the transport nvme_ls_req structure for LS to be aborted + * + * Driver registers this routine to abort an NVME LS request that is + * in progress (from the transports perspective). + **/ +static void +lpfc_nvmet_ls_abort(struct nvmet_fc_target_port *targetport, + void *hosthandle, + struct nvmefc_ls_req *pnvme_lsreq) +{ + struct lpfc_nvmet_tgtport *lpfc_nvmet = targetport->private; + struct lpfc_hba *phba; + struct lpfc_nodelist *ndlp; + int ret; + + phba = lpfc_nvmet->phba; + if (phba->pport->load_flag & FC_UNLOADING) + return; + + ndlp = (struct lpfc_nodelist *)hosthandle; + + ret = __lpfc_nvme_ls_abort(phba->pport, ndlp, pnvme_lsreq); + if (!ret) + atomic_inc(&lpfc_nvmet->xmt_ls_abort); +} + static void lpfc_nvmet_host_release(void *hosthandle) { @@ -1324,6 +1416,8 @@ static struct nvmet_fc_target_template lpfc_tgttemplate = { .fcp_req_release = lpfc_nvmet_xmt_fcp_release, .defer_rcv = lpfc_nvmet_defer_rcv, .discovery_event = lpfc_nvmet_discovery_event, + .ls_req = lpfc_nvmet_ls_req, + .ls_abort = lpfc_nvmet_ls_abort, .host_release = lpfc_nvmet_host_release, .max_hw_queues = 1, @@ -1335,6 +1429,7 @@ static struct nvmet_fc_target_template lpfc_tgttemplate = { .target_features = 0, /* sizes of additional private data for data structures */ .target_priv_sz = sizeof(struct lpfc_nvmet_tgtport), + .lsrqst_priv_sz = 0, }; static void From 71fb90eb71d76089115aedf942533bcb3fd5c7f9 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Fri, 3 Apr 2020 09:24:01 -0700 Subject: [PATCH 0332/1043] nvme: provide num dword helper Various nvme commands use a zeroes based number of dwords field. Create a helper function to convert byte lengths to this format. Signed-off-by: Keith Busch Reviewed-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 4 ++-- drivers/nvme/host/nvme.h | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index f3c037f5a9ba..6da941b6c67c 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -530,7 +530,7 @@ static int nvme_get_stream_params(struct nvme_ctrl *ctrl, c.directive.opcode = nvme_admin_directive_recv; c.directive.nsid = cpu_to_le32(nsid); - c.directive.numd = cpu_to_le32((sizeof(*s) >> 2) - 1); + c.directive.numd = cpu_to_le32(nvme_bytes_to_numd(sizeof(*s))); c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM; c.directive.dtype = NVME_DIR_STREAMS; @@ -2746,7 +2746,7 @@ int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, void *log, size_t size, u64 offset) { struct nvme_command c = { }; - unsigned long dwlen = size / 4 - 1; + u32 dwlen = nvme_bytes_to_numd(size); c.get_log_page.opcode = nvme_admin_get_log_page; c.get_log_page.nsid = cpu_to_le32(nsid); diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 2e04a36296d9..36f44b79bb3b 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -449,6 +449,14 @@ static inline sector_t nvme_lba_to_sect(struct nvme_ns *ns, u64 lba) return lba << (ns->lba_shift - SECTOR_SHIFT); } +/* + * Convert byte length to nvme's 0-based num dwords + */ +static inline u32 nvme_bytes_to_numd(size_t len) +{ + return (len >> 2) - 1; +} + static inline void nvme_end_request(struct request *req, __le16 status, union nvme_result result) { From 03f8cebc127fa285874074ec314b76b1333f6c43 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Fri, 3 Apr 2020 09:24:09 -0700 Subject: [PATCH 0333/1043] nvme: remove unused parameter nvme_alloc_ns_head() doesn't use the 'struct nvme_id_ns' parameter. Remove it, and update caller accordingly. Signed-off-by: Keith Busch Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 6da941b6c67c..3d0c7d90ad25 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3410,8 +3410,7 @@ static int __nvme_check_ids(struct nvme_subsystem *subsys, } static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, - unsigned nsid, struct nvme_id_ns *id, - struct nvme_ns_ids *ids) + unsigned nsid, struct nvme_ns_ids *ids) { struct nvme_ns_head *head; size_t size = sizeof(*head); @@ -3482,7 +3481,7 @@ static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid, if (is_shared) head = nvme_find_ns_head(ctrl->subsys, nsid); if (!head) { - head = nvme_alloc_ns_head(ctrl, nsid, id, &ids); + head = nvme_alloc_ns_head(ctrl, nsid, &ids); if (IS_ERR(head)) { ret = PTR_ERR(head); goto out_unlock; From b0012dd397155438c61b0c1b52ceec1f1366b3cc Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Wed, 19 Apr 2017 11:56:57 +0300 Subject: [PATCH 0334/1043] nvmet-rdma: use SRQ per completion vector In order to save resource allocation and utilize the completion locality in a better way (compared to SRQ per device that exist today), allocate Shared Receive Queues (SRQs) per completion vector. Associate each created QP/CQ with an appropriate SRQ according to the queue index. This association will reduce the lock contention in the fast path (compared to SRQ per device solution) and increase the locality in memory buffers. Add new module parameter for SRQ size to adjust it according to the expected load. User should make sure the size is >= 256 to avoid lack of resources. Also reduce the debug level of "last WQE reached" event that is raised when a QP is using SRQ during destruction process to relief the log. Signed-off-by: Max Gurtovoy Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/rdma.c | 182 ++++++++++++++++++++++++++++--------- 1 file changed, 138 insertions(+), 44 deletions(-) diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index fd47de0e4e4e..7a90b10359bb 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -34,6 +34,8 @@ /* Assume mpsmin == device_page_size == 4KB */ #define NVMET_RDMA_MAX_MDTS 8 +struct nvmet_rdma_srq; + struct nvmet_rdma_cmd { struct ib_sge sge[NVMET_RDMA_MAX_INLINE_SGE + 1]; struct ib_cqe cqe; @@ -41,6 +43,7 @@ struct nvmet_rdma_cmd { struct scatterlist inline_sg[NVMET_RDMA_MAX_INLINE_SGE]; struct nvme_command *nvme_cmd; struct nvmet_rdma_queue *queue; + struct nvmet_rdma_srq *nsrq; }; enum { @@ -83,6 +86,7 @@ struct nvmet_rdma_queue { struct ib_cq *cq; atomic_t sq_wr_avail; struct nvmet_rdma_device *dev; + struct nvmet_rdma_srq *nsrq; spinlock_t state_lock; enum nvmet_rdma_queue_state state; struct nvmet_cq nvme_cq; @@ -100,6 +104,7 @@ struct nvmet_rdma_queue { int idx; int host_qid; + int comp_vector; int recv_queue_size; int send_queue_size; @@ -113,11 +118,17 @@ struct nvmet_rdma_port { struct delayed_work repair_work; }; +struct nvmet_rdma_srq { + struct ib_srq *srq; + struct nvmet_rdma_cmd *cmds; + struct nvmet_rdma_device *ndev; +}; + struct nvmet_rdma_device { struct ib_device *device; struct ib_pd *pd; - struct ib_srq *srq; - struct nvmet_rdma_cmd *srq_cmds; + struct nvmet_rdma_srq **srqs; + int srq_count; size_t srq_size; struct kref ref; struct list_head entry; @@ -129,6 +140,16 @@ static bool nvmet_rdma_use_srq; module_param_named(use_srq, nvmet_rdma_use_srq, bool, 0444); MODULE_PARM_DESC(use_srq, "Use shared receive queue."); +static int srq_size_set(const char *val, const struct kernel_param *kp); +static const struct kernel_param_ops srq_size_ops = { + .set = srq_size_set, + .get = param_get_int, +}; + +static int nvmet_rdma_srq_size = 1024; +module_param_cb(srq_size, &srq_size_ops, &nvmet_rdma_srq_size, 0644); +MODULE_PARM_DESC(srq_size, "set Shared Receive Queue (SRQ) size, should >= 256 (default: 1024)"); + static DEFINE_IDA(nvmet_rdma_queue_ida); static LIST_HEAD(nvmet_rdma_queue_list); static DEFINE_MUTEX(nvmet_rdma_queue_mutex); @@ -149,6 +170,17 @@ static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device *ndev, static const struct nvmet_fabrics_ops nvmet_rdma_ops; +static int srq_size_set(const char *val, const struct kernel_param *kp) +{ + int n = 0, ret; + + ret = kstrtoint(val, 10, &n); + if (ret != 0 || n < 256) + return -EINVAL; + + return param_set_int(val, kp); +} + static int num_pages(int len) { return 1 + (((len - 1) & PAGE_MASK) >> PAGE_SHIFT); @@ -466,8 +498,8 @@ static int nvmet_rdma_post_recv(struct nvmet_rdma_device *ndev, cmd->sge[0].addr, cmd->sge[0].length, DMA_FROM_DEVICE); - if (ndev->srq) - ret = ib_post_srq_recv(ndev->srq, &cmd->wr, NULL); + if (cmd->nsrq) + ret = ib_post_srq_recv(cmd->nsrq->srq, &cmd->wr, NULL); else ret = ib_post_recv(cmd->queue->qp, &cmd->wr, NULL); @@ -845,23 +877,40 @@ static void nvmet_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc) nvmet_rdma_handle_command(queue, rsp); } -static void nvmet_rdma_destroy_srq(struct nvmet_rdma_device *ndev) +static void nvmet_rdma_destroy_srq(struct nvmet_rdma_srq *nsrq) { - if (!ndev->srq) - return; + nvmet_rdma_free_cmds(nsrq->ndev, nsrq->cmds, nsrq->ndev->srq_size, + false); + ib_destroy_srq(nsrq->srq); - nvmet_rdma_free_cmds(ndev, ndev->srq_cmds, ndev->srq_size, false); - ib_destroy_srq(ndev->srq); + kfree(nsrq); } -static int nvmet_rdma_init_srq(struct nvmet_rdma_device *ndev) +static void nvmet_rdma_destroy_srqs(struct nvmet_rdma_device *ndev) +{ + int i; + + if (!ndev->srqs) + return; + + for (i = 0; i < ndev->srq_count; i++) + nvmet_rdma_destroy_srq(ndev->srqs[i]); + + kfree(ndev->srqs); +} + +static struct nvmet_rdma_srq * +nvmet_rdma_init_srq(struct nvmet_rdma_device *ndev) { struct ib_srq_init_attr srq_attr = { NULL, }; + size_t srq_size = ndev->srq_size; + struct nvmet_rdma_srq *nsrq; struct ib_srq *srq; - size_t srq_size; int ret, i; - srq_size = 4095; /* XXX: tune */ + nsrq = kzalloc(sizeof(*nsrq), GFP_KERNEL); + if (!nsrq) + return ERR_PTR(-ENOMEM); srq_attr.attr.max_wr = srq_size; srq_attr.attr.max_sge = 1 + ndev->inline_page_count; @@ -869,6 +918,42 @@ static int nvmet_rdma_init_srq(struct nvmet_rdma_device *ndev) srq_attr.srq_type = IB_SRQT_BASIC; srq = ib_create_srq(ndev->pd, &srq_attr); if (IS_ERR(srq)) { + ret = PTR_ERR(srq); + goto out_free; + } + + nsrq->cmds = nvmet_rdma_alloc_cmds(ndev, srq_size, false); + if (IS_ERR(nsrq->cmds)) { + ret = PTR_ERR(nsrq->cmds); + goto out_destroy_srq; + } + + nsrq->srq = srq; + nsrq->ndev = ndev; + + for (i = 0; i < srq_size; i++) { + nsrq->cmds[i].nsrq = nsrq; + ret = nvmet_rdma_post_recv(ndev, &nsrq->cmds[i]); + if (ret) + goto out_free_cmds; + } + + return nsrq; + +out_free_cmds: + nvmet_rdma_free_cmds(ndev, nsrq->cmds, srq_size, false); +out_destroy_srq: + ib_destroy_srq(srq); +out_free: + kfree(nsrq); + return ERR_PTR(ret); +} + +static int nvmet_rdma_init_srqs(struct nvmet_rdma_device *ndev) +{ + int i, ret; + + if (!ndev->device->attrs.max_srq_wr || !ndev->device->attrs.max_srq) { /* * If SRQs aren't supported we just go ahead and use normal * non-shared receive queues. @@ -877,27 +962,29 @@ static int nvmet_rdma_init_srq(struct nvmet_rdma_device *ndev) return 0; } - ndev->srq_cmds = nvmet_rdma_alloc_cmds(ndev, srq_size, false); - if (IS_ERR(ndev->srq_cmds)) { - ret = PTR_ERR(ndev->srq_cmds); - goto out_destroy_srq; - } + ndev->srq_size = min(ndev->device->attrs.max_srq_wr, + nvmet_rdma_srq_size); + ndev->srq_count = min(ndev->device->num_comp_vectors, + ndev->device->attrs.max_srq); - ndev->srq = srq; - ndev->srq_size = srq_size; + ndev->srqs = kcalloc(ndev->srq_count, sizeof(*ndev->srqs), GFP_KERNEL); + if (!ndev->srqs) + return -ENOMEM; - for (i = 0; i < srq_size; i++) { - ret = nvmet_rdma_post_recv(ndev, &ndev->srq_cmds[i]); - if (ret) - goto out_free_cmds; + for (i = 0; i < ndev->srq_count; i++) { + ndev->srqs[i] = nvmet_rdma_init_srq(ndev); + if (IS_ERR(ndev->srqs[i])) { + ret = PTR_ERR(ndev->srqs[i]); + goto err_srq; + } } return 0; -out_free_cmds: - nvmet_rdma_free_cmds(ndev, ndev->srq_cmds, ndev->srq_size, false); -out_destroy_srq: - ib_destroy_srq(srq); +err_srq: + while (--i >= 0) + nvmet_rdma_destroy_srq(ndev->srqs[i]); + kfree(ndev->srqs); return ret; } @@ -910,7 +997,7 @@ static void nvmet_rdma_free_dev(struct kref *ref) list_del(&ndev->entry); mutex_unlock(&device_list_mutex); - nvmet_rdma_destroy_srq(ndev); + nvmet_rdma_destroy_srqs(ndev); ib_dealloc_pd(ndev->pd); kfree(ndev); @@ -957,7 +1044,7 @@ nvmet_rdma_find_get_device(struct rdma_cm_id *cm_id) goto out_free_dev; if (nvmet_rdma_use_srq) { - ret = nvmet_rdma_init_srq(ndev); + ret = nvmet_rdma_init_srqs(ndev); if (ret) goto out_free_pd; } @@ -981,14 +1068,7 @@ static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue *queue) { struct ib_qp_init_attr qp_attr; struct nvmet_rdma_device *ndev = queue->dev; - int comp_vector, nr_cqe, ret, i, factor; - - /* - * Spread the io queues across completion vectors, - * but still keep all admin queues on vector 0. - */ - comp_vector = !queue->host_qid ? 0 : - queue->idx % ndev->device->num_comp_vectors; + int nr_cqe, ret, i, factor; /* * Reserve CQ slots for RECV + RDMA_READ/RDMA_WRITE + RDMA_SEND. @@ -996,7 +1076,7 @@ static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue *queue) nr_cqe = queue->recv_queue_size + 2 * queue->send_queue_size; queue->cq = ib_alloc_cq(ndev->device, queue, - nr_cqe + 1, comp_vector, + nr_cqe + 1, queue->comp_vector, IB_POLL_WORKQUEUE); if (IS_ERR(queue->cq)) { ret = PTR_ERR(queue->cq); @@ -1020,8 +1100,8 @@ static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue *queue) qp_attr.cap.max_send_sge = max(ndev->device->attrs.max_sge_rd, ndev->device->attrs.max_send_sge); - if (ndev->srq) { - qp_attr.srq = ndev->srq; + if (queue->nsrq) { + qp_attr.srq = queue->nsrq->srq; } else { /* +1 for drain */ qp_attr.cap.max_recv_wr = 1 + queue->recv_queue_size; @@ -1041,7 +1121,7 @@ static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue *queue) __func__, queue->cq->cqe, qp_attr.cap.max_send_sge, qp_attr.cap.max_send_wr, queue->cm_id); - if (!ndev->srq) { + if (!queue->nsrq) { for (i = 0; i < queue->recv_queue_size; i++) { queue->cmds[i].queue = queue; ret = nvmet_rdma_post_recv(ndev, &queue->cmds[i]); @@ -1076,7 +1156,7 @@ static void nvmet_rdma_free_queue(struct nvmet_rdma_queue *queue) nvmet_sq_destroy(&queue->nvme_sq); nvmet_rdma_destroy_queue_ib(queue); - if (!queue->dev->srq) { + if (!queue->nsrq) { nvmet_rdma_free_cmds(queue->dev, queue->cmds, queue->recv_queue_size, !queue->host_qid); @@ -1188,13 +1268,23 @@ nvmet_rdma_alloc_queue(struct nvmet_rdma_device *ndev, goto out_destroy_sq; } + /* + * Spread the io queues across completion vectors, + * but still keep all admin queues on vector 0. + */ + queue->comp_vector = !queue->host_qid ? 0 : + queue->idx % ndev->device->num_comp_vectors; + + ret = nvmet_rdma_alloc_rsps(queue); if (ret) { ret = NVME_RDMA_CM_NO_RSC; goto out_ida_remove; } - if (!ndev->srq) { + if (ndev->srqs) { + queue->nsrq = ndev->srqs[queue->comp_vector % ndev->srq_count]; + } else { queue->cmds = nvmet_rdma_alloc_cmds(ndev, queue->recv_queue_size, !queue->host_qid); @@ -1215,7 +1305,7 @@ nvmet_rdma_alloc_queue(struct nvmet_rdma_device *ndev, return queue; out_free_cmds: - if (!ndev->srq) { + if (!queue->nsrq) { nvmet_rdma_free_cmds(queue->dev, queue->cmds, queue->recv_queue_size, !queue->host_qid); @@ -1241,6 +1331,10 @@ static void nvmet_rdma_qp_event(struct ib_event *event, void *priv) case IB_EVENT_COMM_EST: rdma_notify(queue->cm_id, event->event); break; + case IB_EVENT_QP_LAST_WQE_REACHED: + pr_debug("received last WQE reached event for queue=0x%p\n", + queue); + break; default: pr_err("received IB QP event: %s (%d)\n", ib_event_msg(event->event), event->event); From e4fcc72c1a420bdbe425530dd19724214ceb44ec Mon Sep 17 00:00:00 2001 From: James Smart Date: Mon, 6 Apr 2020 16:55:34 -0700 Subject: [PATCH 0335/1043] nvmet-fc: slight cleanup for kbuild test warnings The kbuild tst robot flagged the following 3 issues: Case 1) >> drivers/nvme/target/fc.c:1201:37: warning: Either the condition >> '!assoc' is redundant or there is possible null pointer dereference: >> assoc. [nullPointerRedundantCheck] >> struct nvmet_fc_tgtport *tgtport = assoc->tgtport; ^ >> drivers/nvme/target/fc.c:1853:7: note: Assuming that condition '!assoc' >> is not redundant >> if (!assoc) ^ >> drivers/nvme/target/fc.c:1850:37: note: Assignment >> 'assoc=nvmet_fc_find_target_assoc(tgtport,be64_to_cpu( >> rqst->associd.association_id))', assigned value is 0 >> assoc = nvmet_fc_find_target_assoc(tgtport, ^ >> drivers/nvme/target/fc.c:1896:31: note: Calling function >> 'nvmet_fc_delete_target_assoc', 1st argument 'assoc' value is 0 >> nvmet_fc_delete_target_assoc(assoc); ^ The tool isn't smart enough to see that line 1854 sets a ret value which thereafter causes the routine to exit. This occurs before any of the assoc references, so it is not an issue. There are 2 more reportings of this same failure. To quiet the tool - rework the if test that does the exit to also reference assoc. No change in logic otherwise. Case 2) drivers/nvme/target/fc.c:1202:29: warning: The scope of the variable 'queue' can be reduced. [variableScope] struct nvmet_fc_tgt_queue *queue; ^ The tool is requesting the variable be declared within the code block that utilizes it. Ignoring this report as existing code style is fine. Case 3) drivers/nvme/target/fc.c:1137:16: warning: Variable 'needrandom' is assigned a value that is never used. [unreadVariable] needrandom = true; ^ Another parsing issue with the tool. Given that parens were not used with the list_for_each_entry() check, it inadvertantly thinks the break exited the outer while loop not the inner for loop. This is not an error. But, added parens to the inner list_for_each_entry() to quiet the tool and as it is better coding style. -- james Signed-off-by: James Smart Reported-by: kbuild test robot CC: kbuild test robot CC: Christoph Hellwig Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/fc.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index 02d9751bb7ee..27fd3b5aa621 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -1132,11 +1132,12 @@ nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport, void *hosthandle) spin_lock_irqsave(&tgtport->lock, flags); needrandom = false; - list_for_each_entry(tmpassoc, &tgtport->assoc_list, a_list) + list_for_each_entry(tmpassoc, &tgtport->assoc_list, a_list) { if (ran == tmpassoc->association_id) { needrandom = true; break; } + } if (!needrandom) { assoc->association_id = ran; list_add_tail(&assoc->a_list, &tgtport->assoc_list); @@ -1837,7 +1838,7 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, &iod->rqstbuf->rq_dis_assoc; struct fcnvme_ls_disconnect_assoc_acc *acc = &iod->rspbuf->rsp_dis_assoc; - struct nvmet_fc_tgt_assoc *assoc; + struct nvmet_fc_tgt_assoc *assoc = NULL; struct nvmet_fc_ls_iod *oldls = NULL; unsigned long flags; int ret = 0; @@ -1854,7 +1855,7 @@ nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, ret = VERR_NO_ASSOC; } - if (ret) { + if (ret || !assoc) { dev_err(tgtport->dev, "Disconnect LS failed: %s\n", validation_errors[ret]); From b9a5c3d4c34d8bd9fd75f7f28d18a57cb68da237 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 4 Apr 2020 10:11:28 +0200 Subject: [PATCH 0336/1043] nvme: refine the Qemu Identify CNS quirk Add a helper to check if we can use Identify CNS values > 1, and refine the Qemu quirk to not apply to reported versions larger than 1.1, as the Qemu implementation had been fixed by then. Signed-off-by: Christoph Hellwig Reviewed-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 3d0c7d90ad25..a9af83918afe 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1027,6 +1027,19 @@ void nvme_stop_keep_alive(struct nvme_ctrl *ctrl) } EXPORT_SYMBOL_GPL(nvme_stop_keep_alive); +/* + * In NVMe 1.0 the CNS field was just a binary controller or namespace + * flag, thus sending any new CNS opcodes has a big chance of not working. + * Qemu unfortunately had that bug after reporting a 1.1 version compliance + * (but not for any later version). + */ +static bool nvme_ctrl_limited_cns(struct nvme_ctrl *ctrl) +{ + if (ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS) + return ctrl->vs < NVME_VS(1, 2, 0); + return ctrl->vs < NVME_VS(1, 1, 0); +} + static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id) { struct nvme_command c = { }; @@ -3814,8 +3827,7 @@ static void nvme_scan_work(struct work_struct *work) mutex_lock(&ctrl->scan_lock); nn = le32_to_cpu(id->nn); - if (ctrl->vs >= NVME_VS(1, 1, 0) && - !(ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)) { + if (!nvme_ctrl_limited_cns(ctrl)) { if (!nvme_scan_ns_list(ctrl, nn)) goto out_free_id; } From 25dcaa9292afc0689749099a5ba56fdb264eda7a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 4 Apr 2020 10:16:03 +0200 Subject: [PATCH 0337/1043] nvme: clean up nvme_scan_work Move the check for the supported CNS value into nvme_scan_ns_list, and limit the life time of the identify controller allocation. Signed-off-by: Christoph Hellwig Reviewed-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index a9af83918afe..025a8a6d81c7 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3738,6 +3738,9 @@ static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn) unsigned num_lists = DIV_ROUND_UP_ULL((u64)nn, 1024); int ret = 0; + if (nvme_ctrl_limited_cns(ctrl)) + return -EOPNOTSUPP; + ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL); if (!ns_list) return -ENOMEM; @@ -3824,17 +3827,14 @@ static void nvme_scan_work(struct work_struct *work) if (nvme_identify_ctrl(ctrl, &id)) return; + nn = le32_to_cpu(id->nn); + kfree(id); mutex_lock(&ctrl->scan_lock); - nn = le32_to_cpu(id->nn); - if (!nvme_ctrl_limited_cns(ctrl)) { - if (!nvme_scan_ns_list(ctrl, nn)) - goto out_free_id; - } - nvme_scan_ns_sequential(ctrl, nn); -out_free_id: + if (nvme_scan_ns_list(ctrl, nn) != 0) + nvme_scan_ns_sequential(ctrl, nn); mutex_unlock(&ctrl->scan_lock); - kfree(id); + down_write(&ctrl->namespaces_rwsem); list_sort(NULL, &ctrl->namespaces, ns_cmp); up_write(&ctrl->namespaces_rwsem); From 4450ba3bbb47d5fe852b730ab6773d2c18cfbfd6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 4 Apr 2020 10:30:32 +0200 Subject: [PATCH 0338/1043] nvme: factor out a nvme_ns_remove_by_nsid helper Factor out a piece of deeply indented and logicaly separate code from nvme_scan_ns_list into a new helper. Signed-off-by: Christoph Hellwig Reviewed-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 025a8a6d81c7..42102b81d540 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3699,6 +3699,16 @@ static void nvme_ns_remove(struct nvme_ns *ns) nvme_put_ns(ns); } +static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid) +{ + struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid); + + if (ns) { + nvme_ns_remove(ns); + nvme_put_ns(ns); + } +} + static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid) { struct nvme_ns *ns; @@ -3732,7 +3742,6 @@ static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl, static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn) { - struct nvme_ns *ns; __le32 *ns_list; unsigned i, j, nsid, prev = 0; unsigned num_lists = DIV_ROUND_UP_ULL((u64)nn, 1024); @@ -3757,13 +3766,8 @@ static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn) nvme_validate_ns(ctrl, nsid); - while (++prev < nsid) { - ns = nvme_find_get_ns(ctrl, prev); - if (ns) { - nvme_ns_remove(ns); - nvme_put_ns(ns); - } - } + while (++prev < nsid) + nvme_ns_remove_by_nsid(ctrl, prev); } nn -= j; } From 4005f28d25cbd3d4f85529a31e46a0b78e293082 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 4 Apr 2020 10:31:35 +0200 Subject: [PATCH 0339/1043] nvme: avoid an Identify Controller command for each namespace scan The namespace lists are 0-terminated, so we don't really need the NN value execept for the legacy sequential scan. Signed-off-by: Christoph Hellwig Reviewed-by: Keith Busch Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 42102b81d540..15b63426a113 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3740,12 +3740,11 @@ static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl, } -static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn) +static int nvme_scan_ns_list(struct nvme_ctrl *ctrl) { __le32 *ns_list; - unsigned i, j, nsid, prev = 0; - unsigned num_lists = DIV_ROUND_UP_ULL((u64)nn, 1024); - int ret = 0; + u32 prev = 0; + int ret = 0, i; if (nvme_ctrl_limited_cns(ctrl)) return -EOPNOTSUPP; @@ -3754,22 +3753,20 @@ static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn) if (!ns_list) return -ENOMEM; - for (i = 0; i < num_lists; i++) { + for (;;) { ret = nvme_identify_ns_list(ctrl, prev, ns_list); if (ret) goto free; - for (j = 0; j < min(nn, 1024U); j++) { - nsid = le32_to_cpu(ns_list[j]); - if (!nsid) + for (i = 0; i < 1024; i++) { + u32 nsid = le32_to_cpu(ns_list[i]); + + if (!nsid) /* end of the list? */ goto out; - nvme_validate_ns(ctrl, nsid); - while (++prev < nsid) nvme_ns_remove_by_nsid(ctrl, prev); } - nn -= j; } out: nvme_remove_invalid_namespaces(ctrl, prev); @@ -3778,9 +3775,15 @@ static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn) return ret; } -static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl, unsigned nn) +static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl) { - unsigned i; + struct nvme_id_ctrl *id; + u32 nn, i; + + if (nvme_identify_ctrl(ctrl, &id)) + return; + nn = le32_to_cpu(id->nn); + kfree(id); for (i = 1; i <= nn; i++) nvme_validate_ns(ctrl, i); @@ -3817,8 +3820,6 @@ static void nvme_scan_work(struct work_struct *work) { struct nvme_ctrl *ctrl = container_of(work, struct nvme_ctrl, scan_work); - struct nvme_id_ctrl *id; - unsigned nn; /* No tagset on a live ctrl means IO queues could not created */ if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset) @@ -3829,14 +3830,9 @@ static void nvme_scan_work(struct work_struct *work) nvme_clear_changed_ns_log(ctrl); } - if (nvme_identify_ctrl(ctrl, &id)) - return; - nn = le32_to_cpu(id->nn); - kfree(id); - mutex_lock(&ctrl->scan_lock); - if (nvme_scan_ns_list(ctrl, nn) != 0) - nvme_scan_ns_sequential(ctrl, nn); + if (nvme_scan_ns_list(ctrl) != 0) + nvme_scan_ns_sequential(ctrl); mutex_unlock(&ctrl->scan_lock); down_write(&ctrl->namespaces_rwsem); From aec459b484b8356d6abe862428787d98ffb1cbde Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 4 Apr 2020 10:34:21 +0200 Subject: [PATCH 0340/1043] nvme: remove the magic 1024 constant in nvme_scan_ns_list Replace it with a value derived from the identify data and nsid sizes. Signed-off-by: Christoph Hellwig Reviewed-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 15b63426a113..d0f65322156c 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3742,6 +3742,7 @@ static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl, static int nvme_scan_ns_list(struct nvme_ctrl *ctrl) { + const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32); __le32 *ns_list; u32 prev = 0; int ret = 0, i; @@ -3758,7 +3759,7 @@ static int nvme_scan_ns_list(struct nvme_ctrl *ctrl) if (ret) goto free; - for (i = 0; i < 1024; i++) { + for (i = 0; i < nr_entries; i++) { u32 nsid = le32_to_cpu(ns_list[i]); if (!nsid) /* end of the list? */ From d567572906d986dedb78b37f111c44eba033f3ef Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 9 Apr 2020 09:08:59 -0700 Subject: [PATCH 0341/1043] nvme: unlink head after removing last namespace The driver had been unlinking the namespace head from the subsystem's list only after the last reference was released, and outside of the list's subsys->lock protection. There is no reason to track an empty head, so unlink the entry from the subsystem's list when the last namespace using that head is removed and with the mutex lock protecting the list update. The next namespace to attach reusing the previous NSID will allocate a new head rather than find the old head with mismatched identifiers. Signed-off-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index d0f65322156c..15b9e60be204 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -433,7 +433,6 @@ static void nvme_free_ns_head(struct kref *ref) nvme_mpath_remove_disk(head); ida_simple_remove(&head->subsys->ns_ida, head->instance); - list_del_init(&head->entry); cleanup_srcu_struct(&head->srcu); nvme_put_subsystem(head->subsys); kfree(head); @@ -3414,7 +3413,6 @@ static int __nvme_check_ids(struct nvme_subsystem *subsys, list_for_each_entry(h, &subsys->nsheads, entry) { if (nvme_ns_ids_valid(&new->ids) && - !list_empty(&h->list) && nvme_ns_ids_equal(&new->ids, &h->ids)) return -EINVAL; } @@ -3660,6 +3658,8 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid) out_unlink_ns: mutex_lock(&ctrl->subsys->lock); list_del_rcu(&ns->siblings); + if (list_empty(&ns->head->list)) + list_del_init(&ns->head->entry); mutex_unlock(&ctrl->subsys->lock); nvme_put_ns_head(ns->head); out_free_id: @@ -3679,7 +3679,10 @@ static void nvme_ns_remove(struct nvme_ns *ns) mutex_lock(&ns->ctrl->subsys->lock); list_del_rcu(&ns->siblings); + if (list_empty(&ns->head->list)) + list_del_init(&ns->head->entry); mutex_unlock(&ns->ctrl->subsys->lock); + synchronize_rcu(); /* guarantee not available in head->list */ nvme_mpath_clear_current_path(ns); synchronize_srcu(&ns->head->srcu); /* wait for concurrent submissions */ From ac262508daa88fb12c5dc53cf30bde163f9f26c9 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 9 Apr 2020 09:09:00 -0700 Subject: [PATCH 0342/1043] nvme: release namespace head reference on error If a namespace identification does not match the subsystem's head for that NSID, release the reference that was taken when the matching head was initially found. Signed-off-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 15b9e60be204..f2dc89dd2e1c 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3503,6 +3503,7 @@ static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid, "IDs don't match for shared namespace %d\n", nsid); ret = -EINVAL; + nvme_put_ns_head(head); goto out_unlock; } } From 9ad1927a3bc2735996ccc74e31b68a41ae9a3d33 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 9 Apr 2020 09:09:01 -0700 Subject: [PATCH 0343/1043] nvme: always search for namespace head Even if a namespace reports it is not capable of sharing, search the subsystem for a matching namespace head. If found, the driver should reject that namespace since it's coming from an invalid configuration. Signed-off-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index f2dc89dd2e1c..6093c8baf809 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3489,8 +3489,7 @@ static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid, goto out; mutex_lock(&ctrl->subsys->lock); - if (is_shared) - head = nvme_find_ns_head(ctrl->subsys, nsid); + head = nvme_find_ns_head(ctrl->subsys, nsid); if (!head) { head = nvme_alloc_ns_head(ctrl, nsid, &ids); if (IS_ERR(head)) { @@ -3498,6 +3497,14 @@ static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid, goto out_unlock; } } else { + if (!is_shared) { + dev_err(ctrl->device, + "Duplicate unshared namespace %d\n", + nsid); + ret = -EINVAL; + nvme_put_ns_head(head); + goto out_unlock; + } if (!nvme_ns_ids_equal(&head->ids, &ids)) { dev_err(ctrl->device, "IDs don't match for shared namespace %d\n", From 0c284db7f23571f6428c44ca714f13a1fc5f70df Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 9 Apr 2020 09:09:02 -0700 Subject: [PATCH 0344/1043] nvme: check namespace head shared property Reject a new shared namespace if a duplicate unshared namespace exists. Signed-off-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 3 ++- drivers/nvme/host/nvme.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 6093c8baf809..6437f2273785 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3496,8 +3496,9 @@ static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid, ret = PTR_ERR(head); goto out_unlock; } + head->shared = is_shared; } else { - if (!is_shared) { + if (!is_shared || !head->shared) { dev_err(ctrl->device, "Duplicate unshared namespace %d\n", nsid); diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 36f44b79bb3b..6222439a0776 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -352,6 +352,7 @@ struct nvme_ns_head { struct nvme_ns_ids ids; struct list_head entry; struct kref ref; + bool shared; int instance; #ifdef CONFIG_NVME_MULTIPATH struct gendisk *disk; From b2ce4d90690bd29ce5b554e203cd03682dd59697 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 9 Apr 2020 09:09:04 -0700 Subject: [PATCH 0345/1043] nvme-multipath: set bdi capabilities once The queues' backing device info capabilities don't change with each namespace revalidation. Set it only when each path's request_queue is initially added to a multipath queue. Signed-off-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 7 ------- drivers/nvme/host/multipath.c | 8 ++++++++ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 6437f2273785..0624393d95e2 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1909,13 +1909,6 @@ static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) if (ns->head->disk) { nvme_update_disk_info(ns->head->disk, ns, id); blk_queue_stack_limits(ns->head->disk->queue, ns->queue); - if (bdi_cap_stable_pages_required(ns->queue->backing_dev_info)) { - struct backing_dev_info *info = - ns->head->disk->queue->backing_dev_info; - - info->capabilities |= BDI_CAP_STABLE_WRITES; - } - revalidate_disk(ns->head->disk); } #endif diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 54603bd3e02d..9f2844935fdf 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -3,6 +3,7 @@ * Copyright (c) 2017-2018 Christoph Hellwig. */ +#include #include #include #include "nvme.h" @@ -666,6 +667,13 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id) nvme_mpath_set_live(ns); mutex_unlock(&ns->head->lock); } + + if (bdi_cap_stable_pages_required(ns->queue->backing_dev_info)) { + struct backing_dev_info *info = + ns->head->disk->queue->backing_dev_info; + + info->capabilities |= BDI_CAP_STABLE_WRITES; + } } void nvme_mpath_remove_disk(struct nvme_ns_head *head) From b2b2de7c5a0127882848f9c5e9379e84d2e02041 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 9 Apr 2020 09:09:05 -0700 Subject: [PATCH 0346/1043] nvme: revalidate after verifying identifiers If the namespace identifiers have changed, skip updating the disk information, as that will register parameters from a mismatched namespace. Signed-off-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 0624393d95e2..b9e6e6de58cf 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1936,7 +1936,6 @@ static int nvme_revalidate_disk(struct gendisk *disk) goto free_id; } - __nvme_revalidate_disk(disk, id); ret = nvme_report_ns_ids(ctrl, ns->head->ns_id, id, &ids); if (ret) goto free_id; @@ -1945,8 +1944,10 @@ static int nvme_revalidate_disk(struct gendisk *disk) dev_err(ctrl->device, "identifiers changed for nsid %d\n", ns->head->ns_id); ret = -ENODEV; + goto free_id; } + __nvme_revalidate_disk(disk, id); free_id: kfree(id); out: From 38adf94e166e3cb4eb89683458ca578051e8218d Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 9 Apr 2020 09:09:06 -0700 Subject: [PATCH 0347/1043] nvme: consolidate chunk_sectors settings Move the quirked chunk_sectors setting to the same location as noiob so one place registers this setting. And since the noiob value is only used locally, remove the member from struct nvme_ns. Signed-off-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 22 ++++++++++------------ drivers/nvme/host/nvme.h | 1 - 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index b9e6e6de58cf..2279557d7361 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1725,12 +1725,6 @@ static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type) } #endif /* CONFIG_BLK_DEV_INTEGRITY */ -static void nvme_set_chunk_size(struct nvme_ns *ns) -{ - u32 chunk_size = nvme_lba_to_sect(ns, ns->noiob); - blk_queue_chunk_sectors(ns->queue, rounddown_pow_of_two(chunk_size)); -} - static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns) { struct nvme_ctrl *ctrl = ns->ctrl; @@ -1885,6 +1879,7 @@ static void nvme_update_disk_info(struct gendisk *disk, static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) { struct nvme_ns *ns = disk->private_data; + u32 iob; /* * If identify namespace failed, use default 512 byte block size so @@ -1893,7 +1888,13 @@ static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) ns->lba_shift = id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ds; if (ns->lba_shift == 0) ns->lba_shift = 9; - ns->noiob = le16_to_cpu(id->noiob); + + if ((ns->ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) && + is_power_of_2(ns->ctrl->max_hw_sectors)) + iob = ns->ctrl->max_hw_sectors; + else + iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob)); + ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms); ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT); /* the PI implementation requires metadata equal t10 pi tuple size */ @@ -1902,8 +1903,8 @@ static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) else ns->pi_type = 0; - if (ns->noiob) - nvme_set_chunk_size(ns); + if (iob) + blk_queue_chunk_sectors(ns->queue, rounddown_pow_of_two(iob)); nvme_update_disk_info(disk, ns, id); #ifdef CONFIG_NVME_MULTIPATH if (ns->head->disk) { @@ -2255,9 +2256,6 @@ static void nvme_set_queue_limits(struct nvme_ctrl *ctrl, blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors); blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX)); } - if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) && - is_power_of_2(ctrl->max_hw_sectors)) - blk_queue_chunk_sectors(q, ctrl->max_hw_sectors); blk_queue_virt_boundary(q, ctrl->page_size - 1); if (ctrl->vwc & NVME_CTRL_VWC_PRESENT) vwc = true; diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 6222439a0776..f3ab17778349 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -389,7 +389,6 @@ struct nvme_ns { #define NVME_NS_REMOVING 0 #define NVME_NS_DEAD 1 #define NVME_NS_ANA_PENDING 2 - u16 noiob; struct nvme_fault_inject fault_inject; From bc1af009a8ed99aac86ee44a711c4a406ed74263 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 9 Apr 2020 09:09:07 -0700 Subject: [PATCH 0348/1043] nvme: revalidate namespace stream parameters The stream parameters are based on the currently formatted logical block size. Recheck these parameters on namespace revalidation so the registered constraints will be accurate. Signed-off-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 54 ++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 2279557d7361..86ca33b3873f 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1810,6 +1810,32 @@ static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b) memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0; } +static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns) +{ + struct streams_directive_params s; + int ret; + + if (!ctrl->nr_streams) + return 0; + + ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id); + if (ret) + return ret; + + ns->sws = le32_to_cpu(s.sws); + ns->sgs = le16_to_cpu(s.sgs); + + if (ns->sws) { + unsigned int bs = 1 << ns->lba_shift; + + blk_queue_io_min(ns->queue, bs * ns->sws); + if (ns->sgs) + blk_queue_io_opt(ns->queue, bs * ns->sws * ns->sgs); + } + + return 0; +} + static void nvme_update_disk_info(struct gendisk *disk, struct nvme_ns *ns, struct nvme_id_ns *id) { @@ -1824,6 +1850,7 @@ static void nvme_update_disk_info(struct gendisk *disk, blk_mq_freeze_queue(disk->queue); blk_integrity_unregister(disk); + nvme_setup_streams_ns(ns->ctrl, ns); if (id->nabo == 0) { /* * Bit 1 indicates whether NAWUPF is defined for this namespace @@ -3546,32 +3573,6 @@ static struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid) return ret; } -static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns) -{ - struct streams_directive_params s; - int ret; - - if (!ctrl->nr_streams) - return 0; - - ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id); - if (ret) - return ret; - - ns->sws = le32_to_cpu(s.sws); - ns->sgs = le16_to_cpu(s.sgs); - - if (ns->sws) { - unsigned int bs = 1 << ns->lba_shift; - - blk_queue_io_min(ns->queue, bs * ns->sws); - if (ns->sgs) - blk_queue_io_opt(ns->queue, bs * ns->sws * ns->sgs); - } - - return 0; -} - static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid) { struct nvme_ns *ns; @@ -3615,7 +3616,6 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid) ret = nvme_init_ns_head(ns, nsid, id); if (ret) goto out_free_id; - nvme_setup_streams_ns(ctrl, ns); nvme_set_disk_name(disk_name, ns, ctrl, &flags); disk = alloc_disk_node(0, node); From 31fdad7be18992606078caed6ff71741fa76310a Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 9 Apr 2020 09:09:08 -0700 Subject: [PATCH 0349/1043] nvme: consolodate io settings The stream parameters indicating optimal io settings were just getting overwritten later. Rearrange the settings so the streams parameters can be preserved if provided. Signed-off-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 86ca33b3873f..0231f61f37d0 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1810,7 +1810,8 @@ static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b) memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0; } -static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns) +static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns, + u32 *phys_bs, u32 *io_opt) { struct streams_directive_params s; int ret; @@ -1826,11 +1827,9 @@ static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns) ns->sgs = le16_to_cpu(s.sgs); if (ns->sws) { - unsigned int bs = 1 << ns->lba_shift; - - blk_queue_io_min(ns->queue, bs * ns->sws); + *phys_bs = ns->sws * (1 << ns->lba_shift); if (ns->sgs) - blk_queue_io_opt(ns->queue, bs * ns->sws * ns->sgs); + *io_opt = *phys_bs * ns->sgs; } return 0; @@ -1850,7 +1849,8 @@ static void nvme_update_disk_info(struct gendisk *disk, blk_mq_freeze_queue(disk->queue); blk_integrity_unregister(disk); - nvme_setup_streams_ns(ns->ctrl, ns); + atomic_bs = phys_bs = io_opt = bs; + nvme_setup_streams_ns(ns->ctrl, ns, &phys_bs, &io_opt); if (id->nabo == 0) { /* * Bit 1 indicates whether NAWUPF is defined for this namespace @@ -1861,16 +1861,13 @@ static void nvme_update_disk_info(struct gendisk *disk, atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs; else atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs; - } else { - atomic_bs = bs; } - phys_bs = bs; - io_opt = bs; + if (id->nsfeat & (1 << 4)) { /* NPWG = Namespace Preferred Write Granularity */ - phys_bs *= 1 + le16_to_cpu(id->npwg); + phys_bs = bs * (1 + le16_to_cpu(id->npwg)); /* NOWS = Namespace Optimal Write Size */ - io_opt *= 1 + le16_to_cpu(id->nows); + io_opt = bs * (1 + le16_to_cpu(id->nows)); } blk_queue_logical_block_size(disk->queue, bs); From e8cd1ff11d58a21242ea2b85450298f3681768a1 Mon Sep 17 00:00:00 2001 From: Anthony Iliopoulos Date: Sun, 19 Apr 2020 16:48:50 -0700 Subject: [PATCH 0350/1043] nvmet: add ns revalidation support Add support for detecting capacity changes on nvmet blockdev and file backed namespaces. This allows for emulating and testing online resizing of nvme devices and filesystems on top. Signed-off-by: Anthony Iliopoulos [chaitanya: Fix comments posted on V1] Signed-off-by: Chaitanya Kulkarni [hch: reuse code a bit more] Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/admin-cmd.c | 5 +++++ drivers/nvme/target/io-cmd-bdev.c | 5 +++++ drivers/nvme/target/io-cmd-file.c | 17 +++++++++++++---- drivers/nvme/target/nvmet.h | 2 ++ 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 9d6f75cfa77c..4c79aa804887 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -486,6 +486,11 @@ static void nvmet_execute_identify_ns(struct nvmet_req *req) if (!ns) goto done; + if (ns->bdev) + nvmet_bdev_ns_revalidate(ns); + else + nvmet_file_ns_revalidate(ns); + /* * nuse = ncap = nsze isn't always true, but we have no way to find * that out from the underlying device. diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index ea0e596be15d..0427e040e3dd 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -75,6 +75,11 @@ void nvmet_bdev_ns_disable(struct nvmet_ns *ns) } } +void nvmet_bdev_ns_revalidate(struct nvmet_ns *ns) +{ + ns->size = i_size_read(ns->bdev->bd_inode); +} + static u16 blk_to_nvme_status(struct nvmet_req *req, blk_status_t blk_sts) { u16 status = NVME_SC_SUCCESS; diff --git a/drivers/nvme/target/io-cmd-file.c b/drivers/nvme/target/io-cmd-file.c index cd5670b83118..f0bd08d86ac0 100644 --- a/drivers/nvme/target/io-cmd-file.c +++ b/drivers/nvme/target/io-cmd-file.c @@ -13,6 +13,18 @@ #define NVMET_MAX_MPOOL_BVEC 16 #define NVMET_MIN_MPOOL_OBJ 16 +int nvmet_file_ns_revalidate(struct nvmet_ns *ns) +{ + struct kstat stat; + int ret; + + ret = vfs_getattr(&ns->file->f_path, &stat, STATX_SIZE, + AT_STATX_FORCE_SYNC); + if (!ret) + ns->size = stat.size; + return ret; +} + void nvmet_file_ns_disable(struct nvmet_ns *ns) { if (ns->file) { @@ -30,7 +42,6 @@ void nvmet_file_ns_disable(struct nvmet_ns *ns) int nvmet_file_ns_enable(struct nvmet_ns *ns) { int flags = O_RDWR | O_LARGEFILE; - struct kstat stat; int ret; if (!ns->buffered_io) @@ -43,12 +54,10 @@ int nvmet_file_ns_enable(struct nvmet_ns *ns) return PTR_ERR(ns->file); } - ret = vfs_getattr(&ns->file->f_path, - &stat, STATX_SIZE, AT_STATX_FORCE_SYNC); + ret = nvmet_file_ns_revalidate(ns); if (ret) goto err; - ns->size = stat.size; /* * i_blkbits can be greater than the universally accepted upper bound, * so make sure we export a sane namespace lba_shift. diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 421dff3ea143..3d981eb6e100 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -498,6 +498,8 @@ void nvmet_file_ns_disable(struct nvmet_ns *ns); u16 nvmet_bdev_flush(struct nvmet_req *req); u16 nvmet_file_flush(struct nvmet_req *req); void nvmet_ns_changed(struct nvmet_subsys *subsys, u32 nsid); +void nvmet_bdev_ns_revalidate(struct nvmet_ns *ns); +int nvmet_file_ns_revalidate(struct nvmet_ns *ns); static inline u32 nvmet_rw_len(struct nvmet_req *req) { From 3add1d93d9919b6de94aa47900d4904adffbc976 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Thu, 30 Apr 2020 23:30:57 +0200 Subject: [PATCH 0351/1043] nvme-fc: avoid gcc-10 zero-length-bounds warning When CONFIG_ARCH_NO_SG_CHAIN is set, op->sgl[0] cannot be dereferenced, as gcc-10 now points out: drivers/nvme/host/fc.c: In function 'nvme_fc_init_request': drivers/nvme/host/fc.c:1774:29: warning: array subscript 0 is outside the bounds of an interior zero-length array 'struct scatterlist[0]' [-Wzero-length-bounds] 1774 | op->op.fcp_req.first_sgl = &op->sgl[0]; | ^~~~~~~~~~~ drivers/nvme/host/fc.c:98:21: note: while referencing 'sgl' 98 | struct scatterlist sgl[NVME_INLINE_SG_CNT]; | ^~~ I don't know if this is a legitimate warning or a false-positive. If this is just a false alarm, the warning is easily suppressed by interpreting the array as a pointer. Fixes: b1ae1a238900 ("nvme-fc: Avoid preallocating big SGL for data") Signed-off-by: Arnd Bergmann Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/fc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 1921d2195541..0b3ab3355e25 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2106,7 +2106,7 @@ nvme_fc_init_request(struct blk_mq_tag_set *set, struct request *rq, res = __nvme_fc_init_request(ctrl, queue, &op->op, rq, queue->rqcnt++); if (res) return res; - op->op.fcp_req.first_sgl = &op->sgl[0]; + op->op.fcp_req.first_sgl = op->sgl; op->op.fcp_req.private = &op->priv[0]; nvme_req(rq)->ctrl = &ctrl->ctrl; return res; From 6623c5b3dfa5513190d729a8516db7a5163ec7de Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 22 Apr 2020 09:59:08 +0200 Subject: [PATCH 0352/1043] nvme: clean up error handling in nvme_init_ns_head Use a common label for putting the nshead if needed and only convert nvme status codes for the one case where it actually is needed. Signed-off-by: Christoph Hellwig Reviewed-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 0231f61f37d0..35e279261c7d 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3501,8 +3501,11 @@ static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid, int ret = 0; ret = nvme_report_ns_ids(ctrl, nsid, id, &ids); - if (ret) - goto out; + if (ret) { + if (ret < 0) + return ret; + return blk_status_to_errno(nvme_error_status(ret)); + } mutex_lock(&ctrl->subsys->lock); head = nvme_find_ns_head(ctrl->subsys, nsid); @@ -3514,32 +3517,29 @@ static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid, } head->shared = is_shared; } else { + ret = -EINVAL; if (!is_shared || !head->shared) { dev_err(ctrl->device, - "Duplicate unshared namespace %d\n", - nsid); - ret = -EINVAL; - nvme_put_ns_head(head); - goto out_unlock; + "Duplicate unshared namespace %d\n", nsid); + goto out_put_ns_head; } if (!nvme_ns_ids_equal(&head->ids, &ids)) { dev_err(ctrl->device, "IDs don't match for shared namespace %d\n", nsid); - ret = -EINVAL; - nvme_put_ns_head(head); - goto out_unlock; + goto out_put_ns_head; } } list_add_tail(&ns->siblings, &head->list); ns->head = head; + mutex_unlock(&ctrl->subsys->lock); + return 0; +out_put_ns_head: + nvme_put_ns_head(head); out_unlock: mutex_unlock(&ctrl->subsys->lock); -out: - if (ret > 0) - ret = blk_status_to_errno(nvme_error_status(ret)); return ret; } From b04df85d9a05d7ee4b3f93c1a22312b41a949ec1 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 30 Apr 2020 05:31:23 +0900 Subject: [PATCH 0353/1043] nvme: flush scan work on passthrough commands If a passthrough command causes the namespace inventory or capabilities to change, flush the scan work that handles these changes so the driver synchronizes with the user command's effects before returning the result to user space. Signed-off-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 35e279261c7d..4a124a28118f 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1404,8 +1404,10 @@ static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects) } if (effects & NVME_CMD_EFFECTS_CCC) nvme_init_identify(ctrl); - if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) + if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) { nvme_queue_scan(ctrl); + flush_work(&ctrl->scan_work); + } } static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns, From 74943d45eef4db64b1e5c9f7ad1d018576e113c5 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Tue, 28 Apr 2020 07:21:56 -0700 Subject: [PATCH 0354/1043] nvme-pci: remove volatile cqes The completion queue entry is not volatile once the phase is confirmed. Remove the volatile keywords and check the phase using the appropriate READ_ONCE() accessor, allowing the compiler to optimize the remaining completion path. Signed-off-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/pci.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index e13c370de830..e95c7465c7bd 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -166,7 +166,7 @@ struct nvme_queue { void *sq_cmds; /* only used for poll queues: */ spinlock_t cq_poll_lock ____cacheline_aligned_in_smp; - volatile struct nvme_completion *cqes; + struct nvme_completion *cqes; dma_addr_t sq_dma_addr; dma_addr_t cq_dma_addr; u32 __iomem *q_db; @@ -922,8 +922,9 @@ static void nvme_pci_complete_rq(struct request *req) /* We read the CQE phase first to check if the rest of the entry is valid */ static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq) { - return (le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) == - nvmeq->cq_phase; + struct nvme_completion *hcqe = &nvmeq->cqes[nvmeq->cq_head]; + + return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == nvmeq->cq_phase; } static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq) @@ -944,7 +945,7 @@ static inline struct blk_mq_tags *nvme_queue_tagset(struct nvme_queue *nvmeq) static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx) { - volatile struct nvme_completion *cqe = &nvmeq->cqes[idx]; + struct nvme_completion *cqe = &nvmeq->cqes[idx]; struct request *req; if (unlikely(cqe->command_id >= nvmeq->q_depth)) { From 54b2fcee1db041a83b52b51752dade6090cf952f Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Mon, 27 Apr 2020 11:54:46 -0700 Subject: [PATCH 0355/1043] nvme-pci: remove last_sq_tail The nvme driver does not have enough tags to wrap the queue, and blk-mq will no longer call commit_rqs() when there are no new submissions to notify. Signed-off-by: Keith Busch Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/pci.c | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index e95c7465c7bd..b945e9a89883 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -173,7 +173,6 @@ struct nvme_queue { u16 q_depth; u16 cq_vector; u16 sq_tail; - u16 last_sq_tail; u16 cq_head; u16 qid; u8 cq_phase; @@ -446,24 +445,11 @@ static int nvme_pci_map_queues(struct blk_mq_tag_set *set) return 0; } -/* - * Write sq tail if we are asked to, or if the next command would wrap. - */ -static inline void nvme_write_sq_db(struct nvme_queue *nvmeq, bool write_sq) +static inline void nvme_write_sq_db(struct nvme_queue *nvmeq) { - if (!write_sq) { - u16 next_tail = nvmeq->sq_tail + 1; - - if (next_tail == nvmeq->q_depth) - next_tail = 0; - if (next_tail != nvmeq->last_sq_tail) - return; - } - if (nvme_dbbuf_update_and_check_event(nvmeq->sq_tail, nvmeq->dbbuf_sq_db, nvmeq->dbbuf_sq_ei)) writel(nvmeq->sq_tail, nvmeq->q_db); - nvmeq->last_sq_tail = nvmeq->sq_tail; } /** @@ -480,7 +466,8 @@ static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd, cmd, sizeof(*cmd)); if (++nvmeq->sq_tail == nvmeq->q_depth) nvmeq->sq_tail = 0; - nvme_write_sq_db(nvmeq, write_sq); + if (write_sq) + nvme_write_sq_db(nvmeq); spin_unlock(&nvmeq->sq_lock); } @@ -489,8 +476,7 @@ static void nvme_commit_rqs(struct blk_mq_hw_ctx *hctx) struct nvme_queue *nvmeq = hctx->driver_data; spin_lock(&nvmeq->sq_lock); - if (nvmeq->sq_tail != nvmeq->last_sq_tail) - nvme_write_sq_db(nvmeq, true); + nvme_write_sq_db(nvmeq); spin_unlock(&nvmeq->sq_lock); } @@ -1494,7 +1480,6 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) struct nvme_dev *dev = nvmeq->dev; nvmeq->sq_tail = 0; - nvmeq->last_sq_tail = 0; nvmeq->cq_head = 0; nvmeq->cq_phase = 1; nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride]; From 2a5bcfdd41d68559567cec3c124a75e093506cc1 Mon Sep 17 00:00:00 2001 From: Weiping Zhang Date: Sat, 2 May 2020 15:29:41 +0800 Subject: [PATCH 0356/1043] nvme-pci: align io queue count with allocted nvme_queue in nvme_probe Since commit 147b27e4bd08 ("nvme-pci: allocate device queues storage space at probe"), nvme_alloc_queue does not alloc the nvme queues itself anymore. If the write/poll_queues module parameters are changed at runtime to values larger than the number of allocated queues in nvme_probe, nvme_alloc_queue will access unallocated memory. Add a new nr_allocated_queues member to struct nvme_dev to record how many queues were alloctated in nvme_probe to avoid using more than the allocated queues after a reset following a change to the write/poll_queues module parameters. Also add nr_write_queues and nr_poll_queues members to allow refreshing the number of write and poll queues based on a change to the module parameters when resetting the controller. Fixes: 147b27e4bd08 ("nvme-pci: allocate device queues storage space at probe") Signed-off-by: Weiping Zhang Reviewed-by: Keith Busch Reviewed-by: Max Gurtovoy [hch: add nvme_max_io_queues, update the commit message] Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/pci.c | 57 ++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index b945e9a89883..b0978ac554d5 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -128,6 +128,9 @@ struct nvme_dev { dma_addr_t host_mem_descs_dma; struct nvme_host_mem_buf_desc *host_mem_descs; void **host_mem_desc_bufs; + unsigned int nr_allocated_queues; + unsigned int nr_write_queues; + unsigned int nr_poll_queues; }; static int io_queue_depth_set(const char *val, const struct kernel_param *kp) @@ -208,25 +211,14 @@ struct nvme_iod { struct scatterlist *sg; }; -static unsigned int max_io_queues(void) +static inline unsigned int nvme_dbbuf_size(struct nvme_dev *dev) { - return num_possible_cpus() + write_queues + poll_queues; -} - -static unsigned int max_queue_count(void) -{ - /* IO queues + admin queue */ - return 1 + max_io_queues(); -} - -static inline unsigned int nvme_dbbuf_size(u32 stride) -{ - return (max_queue_count() * 8 * stride); + return dev->nr_allocated_queues * 8 * dev->db_stride; } static int nvme_dbbuf_dma_alloc(struct nvme_dev *dev) { - unsigned int mem_size = nvme_dbbuf_size(dev->db_stride); + unsigned int mem_size = nvme_dbbuf_size(dev); if (dev->dbbuf_dbs) return 0; @@ -251,7 +243,7 @@ static int nvme_dbbuf_dma_alloc(struct nvme_dev *dev) static void nvme_dbbuf_dma_free(struct nvme_dev *dev) { - unsigned int mem_size = nvme_dbbuf_size(dev->db_stride); + unsigned int mem_size = nvme_dbbuf_size(dev); if (dev->dbbuf_dbs) { dma_free_coherent(dev->dev, mem_size, @@ -1981,7 +1973,7 @@ static int nvme_setup_host_mem(struct nvme_dev *dev) static void nvme_calc_irq_sets(struct irq_affinity *affd, unsigned int nrirqs) { struct nvme_dev *dev = affd->priv; - unsigned int nr_read_queues; + unsigned int nr_read_queues, nr_write_queues = dev->nr_write_queues; /* * If there is no interupt available for queues, ensure that @@ -1997,12 +1989,12 @@ static void nvme_calc_irq_sets(struct irq_affinity *affd, unsigned int nrirqs) if (!nrirqs) { nrirqs = 1; nr_read_queues = 0; - } else if (nrirqs == 1 || !write_queues) { + } else if (nrirqs == 1 || !nr_write_queues) { nr_read_queues = 0; - } else if (write_queues >= nrirqs) { + } else if (nr_write_queues >= nrirqs) { nr_read_queues = 1; } else { - nr_read_queues = nrirqs - write_queues; + nr_read_queues = nrirqs - nr_write_queues; } dev->io_queues[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues; @@ -2026,7 +2018,7 @@ static int nvme_setup_irqs(struct nvme_dev *dev, unsigned int nr_io_queues) * Poll queues don't need interrupts, but we need at least one IO * queue left over for non-polled IO. */ - this_p_queues = poll_queues; + this_p_queues = dev->nr_poll_queues; if (this_p_queues >= nr_io_queues) { this_p_queues = nr_io_queues - 1; irq_queues = 1; @@ -2056,14 +2048,25 @@ static void nvme_disable_io_queues(struct nvme_dev *dev) __nvme_disable_io_queues(dev, nvme_admin_delete_cq); } +static unsigned int nvme_max_io_queues(struct nvme_dev *dev) +{ + return num_possible_cpus() + dev->nr_write_queues + dev->nr_poll_queues; +} + static int nvme_setup_io_queues(struct nvme_dev *dev) { struct nvme_queue *adminq = &dev->queues[0]; struct pci_dev *pdev = to_pci_dev(dev->dev); - int result, nr_io_queues; + unsigned int nr_io_queues; unsigned long size; + int result; - nr_io_queues = max_io_queues(); + /* + * Sample the module parameters once at reset time so that we have + * stable values to work with. + */ + dev->nr_write_queues = write_queues; + dev->nr_poll_queues = poll_queues; /* * If tags are shared with admin queue (Apple bug), then @@ -2071,6 +2074,9 @@ static int nvme_setup_io_queues(struct nvme_dev *dev) */ if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) nr_io_queues = 1; + else + nr_io_queues = min(nvme_max_io_queues(dev), + dev->nr_allocated_queues - 1); result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues); if (result < 0) @@ -2745,8 +2751,11 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!dev) return -ENOMEM; - dev->queues = kcalloc_node(max_queue_count(), sizeof(struct nvme_queue), - GFP_KERNEL, node); + dev->nr_write_queues = write_queues; + dev->nr_poll_queues = poll_queues; + dev->nr_allocated_queues = nvme_max_io_queues(dev) + 1; + dev->queues = kcalloc_node(dev->nr_allocated_queues, + sizeof(struct nvme_queue), GFP_KERNEL, node); if (!dev->queues) goto free; From 386e5e6e1aa90b479fcf0467935922df8524393d Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Thu, 30 Apr 2020 13:59:32 -0700 Subject: [PATCH 0357/1043] nvme-tcp: use bh_lock in data_ready data_ready may be invoked from send context or from softirq, so need bh locking for that. Fixes: 3f2304f8c6d6 ("nvme-tcp: add NVMe over TCP host driver") Signed-off-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/tcp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index c15a92163c1f..4862fa962011 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -794,11 +794,11 @@ static void nvme_tcp_data_ready(struct sock *sk) { struct nvme_tcp_queue *queue; - read_lock(&sk->sk_callback_lock); + read_lock_bh(&sk->sk_callback_lock); queue = sk->sk_user_data; if (likely(queue && queue->rd_enabled)) queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); - read_unlock(&sk->sk_callback_lock); + read_unlock_bh(&sk->sk_callback_lock); } static void nvme_tcp_write_space(struct sock *sk) From 72e5d757c62029664c0287d14519ec4451901b5e Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Fri, 1 May 2020 14:25:44 -0700 Subject: [PATCH 0358/1043] nvme-tcp: avoid scheduling io_work if we are already polling When the user runs polled I/O, we shouldn't have to trigger the workqueue to generate the receive work upon the .data_ready upcall. This prevents a redundant context switch when the application is already polling for completions. Proposed-by: Mark Wunderlich Signed-off-by: Mark Wunderlich Signed-off-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/tcp.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 4862fa962011..b28f91d0f083 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -60,6 +60,7 @@ struct nvme_tcp_request { enum nvme_tcp_queue_flags { NVME_TCP_Q_ALLOCATED = 0, NVME_TCP_Q_LIVE = 1, + NVME_TCP_Q_POLLING = 2, }; enum nvme_tcp_recv_state { @@ -796,7 +797,8 @@ static void nvme_tcp_data_ready(struct sock *sk) read_lock_bh(&sk->sk_callback_lock); queue = sk->sk_user_data; - if (likely(queue && queue->rd_enabled)) + if (likely(queue && queue->rd_enabled) && + !test_bit(NVME_TCP_Q_POLLING, &queue->flags)) queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); read_unlock_bh(&sk->sk_callback_lock); } @@ -2302,9 +2304,11 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx) if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) return 0; + set_bit(NVME_TCP_Q_POLLING, &queue->flags); if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue)) sk_busy_loop(sk, true); nvme_tcp_try_recv(queue); + clear_bit(NVME_TCP_Q_POLLING, &queue->flags); return queue->nr_cqe; } From db5ad6b7f8cdd6d78efef62a1557461d0cbaee54 Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Fri, 1 May 2020 14:25:45 -0700 Subject: [PATCH 0359/1043] nvme-tcp: try to send request in queue_rq context Today, nvme-tcp automatically schedules a send request to a workqueue context, which is 1 more than we'd need in case the socket buffer is wide open. However, because we have async send activity (as a result of r2t, or write_space callbacks), we need to synchronize sends from possibly multiple contexts (ideally all running on the same cpu though). Thus, we only try to send directly from queue_rq in cases: 1. the send_list is empty 2. we can send it synchronously (i.e. not from the RX path) 3. we run on the same cpu as the queue->io_cpu to avoid contention on the send operation. Proposed-by: Mark Wunderlich Signed-off-by: Mark Wunderlich Signed-off-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/tcp.c | 43 ++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index b28f91d0f083..c79e248b9f43 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -76,6 +76,7 @@ struct nvme_tcp_queue { int io_cpu; spinlock_t lock; + struct mutex send_mutex; struct list_head send_list; /* recv state */ @@ -132,6 +133,7 @@ static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); static struct workqueue_struct *nvme_tcp_wq; static struct blk_mq_ops nvme_tcp_mq_ops; static struct blk_mq_ops nvme_tcp_admin_mq_ops; +static int nvme_tcp_try_send(struct nvme_tcp_queue *queue); static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl) { @@ -258,15 +260,29 @@ static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, } } -static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req) +static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, + bool sync) { struct nvme_tcp_queue *queue = req->queue; + bool empty; spin_lock(&queue->lock); + empty = list_empty(&queue->send_list) && !queue->request; list_add_tail(&req->entry, &queue->send_list); spin_unlock(&queue->lock); - queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); + /* + * if we're the first on the send_list and we can try to send + * directly, otherwise queue io_work. Also, only do that if we + * are on the same cpu, so we don't introduce contention. + */ + if (queue->io_cpu == smp_processor_id() && + sync && empty && mutex_trylock(&queue->send_mutex)) { + nvme_tcp_try_send(queue); + mutex_unlock(&queue->send_mutex); + } else { + queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); + } } static inline struct nvme_tcp_request * @@ -579,7 +595,7 @@ static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, req->state = NVME_TCP_SEND_H2C_PDU; req->offset = 0; - nvme_tcp_queue_request(req); + nvme_tcp_queue_request(req, false); return 0; } @@ -1065,11 +1081,14 @@ static void nvme_tcp_io_work(struct work_struct *w) bool pending = false; int result; - result = nvme_tcp_try_send(queue); - if (result > 0) - pending = true; - else if (unlikely(result < 0)) - break; + if (mutex_trylock(&queue->send_mutex)) { + result = nvme_tcp_try_send(queue); + mutex_unlock(&queue->send_mutex); + if (result > 0) + pending = true; + else if (unlikely(result < 0)) + break; + } result = nvme_tcp_try_recv(queue); if (result > 0) @@ -1321,6 +1340,7 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, queue->ctrl = ctrl; INIT_LIST_HEAD(&queue->send_list); spin_lock_init(&queue->lock); + mutex_init(&queue->send_mutex); INIT_WORK(&queue->io_work, nvme_tcp_io_work); queue->queue_size = queue_size; @@ -1545,6 +1565,7 @@ static struct blk_mq_tag_set *nvme_tcp_alloc_tagset(struct nvme_ctrl *nctrl, set->queue_depth = NVME_AQ_MQ_TAG_DEPTH; set->reserved_tags = 2; /* connect + keep-alive */ set->numa_node = NUMA_NO_NODE; + set->flags = BLK_MQ_F_BLOCKING; set->cmd_size = sizeof(struct nvme_tcp_request); set->driver_data = ctrl; set->nr_hw_queues = 1; @@ -1556,7 +1577,7 @@ static struct blk_mq_tag_set *nvme_tcp_alloc_tagset(struct nvme_ctrl *nctrl, set->queue_depth = nctrl->sqsize + 1; set->reserved_tags = 1; /* fabric connect */ set->numa_node = NUMA_NO_NODE; - set->flags = BLK_MQ_F_SHOULD_MERGE; + set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING; set->cmd_size = sizeof(struct nvme_tcp_request); set->driver_data = ctrl; set->nr_hw_queues = nctrl->queue_count - 1; @@ -2115,7 +2136,7 @@ static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg) ctrl->async_req.curr_bio = NULL; ctrl->async_req.data_len = 0; - nvme_tcp_queue_request(&ctrl->async_req); + nvme_tcp_queue_request(&ctrl->async_req, true); } static enum blk_eh_timer_return @@ -2246,7 +2267,7 @@ static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx, blk_mq_start_request(rq); - nvme_tcp_queue_request(req); + nvme_tcp_queue_request(req, true); return BLK_STS_OK; } From 7890b9701b792a4d75b4adef4abe325383ccfca4 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 29 Mar 2020 19:41:38 +0200 Subject: [PATCH 0360/1043] nvme-multipath: stop using ->queuedata nvme-multipath already uses the gendisk private data, not need to also set up the request_queue queuedata and use it in one place only. Signed-off-by: Christoph Hellwig Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- drivers/nvme/host/multipath.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 9f2844935fdf..5f5537d0a024 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -294,7 +294,7 @@ static bool nvme_available_path(struct nvme_ns_head *head) static blk_qc_t nvme_ns_head_make_request(struct request_queue *q, struct bio *bio) { - struct nvme_ns_head *head = q->queuedata; + struct nvme_ns_head *head = bio->bi_disk->private_data; struct device *dev = disk_to_dev(head->disk); struct nvme_ns *ns; blk_qc_t ret = BLK_QC_T_NONE; @@ -378,7 +378,6 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) q = blk_alloc_queue(nvme_ns_head_make_request, ctrl->numa_node); if (!q) goto out; - q->queuedata = head; blk_queue_flag_set(QUEUE_FLAG_NONROT, q); /* set to a default value for 512 until disk is validated */ blk_queue_logical_block_size(q, 512); From 45e2f3c2d2f5f39b47745cba41e9e2b0c58d7d94 Mon Sep 17 00:00:00 2001 From: Chaitanya Kulkarni Date: Mon, 4 May 2020 01:56:43 -0700 Subject: [PATCH 0361/1043] nvmet: add generic type-name mapping This patch adds a new type to name mapping generic structure. It replaces nvmet_transport_name with new generic mapping structure nvmet_transport. Signed-off-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/configfs.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 58cabd7b6fc5..143f3d02f334 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -20,10 +20,12 @@ static const struct config_item_type nvmet_subsys_type; static LIST_HEAD(nvmet_ports_list); struct list_head *nvmet_ports = &nvmet_ports_list; -static const struct nvmet_transport_name { +struct nvmet_type_name_map { u8 type; const char *name; -} nvmet_transport_names[] = { +}; + +static struct nvmet_type_name_map nvmet_transport[] = { { NVMF_TRTYPE_RDMA, "rdma" }, { NVMF_TRTYPE_FC, "fc" }, { NVMF_TRTYPE_TCP, "tcp" }, @@ -254,10 +256,9 @@ static ssize_t nvmet_addr_trtype_show(struct config_item *item, struct nvmet_port *port = to_nvmet_port(item); int i; - for (i = 0; i < ARRAY_SIZE(nvmet_transport_names); i++) { - if (port->disc_addr.trtype != nvmet_transport_names[i].type) - continue; - return sprintf(page, "%s\n", nvmet_transport_names[i].name); + for (i = 0; i < ARRAY_SIZE(nvmet_transport); i++) { + if (port->disc_addr.trtype == nvmet_transport[i].type) + return sprintf(page, "%s\n", nvmet_transport[i].name); } return sprintf(page, "\n"); @@ -282,8 +283,8 @@ static ssize_t nvmet_addr_trtype_store(struct config_item *item, return -EACCES; } - for (i = 0; i < ARRAY_SIZE(nvmet_transport_names); i++) { - if (sysfs_streq(page, nvmet_transport_names[i].name)) + for (i = 0; i < ARRAY_SIZE(nvmet_transport); i++) { + if (sysfs_streq(page, nvmet_transport[i].name)) goto found; } @@ -291,7 +292,7 @@ static ssize_t nvmet_addr_trtype_store(struct config_item *item, return -EINVAL; found: memset(&port->disc_addr.tsas, 0, NVMF_TSAS_SIZE); - port->disc_addr.trtype = nvmet_transport_names[i].type; + port->disc_addr.trtype = nvmet_transport[i].type; if (port->disc_addr.trtype == NVMF_TRTYPE_RDMA) nvmet_port_init_tsas_rdma(port); return count; From 7e764179c86784594d54764556c4d3973645e6a5 Mon Sep 17 00:00:00 2001 From: Chaitanya Kulkarni Date: Mon, 4 May 2020 01:56:44 -0700 Subject: [PATCH 0362/1043] nvmet: use type-name map for address family Right now nvmet_addr_adrfam_[store|show]() uses switch and if else ladder for address family to string and reverse mapping which also repeats the strings in show and store function. With addition of generic nvmet_type_name_map structure we can now get rid of the switch and if else ladder and string duplication. Also, we add a newline in before found label in nvmet_addr_trtype_store() which keeps goto label code consistent with nvmet_allowed_hosts_drop_link(), nvmet_port_subsys_drop_link() and nvmet_ana_group_ana_state_store(). Signed-off-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/configfs.c | 51 ++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 143f3d02f334..8a5d99e25192 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -32,31 +32,36 @@ static struct nvmet_type_name_map nvmet_transport[] = { { NVMF_TRTYPE_LOOP, "loop" }, }; +static const struct nvmet_type_name_map nvmet_addr_family[] = { + { NVMF_ADDR_FAMILY_PCI, "pcie" }, + { NVMF_ADDR_FAMILY_IP4, "ipv4" }, + { NVMF_ADDR_FAMILY_IP6, "ipv6" }, + { NVMF_ADDR_FAMILY_IB, "ib" }, + { NVMF_ADDR_FAMILY_FC, "fc" }, +}; + /* * nvmet_port Generic ConfigFS definitions. * Used in any place in the ConfigFS tree that refers to an address. */ -static ssize_t nvmet_addr_adrfam_show(struct config_item *item, - char *page) +static ssize_t nvmet_addr_adrfam_show(struct config_item *item, char *page) { - switch (to_nvmet_port(item)->disc_addr.adrfam) { - case NVMF_ADDR_FAMILY_IP4: - return sprintf(page, "ipv4\n"); - case NVMF_ADDR_FAMILY_IP6: - return sprintf(page, "ipv6\n"); - case NVMF_ADDR_FAMILY_IB: - return sprintf(page, "ib\n"); - case NVMF_ADDR_FAMILY_FC: - return sprintf(page, "fc\n"); - default: - return sprintf(page, "\n"); + u8 adrfam = to_nvmet_port(item)->disc_addr.adrfam; + int i; + + for (i = 1; i < ARRAY_SIZE(nvmet_addr_family); i++) { + if (nvmet_addr_family[i].type == adrfam) + return sprintf(page, "%s\n", nvmet_addr_family[i].name); } + + return sprintf(page, "\n"); } static ssize_t nvmet_addr_adrfam_store(struct config_item *item, const char *page, size_t count) { struct nvmet_port *port = to_nvmet_port(item); + int i; if (port->enabled) { pr_err("Cannot modify address while enabled\n"); @@ -64,19 +69,16 @@ static ssize_t nvmet_addr_adrfam_store(struct config_item *item, return -EACCES; } - if (sysfs_streq(page, "ipv4")) { - port->disc_addr.adrfam = NVMF_ADDR_FAMILY_IP4; - } else if (sysfs_streq(page, "ipv6")) { - port->disc_addr.adrfam = NVMF_ADDR_FAMILY_IP6; - } else if (sysfs_streq(page, "ib")) { - port->disc_addr.adrfam = NVMF_ADDR_FAMILY_IB; - } else if (sysfs_streq(page, "fc")) { - port->disc_addr.adrfam = NVMF_ADDR_FAMILY_FC; - } else { - pr_err("Invalid value '%s' for adrfam\n", page); - return -EINVAL; + for (i = 1; i < ARRAY_SIZE(nvmet_addr_family); i++) { + if (sysfs_streq(page, nvmet_addr_family[i].name)) + goto found; } + pr_err("Invalid value '%s' for adrfam\n", page); + return -EINVAL; + +found: + port->disc_addr.adrfam = i; return count; } @@ -290,6 +292,7 @@ static ssize_t nvmet_addr_trtype_store(struct config_item *item, pr_err("Invalid value '%s' for trtype\n", page); return -EINVAL; + found: memset(&port->disc_addr.tsas, 0, NVMF_TSAS_SIZE); port->disc_addr.trtype = nvmet_transport[i].type; From 84b8d0d7aa159652dc191d58c4d353b6c9173c54 Mon Sep 17 00:00:00 2001 From: Chaitanya Kulkarni Date: Mon, 4 May 2020 01:56:45 -0700 Subject: [PATCH 0363/1043] nvmet: use type-name map for ana states Now that we have a generic type to name map for configfs, get rid of the nvmet_ana_state_names structure and replace it with newly added nvmet_type_name_map. Signed-off-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/configfs.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 8a5d99e25192..78eb822a20d5 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -1153,10 +1153,7 @@ static const struct config_item_type nvmet_referrals_type = { .ct_group_ops = &nvmet_referral_group_ops, }; -static struct { - enum nvme_ana_state state; - const char *name; -} nvmet_ana_state_names[] = { +struct nvmet_type_name_map nvmet_ana_state[] = { { NVME_ANA_OPTIMIZED, "optimized" }, { NVME_ANA_NONOPTIMIZED, "non-optimized" }, { NVME_ANA_INACCESSIBLE, "inaccessible" }, @@ -1171,10 +1168,9 @@ static ssize_t nvmet_ana_group_ana_state_show(struct config_item *item, enum nvme_ana_state state = grp->port->ana_state[grp->grpid]; int i; - for (i = 0; i < ARRAY_SIZE(nvmet_ana_state_names); i++) { - if (state != nvmet_ana_state_names[i].state) - continue; - return sprintf(page, "%s\n", nvmet_ana_state_names[i].name); + for (i = 0; i < ARRAY_SIZE(nvmet_ana_state); i++) { + if (state == nvmet_ana_state[i].type) + return sprintf(page, "%s\n", nvmet_ana_state[i].name); } return sprintf(page, "\n"); @@ -1184,10 +1180,11 @@ static ssize_t nvmet_ana_group_ana_state_store(struct config_item *item, const char *page, size_t count) { struct nvmet_ana_group *grp = to_ana_group(item); + enum nvme_ana_state *ana_state = grp->port->ana_state; int i; - for (i = 0; i < ARRAY_SIZE(nvmet_ana_state_names); i++) { - if (sysfs_streq(page, nvmet_ana_state_names[i].name)) + for (i = 0; i < ARRAY_SIZE(nvmet_ana_state); i++) { + if (sysfs_streq(page, nvmet_ana_state[i].name)) goto found; } @@ -1196,10 +1193,9 @@ static ssize_t nvmet_ana_group_ana_state_store(struct config_item *item, found: down_write(&nvmet_ana_sem); - grp->port->ana_state[grp->grpid] = nvmet_ana_state_names[i].state; + ana_state[grp->grpid] = (enum nvme_ana_state) nvmet_ana_state[i].type; nvmet_ana_chgcnt++; up_write(&nvmet_ana_sem); - nvmet_port_send_ana_event(grp->port); return count; } From 87628e2851008d5b0e53859f6062988c6d66d7f5 Mon Sep 17 00:00:00 2001 From: Chaitanya Kulkarni Date: Mon, 4 May 2020 01:56:46 -0700 Subject: [PATCH 0364/1043] nvmet: use type-name map for address treq Currently nvmet_addr_treq_[store|show]() uses switch and if else ladder for address transport requirements to string and reverse mapping. With addtion of the generic nvmet_type_name_map structure we can get rid of the switch and if else ladder with string duplication. Signed-off-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/configfs.c | 48 ++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 78eb822a20d5..9c808f4185a0 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -147,20 +147,24 @@ static ssize_t nvmet_addr_traddr_store(struct config_item *item, CONFIGFS_ATTR(nvmet_, addr_traddr); -static ssize_t nvmet_addr_treq_show(struct config_item *item, - char *page) +static const struct nvmet_type_name_map nvmet_addr_treq[] = { + { NVMF_TREQ_NOT_SPECIFIED, "not specified" }, + { NVMF_TREQ_REQUIRED, "required" }, + { NVMF_TREQ_NOT_REQUIRED, "not required" }, +}; + +static ssize_t nvmet_addr_treq_show(struct config_item *item, char *page) { - switch (to_nvmet_port(item)->disc_addr.treq & - NVME_TREQ_SECURE_CHANNEL_MASK) { - case NVMF_TREQ_NOT_SPECIFIED: - return sprintf(page, "not specified\n"); - case NVMF_TREQ_REQUIRED: - return sprintf(page, "required\n"); - case NVMF_TREQ_NOT_REQUIRED: - return sprintf(page, "not required\n"); - default: - return sprintf(page, "\n"); + u8 treq = to_nvmet_port(item)->disc_addr.treq & + NVME_TREQ_SECURE_CHANNEL_MASK; + int i; + + for (i = 0; i < ARRAY_SIZE(nvmet_addr_treq); i++) { + if (treq == nvmet_addr_treq[i].type) + return sprintf(page, "%s\n", nvmet_addr_treq[i].name); } + + return sprintf(page, "\n"); } static ssize_t nvmet_addr_treq_store(struct config_item *item, @@ -168,6 +172,7 @@ static ssize_t nvmet_addr_treq_store(struct config_item *item, { struct nvmet_port *port = to_nvmet_port(item); u8 treq = port->disc_addr.treq & ~NVME_TREQ_SECURE_CHANNEL_MASK; + int i; if (port->enabled) { pr_err("Cannot modify address while enabled\n"); @@ -175,18 +180,17 @@ static ssize_t nvmet_addr_treq_store(struct config_item *item, return -EACCES; } - if (sysfs_streq(page, "not specified")) { - treq |= NVMF_TREQ_NOT_SPECIFIED; - } else if (sysfs_streq(page, "required")) { - treq |= NVMF_TREQ_REQUIRED; - } else if (sysfs_streq(page, "not required")) { - treq |= NVMF_TREQ_NOT_REQUIRED; - } else { - pr_err("Invalid value '%s' for treq\n", page); - return -EINVAL; + for (i = 0; i < ARRAY_SIZE(nvmet_addr_treq); i++) { + if (sysfs_streq(page, nvmet_addr_treq[i].name)) + goto found; } - port->disc_addr.treq = treq; + pr_err("Invalid value '%s' for treq\n", page); + return -EINVAL; + +found: + treq |= nvmet_addr_treq[i].type; + port->disc_addr.treq = treq; return count; } From 3ecb5faa07c7fd33a3ce1a8340841aa368df7d43 Mon Sep 17 00:00:00 2001 From: Chaitanya Kulkarni Date: Mon, 4 May 2020 01:56:47 -0700 Subject: [PATCH 0365/1043] nvmet: centralize port enable access for configfs The configfs attributes which are supposed to set when port is disable such as addr[addrfam|portid|traddr|treq|trsvcid|inline_data_size|trtype] has repetitive check and generic error message printing. This patch creates centralize helper to check and print an error message that also accepts caller as a parameter. This makes error message easy to parse for the user, removes the duplicate code and makes it available for futures such scenarios. Signed-off-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/configfs.c | 44 +++++++++++++--------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 9c808f4185a0..20779587eefe 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -40,6 +40,14 @@ static const struct nvmet_type_name_map nvmet_addr_family[] = { { NVMF_ADDR_FAMILY_FC, "fc" }, }; +static bool nvmet_is_port_enabled(struct nvmet_port *p, const char *caller) +{ + if (p->enabled) + pr_err("Disable port '%u' before changing attribute in %s\n", + le16_to_cpu(p->disc_addr.portid), caller); + return p->enabled; +} + /* * nvmet_port Generic ConfigFS definitions. * Used in any place in the ConfigFS tree that refers to an address. @@ -63,11 +71,8 @@ static ssize_t nvmet_addr_adrfam_store(struct config_item *item, struct nvmet_port *port = to_nvmet_port(item); int i; - if (port->enabled) { - pr_err("Cannot modify address while enabled\n"); - pr_err("Disable the address before modifying\n"); + if (nvmet_is_port_enabled(port, __func__)) return -EACCES; - } for (i = 1; i < ARRAY_SIZE(nvmet_addr_family); i++) { if (sysfs_streq(page, nvmet_addr_family[i].name)) @@ -104,11 +109,9 @@ static ssize_t nvmet_addr_portid_store(struct config_item *item, return -EINVAL; } - if (port->enabled) { - pr_err("Cannot modify address while enabled\n"); - pr_err("Disable the address before modifying\n"); + if (nvmet_is_port_enabled(port, __func__)) return -EACCES; - } + port->disc_addr.portid = cpu_to_le16(portid); return count; } @@ -134,11 +137,8 @@ static ssize_t nvmet_addr_traddr_store(struct config_item *item, return -EINVAL; } - if (port->enabled) { - pr_err("Cannot modify address while enabled\n"); - pr_err("Disable the address before modifying\n"); + if (nvmet_is_port_enabled(port, __func__)) return -EACCES; - } if (sscanf(page, "%s\n", port->disc_addr.traddr) != 1) return -EINVAL; @@ -174,11 +174,8 @@ static ssize_t nvmet_addr_treq_store(struct config_item *item, u8 treq = port->disc_addr.treq & ~NVME_TREQ_SECURE_CHANNEL_MASK; int i; - if (port->enabled) { - pr_err("Cannot modify address while enabled\n"); - pr_err("Disable the address before modifying\n"); + if (nvmet_is_port_enabled(port, __func__)) return -EACCES; - } for (i = 0; i < ARRAY_SIZE(nvmet_addr_treq); i++) { if (sysfs_streq(page, nvmet_addr_treq[i].name)) @@ -214,11 +211,8 @@ static ssize_t nvmet_addr_trsvcid_store(struct config_item *item, pr_err("Invalid value '%s' for trsvcid\n", page); return -EINVAL; } - if (port->enabled) { - pr_err("Cannot modify address while enabled\n"); - pr_err("Disable the address before modifying\n"); + if (nvmet_is_port_enabled(port, __func__)) return -EACCES; - } if (sscanf(page, "%s\n", port->disc_addr.trsvcid) != 1) return -EINVAL; @@ -241,11 +235,8 @@ static ssize_t nvmet_param_inline_data_size_store(struct config_item *item, struct nvmet_port *port = to_nvmet_port(item); int ret; - if (port->enabled) { - pr_err("Cannot modify inline_data_size while port enabled\n"); - pr_err("Disable the port before modifying\n"); + if (nvmet_is_port_enabled(port, __func__)) return -EACCES; - } ret = kstrtoint(page, 0, &port->inline_data_size); if (ret) { pr_err("Invalid value '%s' for inline_data_size\n", page); @@ -283,11 +274,8 @@ static ssize_t nvmet_addr_trtype_store(struct config_item *item, struct nvmet_port *port = to_nvmet_port(item); int i; - if (port->enabled) { - pr_err("Cannot modify address while enabled\n"); - pr_err("Disable the address before modifying\n"); + if (nvmet_is_port_enabled(port, __func__)) return -EACCES; - } for (i = 0; i < ARRAY_SIZE(nvmet_transport); i++) { if (sysfs_streq(page, nvmet_transport[i].name)) From d02abd198633a4c40411b9a5994111452720470d Mon Sep 17 00:00:00 2001 From: Chaitanya Kulkarni Date: Mon, 4 May 2020 01:56:48 -0700 Subject: [PATCH 0366/1043] nvmet: align addrfam list to spec With reference to the NVMeOF Specification (page 44, Figure 38) discovery log page entry provides address family field. We do set the transport type field but the adrfam field is not set when using loop transport and also it doesn't have support in the nvme-cli. So when reading discovery log page with a loop transport it leads to confusing output. As per the spec for adrfam value 254 is reserved for Intra Host Transport i.e. loopback), we add a required macro in the protocol header file, set default port disc addr entry's adrfam to NVMF_ADDR_FAMILY_MAX, and update nvmet_addr_family configfs array for show/store attribute. Without this patch, setting adrfam to (ipv4/ipv6/ib/fc/loop/" ") we get following output for nvme discover command from nvme-cli which is confusing. trtype: loop adrfam: ipv4 trtype: loop adrfam: ipv6 trtype: loop adrfam: infiniband trtype: loop adrfam: fibre-channel trtype: loop # ${CFGFS_HOME}/nvmet/ports/1/addr_adrfam = loop adrfam: pci # <----- pci for loop trtype: loop # ${CFGFS_HOME}/nvmet/ports/1/addr_adrfam = " " adrfam: pci # <----- pci for unrecognized This patch fixes above output :- trtype: loop adrfam: ipv4 trtype: loop adrfam: ipv6 trtype: loop adrfam: infiniband trtype: loop adrfam: fibre-channel trtype: loop # ${CFGFS_HOME}/nvmet/ports/1/addr_adrfam = loop adrfam: loop # <----- loop for loop trtype: loop # ${CFGFS_HOME}/config/nvmet/ports/adrfam = " " adrfam: unrecognized # <----- unrecognized when invalid value Signed-off-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/target/configfs.c | 14 ++++++++------ include/linux/nvme.h | 2 ++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 20779587eefe..ae8fb4489a10 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -33,11 +33,12 @@ static struct nvmet_type_name_map nvmet_transport[] = { }; static const struct nvmet_type_name_map nvmet_addr_family[] = { - { NVMF_ADDR_FAMILY_PCI, "pcie" }, - { NVMF_ADDR_FAMILY_IP4, "ipv4" }, - { NVMF_ADDR_FAMILY_IP6, "ipv6" }, - { NVMF_ADDR_FAMILY_IB, "ib" }, - { NVMF_ADDR_FAMILY_FC, "fc" }, + { NVMF_ADDR_FAMILY_PCI, "pcie" }, + { NVMF_ADDR_FAMILY_IP4, "ipv4" }, + { NVMF_ADDR_FAMILY_IP6, "ipv6" }, + { NVMF_ADDR_FAMILY_IB, "ib" }, + { NVMF_ADDR_FAMILY_FC, "fc" }, + { NVMF_ADDR_FAMILY_LOOP, "loop" }, }; static bool nvmet_is_port_enabled(struct nvmet_port *p, const char *caller) @@ -83,7 +84,7 @@ static ssize_t nvmet_addr_adrfam_store(struct config_item *item, return -EINVAL; found: - port->disc_addr.adrfam = i; + port->disc_addr.adrfam = nvmet_addr_family[i].type; return count; } @@ -1338,6 +1339,7 @@ static struct config_group *nvmet_ports_make(struct config_group *group, port->inline_data_size = -1; /* < 0 == let the transport choose */ port->disc_addr.portid = cpu_to_le16(portid); + port->disc_addr.adrfam = NVMF_ADDR_FAMILY_MAX; port->disc_addr.treq = NVMF_TREQ_DISABLE_SQFLOW; config_group_init_type_name(&port->group, name, &nvmet_port_type); diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 3d5189f46cb1..2d978d0cbde6 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -38,6 +38,8 @@ enum { NVMF_ADDR_FAMILY_IP6 = 2, /* IP6 */ NVMF_ADDR_FAMILY_IB = 3, /* InfiniBand */ NVMF_ADDR_FAMILY_FC = 4, /* Fibre Channel */ + NVMF_ADDR_FAMILY_LOOP = 254, /* Reserved for host usage */ + NVMF_ADDR_FAMILY_MAX, }; /* Transport Type codes for Discovery Log Page entry TRTYPE field */ From 92decf118f1da4c866515f80387f9cf4d48611d6 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Fri, 3 Apr 2020 10:53:46 -0700 Subject: [PATCH 0367/1043] nvme: define constants for identification values Improve code readability by defining the specification's constants that the driver is using when decoding identification payloads. Signed-off-by: Keith Busch Reviewed-by: Bart van Assche Reviewed-by: Chaitanya Kulkarni Acked-by: Sagi Grimberg Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 10 +++++----- drivers/nvme/host/multipath.c | 5 +++-- include/linux/nvme.h | 6 ++++++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 4a124a28118f..805d289e6cd9 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1859,13 +1859,13 @@ static void nvme_update_disk_info(struct gendisk *disk, * and whether it should be used instead of AWUPF. If NAWUPF == * 0 then AWUPF must be used instead. */ - if (id->nsfeat & (1 << 1) && id->nawupf) + if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf) atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs; else atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs; } - if (id->nsfeat & (1 << 4)) { + if (id->nsfeat & NVME_NS_FEAT_IO_OPT) { /* NPWG = Namespace Preferred Write Granularity */ phys_bs = bs * (1 + le16_to_cpu(id->npwg)); /* NOWS = Namespace Optimal Write Size */ @@ -1894,7 +1894,7 @@ static void nvme_update_disk_info(struct gendisk *disk, nvme_config_discard(disk, ns); nvme_config_write_zeroes(disk, ns); - if (id->nsattr & (1 << 0)) + if (id->nsattr & NVME_NS_ATTR_RO) set_disk_ro(disk, true); else set_disk_ro(disk, false); @@ -2685,7 +2685,7 @@ static bool nvme_validate_cntlid(struct nvme_subsystem *subsys, return false; } - if ((id->cmic & (1 << 1)) || + if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || (ctrl->opts && ctrl->opts->discovery_nqn)) continue; @@ -3497,7 +3497,7 @@ static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid, struct nvme_id_ns *id) { struct nvme_ctrl *ctrl = ns->ctrl; - bool is_shared = id->nmic & (1 << 0); + bool is_shared = id->nmic & NVME_NS_NMIC_SHARED; struct nvme_ns_head *head = NULL; struct nvme_ns_ids ids; int ret = 0; diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 5f5537d0a024..da78e499947a 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -372,7 +372,7 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) * We also do this for private namespaces as the namespace sharing data could * change after a rescan. */ - if (!(ctrl->subsys->cmic & (1 << 1)) || !multipath) + if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || !multipath) return 0; q = blk_alloc_queue(nvme_ns_head_make_request, ctrl->numa_node); @@ -694,7 +694,8 @@ int nvme_mpath_init(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) int error; /* check if multipath is enabled and we have the capability */ - if (!multipath || !ctrl->subsys || !(ctrl->subsys->cmic & (1 << 3))) + if (!multipath || !ctrl->subsys || + !(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA)) return 0; ctrl->anacap = id->anacap; diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 2d978d0cbde6..b235a48eac8c 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -301,6 +301,8 @@ struct nvme_id_ctrl { }; enum { + NVME_CTRL_CMIC_MULTI_CTRL = 1 << 1, + NVME_CTRL_CMIC_ANA = 1 << 3, NVME_CTRL_ONCS_COMPARE = 1 << 0, NVME_CTRL_ONCS_WRITE_UNCORRECTABLE = 1 << 1, NVME_CTRL_ONCS_DSM = 1 << 2, @@ -396,8 +398,12 @@ enum { enum { NVME_NS_FEAT_THIN = 1 << 0, + NVME_NS_FEAT_ATOMICS = 1 << 1, + NVME_NS_FEAT_IO_OPT = 1 << 4, + NVME_NS_ATTR_RO = 1 << 0, NVME_NS_FLBAS_LBA_MASK = 0xf, NVME_NS_FLBAS_META_EXT = 0x10, + NVME_NS_NMIC_SHARED = 1 << 0, NVME_LBAF_RP_BEST = 0, NVME_LBAF_RP_BETTER = 1, NVME_LBAF_RP_GOOD = 2, From 0d341e0ddb1a3444716420ad9151949c02adbe4e Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Sun, 10 May 2020 22:43:33 +0200 Subject: [PATCH 0368/1043] parisc: suppress error messages for 'make clean' Avoid error messages when running 'make ARCH=parisc clean'. Noticed-by: Masahiro Yamada Signed-off-by: Helge Deller --- arch/parisc/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/parisc/Makefile b/arch/parisc/Makefile index 628cd8bb7ad8..fadbbd010337 100644 --- a/arch/parisc/Makefile +++ b/arch/parisc/Makefile @@ -21,8 +21,6 @@ KBUILD_IMAGE := vmlinuz NM = sh $(srctree)/arch/parisc/nm CHECKFLAGS += -D__hppa__=1 -LIBGCC = $(shell $(CC) $(KBUILD_CFLAGS) -print-libgcc-file-name) -export LIBGCC ifdef CONFIG_64BIT UTS_MACHINE := parisc64 @@ -110,6 +108,8 @@ cflags-$(CONFIG_PA8X00) += -march=2.0 -mschedule=8000 head-y := arch/parisc/kernel/head.o KBUILD_CFLAGS += $(cflags-y) +LIBGCC := $(shell $(CC) -print-libgcc-file-name) +export LIBGCC kernel-y := mm/ kernel/ math-emu/ From 3fd84a4fc063f36855ddf2408c66e6e5d219b223 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 8 May 2020 11:35:43 +0200 Subject: [PATCH 0369/1043] parisc: use -fno-strict-aliasing for decompressor An experimental patch series of mine reworks how warnings are processed in Kbuild. A side effect is a new warning about a harmless aliasing rule violation in an inline function: In file included from include/linux/rhashtable-types.h:15:0, from include/linux/ipc.h:7, from include/uapi/linux/sem.h:5, from include/linux/sem.h:5, from include/linux/sched.h:15, from include/linux/uaccess.h:6, from arch/parisc/boot/compressed/misc.c:7: include/linux/workqueue.h: In function 'work_static': include/linux/workqueue.h:212:2: warning: dereferencing type-punned pointer will break strict-aliasing rules [-Wstrict-aliasing] return *work_data_bits(work) & WORK_STRUCT_STATIC; Make the decompressor use -fno-strict-aliasing like the rest of the kernel for consistency, and to ensure this warning never makes it into a release. Reported-by: kbuild test robot Signed-off-by: Arnd Bergmann Signed-off-by: Helge Deller --- arch/parisc/boot/compressed/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/parisc/boot/compressed/Makefile b/arch/parisc/boot/compressed/Makefile index 1e5879c6a752..dff453687530 100644 --- a/arch/parisc/boot/compressed/Makefile +++ b/arch/parisc/boot/compressed/Makefile @@ -16,6 +16,7 @@ targets += real2.S firmware.c KBUILD_CFLAGS := -D__KERNEL__ -O2 -DBOOTLOADER KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING +KBUILD_CFLAGS += -fno-strict-aliasing KBUILD_CFLAGS += $(cflags-y) -fno-delete-null-pointer-checks -fno-builtin-printf KBUILD_CFLAGS += -fno-PIE -mno-space-regs -mdisable-fpregs -Os ifndef CONFIG_64BIT From b6522fa409cfafbc3968679b09e4028f0609f2b9 Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Sat, 11 Apr 2020 21:06:19 +0800 Subject: [PATCH 0370/1043] parisc: add sysctl file interface panic_on_stackoverflow The variable sysctl_panic_on_stackoverflow is used in arch/parisc/kernel/irq.c and arch/x86/kernel/irq_32.c, but the sysctl file interface panic_on_stackoverflow only exists on x86. Add sysctl file interface panic_on_stackoverflow for parisc Signed-off-by: Xiaoming Ni Reviewed-by: Luis Chamberlain Signed-off-by: Helge Deller --- kernel/sysctl.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 8a176d8727a3..b9ff323e1d26 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -994,6 +994,17 @@ static struct ctl_table kern_table[] = { .proc_handler = proc_dointvec, }, #endif + +#if (defined(CONFIG_X86_32) || defined(CONFIG_PARISC)) && \ + defined(CONFIG_DEBUG_STACKOVERFLOW) + { + .procname = "panic_on_stackoverflow", + .data = &sysctl_panic_on_stackoverflow, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif #if defined(CONFIG_X86) { .procname = "panic_on_unrecovered_nmi", @@ -1009,15 +1020,6 @@ static struct ctl_table kern_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, -#ifdef CONFIG_DEBUG_STACKOVERFLOW - { - .procname = "panic_on_stackoverflow", - .data = &sysctl_panic_on_stackoverflow, - .maxlen = sizeof(int), - .mode = 0644, - .proc_handler = proc_dointvec, - }, -#endif { .procname = "bootloader_type", .data = &bootloader_type, From 8469508951d4a324b2df3b5bad75e99922c3b798 Mon Sep 17 00:00:00 2001 From: Xiaoming Ni Date: Mon, 11 May 2020 19:25:43 +0800 Subject: [PATCH 0371/1043] io_uring: remove duplicate semicolon at the end of line Remove duplicate semicolon at the end of line in io_file_from_index() Signed-off-by: Xiaoming Ni Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 94870d63b16a..d2e37215d05a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5364,7 +5364,7 @@ static inline struct file *io_file_from_index(struct io_ring_ctx *ctx, struct fixed_file_table *table; table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT]; - return table->files[index & IORING_FILE_TABLE_MASK];; + return table->files[index & IORING_FILE_TABLE_MASK]; } static int io_file_get(struct io_submit_state *state, struct io_kiocb *req, From 1bec48982c7abb5d1f065ed72181c134a0b67d9b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sun, 10 May 2020 09:54:42 +0200 Subject: [PATCH 0372/1043] MIPS: unexport __flush_icache_user_range __flush_icache_user_range is not used in modular code, so unexport it. Signed-off-by: Christoph Hellwig Signed-off-by: Thomas Bogendoerfer --- arch/mips/mm/cache.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c index 33b409391ddb..ad6df1cea866 100644 --- a/arch/mips/mm/cache.c +++ b/arch/mips/mm/cache.c @@ -36,7 +36,6 @@ EXPORT_SYMBOL_GPL(flush_icache_range); void (*local_flush_icache_range)(unsigned long start, unsigned long end); EXPORT_SYMBOL_GPL(local_flush_icache_range); void (*__flush_icache_user_range)(unsigned long start, unsigned long end); -EXPORT_SYMBOL_GPL(__flush_icache_user_range); void (*__local_flush_icache_user_range)(unsigned long start, unsigned long end); EXPORT_SYMBOL_GPL(__local_flush_icache_user_range); From 27acbf41be3928999b3a291fceee2a4b50218f00 Mon Sep 17 00:00:00 2001 From: Zou Wei Date: Tue, 14 Apr 2020 16:19:48 +0800 Subject: [PATCH 0373/1043] tomoyo: use true for bool variable Fixes coccicheck warning: security/tomoyo/common.c:1028:2-13: WARNING: Assignment of 0/1 to bool variable Reported-by: Hulk Robot Signed-off-by: Zou Wei Signed-off-by: Tetsuo Handa --- security/tomoyo/common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security/tomoyo/common.c b/security/tomoyo/common.c index 1b467381986f..c577525e4334 100644 --- a/security/tomoyo/common.c +++ b/security/tomoyo/common.c @@ -1025,7 +1025,7 @@ static bool tomoyo_select_domain(struct tomoyo_io_buffer *head, if (domain) head->r.domain = &domain->list; else - head->r.eof = 1; + head->r.eof = true; tomoyo_io_printf(head, "# select %s\n", data); if (domain && domain->is_deleted) tomoyo_io_printf(head, "# This is a deleted domain.\n"); From b744b43f79cc758127042e71f9ad7b1afda30f84 Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Tue, 28 Apr 2020 15:14:15 -0700 Subject: [PATCH 0374/1043] kbuild: add CONFIG_LD_IS_LLD Similarly to the CC_IS_CLANG config, add LD_IS_LLD to avoid GNU ld specific logic such as ld-version or ld-ifversion and gain the ability to select potential features that depend on the linker at configuration time such as LTO. Signed-off-by: Sami Tolvanen Acked-by: Masahiro Yamada [nc: Reword commit message] Signed-off-by: Nathan Chancellor Tested-by: Sedat Dilek Reviewed-by: Sedat Dilek Signed-off-by: Thomas Bogendoerfer --- init/Kconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/init/Kconfig b/init/Kconfig index 9e22ee8fbd75..c15ee42b8272 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -23,6 +23,9 @@ config LD_VERSION config CC_IS_CLANG def_bool $(success,$(CC) --version | head -n 1 | grep -q clang) +config LD_IS_LLD + def_bool $(success,$(LD) -v | head -n 1 | grep -q LLD) + config CLANG_VERSION int default $(shell,$(srctree)/scripts/clang-version.sh $(CC)) From e91946d6d93ef6167bd3b1456f163d1585095ea1 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 28 Apr 2020 15:14:16 -0700 Subject: [PATCH 0375/1043] MIPS: VDSO: Move disabling the VDSO logic to Kconfig After commit 9553d16fa671 ("init/kconfig: Add LD_VERSION Kconfig"), we have access to GNU ld's version at configuration time. As a result, we can make it clearer under what configuration circumstances the MIPS VDSO needs to be disabled. This is a prerequisite for getting rid of the MIPS VDSO binutils warning and linking the VDSO when LD is ld.lld. Wrapping the call to ld-ifversion with CONFIG_LD_IS_LLD does not work because the config values are wiped away during 'make clean'. Signed-off-by: Nathan Chancellor Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kconfig | 2 ++ arch/mips/vdso/Kconfig | 18 ++++++++++++++++++ arch/mips/vdso/Makefile | 30 ++---------------------------- arch/mips/vdso/vdso.lds.S | 2 +- 4 files changed, 23 insertions(+), 29 deletions(-) create mode 100644 arch/mips/vdso/Kconfig diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index ce1aacc2ee9c..bfa9cd962b06 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -3214,3 +3214,5 @@ endmenu source "drivers/firmware/Kconfig" source "arch/mips/kvm/Kconfig" + +source "arch/mips/vdso/Kconfig" diff --git a/arch/mips/vdso/Kconfig b/arch/mips/vdso/Kconfig new file mode 100644 index 000000000000..36a52158d849 --- /dev/null +++ b/arch/mips/vdso/Kconfig @@ -0,0 +1,18 @@ +# For the pre-R6 code in arch/mips/vdso/vdso.h for locating +# the base address of VDSO, the linker will emit a R_MIPS_PC32 +# relocation in binutils > 2.25 but it will fail with older versions +# because that relocation is not supported for that symbol. As a result +# of which we are forced to disable the VDSO symbols when building +# with < 2.25 binutils on pre-R6 kernels. For more references on why we +# can't use other methods to get the base address of VDSO please refer to +# the comments on that file. +# +# GCC (at least up to version 9.2) appears to emit function calls that make use +# of the GOT when targeting microMIPS, which we can't use in the VDSO due to +# the lack of relocations. As such, we disable the VDSO for microMIPS builds. + +config MIPS_LD_CAN_LINK_VDSO + def_bool LD_VERSION >= 225000000 + +config MIPS_DISABLE_VDSO + def_bool CPU_MICROMIPS || (!CPU_MIPSR6 && !MIPS_LD_CAN_LINK_VDSO) diff --git a/arch/mips/vdso/Makefile b/arch/mips/vdso/Makefile index d7fe8408603e..92b53d1df42c 100644 --- a/arch/mips/vdso/Makefile +++ b/arch/mips/vdso/Makefile @@ -52,37 +52,11 @@ endif CFLAGS_REMOVE_vgettimeofday.o = -pg -DISABLE_VDSO := n - -# -# For the pre-R6 code in arch/mips/vdso/vdso.h for locating -# the base address of VDSO, the linker will emit a R_MIPS_PC32 -# relocation in binutils > 2.25 but it will fail with older versions -# because that relocation is not supported for that symbol. As a result -# of which we are forced to disable the VDSO symbols when building -# with < 2.25 binutils on pre-R6 kernels. For more references on why we -# can't use other methods to get the base address of VDSO please refer to -# the comments on that file. -# -ifndef CONFIG_CPU_MIPSR6 - ifeq ($(call ld-ifversion, -lt, 225000000, y),y) +ifdef CONFIG_MIPS_DISABLE_VDSO + ifndef CONFIG_MIPS_LD_CAN_LINK_VDSO $(warning MIPS VDSO requires binutils >= 2.25) - DISABLE_VDSO := y endif -endif - -# -# GCC (at least up to version 9.2) appears to emit function calls that make use -# of the GOT when targeting microMIPS, which we can't use in the VDSO due to -# the lack of relocations. As such, we disable the VDSO for microMIPS builds. -# -ifdef CONFIG_CPU_MICROMIPS - DISABLE_VDSO := y -endif - -ifeq ($(DISABLE_VDSO),y) obj-vdso-y := $(filter-out vgettimeofday.o, $(obj-vdso-y)) - ccflags-vdso += -DDISABLE_MIPS_VDSO endif # VDSO linker flags. diff --git a/arch/mips/vdso/vdso.lds.S b/arch/mips/vdso/vdso.lds.S index da4627430aba..d90b65724d78 100644 --- a/arch/mips/vdso/vdso.lds.S +++ b/arch/mips/vdso/vdso.lds.S @@ -91,7 +91,7 @@ PHDRS VERSION { LINUX_2.6 { -#ifndef DISABLE_MIPS_VDSO +#ifndef CONFIG_MIPS_DISABLE_VDSO global: __vdso_clock_gettime; __vdso_gettimeofday; From fd9d0ca2cc4f3c26877eb336a7d26d705ff42b8a Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 28 Apr 2020 15:14:17 -0700 Subject: [PATCH 0376/1043] MIPS: Unconditionally specify '-EB' or '-EL' This was all done to work around a GCC bug that has been fixed after 4.2. The kernel requires GCC 4.6 or newer so remove all of these hacks and just use the traditional flags. $ mips64-linux-gcc --version | head -n1 mips64-linux-gcc (GCC) 4.6.3 $ mips64-linux-gcc -EB -dM -E -C -x c /dev/null | grep MIPSE #define MIPSEB 1 #define __MIPSEB__ 1 #define _MIPSEB 1 #define __MIPSEB 1 $ mips64-linux-gcc -EL -dM -E -C -x c /dev/null | grep MIPSE #define __MIPSEL__ 1 #define MIPSEL 1 #define _MIPSEL 1 #define __MIPSEL 1 This is necessary when converting the MIPS VDSO to use $(LD) instead of $(CC) to link because the OUTPUT_FORMAT is defaulted to little endian and only flips to big endian when '-EB' is set on the command line. There is no issue currently because the compiler explicitly passes '-EB' or '-EL' to the linker regardless of whether or not it was provided by the user. Passing '-v' to VDSO_LDFLAGS shows: /libexec/gcc/mips64-linux/9.3.0/collect2 ... -EB ... even though '-EB' is nowhere to be found in KBUILD_CFLAGS. The VDSO Makefile already supports getting '-EB' or '-EL' from KBUILD_CFLAGS through a filter directive but '-EB' or '-EL' is not always present. If we do not do this, we will see the following error when compiling for big endian: $ make -j$(nproc) ARCH=mips CROSS_COMPILE=mips64-linux- \ 64r2el_defconfig arch/mips/vdso/ ... mips64-linux-ld: arch/mips/vdso/elf.o: compiled for a big endian system and target is little endian mips64-linux-ld: arch/mips/vdso/elf.o: endianness incompatible with that of the selected emulation mips64-linux-ld: failed to merge target specific data of file arch/mips/vdso/elf.o ... Remove this legacy hack and just use '-EB' and '-EL' unconditionally. Reported-by: Thomas Bogendoerfer Signed-off-by: Nathan Chancellor Signed-off-by: Thomas Bogendoerfer --- arch/mips/Makefile | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/arch/mips/Makefile b/arch/mips/Makefile index ef15caa4d320..b50377ec3ab5 100644 --- a/arch/mips/Makefile +++ b/arch/mips/Makefile @@ -116,33 +116,8 @@ endif cflags-y += -ffreestanding -# -# We explicitly add the endianness specifier if needed, this allows -# to compile kernels with a toolchain for the other endianness. We -# carefully avoid to add it redundantly because gcc 3.3/3.4 complains -# when fed the toolchain default! -# -# Certain gcc versions up to gcc 4.1.1 (probably 4.2-subversion as of -# 2006-10-10 don't properly change the predefined symbols if -EB / -EL -# are used, so we kludge that here. A bug has been filed at -# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=29413. -# -# clang doesn't suffer from these issues and our checks against -dumpmachine -# don't work so well when cross compiling, since without providing --target -# clang's output will be based upon the build machine. So for clang we simply -# unconditionally specify -EB or -EL as appropriate. -# -ifdef CONFIG_CC_IS_CLANG cflags-$(CONFIG_CPU_BIG_ENDIAN) += -EB cflags-$(CONFIG_CPU_LITTLE_ENDIAN) += -EL -else -undef-all += -UMIPSEB -U_MIPSEB -U__MIPSEB -U__MIPSEB__ -undef-all += -UMIPSEL -U_MIPSEL -U__MIPSEL -U__MIPSEL__ -predef-be += -DMIPSEB -D_MIPSEB -D__MIPSEB -D__MIPSEB__ -predef-le += -DMIPSEL -D_MIPSEL -D__MIPSEL -D__MIPSEL__ -cflags-$(CONFIG_CPU_BIG_ENDIAN) += $(shell $(CC) -dumpmachine |grep -q 'mips.*el-.*' && echo -EB $(undef-all) $(predef-be)) -cflags-$(CONFIG_CPU_LITTLE_ENDIAN) += $(shell $(CC) -dumpmachine |grep -q 'mips.*el-.*' || echo -EL $(undef-all) $(predef-le)) -endif cflags-$(CONFIG_SB1XXX_CORELIS) += $(call cc-option,-mno-sched-prolog) \ -fno-omit-frame-pointer From 2ff906994b6c2b949c5bf65330a8abb5dde9c8e5 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 28 Apr 2020 15:14:18 -0700 Subject: [PATCH 0377/1043] MIPS: VDSO: Use $(LD) instead of $(CC) to link VDSO Currently, the VDSO is being linked through $(CC). This does not match how the rest of the kernel links objects, which is through the $(LD) variable. When clang is built in a default configuration, it first attempts to use the target triple's default linker then the system's default linker, unless told otherwise through -fuse-ld=... We do not use -fuse-ld= because it can be brittle and we have support for invoking $(LD) directly. See commit fe00e50b2db8c ("ARM: 8858/1: vdso: use $(LD) instead of $(CC) to link VDSO") and commit 691efbedc60d2 ("arm64: vdso: use $(LD) instead of $(CC) to link VDSO") for examples of doing this in the VDSO. Do the same thing here. Replace the custom linking logic with $(cmd_ld) and ldflags-y so that $(LD) is respected. We need to explicitly add two flags to the linker that were implicitly passed by the compiler: -G 0 (which comes from ccflags-vdso) and --eh-frame-hdr. Before this patch (generated by adding '-v' to VDSO_LDFLAGS): /libexec/gcc/mips64-linux/9.3.0/collect2 \ -plugin /libexec/gcc/mips64-linux/9.3.0/liblto_plugin.so \ -plugin-opt=/libexec/gcc/mips64-linux/9.3.0/lto-wrapper \ -plugin-opt=-fresolution=/tmp/ccGEi5Ka.res \ --eh-frame-hdr \ -G 0 \ -EB \ -mips64r2 \ -shared \ -melf64btsmip \ -o arch/mips/vdso/vdso.so.dbg.raw \ -L/lib/gcc/mips64-linux/9.3.0/64 \ -L/lib/gcc/mips64-linux/9.3.0 \ -L/lib/gcc/mips64-linux/9.3.0/../../../../mips64-linux/lib \ -Bsymbolic \ --no-undefined \ -soname=linux-vdso.so.1 \ -EB \ --hash-style=sysv \ --build-id \ -T arch/mips/vdso/vdso.lds \ arch/mips/vdso/elf.o \ arch/mips/vdso/vgettimeofday.o \ arch/mips/vdso/sigreturn.o After this patch: /bin/mips64-linux-ld \ -m elf64btsmip \ -Bsymbolic \ --no-undefined \ -soname=linux-vdso.so.1 \ -EB \ -nostdlib \ -shared \ -G 0 \ --eh-frame-hdr \ --hash-style=sysv \ --build-id \ -T arch/mips/vdso/vdso.lds \ arch/mips/vdso/elf.o \ arch/mips/vdso/vgettimeofday.o arch/mips/vdso/sigreturn.o \ -o arch/mips/vdso/vdso.so.dbg.raw Note that we leave behind -mips64r2. Turns out that ld ignores it (see get_emulation in ld/ldmain.c). This is true of current trunk and 2.23, which is the minimum supported version for the kernel: https://sourceware.org/git/?p=binutils-gdb.git;a=blob;f=ld/ldmain.c;hb=aa4209e7b679afd74a3860ce25659e71cc4847d5#l593 https://sourceware.org/git/?p=binutils-gdb.git;a=blob;f=ld/ldmain.c;hb=a55e30b51bc6227d8d41f707654d0a5620978dcf#l641 Before this patch, LD=ld.lld did nothing: $ llvm-readelf -p.comment arch/mips/vdso/vdso.so.dbg | sed 's/(.*//' String dump of section '.comment': [ 0] ClangBuiltLinux clang version 11.0.0 After this patch, it does: $ llvm-readelf -p.comment arch/mips/vdso/vdso.so.dbg | sed 's/(.*//' String dump of section '.comment': [ 0] Linker: LLD 11.0.0 [ 62] ClangBuiltLinux clang version 11.0.0 Link: https://github.com/ClangBuiltLinux/linux/issues/785 Signed-off-by: Nathan Chancellor Signed-off-by: Thomas Bogendoerfer --- arch/mips/vdso/Makefile | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/arch/mips/vdso/Makefile b/arch/mips/vdso/Makefile index 92b53d1df42c..2e64c7600eea 100644 --- a/arch/mips/vdso/Makefile +++ b/arch/mips/vdso/Makefile @@ -60,10 +60,9 @@ ifdef CONFIG_MIPS_DISABLE_VDSO endif # VDSO linker flags. -VDSO_LDFLAGS := \ - -Wl,-Bsymbolic -Wl,--no-undefined -Wl,-soname=linux-vdso.so.1 \ - $(addprefix -Wl$(comma),$(filter -E%,$(KBUILD_CFLAGS))) \ - -nostdlib -shared -Wl,--hash-style=sysv -Wl,--build-id +ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \ + $(filter -E%,$(KBUILD_CFLAGS)) -nostdlib -shared \ + -G 0 --eh-frame-hdr --hash-style=sysv --build-id -T CFLAGS_REMOVE_vdso.o = -pg @@ -82,11 +81,7 @@ quiet_cmd_vdso_mips_check = VDSOCHK $@ # quiet_cmd_vdsold_and_vdso_check = LD $@ - cmd_vdsold_and_vdso_check = $(cmd_vdsold); $(cmd_vdso_check); $(cmd_vdso_mips_check) - -quiet_cmd_vdsold = VDSO $@ - cmd_vdsold = $(CC) $(c_flags) $(VDSO_LDFLAGS) \ - -Wl,-T $(filter %.lds,$^) $(filter %.o,$^) -o $@ + cmd_vdsold_and_vdso_check = $(cmd_ld); $(cmd_vdso_check); $(cmd_vdso_mips_check) quiet_cmd_vdsoas_o_S = AS $@ cmd_vdsoas_o_S = $(CC) $(a_flags) -c -o $@ $< From 22235ef34a9745c4964c5aa11d57d5ce0b0117de Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 28 Apr 2020 15:14:19 -0700 Subject: [PATCH 0378/1043] MIPS: VDSO: Allow ld.lld to link the VDSO Currently, when linking with ld.lld, this warning pops up: arch/mips/vdso/Makefile:70: MIPS VDSO requires binutils >= 2.25 CONFIG_LD_VERSION is set with scripts/ld-version.sh, which is specific to GNU ld. It returns 0 for ld.lld so CONFIG_MIPS_LD_CAN_LINK_VDSO does not set. ld.lld has a completely different versioning scheme (as it follows LLVM's versioning) and it does not have the issue mentioned in the comment block so it should be allowed to link the VDSO. With this patch, the VDSO successfully links and shows P_MIPS_PC32 in vgettimeofday.o. $ llvm-objdump -Dr arch/mips/vdso/vgettimeofday.o | grep R_MIPS_PC32 00000024: R_MIPS_PC32 _start 000000b0: R_MIPS_PC32 _start 000002bc: R_MIPS_PC32 _start 0000036c: R_MIPS_PC32 _start 00000468: R_MIPS_PC32 _start Reported-by: Dmitry Golovin Signed-off-by: Nathan Chancellor Link: https://github.com/ClangBuiltLinux/linux/issues/785 Link: https://github.com/llvm/llvm-project/commit/e364e2e9ce50c12eb2bf093560e1a1a8544d455a Signed-off-by: Thomas Bogendoerfer --- arch/mips/vdso/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/vdso/Kconfig b/arch/mips/vdso/Kconfig index 36a52158d849..7aec721398d5 100644 --- a/arch/mips/vdso/Kconfig +++ b/arch/mips/vdso/Kconfig @@ -12,7 +12,7 @@ # the lack of relocations. As such, we disable the VDSO for microMIPS builds. config MIPS_LD_CAN_LINK_VDSO - def_bool LD_VERSION >= 225000000 + def_bool LD_VERSION >= 225000000 || LD_IS_LLD config MIPS_DISABLE_VDSO def_bool CPU_MICROMIPS || (!CPU_MIPSR6 && !MIPS_LD_CAN_LINK_VDSO) From 90b5363acd4739769c3f38c1aff16171bd133e8c Mon Sep 17 00:00:00 2001 From: "Peter Zijlstra (Intel)" Date: Fri, 27 Mar 2020 11:44:56 +0100 Subject: [PATCH 0379/1043] sched: Clean up scheduler_ipi() The scheduler IPI has grown weird and wonderful over the years, time for spring cleaning. Move all the non-trivial stuff out of it and into a regular smp function call IPI. This then reduces the schedule_ipi() to most of it's former NOP glory and ensures to keep the interrupt vector lean and mean. Aside of that avoiding the full irq_enter() in the x86 IPI implementation is incorrect as scheduler_ipi() can be instrumented. To work around that scheduler_ipi() had an irq_enter/exit() hack when heavy work was pending. This is gone now. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Link: https://lkml.kernel.org/r/20200505134058.361859938@linutronix.de --- kernel/sched/core.c | 64 +++++++++++++++++++++----------------------- kernel/sched/fair.c | 5 ++-- kernel/sched/sched.h | 6 +++-- 3 files changed, 36 insertions(+), 39 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b58efb1156eb..cd2070d6f1e4 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -219,6 +219,13 @@ void update_rq_clock(struct rq *rq) update_rq_clock_task(rq, delta); } +static inline void +rq_csd_init(struct rq *rq, call_single_data_t *csd, smp_call_func_t func) +{ + csd->flags = 0; + csd->func = func; + csd->info = rq; +} #ifdef CONFIG_SCHED_HRTICK /* @@ -314,16 +321,14 @@ void hrtick_start(struct rq *rq, u64 delay) hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), HRTIMER_MODE_REL_PINNED_HARD); } + #endif /* CONFIG_SMP */ static void hrtick_rq_init(struct rq *rq) { #ifdef CONFIG_SMP - rq->hrtick_csd.flags = 0; - rq->hrtick_csd.func = __hrtick_start; - rq->hrtick_csd.info = rq; + rq_csd_init(rq, &rq->hrtick_csd, __hrtick_start); #endif - hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); rq->hrtick_timer.function = hrtick; } @@ -650,6 +655,16 @@ static inline bool got_nohz_idle_kick(void) return false; } +static void nohz_csd_func(void *info) +{ + struct rq *rq = info; + + if (got_nohz_idle_kick()) { + rq->idle_balance = 1; + raise_softirq_irqoff(SCHED_SOFTIRQ); + } +} + #else /* CONFIG_NO_HZ_COMMON */ static inline bool got_nohz_idle_kick(void) @@ -2292,6 +2307,11 @@ void sched_ttwu_pending(void) rq_unlock_irqrestore(rq, &rf); } +static void wake_csd_func(void *info) +{ + sched_ttwu_pending(); +} + void scheduler_ipi(void) { /* @@ -2300,34 +2320,6 @@ void scheduler_ipi(void) * this IPI. */ preempt_fold_need_resched(); - - if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()) - return; - - /* - * Not all reschedule IPI handlers call irq_enter/irq_exit, since - * traditionally all their work was done from the interrupt return - * path. Now that we actually do some work, we need to make sure - * we do call them. - * - * Some archs already do call them, luckily irq_enter/exit nest - * properly. - * - * Arguably we should visit all archs and update all handlers, - * however a fair share of IPIs are still resched only so this would - * somewhat pessimize the simple resched case. - */ - irq_enter(); - sched_ttwu_pending(); - - /* - * Check if someone kicked us for doing the nohz idle load balance. - */ - if (unlikely(got_nohz_idle_kick())) { - this_rq()->idle_balance = 1; - raise_softirq_irqoff(SCHED_SOFTIRQ); - } - irq_exit(); } static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags) @@ -2336,9 +2328,9 @@ static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags) p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); - if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) { + if (llist_add(&p->wake_entry, &rq->wake_list)) { if (!set_nr_if_polling(rq->idle)) - smp_send_reschedule(cpu); + smp_call_function_single_async(cpu, &rq->wake_csd); else trace_sched_wake_idle_without_ipi(cpu); } @@ -6693,12 +6685,16 @@ void __init sched_init(void) rq->avg_idle = 2*sysctl_sched_migration_cost; rq->max_idle_balance_cost = sysctl_sched_migration_cost; + rq_csd_init(rq, &rq->wake_csd, wake_csd_func); + INIT_LIST_HEAD(&rq->cfs_tasks); rq_attach_root(rq, &def_root_domain); #ifdef CONFIG_NO_HZ_COMMON rq->last_blocked_load_update_tick = jiffies; atomic_set(&rq->nohz_flags, 0); + + rq_csd_init(rq, &rq->nohz_csd, nohz_csd_func); #endif #endif /* CONFIG_SMP */ hrtick_rq_init(rq); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 46b7bd41573f..6b7f1474e2d6 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10000,12 +10000,11 @@ static void kick_ilb(unsigned int flags) return; /* - * Use smp_send_reschedule() instead of resched_cpu(). - * This way we generate a sched IPI on the target CPU which + * This way we generate an IPI on the target CPU which * is idle. And the softirq performing nohz idle load balance * will be run before returning from the IPI. */ - smp_send_reschedule(ilb_cpu); + smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd); } /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 978c6fac8cb8..21416b30c520 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -889,9 +889,10 @@ struct rq { #ifdef CONFIG_SMP unsigned long last_blocked_load_update_tick; unsigned int has_blocked_load; + call_single_data_t nohz_csd; #endif /* CONFIG_SMP */ unsigned int nohz_tick_stopped; - atomic_t nohz_flags; + atomic_t nohz_flags; #endif /* CONFIG_NO_HZ_COMMON */ unsigned long nr_load_updates; @@ -978,7 +979,7 @@ struct rq { /* This is used to determine avg_idle's max value */ u64 max_idle_balance_cost; -#endif +#endif /* CONFIG_SMP */ #ifdef CONFIG_IRQ_TIME_ACCOUNTING u64 prev_irq_time; @@ -1020,6 +1021,7 @@ struct rq { #endif #ifdef CONFIG_SMP + call_single_data_t wake_csd; struct llist_head wake_list; #endif From 2a0a24ebb499c9d499eea948d3fc108f936e36d4 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 27 Mar 2020 12:42:00 +0100 Subject: [PATCH 0380/1043] sched: Make scheduler_ipi inline Now that the scheduler IPI is trivial and simple again there is no point to have the little function out of line. This simplifies the effort of constraining the instrumentation nicely. Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Peter Zijlstra Link: https://lkml.kernel.org/r/20200505134058.453581595@linutronix.de --- include/linux/sched.h | 10 +++++++++- kernel/sched/core.c | 10 ---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 4418f5cb8324..d4ea4407cd6d 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1715,7 +1715,15 @@ extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk); }) #ifdef CONFIG_SMP -void scheduler_ipi(void); +static __always_inline void scheduler_ipi(void) +{ + /* + * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting + * TIF_NEED_RESCHED remotely (for the first time) will also send + * this IPI. + */ + preempt_fold_need_resched(); +} extern unsigned long wait_task_inactive(struct task_struct *, long match_state); #else static inline void scheduler_ipi(void) { } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index cd2070d6f1e4..74fb89b5ce3e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2312,16 +2312,6 @@ static void wake_csd_func(void *info) sched_ttwu_pending(); } -void scheduler_ipi(void) -{ - /* - * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting - * TIF_NEED_RESCHED remotely (for the first time) will also send - * this IPI. - */ - preempt_fold_need_resched(); -} - static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags) { struct rq *rq = cpu_rq(cpu); From e72e8bf1c9847a12de74f2fd3ea1f5511866526b Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:32 +0200 Subject: [PATCH 0381/1043] floppy: split the base port from the register in I/O accesses Currently we have architecture-specific fd_inb() and fd_outb() functions or macros, taking just a port which is in fact made of a base address and a register. The base address is FDC-specific and derived from the local or global "fdc" variable through the FD_IOPORT macro used in the base address calculation. This change splits this by explicitly passing the FDC's base address and the register separately to fd_outb() and fd_inb(). It affects the following archs: - x86, alpha, mips, powerpc, parisc, arm, m68k: simple remap of port -> base+reg - sparc32: use of reg only, since the base address was already masked out and the FDC controller is known from a static struct. - sparc64: like x86 for PCI, like sparc32 for 82077 Some archs use inline functions and others macros. This was not unified in order to minimize the number of changes to review. For the same reason checkpatch still spews a few warnings about things that were already there before. The parisc still uses hard-coded register values and could be cleaned up by taking the register definitions. The sparc per-controller inb/outb functions could further be refined to explicitly take an FDC register instead of a port in argument but it was not needed yet and may be cleaned later. Link: https://lore.kernel.org/r/20200331094054.24441-2-w@1wt.eu Cc: Ivan Kokshaysky Cc: Richard Henderson Cc: Matt Turner Cc: Ian Molton Cc: Russell King Cc: Geert Uytterhoeven Cc: Thomas Bogendoerfer Cc: Helge Deller Cc: Benjamin Herrenschmidt Cc: "David S. Miller" Cc: x86@kernel.org Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- arch/alpha/include/asm/floppy.h | 4 ++-- arch/arm/include/asm/floppy.h | 8 ++++---- arch/m68k/include/asm/floppy.h | 12 ++++++------ arch/mips/include/asm/mach-generic/floppy.h | 8 ++++---- arch/mips/include/asm/mach-jazz/floppy.h | 8 ++++---- arch/parisc/include/asm/floppy.h | 12 ++++++------ arch/powerpc/include/asm/floppy.h | 4 ++-- arch/sparc/include/asm/floppy_32.h | 4 ++-- arch/sparc/include/asm/floppy_64.h | 4 ++-- arch/x86/include/asm/floppy.h | 4 ++-- drivers/block/floppy.c | 4 ++-- 11 files changed, 36 insertions(+), 36 deletions(-) diff --git a/arch/alpha/include/asm/floppy.h b/arch/alpha/include/asm/floppy.h index 942924756cf2..8dfdb3aa1d96 100644 --- a/arch/alpha/include/asm/floppy.h +++ b/arch/alpha/include/asm/floppy.h @@ -11,8 +11,8 @@ #define __ASM_ALPHA_FLOPPY_H -#define fd_inb(port) inb_p(port) -#define fd_outb(value,port) outb_p(value,port) +#define fd_inb(base, reg) inb_p((base) + (reg)) +#define fd_outb(value, base, reg) outb_p(value, (base) + (reg)) #define fd_enable_dma() enable_dma(FLOPPY_DMA) #define fd_disable_dma() disable_dma(FLOPPY_DMA) diff --git a/arch/arm/include/asm/floppy.h b/arch/arm/include/asm/floppy.h index 79fa327238e8..e1cb04ed5008 100644 --- a/arch/arm/include/asm/floppy.h +++ b/arch/arm/include/asm/floppy.h @@ -9,20 +9,20 @@ #ifndef __ASM_ARM_FLOPPY_H #define __ASM_ARM_FLOPPY_H -#define fd_outb(val,port) \ +#define fd_outb(val, base, reg) \ do { \ int new_val = (val); \ - if (((port) & 7) == FD_DOR) { \ + if ((reg) == FD_DOR) { \ if (new_val & 0xf0) \ new_val = (new_val & 0x0c) | \ floppy_selects[new_val & 3]; \ else \ new_val &= 0x0c; \ } \ - outb(new_val, (port)); \ + outb(new_val, (base) + (reg)); \ } while(0) -#define fd_inb(port) inb((port)) +#define fd_inb(base, reg) inb((base) + (reg)) #define fd_request_irq() request_irq(IRQ_FLOPPYDISK,floppy_interrupt,\ 0,"floppy",NULL) #define fd_free_irq() free_irq(IRQ_FLOPPYDISK,NULL) diff --git a/arch/m68k/include/asm/floppy.h b/arch/m68k/include/asm/floppy.h index c3b9ad6732fc..2a6ce29b92aa 100644 --- a/arch/m68k/include/asm/floppy.h +++ b/arch/m68k/include/asm/floppy.h @@ -63,21 +63,21 @@ static __inline__ void release_dma_lock(unsigned long flags) } -static __inline__ unsigned char fd_inb(int port) +static __inline__ unsigned char fd_inb(int base, int reg) { if(MACH_IS_Q40) - return inb_p(port); + return inb_p(base + reg); else if(MACH_IS_SUN3X) - return sun3x_82072_fd_inb(port); + return sun3x_82072_fd_inb(base + reg); return 0; } -static __inline__ void fd_outb(unsigned char value, int port) +static __inline__ void fd_outb(unsigned char value, int base, int reg) { if(MACH_IS_Q40) - outb_p(value, port); + outb_p(value, base + reg); else if(MACH_IS_SUN3X) - sun3x_82072_fd_outb(value, port); + sun3x_82072_fd_outb(value, base + reg); } diff --git a/arch/mips/include/asm/mach-generic/floppy.h b/arch/mips/include/asm/mach-generic/floppy.h index 9ec2f6a5200b..e3f446d54827 100644 --- a/arch/mips/include/asm/mach-generic/floppy.h +++ b/arch/mips/include/asm/mach-generic/floppy.h @@ -26,14 +26,14 @@ /* * How to access the FDC's registers. */ -static inline unsigned char fd_inb(unsigned int port) +static inline unsigned char fd_inb(unsigned int base, unsigned int reg) { - return inb_p(port); + return inb_p(base + reg); } -static inline void fd_outb(unsigned char value, unsigned int port) +static inline void fd_outb(unsigned char value, unsigned int base, unsigned int reg) { - outb_p(value, port); + outb_p(value, base + reg); } /* diff --git a/arch/mips/include/asm/mach-jazz/floppy.h b/arch/mips/include/asm/mach-jazz/floppy.h index 4b86c88a03b7..095000c290e5 100644 --- a/arch/mips/include/asm/mach-jazz/floppy.h +++ b/arch/mips/include/asm/mach-jazz/floppy.h @@ -17,19 +17,19 @@ #include #include -static inline unsigned char fd_inb(unsigned int port) +static inline unsigned char fd_inb(unsigned int base, unsigned int reg) { unsigned char c; - c = *(volatile unsigned char *) port; + c = *(volatile unsigned char *) (base + reg); udelay(1); return c; } -static inline void fd_outb(unsigned char value, unsigned int port) +static inline void fd_outb(unsigned char value, unsigned int base, unsigned int reg) { - *(volatile unsigned char *) port = value; + *(volatile unsigned char *) (base + reg) = value; } /* diff --git a/arch/parisc/include/asm/floppy.h b/arch/parisc/include/asm/floppy.h index 09b6f4c1687e..1aebc23b7744 100644 --- a/arch/parisc/include/asm/floppy.h +++ b/arch/parisc/include/asm/floppy.h @@ -29,8 +29,8 @@ #define CSW fd_routine[can_use_virtual_dma & 1] -#define fd_inb(port) readb(port) -#define fd_outb(value, port) writeb(value, port) +#define fd_inb(base, reg) readb((base) + (reg)) +#define fd_outb(value, base, reg) writeb(value, (base) + (reg)) #define fd_request_dma() CSW._request_dma(FLOPPY_DMA,"floppy") #define fd_free_dma() CSW._free_dma(FLOPPY_DMA) @@ -75,19 +75,19 @@ static void floppy_hardint(int irq, void *dev_id, struct pt_regs * regs) register char *lptr = virtual_dma_addr; for (lcount = virtual_dma_count; lcount; lcount--) { - st = fd_inb(virtual_dma_port+4) & 0xa0 ; + st = fd_inb(virtual_dma_port, 4) & 0xa0; if (st != 0xa0) break; if (virtual_dma_mode) { - fd_outb(*lptr, virtual_dma_port+5); + fd_outb(*lptr, virtual_dma_port, 5); } else { - *lptr = fd_inb(virtual_dma_port+5); + *lptr = fd_inb(virtual_dma_port, 5); } lptr++; } virtual_dma_count = lcount; virtual_dma_addr = lptr; - st = fd_inb(virtual_dma_port+4); + st = fd_inb(virtual_dma_port, 4); } #ifdef TRACE_FLPY_INT diff --git a/arch/powerpc/include/asm/floppy.h b/arch/powerpc/include/asm/floppy.h index 167c44b58848..ed467eb0c4c8 100644 --- a/arch/powerpc/include/asm/floppy.h +++ b/arch/powerpc/include/asm/floppy.h @@ -13,8 +13,8 @@ #include -#define fd_inb(port) inb_p(port) -#define fd_outb(value,port) outb_p(value,port) +#define fd_inb(base, reg) inb_p((base) + (reg)) +#define fd_outb(value, base, reg) outb_p(value, (base) + (reg)) #define fd_enable_dma() enable_dma(FLOPPY_DMA) #define fd_disable_dma() fd_ops->_disable_dma(FLOPPY_DMA) diff --git a/arch/sparc/include/asm/floppy_32.h b/arch/sparc/include/asm/floppy_32.h index b519acf4383d..4d08df4f4deb 100644 --- a/arch/sparc/include/asm/floppy_32.h +++ b/arch/sparc/include/asm/floppy_32.h @@ -59,8 +59,8 @@ struct sun_floppy_ops { static struct sun_floppy_ops sun_fdops; -#define fd_inb(port) sun_fdops.fd_inb(port) -#define fd_outb(value,port) sun_fdops.fd_outb(value,port) +#define fd_inb(base, reg) sun_fdops.fd_inb(reg) +#define fd_outb(value, base, reg) sun_fdops.fd_outb(value, reg) #define fd_enable_dma() sun_fd_enable_dma() #define fd_disable_dma() sun_fd_disable_dma() #define fd_request_dma() (0) /* nothing... */ diff --git a/arch/sparc/include/asm/floppy_64.h b/arch/sparc/include/asm/floppy_64.h index 3729fc35ba83..c0cf157e5b15 100644 --- a/arch/sparc/include/asm/floppy_64.h +++ b/arch/sparc/include/asm/floppy_64.h @@ -62,8 +62,8 @@ struct sun_floppy_ops { static struct sun_floppy_ops sun_fdops; -#define fd_inb(port) sun_fdops.fd_inb(port) -#define fd_outb(value,port) sun_fdops.fd_outb(value,port) +#define fd_inb(base, reg) sun_fdops.fd_inb((base) + (reg)) +#define fd_outb(value, base, reg) sun_fdops.fd_outb(value, (base) + (reg)) #define fd_enable_dma() sun_fdops.fd_enable_dma() #define fd_disable_dma() sun_fdops.fd_disable_dma() #define fd_request_dma() (0) /* nothing... */ diff --git a/arch/x86/include/asm/floppy.h b/arch/x86/include/asm/floppy.h index 7ec59edde154..20088cb08f5e 100644 --- a/arch/x86/include/asm/floppy.h +++ b/arch/x86/include/asm/floppy.h @@ -31,8 +31,8 @@ #define CSW fd_routine[can_use_virtual_dma & 1] -#define fd_inb(port) inb_p(port) -#define fd_outb(value, port) outb_p(value, port) +#define fd_inb(base, reg) inb_p((base) + (reg)) +#define fd_outb(value, base, reg) outb_p(value, (base) + (reg)) #define fd_request_dma() CSW._request_dma(FLOPPY_DMA, "floppy") #define fd_free_dma() CSW._free_dma(FLOPPY_DMA) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index c3daa64cb52c..1cda39098b07 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -595,12 +595,12 @@ static unsigned char in_sector_offset; /* offset within physical sector, static inline unsigned char fdc_inb(int fdc, int reg) { - return fd_inb(fdc_state[fdc].address + reg); + return fd_inb(fdc_state[fdc].address, reg); } static inline void fdc_outb(unsigned char value, int fdc, int reg) { - fd_outb(value, fdc_state[fdc].address + reg); + fd_outb(value, fdc_state[fdc].address, reg); } static inline bool drive_no_geom(int drive) From 7d33850abdb9048c6aa421440a64905eb4ad07a2 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:33 +0200 Subject: [PATCH 0382/1043] floppy: add references to 82077's extra registers This controller provides extra status registers SRA and SRB as well as a tape drive register (TDR) and a data rate select register (DSR), which are referenced in the sparc port, so let's have their symbolic definitions centralized. Link: https://lore.kernel.org/r/20200331094054.24441-3-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- include/uapi/linux/fdreg.h | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/include/uapi/linux/fdreg.h b/include/uapi/linux/fdreg.h index 1318881954e1..10d33632939d 100644 --- a/include/uapi/linux/fdreg.h +++ b/include/uapi/linux/fdreg.h @@ -7,13 +7,23 @@ * Handbook", Sanches and Canton. */ -/* Fd controller regs. S&C, about page 340 */ -#define FD_STATUS 4 -#define FD_DATA 5 +/* 82077's auxiliary status registers A & B (R) */ +#define FD_SRA 0 +#define FD_SRB 1 /* Digital Output Register */ #define FD_DOR 2 +/* 82077's tape drive register (R/W) */ +#define FD_TDR 3 + +/* 82077's data rate select register (W) */ +#define FD_DSR 4 + +/* Fd controller regs. S&C, about page 340 */ +#define FD_STATUS 4 +#define FD_DATA 5 + /* Digital Input Register (read) */ #define FD_DIR 7 From 76373fc666a1c62d422f5e6fc9e4927877934045 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:34 +0200 Subject: [PATCH 0383/1043] floppy: use symbolic register names in the m68k port Now we can use FD_STATUS and FD_DATA instead of 4 or 5, let's do this, and also use STATUS_DMA and STATUS_READY for the status bits. Link: https://lore.kernel.org/r/20200331094054.24441-4-w@1wt.eu Cc: Geert Uytterhoeven Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- arch/m68k/include/asm/floppy.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/arch/m68k/include/asm/floppy.h b/arch/m68k/include/asm/floppy.h index 2a6ce29b92aa..a4d0fea47c6b 100644 --- a/arch/m68k/include/asm/floppy.h +++ b/arch/m68k/include/asm/floppy.h @@ -211,26 +211,27 @@ asmlinkage irqreturn_t floppy_hardint(int irq, void *dev_id) st=1; for(lcount=virtual_dma_count, lptr=virtual_dma_addr; lcount; lcount--, lptr++) { - st=inb(virtual_dma_port+4) & 0xa0 ; - if(st != 0xa0) + st = inb(virtual_dma_port + FD_STATUS); + st &= STATUS_DMA | STATUS_READY; + if (st != (STATUS_DMA | STATUS_READY)) break; if(virtual_dma_mode) - outb_p(*lptr, virtual_dma_port+5); + outb_p(*lptr, virtual_dma_port + FD_DATA); else - *lptr = inb_p(virtual_dma_port+5); + *lptr = inb_p(virtual_dma_port + FD_DATA); } virtual_dma_count = lcount; virtual_dma_addr = lptr; - st = inb(virtual_dma_port+4); + st = inb(virtual_dma_port + FD_STATUS); } #ifdef TRACE_FLPY_INT calls++; #endif - if(st == 0x20) + if (st == STATUS_DMA) return IRQ_HANDLED; - if(!(st & 0x20)) { + if (!(st & STATUS_DMA)) { virtual_dma_residue += virtual_dma_count; virtual_dma_count=0; #ifdef TRACE_FLPY_INT From 40b7d1b69093b3592ba8b877369d371d65ab7059 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:35 +0200 Subject: [PATCH 0384/1043] floppy: use symbolic register names in the parisc port Now we can use FD_STATUS and FD_DATA instead of 4 or 5, let's do this, and also use STATUS_DMA and STATUS_READY for the status bits. Link: https://lore.kernel.org/r/20200331094054.24441-5-w@1wt.eu Cc: Helge Deller Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- arch/parisc/include/asm/floppy.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/arch/parisc/include/asm/floppy.h b/arch/parisc/include/asm/floppy.h index 1aebc23b7744..762cfe7778c0 100644 --- a/arch/parisc/include/asm/floppy.h +++ b/arch/parisc/include/asm/floppy.h @@ -75,27 +75,28 @@ static void floppy_hardint(int irq, void *dev_id, struct pt_regs * regs) register char *lptr = virtual_dma_addr; for (lcount = virtual_dma_count; lcount; lcount--) { - st = fd_inb(virtual_dma_port, 4) & 0xa0; - if (st != 0xa0) + st = fd_inb(virtual_dma_port, FD_STATUS); + st &= STATUS_DMA | STATUS_READY; + if (st != (STATUS_DMA | STATUS_READY)) break; if (virtual_dma_mode) { - fd_outb(*lptr, virtual_dma_port, 5); + fd_outb(*lptr, virtual_dma_port, FD_DATA); } else { - *lptr = fd_inb(virtual_dma_port, 5); + *lptr = fd_inb(virtual_dma_port, FD_DATA); } lptr++; } virtual_dma_count = lcount; virtual_dma_addr = lptr; - st = fd_inb(virtual_dma_port, 4); + st = fd_inb(virtual_dma_port, FD_STATUS); } #ifdef TRACE_FLPY_INT calls++; #endif - if (st == 0x20) + if (st == STATUS_DMA) return; - if (!(st & 0x20)) { + if (!(st & STATUS_DMA)) { virtual_dma_residue += virtual_dma_count; virtual_dma_count = 0; #ifdef TRACE_FLPY_INT From 7fd346318847aa967593b25153ccbeb6cfe1daf1 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:36 +0200 Subject: [PATCH 0385/1043] floppy: use symbolic register names in the powerpc port Now we can use FD_STATUS and FD_DATA instead of 4 or 5, let's do this, and also use STATUS_DMA and STATUS_READY for the status bits. Link: https://lore.kernel.org/r/20200331094054.24441-6-w@1wt.eu Cc: Benjamin Herrenschmidt Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- arch/powerpc/include/asm/floppy.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/arch/powerpc/include/asm/floppy.h b/arch/powerpc/include/asm/floppy.h index ed467eb0c4c8..7af9a68fd949 100644 --- a/arch/powerpc/include/asm/floppy.h +++ b/arch/powerpc/include/asm/floppy.h @@ -61,21 +61,22 @@ static irqreturn_t floppy_hardint(int irq, void *dev_id) st = 1; for (lcount=virtual_dma_count, lptr=virtual_dma_addr; lcount; lcount--, lptr++) { - st=inb(virtual_dma_port+4) & 0xa0 ; - if (st != 0xa0) + st = inb(virtual_dma_port + FD_STATUS); + st &= STATUS_DMA | STATUS_READY; + if (st != (STATUS_DMA | STATUS_READY)) break; if (virtual_dma_mode) - outb_p(*lptr, virtual_dma_port+5); + outb_p(*lptr, virtual_dma_port + FD_DATA); else - *lptr = inb_p(virtual_dma_port+5); + *lptr = inb_p(virtual_dma_port + FD_DATA); } virtual_dma_count = lcount; virtual_dma_addr = lptr; - st = inb(virtual_dma_port+4); + st = inb(virtual_dma_port + FD_STATUS); - if (st == 0x20) + if (st == STATUS_DMA) return IRQ_HANDLED; - if (!(st & 0x20)) { + if (!(st & STATUS_DMA)) { virtual_dma_residue += virtual_dma_count; virtual_dma_count=0; doing_vdma = 0; From 6d362018c66a0ad13117357422e39a8821a812ad Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:37 +0200 Subject: [PATCH 0386/1043] floppy: use symbolic register names in the sparc32 port The sparc port used to be forced to rely on numeric register indexes with their equivalent in comments. Now that they don't depend on the IO port we can use their symbolic names. Link: https://lore.kernel.org/r/20200331094054.24441-7-w@1wt.eu Cc: "David S. Miller" Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- arch/sparc/include/asm/floppy_32.h | 46 +++++++++++++++--------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/arch/sparc/include/asm/floppy_32.h b/arch/sparc/include/asm/floppy_32.h index 4d08df4f4deb..946dbcbf3a83 100644 --- a/arch/sparc/include/asm/floppy_32.h +++ b/arch/sparc/include/asm/floppy_32.h @@ -114,15 +114,15 @@ static unsigned char sun_read_dir(void) static unsigned char sun_82072_fd_inb(int port) { udelay(5); - switch(port & 7) { + switch (port) { default: printk("floppy: Asked to read unknown port %d\n", port); panic("floppy: Port bolixed."); - case 4: /* FD_STATUS */ + case FD_STATUS: return sun_fdc->status_82072 & ~STATUS_DMA; - case 5: /* FD_DATA */ + case FD_DATA: return sun_fdc->data_82072; - case 7: /* FD_DIR */ + case FD_DIR: return sun_read_dir(); } panic("sun_82072_fd_inb: How did I get here?"); @@ -131,20 +131,20 @@ static unsigned char sun_82072_fd_inb(int port) static void sun_82072_fd_outb(unsigned char value, int port) { udelay(5); - switch(port & 7) { + switch (port) { default: printk("floppy: Asked to write to unknown port %d\n", port); panic("floppy: Port bolixed."); - case 2: /* FD_DOR */ + case FD_DOR: sun_set_dor(value, 0); break; - case 5: /* FD_DATA */ + case FD_DATA: sun_fdc->data_82072 = value; break; - case 7: /* FD_DCR */ + case FD_DCR: sun_fdc->dcr_82072 = value; break; - case 4: /* FD_STATUS */ + case FD_DSR: sun_fdc->status_82072 = value; break; } @@ -154,23 +154,23 @@ static void sun_82072_fd_outb(unsigned char value, int port) static unsigned char sun_82077_fd_inb(int port) { udelay(5); - switch(port & 7) { + switch (port) { default: printk("floppy: Asked to read unknown port %d\n", port); panic("floppy: Port bolixed."); - case 0: /* FD_STATUS_0 */ + case FD_SRA: return sun_fdc->status1_82077; - case 1: /* FD_STATUS_1 */ + case FD_SRB: return sun_fdc->status2_82077; - case 2: /* FD_DOR */ + case FD_DOR: return sun_fdc->dor_82077; - case 3: /* FD_TDR */ + case FD_TDR: return sun_fdc->tapectl_82077; - case 4: /* FD_STATUS */ + case FD_STATUS: return sun_fdc->status_82077 & ~STATUS_DMA; - case 5: /* FD_DATA */ + case FD_DATA: return sun_fdc->data_82077; - case 7: /* FD_DIR */ + case FD_DIR: return sun_read_dir(); } panic("sun_82077_fd_inb: How did I get here?"); @@ -179,23 +179,23 @@ static unsigned char sun_82077_fd_inb(int port) static void sun_82077_fd_outb(unsigned char value, int port) { udelay(5); - switch(port & 7) { + switch (port) { default: printk("floppy: Asked to write to unknown port %d\n", port); panic("floppy: Port bolixed."); - case 2: /* FD_DOR */ + case FD_DOR: sun_set_dor(value, 1); break; - case 5: /* FD_DATA */ + case FD_DATA: sun_fdc->data_82077 = value; break; - case 7: /* FD_DCR */ + case FD_DCR: sun_fdc->dcr_82077 = value; break; - case 4: /* FD_STATUS */ + case FD_DSR: sun_fdc->status_82077 = value; break; - case 3: /* FD_TDR */ + case FD_TDR: sun_fdc->tapectl_82077 = value; break; } From 6cb7e69671843c6f10fd25fc93eecadd4d52ac37 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:38 +0200 Subject: [PATCH 0387/1043] floppy: use symbolic register names in the sparc64 port Now by splitting the base address from the register index we can use the symbolic register names instead of the hard-coded numeric values. Link: https://lore.kernel.org/r/20200331094054.24441-8-w@1wt.eu Cc: "David S. Miller" [willy: fix printk warnings s/%lx/%x/g in sun_82077_fd_{inb,outb}()] Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- arch/sparc/include/asm/floppy_64.h | 59 ++++++++++++++++-------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/arch/sparc/include/asm/floppy_64.h b/arch/sparc/include/asm/floppy_64.h index c0cf157e5b15..070c8c1f5c8f 100644 --- a/arch/sparc/include/asm/floppy_64.h +++ b/arch/sparc/include/asm/floppy_64.h @@ -47,8 +47,9 @@ unsigned long fdc_status; static struct platform_device *floppy_op = NULL; struct sun_floppy_ops { - unsigned char (*fd_inb) (unsigned long port); - void (*fd_outb) (unsigned char value, unsigned long port); + unsigned char (*fd_inb) (unsigned long port, unsigned int reg); + void (*fd_outb) (unsigned char value, unsigned long base, + unsigned int reg); void (*fd_enable_dma) (void); void (*fd_disable_dma) (void); void (*fd_set_dma_mode) (int); @@ -62,8 +63,8 @@ struct sun_floppy_ops { static struct sun_floppy_ops sun_fdops; -#define fd_inb(base, reg) sun_fdops.fd_inb((base) + (reg)) -#define fd_outb(value, base, reg) sun_fdops.fd_outb(value, (base) + (reg)) +#define fd_inb(base, reg) sun_fdops.fd_inb(base, reg) +#define fd_outb(value, base, reg) sun_fdops.fd_outb(value, base, reg) #define fd_enable_dma() sun_fdops.fd_enable_dma() #define fd_disable_dma() sun_fdops.fd_disable_dma() #define fd_request_dma() (0) /* nothing... */ @@ -97,42 +98,43 @@ static int sun_floppy_types[2] = { 0, 0 }; /* No 64k boundary crossing problems on the Sparc. */ #define CROSS_64KB(a,s) (0) -static unsigned char sun_82077_fd_inb(unsigned long port) +static unsigned char sun_82077_fd_inb(unsigned long base, unsigned int reg) { udelay(5); - switch(port & 7) { + switch (reg) { default: - printk("floppy: Asked to read unknown port %lx\n", port); + printk("floppy: Asked to read unknown port %x\n", reg); panic("floppy: Port bolixed."); - case 4: /* FD_STATUS */ + case FD_STATUS: return sbus_readb(&sun_fdc->status_82077) & ~STATUS_DMA; - case 5: /* FD_DATA */ + case FD_DATA: return sbus_readb(&sun_fdc->data_82077); - case 7: /* FD_DIR */ + case FD_DIR: /* XXX: Is DCL on 0x80 in sun4m? */ return sbus_readb(&sun_fdc->dir_82077); } panic("sun_82072_fd_inb: How did I get here?"); } -static void sun_82077_fd_outb(unsigned char value, unsigned long port) +static void sun_82077_fd_outb(unsigned char value, unsigned long base, + unsigned int reg) { udelay(5); - switch(port & 7) { + switch (reg) { default: - printk("floppy: Asked to write to unknown port %lx\n", port); + printk("floppy: Asked to write to unknown port %x\n", reg); panic("floppy: Port bolixed."); - case 2: /* FD_DOR */ + case FD_DOR: /* Happily, the 82077 has a real DOR register. */ sbus_writeb(value, &sun_fdc->dor_82077); break; - case 5: /* FD_DATA */ + case FD_DATA: sbus_writeb(value, &sun_fdc->data_82077); break; - case 7: /* FD_DCR */ + case FD_DCR: sbus_writeb(value, &sun_fdc->dcr_82077); break; - case 4: /* FD_STATUS */ + case FD_DSR: sbus_writeb(value, &sun_fdc->status_82077); break; } @@ -298,19 +300,21 @@ static struct sun_pci_dma_op sun_pci_dma_pending = { -1U, 0, 0, NULL}; irqreturn_t floppy_interrupt(int irq, void *dev_id); -static unsigned char sun_pci_fd_inb(unsigned long port) +static unsigned char sun_pci_fd_inb(unsigned long base, unsigned int reg) { udelay(5); - return inb(port); + return inb(base + reg); } -static void sun_pci_fd_outb(unsigned char val, unsigned long port) +static void sun_pci_fd_outb(unsigned char val, unsigned long base, + unsigned int reg) { udelay(5); - outb(val, port); + outb(val, base + reg); } -static void sun_pci_fd_broken_outb(unsigned char val, unsigned long port) +static void sun_pci_fd_broken_outb(unsigned char val, unsigned long base, + unsigned int reg) { udelay(5); /* @@ -320,16 +324,17 @@ static void sun_pci_fd_broken_outb(unsigned char val, unsigned long port) * this does not hurt correct hardware like the AXmp. * (Eddie, Sep 12 1998). */ - if (port == ((unsigned long)sun_fdc) + 2) { + if (reg == FD_DOR) { if (((val & 0x03) == sun_pci_broken_drive) && (val & 0x20)) { val |= 0x10; } } - outb(val, port); + outb(val, base + reg); } #ifdef PCI_FDC_SWAP_DRIVES -static void sun_pci_fd_lde_broken_outb(unsigned char val, unsigned long port) +static void sun_pci_fd_lde_broken_outb(unsigned char val, unsigned long base, + unsigned int reg) { udelay(5); /* @@ -339,13 +344,13 @@ static void sun_pci_fd_lde_broken_outb(unsigned char val, unsigned long port) * this does not hurt correct hardware like the AXmp. * (Eddie, Sep 12 1998). */ - if (port == ((unsigned long)sun_fdc) + 2) { + if (reg == FD_DOR) { if (((val & 0x03) == sun_pci_broken_drive) && (val & 0x10)) { val &= ~(0x03); val |= 0x21; } } - outb(val, port); + outb(val, base + reg); } #endif /* PCI_FDC_SWAP_DRIVES */ From 38ede90831c7f3e931b58c2b2790d02d5d061592 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:39 +0200 Subject: [PATCH 0388/1043] floppy: use symbolic register names in the x86 port Now we can use FD_STATUS and FD_DATA instead of 4 or 5, let's do this, and also use STATUS_DMA and STATUS_READY for the status bits. Link: https://lore.kernel.org/r/20200331094054.24441-9-w@1wt.eu Cc: x86@kernel.org Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- arch/x86/include/asm/floppy.h | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/floppy.h b/arch/x86/include/asm/floppy.h index 20088cb08f5e..d43717b423cb 100644 --- a/arch/x86/include/asm/floppy.h +++ b/arch/x86/include/asm/floppy.h @@ -77,25 +77,26 @@ static irqreturn_t floppy_hardint(int irq, void *dev_id) st = 1; for (lcount = virtual_dma_count, lptr = virtual_dma_addr; lcount; lcount--, lptr++) { - st = inb(virtual_dma_port + 4) & 0xa0; - if (st != 0xa0) + st = inb(virtual_dma_port + FD_STATUS); + st &= STATUS_DMA | STATUS_READY; + if (st != (STATUS_DMA | STATUS_READY)) break; if (virtual_dma_mode) - outb_p(*lptr, virtual_dma_port + 5); + outb_p(*lptr, virtual_dma_port + FD_DATA); else - *lptr = inb_p(virtual_dma_port + 5); + *lptr = inb_p(virtual_dma_port + FD_DATA); } virtual_dma_count = lcount; virtual_dma_addr = lptr; - st = inb(virtual_dma_port + 4); + st = inb(virtual_dma_port + FD_STATUS); } #ifdef TRACE_FLPY_INT calls++; #endif - if (st == 0x20) + if (st == STATUS_DMA) return IRQ_HANDLED; - if (!(st & 0x20)) { + if (!(st & STATUS_DMA)) { virtual_dma_residue += virtual_dma_count; virtual_dma_count = 0; #ifdef TRACE_FLPY_INT From c1f710b5fe8c18d0c2be4514bf509e1a4203ce08 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:40 +0200 Subject: [PATCH 0389/1043] floppy: cleanup: make twaddle() not rely on current_{fdc,drive} anymore Now the fdc and drive are passed in argument so that the function does not use current_fdc nor current_drive anymore. Link: https://lore.kernel.org/r/20200331094054.24441-10-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 1cda39098b07..b1729daa2e2e 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -827,14 +827,14 @@ static int set_dor(int fdc, char mask, char data) return olddor; } -static void twaddle(void) +static void twaddle(int fdc, int drive) { - if (drive_params[current_drive].select_delay) + if (drive_params[drive].select_delay) return; - fdc_outb(fdc_state[current_fdc].dor & ~(0x10 << UNIT(current_drive)), - current_fdc, FD_DOR); - fdc_outb(fdc_state[current_fdc].dor, current_fdc, FD_DOR); - drive_state[current_drive].select_date = jiffies; + fdc_outb(fdc_state[fdc].dor & ~(0x10 << UNIT(drive)), + fdc, FD_DOR); + fdc_outb(fdc_state[fdc].dor, fdc, FD_DOR); + drive_state[drive].select_date = jiffies; } /* @@ -1934,7 +1934,7 @@ static void floppy_ready(void) "calling disk change from floppy_ready\n"); if (!(raw_cmd->flags & FD_RAW_NO_MOTOR) && disk_change(current_drive) && !drive_params[current_drive].select_delay) - twaddle(); /* this clears the dcl on certain + twaddle(current_fdc, current_drive); /* this clears the dcl on certain * drive/controller combinations */ #ifdef fd_chose_dma_mode @@ -2904,7 +2904,7 @@ do_request: } if (test_bit(FD_NEED_TWADDLE_BIT, &drive_state[current_drive].flags)) - twaddle(); + twaddle(current_fdc, current_drive); schedule_bh(floppy_start); debugt(__func__, "queue fd request"); return; @@ -3610,7 +3610,7 @@ static int fd_locked_ioctl(struct block_device *bdev, fmode_t mode, unsigned int case FDTWADDLE: if (lock_fdc(drive)) return -EINTR; - twaddle(); + twaddle(current_fdc, current_drive); process_fd_request(); return 0; default: From f3e0dc1d8b71fa0bdd3a8e24bb129978567fefbb Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:41 +0200 Subject: [PATCH 0390/1043] floppy: cleanup: make reset_fdc_info() not rely on current_fdc anymore Now the fdc is passed in argument so that the function does not use current_fdc anymore. Link: https://lore.kernel.org/r/20200331094054.24441-11-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index b1729daa2e2e..6c98f8d169a9 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -838,19 +838,19 @@ static void twaddle(int fdc, int drive) } /* - * Reset all driver information about the current fdc. + * Reset all driver information about the specified fdc. * This is needed after a reset, and after a raw command. */ -static void reset_fdc_info(int mode) +static void reset_fdc_info(int fdc, int mode) { int drive; - fdc_state[current_fdc].spec1 = fdc_state[current_fdc].spec2 = -1; - fdc_state[current_fdc].need_configure = 1; - fdc_state[current_fdc].perp_mode = 1; - fdc_state[current_fdc].rawcmd = 0; + fdc_state[fdc].spec1 = fdc_state[fdc].spec2 = -1; + fdc_state[fdc].need_configure = 1; + fdc_state[fdc].perp_mode = 1; + fdc_state[fdc].rawcmd = 0; for (drive = 0; drive < N_DRIVE; drive++) - if (FDC(drive) == current_fdc && + if (FDC(drive) == fdc && (mode || drive_state[drive].track != NEED_1_RECAL)) drive_state[drive].track = NEED_2_RECAL; } @@ -874,7 +874,7 @@ static void set_fdc(int drive) set_dor(1 - current_fdc, ~8, 0); #endif if (fdc_state[current_fdc].rawcmd == 2) - reset_fdc_info(1); + reset_fdc_info(current_fdc, 1); if (fdc_inb(current_fdc, FD_STATUS) != STATUS_READY) fdc_state[current_fdc].reset = 1; } @@ -1800,7 +1800,7 @@ static void reset_fdc(void) do_floppy = reset_interrupt; fdc_state[current_fdc].reset = 0; - reset_fdc_info(0); + reset_fdc_info(current_fdc, 0); /* Pseudo-DMA may intercept 'reset finished' interrupt. */ /* Irrelevant for systems with true DMA (i386). */ @@ -4890,7 +4890,7 @@ static int floppy_grab_irq_and_dma(void) } for (current_fdc = 0; current_fdc < N_FDC; current_fdc++) { if (fdc_state[current_fdc].address != -1) { - reset_fdc_info(1); + reset_fdc_info(current_fdc, 1); fdc_outb(fdc_state[current_fdc].dor, current_fdc, FD_DOR); } } From 6d494ed03766ead1b180463380511fed9ac779d9 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:42 +0200 Subject: [PATCH 0391/1043] floppy: cleanup: make show_floppy() not rely on current_fdc anymore Now the fdc is passed in argument so that the function does not use current_fdc anymore. Link: https://lore.kernel.org/r/20200331094054.24441-12-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 6c98f8d169a9..dd739594fce7 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -1104,7 +1104,7 @@ static void setup_DMA(void) #endif } -static void show_floppy(void); +static void show_floppy(int fdc); /* waits until the fdc becomes ready */ static int wait_til_ready(void) @@ -1121,7 +1121,7 @@ static int wait_til_ready(void) } if (initialized) { DPRINT("Getstatus times out (%x) on fdc %d\n", status, current_fdc); - show_floppy(); + show_floppy(current_fdc); } fdc_state[current_fdc].reset = 1; return -1; @@ -1147,7 +1147,7 @@ static int output_byte(char byte) if (initialized) { DPRINT("Unable to send byte %x to FDC. Fdc=%x Status=%x\n", byte, current_fdc, status); - show_floppy(); + show_floppy(current_fdc); } return -1; } @@ -1176,7 +1176,7 @@ static int result(void) if (initialized) { DPRINT("get result error. Fdc=%d Last status=%x Read bytes=%d\n", current_fdc, status, i); - show_floppy(); + show_floppy(current_fdc); } fdc_state[current_fdc].reset = 1; return -1; @@ -1819,7 +1819,7 @@ static void reset_fdc(void) } } -static void show_floppy(void) +static void show_floppy(int fdc) { int i; @@ -1842,7 +1842,7 @@ static void show_floppy(void) print_hex_dump(KERN_INFO, "", DUMP_PREFIX_NONE, 16, 1, reply_buffer, resultsize, true); - pr_info("status=%x\n", fdc_inb(current_fdc, FD_STATUS)); + pr_info("status=%x\n", fdc_inb(fdc, FD_STATUS)); pr_info("fdc_busy=%lu\n", fdc_busy); if (do_floppy) pr_info("do_floppy=%ps\n", do_floppy); @@ -1868,7 +1868,7 @@ static void floppy_shutdown(struct work_struct *arg) unsigned long flags; if (initialized) - show_floppy(); + show_floppy(current_fdc); cancel_activity(); flags = claim_dma_lock(); From 5ea00bfc52f428cab828623e2d7084118c25d54b Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:43 +0200 Subject: [PATCH 0392/1043] floppy: cleanup: make wait_til_ready() not rely on current_fdc anymore Now the fdc is passed in argument so that the function does not use current_fdc anymore. Link: https://lore.kernel.org/r/20200331094054.24441-13-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index dd739594fce7..5dfddd4726fb 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -1107,30 +1107,30 @@ static void setup_DMA(void) static void show_floppy(int fdc); /* waits until the fdc becomes ready */ -static int wait_til_ready(void) +static int wait_til_ready(int fdc) { int status; int counter; - if (fdc_state[current_fdc].reset) + if (fdc_state[fdc].reset) return -1; for (counter = 0; counter < 10000; counter++) { - status = fdc_inb(current_fdc, FD_STATUS); + status = fdc_inb(fdc, FD_STATUS); if (status & STATUS_READY) return status; } if (initialized) { - DPRINT("Getstatus times out (%x) on fdc %d\n", status, current_fdc); - show_floppy(current_fdc); + DPRINT("Getstatus times out (%x) on fdc %d\n", status, fdc); + show_floppy(fdc); } - fdc_state[current_fdc].reset = 1; + fdc_state[fdc].reset = 1; return -1; } /* sends a command byte to the fdc */ static int output_byte(char byte) { - int status = wait_til_ready(); + int status = wait_til_ready(current_fdc); if (status < 0) return -1; @@ -1159,7 +1159,7 @@ static int result(void) int status = 0; for (i = 0; i < MAX_REPLIES; i++) { - status = wait_til_ready(); + status = wait_til_ready(current_fdc); if (status < 0) break; status &= STATUS_DIR | STATUS_READY | STATUS_BUSY | STATUS_DMA; @@ -1186,7 +1186,7 @@ static int result(void) /* does the fdc need more output? */ static int need_more_output(void) { - int status = wait_til_ready(); + int status = wait_til_ready(current_fdc); if (status < 0) return -1; From f8a8e0f7a8941bfe105af7a1150c9f6a73ca253d Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:44 +0200 Subject: [PATCH 0393/1043] floppy: cleanup: make output_byte() not rely on current_fdc anymore Now the fdc is passed in argument so that the function does not use current_fdc anymore. Link: https://lore.kernel.org/r/20200331094054.24441-14-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 64 +++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 5dfddd4726fb..81fd06eaea7d 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -1128,26 +1128,26 @@ static int wait_til_ready(int fdc) } /* sends a command byte to the fdc */ -static int output_byte(char byte) +static int output_byte(int fdc, char byte) { - int status = wait_til_ready(current_fdc); + int status = wait_til_ready(fdc); if (status < 0) return -1; if (is_ready_state(status)) { - fdc_outb(byte, current_fdc, FD_DATA); + fdc_outb(byte, fdc, FD_DATA); output_log[output_log_pos].data = byte; output_log[output_log_pos].status = status; output_log[output_log_pos].jiffies = jiffies; output_log_pos = (output_log_pos + 1) % OLOGSIZE; return 0; } - fdc_state[current_fdc].reset = 1; + fdc_state[fdc].reset = 1; if (initialized) { DPRINT("Unable to send byte %x to FDC. Fdc=%x Status=%x\n", - byte, current_fdc, status); - show_floppy(current_fdc); + byte, fdc, status); + show_floppy(fdc); } return -1; } @@ -1229,8 +1229,8 @@ static void perpendicular_mode(void) if (fdc_state[current_fdc].perp_mode == perp_mode) return; if (fdc_state[current_fdc].version >= FDC_82077_ORIG) { - output_byte(FD_PERPENDICULAR); - output_byte(perp_mode); + output_byte(current_fdc, FD_PERPENDICULAR); + output_byte(current_fdc, perp_mode); fdc_state[current_fdc].perp_mode = perp_mode; } else if (perp_mode) { DPRINT("perpendicular mode not supported by this FDC.\n"); @@ -1243,12 +1243,12 @@ static int no_fifo; static int fdc_configure(void) { /* Turn on FIFO */ - output_byte(FD_CONFIGURE); + output_byte(current_fdc, FD_CONFIGURE); if (need_more_output() != MORE_OUTPUT) return 0; - output_byte(0); - output_byte(0x10 | (no_fifo & 0x20) | (fifo_depth & 0xf)); - output_byte(0); /* pre-compensation from track + output_byte(current_fdc, 0); + output_byte(current_fdc, 0x10 | (no_fifo & 0x20) | (fifo_depth & 0xf)); + output_byte(current_fdc, 0); /* pre-compensation from track 0 upwards */ return 1; } @@ -1301,10 +1301,10 @@ static void fdc_specify(void) if (fdc_state[current_fdc].version >= FDC_82078) { /* chose the default rate table, not the one * where 1 = 2 Mbps */ - output_byte(FD_DRIVESPEC); + output_byte(current_fdc, FD_DRIVESPEC); if (need_more_output() == MORE_OUTPUT) { - output_byte(UNIT(current_drive)); - output_byte(0xc0); + output_byte(current_fdc, UNIT(current_drive)); + output_byte(current_fdc, 0xc0); } } break; @@ -1349,9 +1349,9 @@ static void fdc_specify(void) if (fdc_state[current_fdc].spec1 != spec1 || fdc_state[current_fdc].spec2 != spec2) { /* Go ahead and set spec1 and spec2 */ - output_byte(FD_SPECIFY); - output_byte(fdc_state[current_fdc].spec1 = spec1); - output_byte(fdc_state[current_fdc].spec2 = spec2); + output_byte(current_fdc, FD_SPECIFY); + output_byte(current_fdc, fdc_state[current_fdc].spec1 = spec1); + output_byte(current_fdc, fdc_state[current_fdc].spec2 = spec2); } } /* fdc_specify */ @@ -1513,7 +1513,7 @@ static void setup_rw_floppy(void) r = 0; for (i = 0; i < raw_cmd->cmd_count; i++) - r |= output_byte(raw_cmd->cmd[i]); + r |= output_byte(current_fdc, raw_cmd->cmd[i]); debugt(__func__, "rw_command"); @@ -1566,8 +1566,8 @@ static void check_wp(void) { if (test_bit(FD_VERIFY_BIT, &drive_state[current_drive].flags)) { /* check write protection */ - output_byte(FD_GETSTATUS); - output_byte(UNIT(current_drive)); + output_byte(current_fdc, FD_GETSTATUS); + output_byte(current_fdc, UNIT(current_drive)); if (result() != 1) { fdc_state[current_fdc].reset = 1; return; @@ -1639,9 +1639,9 @@ static void seek_floppy(void) } do_floppy = seek_interrupt; - output_byte(FD_SEEK); - output_byte(UNIT(current_drive)); - if (output_byte(track) < 0) { + output_byte(current_fdc, FD_SEEK); + output_byte(current_fdc, UNIT(current_drive)); + if (output_byte(current_fdc, track) < 0) { reset_fdc(); return; } @@ -1748,7 +1748,7 @@ irqreturn_t floppy_interrupt(int irq, void *dev_id) if (inr == 0) { int max_sensei = 4; do { - output_byte(FD_SENSEI); + output_byte(current_fdc, FD_SENSEI); inr = result(); if (do_print) print_result("sensei", inr); @@ -1771,8 +1771,8 @@ static void recalibrate_floppy(void) { debugt(__func__, ""); do_floppy = recal_interrupt; - output_byte(FD_RECALIBRATE); - if (output_byte(UNIT(current_drive)) < 0) + output_byte(current_fdc, FD_RECALIBRATE); + if (output_byte(current_fdc, UNIT(current_drive)) < 0) reset_fdc(); } @@ -4302,7 +4302,7 @@ static char __init get_fdc_version(void) { int r; - output_byte(FD_DUMPREGS); /* 82072 and better know DUMPREGS */ + output_byte(current_fdc, FD_DUMPREGS); /* 82072 and better know DUMPREGS */ if (fdc_state[current_fdc].reset) return FDC_NONE; r = result(); @@ -4323,15 +4323,15 @@ static char __init get_fdc_version(void) return FDC_82072; /* 82072 doesn't know CONFIGURE */ } - output_byte(FD_PERPENDICULAR); + output_byte(current_fdc, FD_PERPENDICULAR); if (need_more_output() == MORE_OUTPUT) { - output_byte(0); + output_byte(current_fdc, 0); } else { pr_info("FDC %d is an 82072A\n", current_fdc); return FDC_82072A; /* 82072A as found on Sparcs. */ } - output_byte(FD_UNLOCK); + output_byte(current_fdc, FD_UNLOCK); r = result(); if ((r == 1) && (reply_buffer[0] == 0x80)) { pr_info("FDC %d is a pre-1991 82077\n", current_fdc); @@ -4343,7 +4343,7 @@ static char __init get_fdc_version(void) current_fdc, r); return FDC_UNKNOWN; } - output_byte(FD_PARTID); + output_byte(current_fdc, FD_PARTID); r = result(); if (r != 1) { pr_info("FDC %d init: PARTID: unexpected return of %d bytes.\n", From 96dad77a6506ceb31eb520f97fbb5c82054f0a73 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:45 +0200 Subject: [PATCH 0394/1043] floppy: cleanup: make result() not rely on current_fdc anymore Now the fdc is passed in argument so that the function does not use current_fdc anymore. It's worth noting that there's still a single reply_buffer[] which will store the result for the current fdc. It may or may not make sense to implement one buffer per fdc in the future. Link: https://lore.kernel.org/r/20200331094054.24441-15-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 81fd06eaea7d..4aaf84217b53 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -1153,13 +1153,13 @@ static int output_byte(int fdc, char byte) } /* gets the response from the fdc */ -static int result(void) +static int result(int fdc) { int i; int status = 0; for (i = 0; i < MAX_REPLIES; i++) { - status = wait_til_ready(current_fdc); + status = wait_til_ready(fdc); if (status < 0) break; status &= STATUS_DIR | STATUS_READY | STATUS_BUSY | STATUS_DMA; @@ -1169,16 +1169,16 @@ static int result(void) return i; } if (status == (STATUS_DIR | STATUS_READY | STATUS_BUSY)) - reply_buffer[i] = fdc_inb(current_fdc, FD_DATA); + reply_buffer[i] = fdc_inb(fdc, FD_DATA); else break; } if (initialized) { DPRINT("get result error. Fdc=%d Last status=%x Read bytes=%d\n", - current_fdc, status, i); - show_floppy(current_fdc); + fdc, status, i); + show_floppy(fdc); } - fdc_state[current_fdc].reset = 1; + fdc_state[fdc].reset = 1; return -1; } @@ -1194,7 +1194,7 @@ static int need_more_output(void) if (is_ready_state(status)) return MORE_OUTPUT; - return result(); + return result(current_fdc); } /* Set perpendicular mode as required, based on data rate, if supported. @@ -1524,7 +1524,7 @@ static void setup_rw_floppy(void) } if (!(flags & FD_RAW_INTR)) { - inr = result(); + inr = result(current_fdc); cont->interrupt(); } else if (flags & FD_RAW_NEED_DISK) fd_watchdog(); @@ -1568,7 +1568,7 @@ static void check_wp(void) /* check write protection */ output_byte(current_fdc, FD_GETSTATUS); output_byte(current_fdc, UNIT(current_drive)); - if (result() != 1) { + if (result(current_fdc) != 1) { fdc_state[current_fdc].reset = 1; return; } @@ -1742,14 +1742,14 @@ irqreturn_t floppy_interrupt(int irq, void *dev_id) do_print = !handler && print_unex && initialized; - inr = result(); + inr = result(current_fdc); if (do_print) print_result("unexpected interrupt", inr); if (inr == 0) { int max_sensei = 4; do { output_byte(current_fdc, FD_SENSEI); - inr = result(); + inr = result(current_fdc); if (do_print) print_result("sensei", inr); max_sensei--; @@ -1782,7 +1782,7 @@ static void recalibrate_floppy(void) static void reset_interrupt(void) { debugt(__func__, ""); - result(); /* get the status ready for set_fdc */ + result(current_fdc); /* get the status ready for set_fdc */ if (fdc_state[current_fdc].reset) { pr_info("reset set in interrupt, calling %ps\n", cont->error); cont->error(); /* a reset just after a reset. BAD! */ @@ -4305,7 +4305,7 @@ static char __init get_fdc_version(void) output_byte(current_fdc, FD_DUMPREGS); /* 82072 and better know DUMPREGS */ if (fdc_state[current_fdc].reset) return FDC_NONE; - r = result(); + r = result(current_fdc); if (r <= 0x00) return FDC_NONE; /* No FDC present ??? */ if ((r == 1) && (reply_buffer[0] == 0x80)) { @@ -4332,7 +4332,7 @@ static char __init get_fdc_version(void) } output_byte(current_fdc, FD_UNLOCK); - r = result(); + r = result(current_fdc); if ((r == 1) && (reply_buffer[0] == 0x80)) { pr_info("FDC %d is a pre-1991 82077\n", current_fdc); return FDC_82077_ORIG; /* Pre-1991 82077, doesn't know @@ -4344,7 +4344,7 @@ static char __init get_fdc_version(void) return FDC_UNKNOWN; } output_byte(current_fdc, FD_PARTID); - r = result(); + r = result(current_fdc); if (r != 1) { pr_info("FDC %d init: PARTID: unexpected return of %d bytes.\n", current_fdc, r); From 3ab12a18209991fa430ea702d5d7d619bbb9ce67 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:46 +0200 Subject: [PATCH 0395/1043] floppy: cleanup: make need_more_output() not rely on current_fdc anymore Now the fdc is passed in argument so that the function does not use current_fdc anymore. Link: https://lore.kernel.org/r/20200331094054.24441-16-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 4aaf84217b53..aa2d840bf06b 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -1184,9 +1184,9 @@ static int result(int fdc) #define MORE_OUTPUT -2 /* does the fdc need more output? */ -static int need_more_output(void) +static int need_more_output(int fdc) { - int status = wait_til_ready(current_fdc); + int status = wait_til_ready(fdc); if (status < 0) return -1; @@ -1194,7 +1194,7 @@ static int need_more_output(void) if (is_ready_state(status)) return MORE_OUTPUT; - return result(current_fdc); + return result(fdc); } /* Set perpendicular mode as required, based on data rate, if supported. @@ -1244,7 +1244,7 @@ static int fdc_configure(void) { /* Turn on FIFO */ output_byte(current_fdc, FD_CONFIGURE); - if (need_more_output() != MORE_OUTPUT) + if (need_more_output(current_fdc) != MORE_OUTPUT) return 0; output_byte(current_fdc, 0); output_byte(current_fdc, 0x10 | (no_fifo & 0x20) | (fifo_depth & 0xf)); @@ -1302,7 +1302,7 @@ static void fdc_specify(void) /* chose the default rate table, not the one * where 1 = 2 Mbps */ output_byte(current_fdc, FD_DRIVESPEC); - if (need_more_output() == MORE_OUTPUT) { + if (need_more_output(current_fdc) == MORE_OUTPUT) { output_byte(current_fdc, UNIT(current_drive)); output_byte(current_fdc, 0xc0); } @@ -4324,7 +4324,7 @@ static char __init get_fdc_version(void) } output_byte(current_fdc, FD_PERPENDICULAR); - if (need_more_output() == MORE_OUTPUT) { + if (need_more_output(current_fdc) == MORE_OUTPUT) { output_byte(current_fdc, 0); } else { pr_info("FDC %d is an 82072A\n", current_fdc); From 197c7ffdb8165854e9e2f11a699d2fcca5adbd5a Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:47 +0200 Subject: [PATCH 0396/1043] floppy: cleanup: make perpendicular_mode() not rely on current_fdc anymore Now the fdc is passed in argument so that the function does not use current_fdc anymore. It's worth noting that there's still a single raw_cmd pointer specific to the current fdc. It may make sense to have one per fdc in the future. In addition, cont->done() still relies on the current drive and current raw_cmd. Link: https://lore.kernel.org/r/20200331094054.24441-17-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index aa2d840bf06b..fcccbb4c143e 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -1200,7 +1200,7 @@ static int need_more_output(int fdc) /* Set perpendicular mode as required, based on data rate, if supported. * 82077 Now tested. 1Mbps data rate only possible with 82077-1. */ -static void perpendicular_mode(void) +static void perpendicular_mode(int fdc) { unsigned char perp_mode; @@ -1215,7 +1215,7 @@ static void perpendicular_mode(void) default: DPRINT("Invalid data rate for perpendicular mode!\n"); cont->done(0); - fdc_state[current_fdc].reset = 1; + fdc_state[fdc].reset = 1; /* * convenient way to return to * redo without too much hassle @@ -1226,12 +1226,12 @@ static void perpendicular_mode(void) } else perp_mode = 0; - if (fdc_state[current_fdc].perp_mode == perp_mode) + if (fdc_state[fdc].perp_mode == perp_mode) return; - if (fdc_state[current_fdc].version >= FDC_82077_ORIG) { - output_byte(current_fdc, FD_PERPENDICULAR); - output_byte(current_fdc, perp_mode); - fdc_state[current_fdc].perp_mode = perp_mode; + if (fdc_state[fdc].version >= FDC_82077_ORIG) { + output_byte(fdc, FD_PERPENDICULAR); + output_byte(fdc, perp_mode); + fdc_state[fdc].perp_mode = perp_mode; } else if (perp_mode) { DPRINT("perpendicular mode not supported by this FDC.\n"); } @@ -1946,7 +1946,7 @@ static void floppy_ready(void) #endif if (raw_cmd->flags & (FD_RAW_NEED_SEEK | FD_RAW_NEED_DISK)) { - perpendicular_mode(); + perpendicular_mode(current_fdc); fdc_specify(); /* must be done here because of hut, hlt ... */ seek_floppy(); } else { From d5da6fa2b892fff23ffd1cb8a04bf618b6072807 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:48 +0200 Subject: [PATCH 0397/1043] floppy: cleanup: make fdc_configure() not rely on current_fdc anymore Now the fdc is passed in argument so that the function does not use current_fdc anymore. Link: https://lore.kernel.org/r/20200331094054.24441-18-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index fcccbb4c143e..c1338c4bb941 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -1240,16 +1240,15 @@ static void perpendicular_mode(int fdc) static int fifo_depth = 0xa; static int no_fifo; -static int fdc_configure(void) +static int fdc_configure(int fdc) { /* Turn on FIFO */ - output_byte(current_fdc, FD_CONFIGURE); - if (need_more_output(current_fdc) != MORE_OUTPUT) + output_byte(fdc, FD_CONFIGURE); + if (need_more_output(fdc) != MORE_OUTPUT) return 0; - output_byte(current_fdc, 0); - output_byte(current_fdc, 0x10 | (no_fifo & 0x20) | (fifo_depth & 0xf)); - output_byte(current_fdc, 0); /* pre-compensation from track - 0 upwards */ + output_byte(fdc, 0); + output_byte(fdc, 0x10 | (no_fifo & 0x20) | (fifo_depth & 0xf)); + output_byte(fdc, 0); /* pre-compensation from track 0 upwards */ return 1; } @@ -1288,7 +1287,7 @@ static void fdc_specify(void) if (fdc_state[current_fdc].need_configure && fdc_state[current_fdc].version >= FDC_82072A) { - fdc_configure(); + fdc_configure(current_fdc); fdc_state[current_fdc].need_configure = 0; } @@ -4318,7 +4317,7 @@ static char __init get_fdc_version(void) return FDC_UNKNOWN; } - if (!fdc_configure()) { + if (!fdc_configure(current_fdc)) { pr_info("FDC %d is an 82072\n", current_fdc); return FDC_82072; /* 82072 doesn't know CONFIGURE */ } From 3631a674a2ed7233905c0a7f37f09eeb83aa4d67 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:49 +0200 Subject: [PATCH 0398/1043] floppy: cleanup: make fdc_specify() not rely on current_{fdc,drive} anymore Now the fdc and drive are passed in argument so that the function does not use current_fdc nor current_drive anymore. Link: https://lore.kernel.org/r/20200331094054.24441-19-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index c1338c4bb941..b929b60afe9b 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -1273,7 +1273,7 @@ static int fdc_configure(int fdc) * * These values are rounded up to the next highest available delay time. */ -static void fdc_specify(void) +static void fdc_specify(int fdc, int drive) { unsigned char spec1; unsigned char spec2; @@ -1285,10 +1285,10 @@ static void fdc_specify(void) int hlt_max_code = 0x7f; int hut_max_code = 0xf; - if (fdc_state[current_fdc].need_configure && - fdc_state[current_fdc].version >= FDC_82072A) { - fdc_configure(current_fdc); - fdc_state[current_fdc].need_configure = 0; + if (fdc_state[fdc].need_configure && + fdc_state[fdc].version >= FDC_82072A) { + fdc_configure(fdc); + fdc_state[fdc].need_configure = 0; } switch (raw_cmd->rate & 0x03) { @@ -1297,13 +1297,13 @@ static void fdc_specify(void) break; case 1: dtr = 300; - if (fdc_state[current_fdc].version >= FDC_82078) { + if (fdc_state[fdc].version >= FDC_82078) { /* chose the default rate table, not the one * where 1 = 2 Mbps */ - output_byte(current_fdc, FD_DRIVESPEC); - if (need_more_output(current_fdc) == MORE_OUTPUT) { - output_byte(current_fdc, UNIT(current_drive)); - output_byte(current_fdc, 0xc0); + output_byte(fdc, FD_DRIVESPEC); + if (need_more_output(fdc) == MORE_OUTPUT) { + output_byte(fdc, UNIT(drive)); + output_byte(fdc, 0xc0); } } break; @@ -1312,14 +1312,14 @@ static void fdc_specify(void) break; } - if (fdc_state[current_fdc].version >= FDC_82072) { + if (fdc_state[fdc].version >= FDC_82072) { scale_dtr = dtr; hlt_max_code = 0x00; /* 0==256msec*dtr0/dtr (not linear!) */ hut_max_code = 0x0; /* 0==256msec*dtr0/dtr (not linear!) */ } /* Convert step rate from microseconds to milliseconds and 4 bits */ - srt = 16 - DIV_ROUND_UP(drive_params[current_drive].srt * scale_dtr / 1000, + srt = 16 - DIV_ROUND_UP(drive_params[drive].srt * scale_dtr / 1000, NOMINAL_DTR); if (slow_floppy) srt = srt / 4; @@ -1327,14 +1327,14 @@ static void fdc_specify(void) SUPBOUND(srt, 0xf); INFBOUND(srt, 0); - hlt = DIV_ROUND_UP(drive_params[current_drive].hlt * scale_dtr / 2, + hlt = DIV_ROUND_UP(drive_params[drive].hlt * scale_dtr / 2, NOMINAL_DTR); if (hlt < 0x01) hlt = 0x01; else if (hlt > 0x7f) hlt = hlt_max_code; - hut = DIV_ROUND_UP(drive_params[current_drive].hut * scale_dtr / 16, + hut = DIV_ROUND_UP(drive_params[drive].hut * scale_dtr / 16, NOMINAL_DTR); if (hut < 0x1) hut = 0x1; @@ -1345,12 +1345,12 @@ static void fdc_specify(void) spec2 = (hlt << 1) | (use_virtual_dma & 1); /* If these parameters did not change, just return with success */ - if (fdc_state[current_fdc].spec1 != spec1 || - fdc_state[current_fdc].spec2 != spec2) { + if (fdc_state[fdc].spec1 != spec1 || + fdc_state[fdc].spec2 != spec2) { /* Go ahead and set spec1 and spec2 */ - output_byte(current_fdc, FD_SPECIFY); - output_byte(current_fdc, fdc_state[current_fdc].spec1 = spec1); - output_byte(current_fdc, fdc_state[current_fdc].spec2 = spec2); + output_byte(fdc, FD_SPECIFY); + output_byte(fdc, fdc_state[fdc].spec1 = spec1); + output_byte(fdc, fdc_state[fdc].spec2 = spec2); } } /* fdc_specify */ @@ -1946,12 +1946,12 @@ static void floppy_ready(void) if (raw_cmd->flags & (FD_RAW_NEED_SEEK | FD_RAW_NEED_DISK)) { perpendicular_mode(current_fdc); - fdc_specify(); /* must be done here because of hut, hlt ... */ + fdc_specify(current_fdc, current_drive); /* must be done here because of hut, hlt ... */ seek_floppy(); } else { if ((raw_cmd->flags & FD_RAW_READ) || (raw_cmd->flags & FD_RAW_WRITE)) - fdc_specify(); + fdc_specify(current_fdc, current_drive); setup_rw_floppy(); } } From c7af70b0fb2535ee3f7165627fc0e73b1934dbfc Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:50 +0200 Subject: [PATCH 0399/1043] floppy: cleanup: make check_wp() not rely on current_{fdc,drive} anymore Now the fdc and drive are passed in argument so that the function does not use current_fdc nor current_drive anymore. Link: https://lore.kernel.org/r/20200331094054.24441-20-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index b929b60afe9b..b9a3a04c2636 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -1561,29 +1561,29 @@ static void seek_interrupt(void) floppy_ready(); } -static void check_wp(void) +static void check_wp(int fdc, int drive) { - if (test_bit(FD_VERIFY_BIT, &drive_state[current_drive].flags)) { + if (test_bit(FD_VERIFY_BIT, &drive_state[drive].flags)) { /* check write protection */ - output_byte(current_fdc, FD_GETSTATUS); - output_byte(current_fdc, UNIT(current_drive)); - if (result(current_fdc) != 1) { - fdc_state[current_fdc].reset = 1; + output_byte(fdc, FD_GETSTATUS); + output_byte(fdc, UNIT(drive)); + if (result(fdc) != 1) { + fdc_state[fdc].reset = 1; return; } - clear_bit(FD_VERIFY_BIT, &drive_state[current_drive].flags); + clear_bit(FD_VERIFY_BIT, &drive_state[drive].flags); clear_bit(FD_NEED_TWADDLE_BIT, - &drive_state[current_drive].flags); - debug_dcl(drive_params[current_drive].flags, + &drive_state[drive].flags); + debug_dcl(drive_params[drive].flags, "checking whether disk is write protected\n"); - debug_dcl(drive_params[current_drive].flags, "wp=%x\n", + debug_dcl(drive_params[drive].flags, "wp=%x\n", reply_buffer[ST3] & 0x40); if (!(reply_buffer[ST3] & 0x40)) set_bit(FD_DISK_WRITABLE_BIT, - &drive_state[current_drive].flags); + &drive_state[drive].flags); else clear_bit(FD_DISK_WRITABLE_BIT, - &drive_state[current_drive].flags); + &drive_state[drive].flags); } } @@ -1627,7 +1627,7 @@ static void seek_floppy(void) track = 1; } } else { - check_wp(); + check_wp(current_fdc, current_drive); if (raw_cmd->track != drive_state[current_drive].track && (raw_cmd->flags & FD_RAW_NEED_SEEK)) track = raw_cmd->track; From 43d81bb6470c431e17f093b3f7adf70fd33ef15a Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:51 +0200 Subject: [PATCH 0400/1043] floppy: cleanup: make next_valid_format() not rely on current_drive anymore Now the drive is passed in argument so that the function does not use current_drive anymore. Link: https://lore.kernel.org/r/20200331094054.24441-21-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index b9a3a04c2636..f53810ba486d 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -2058,18 +2058,18 @@ static void success_and_wakeup(void) * ========================== */ -static int next_valid_format(void) +static int next_valid_format(int drive) { int probed_format; - probed_format = drive_state[current_drive].probed_format; + probed_format = drive_state[drive].probed_format; while (1) { - if (probed_format >= 8 || !drive_params[current_drive].autodetect[probed_format]) { - drive_state[current_drive].probed_format = 0; + if (probed_format >= 8 || !drive_params[drive].autodetect[probed_format]) { + drive_state[drive].probed_format = 0; return 1; } - if (floppy_type[drive_params[current_drive].autodetect[probed_format]].sect) { - drive_state[current_drive].probed_format = probed_format; + if (floppy_type[drive_params[drive].autodetect[probed_format]].sect) { + drive_state[drive].probed_format = probed_format; return 0; } probed_format++; @@ -2082,7 +2082,7 @@ static void bad_flp_intr(void) if (probing) { drive_state[current_drive].probed_format++; - if (!next_valid_format()) + if (!next_valid_format(current_drive)) return; } err_count = ++(*errors); @@ -2884,7 +2884,7 @@ do_request: if (!_floppy) { /* Autodetection */ if (!probing) { drive_state[current_drive].probed_format = 0; - if (next_valid_format()) { + if (next_valid_format(current_drive)) { DPRINT("no autodetectable formats\n"); _floppy = NULL; request_done(0); From e5a9c95f9bdb8ca52ce1ee47bd04f07de0e119ae Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:52 +0200 Subject: [PATCH 0401/1043] floppy: cleanup: make get_fdc_version() not rely on current_fdc anymore Now the fdc is passed in argument so that the function does not use current_fdc anymore. Link: https://lore.kernel.org/r/20200331094054.24441-22-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 52 +++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index f53810ba486d..8850baa3372a 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -4297,79 +4297,79 @@ static const struct block_device_operations floppy_fops = { /* Determine the floppy disk controller type */ /* This routine was written by David C. Niemi */ -static char __init get_fdc_version(void) +static char __init get_fdc_version(int fdc) { int r; - output_byte(current_fdc, FD_DUMPREGS); /* 82072 and better know DUMPREGS */ - if (fdc_state[current_fdc].reset) + output_byte(fdc, FD_DUMPREGS); /* 82072 and better know DUMPREGS */ + if (fdc_state[fdc].reset) return FDC_NONE; - r = result(current_fdc); + r = result(fdc); if (r <= 0x00) return FDC_NONE; /* No FDC present ??? */ if ((r == 1) && (reply_buffer[0] == 0x80)) { - pr_info("FDC %d is an 8272A\n", current_fdc); + pr_info("FDC %d is an 8272A\n", fdc); return FDC_8272A; /* 8272a/765 don't know DUMPREGS */ } if (r != 10) { pr_info("FDC %d init: DUMPREGS: unexpected return of %d bytes.\n", - current_fdc, r); + fdc, r); return FDC_UNKNOWN; } - if (!fdc_configure(current_fdc)) { - pr_info("FDC %d is an 82072\n", current_fdc); + if (!fdc_configure(fdc)) { + pr_info("FDC %d is an 82072\n", fdc); return FDC_82072; /* 82072 doesn't know CONFIGURE */ } - output_byte(current_fdc, FD_PERPENDICULAR); - if (need_more_output(current_fdc) == MORE_OUTPUT) { - output_byte(current_fdc, 0); + output_byte(fdc, FD_PERPENDICULAR); + if (need_more_output(fdc) == MORE_OUTPUT) { + output_byte(fdc, 0); } else { - pr_info("FDC %d is an 82072A\n", current_fdc); + pr_info("FDC %d is an 82072A\n", fdc); return FDC_82072A; /* 82072A as found on Sparcs. */ } - output_byte(current_fdc, FD_UNLOCK); - r = result(current_fdc); + output_byte(fdc, FD_UNLOCK); + r = result(fdc); if ((r == 1) && (reply_buffer[0] == 0x80)) { - pr_info("FDC %d is a pre-1991 82077\n", current_fdc); + pr_info("FDC %d is a pre-1991 82077\n", fdc); return FDC_82077_ORIG; /* Pre-1991 82077, doesn't know * LOCK/UNLOCK */ } if ((r != 1) || (reply_buffer[0] != 0x00)) { pr_info("FDC %d init: UNLOCK: unexpected return of %d bytes.\n", - current_fdc, r); + fdc, r); return FDC_UNKNOWN; } - output_byte(current_fdc, FD_PARTID); - r = result(current_fdc); + output_byte(fdc, FD_PARTID); + r = result(fdc); if (r != 1) { pr_info("FDC %d init: PARTID: unexpected return of %d bytes.\n", - current_fdc, r); + fdc, r); return FDC_UNKNOWN; } if (reply_buffer[0] == 0x80) { - pr_info("FDC %d is a post-1991 82077\n", current_fdc); + pr_info("FDC %d is a post-1991 82077\n", fdc); return FDC_82077; /* Revised 82077AA passes all the tests */ } switch (reply_buffer[0] >> 5) { case 0x0: /* Either a 82078-1 or a 82078SL running at 5Volt */ - pr_info("FDC %d is an 82078.\n", current_fdc); + pr_info("FDC %d is an 82078.\n", fdc); return FDC_82078; case 0x1: - pr_info("FDC %d is a 44pin 82078\n", current_fdc); + pr_info("FDC %d is a 44pin 82078\n", fdc); return FDC_82078; case 0x2: - pr_info("FDC %d is a S82078B\n", current_fdc); + pr_info("FDC %d is a S82078B\n", fdc); return FDC_S82078B; case 0x3: - pr_info("FDC %d is a National Semiconductor PC87306\n", current_fdc); + pr_info("FDC %d is a National Semiconductor PC87306\n", fdc); return FDC_87306; default: pr_info("FDC %d init: 82078 variant with unknown PARTID=%d.\n", - current_fdc, reply_buffer[0] >> 5); + fdc, reply_buffer[0] >> 5); return FDC_82078_UNKN; } } /* get_fdc_version */ @@ -4711,7 +4711,7 @@ static int __init do_floppy_init(void) continue; } /* Try to determine the floppy controller type */ - fdc_state[current_fdc].version = get_fdc_version(); + fdc_state[current_fdc].version = get_fdc_version(current_fdc); if (fdc_state[current_fdc].version == FDC_NONE) { /* free ioports reserved by floppy_grab_irq_and_dma() */ floppy_release_regions(current_fdc); From 82a630105847d7b7657901643810542212082af6 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:53 +0200 Subject: [PATCH 0402/1043] floppy: cleanup: do not iterate on current_fdc in DMA grab/release functions Both floppy_grab_irq_and_dma() and floppy_release_irq_and_dma() used to iterate on the global variable while setting up or freeing resources. Now that they exclusively rely on functions which take the fdc as an argument, so let's not touch the global one anymore. Link: https://lore.kernel.org/r/20200331094054.24441-23-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 8850baa3372a..77bb9a5fcd33 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -4854,6 +4854,8 @@ static void floppy_release_regions(int fdc) static int floppy_grab_irq_and_dma(void) { + int fdc; + if (atomic_inc_return(&usage_count) > 1) return 0; @@ -4881,24 +4883,24 @@ static int floppy_grab_irq_and_dma(void) } } - for (current_fdc = 0; current_fdc < N_FDC; current_fdc++) { - if (fdc_state[current_fdc].address != -1) { - if (floppy_request_regions(current_fdc)) + for (fdc = 0; fdc < N_FDC; fdc++) { + if (fdc_state[fdc].address != -1) { + if (floppy_request_regions(fdc)) goto cleanup; } } - for (current_fdc = 0; current_fdc < N_FDC; current_fdc++) { - if (fdc_state[current_fdc].address != -1) { - reset_fdc_info(current_fdc, 1); - fdc_outb(fdc_state[current_fdc].dor, current_fdc, FD_DOR); + for (fdc = 0; fdc < N_FDC; fdc++) { + if (fdc_state[fdc].address != -1) { + reset_fdc_info(fdc, 1); + fdc_outb(fdc_state[fdc].dor, fdc, FD_DOR); } } - current_fdc = 0; + set_dor(0, ~0, 8); /* avoid immediate interrupt */ - for (current_fdc = 0; current_fdc < N_FDC; current_fdc++) - if (fdc_state[current_fdc].address != -1) - fdc_outb(fdc_state[current_fdc].dor, current_fdc, FD_DOR); + for (fdc = 0; fdc < N_FDC; fdc++) + if (fdc_state[fdc].address != -1) + fdc_outb(fdc_state[fdc].dor, fdc, FD_DOR); /* * The driver will try and free resources and relies on us * to know if they were allocated or not. @@ -4909,15 +4911,16 @@ static int floppy_grab_irq_and_dma(void) cleanup: fd_free_irq(); fd_free_dma(); - while (--current_fdc >= 0) - floppy_release_regions(current_fdc); + while (--fdc >= 0) + floppy_release_regions(fdc); + current_fdc = 0; atomic_dec(&usage_count); return -1; } static void floppy_release_irq_and_dma(void) { - int old_fdc; + int fdc; #ifndef __sparc__ int drive; #endif @@ -4958,11 +4961,9 @@ static void floppy_release_irq_and_dma(void) pr_info("auxiliary floppy timer still active\n"); if (work_pending(&floppy_work)) pr_info("work still pending\n"); - old_fdc = current_fdc; - for (current_fdc = 0; current_fdc < N_FDC; current_fdc++) - if (fdc_state[current_fdc].address != -1) - floppy_release_regions(current_fdc); - current_fdc = old_fdc; + for (fdc = 0; fdc < N_FDC; fdc++) + if (fdc_state[fdc].address != -1) + floppy_release_regions(fdc); } #ifdef MODULE From 12aebfac27ab69b5ed333c94fda45ef31ba2fc2a Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Tue, 31 Mar 2020 11:40:54 +0200 Subject: [PATCH 0403/1043] floppy: cleanup: add a few comments about expectations in certain functions The locking in the driver is far from being obvious, with unlocking automatically happening at end of operations scheduled by interrupt, especially for the error paths where one does not necessarily expect that such an interrupt may be triggered. Let's add a few comments about what to expect at certain places to avoid misdetecting bugs which are not. Link: https://lore.kernel.org/r/20200331094054.24441-24-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 77bb9a5fcd33..07218f8b17f9 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -1791,7 +1791,9 @@ static void reset_interrupt(void) /* * reset is done by pulling bit 2 of DOR low for a while (old FDCs), - * or by setting the self clearing bit 7 of STATUS (newer FDCs) + * or by setting the self clearing bit 7 of STATUS (newer FDCs). + * This WILL trigger an interrupt, causing the handlers in the current + * cont's ->redo() to be called via reset_interrupt(). */ static void reset_fdc(void) { @@ -2003,6 +2005,9 @@ static const struct cont_t intr_cont = { .done = (done_f)empty }; +/* schedules handler, waiting for completion. May be interrupted, will then + * return -EINTR, in which case the driver will automatically be unlocked. + */ static int wait_til_done(void (*handler)(void), bool interruptible) { int ret; @@ -2842,6 +2847,9 @@ static int set_next_request(void) return current_req != NULL; } +/* Starts or continues processing request. Will automatically unlock the + * driver at end of request. + */ static void redo_fd_request(void) { int drive; @@ -2916,6 +2924,7 @@ static const struct cont_t rw_cont = { .done = request_done }; +/* schedule the request and automatically unlock the driver on completion */ static void process_fd_request(void) { cont = &rw_cont; @@ -3005,6 +3014,9 @@ static int user_reset_fdc(int drive, int arg, bool interruptible) if (arg == FD_RESET_ALWAYS) fdc_state[current_fdc].reset = 1; if (fdc_state[current_fdc].reset) { + /* note: reset_fdc will take care of unlocking the driver + * on completion. + */ cont = &reset_cont; ret = wait_til_done(reset_fdc, interruptible); if (ret == -EINTR) From 05f5e319a1eb017442cd0eec87ad52a62d8c3224 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Fri, 10 Apr 2020 11:30:23 +0200 Subject: [PATCH 0404/1043] floppy: cleanup: do not iterate on current_fdc in do_floppy_init() There's no need to iterate on current_fdc in do_floppy_init() anymore, in the first case it's only used as an array index to access fdc_state[], so let's get rid of this confusing assignment. The second case is a bit trickier because user_reset_fdc() needs to already know current_fdc when called with drive==-1 due to this call chain: user_reset_fdc() lock_fdc() set_fdc() drive<0 ==> new_fdc = current_fdc Note that current_drive is not used in this code part and may even not match a unit belonging to current_fdc. Instead of passing -1 we can simply pass the first drive of the FDC being initialized, which is even cleaner as it will allow the function chain above to consistently assign both variables. Link: https://lore.kernel.org/r/20200410093023.14499-1-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 07218f8b17f9..8da7921659f1 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -4657,16 +4657,15 @@ static int __init do_floppy_init(void) config_types(); for (i = 0; i < N_FDC; i++) { - current_fdc = i; - memset(&fdc_state[current_fdc], 0, sizeof(*fdc_state)); - fdc_state[current_fdc].dtr = -1; - fdc_state[current_fdc].dor = 0x4; + memset(&fdc_state[i], 0, sizeof(*fdc_state)); + fdc_state[i].dtr = -1; + fdc_state[i].dor = 0x4; #if defined(__sparc__) || defined(__mc68000__) /*sparcs/sun3x don't have a DOR reset which we can fall back on to */ #ifdef __mc68000__ if (MACH_IS_SUN3X) #endif - fdc_state[current_fdc].version = FDC_82072A; + fdc_state[i].version = FDC_82072A; #endif } @@ -4708,30 +4707,29 @@ static int __init do_floppy_init(void) msleep(10); for (i = 0; i < N_FDC; i++) { - current_fdc = i; - fdc_state[current_fdc].driver_version = FD_DRIVER_VERSION; + fdc_state[i].driver_version = FD_DRIVER_VERSION; for (unit = 0; unit < 4; unit++) - fdc_state[current_fdc].track[unit] = 0; - if (fdc_state[current_fdc].address == -1) + fdc_state[i].track[unit] = 0; + if (fdc_state[i].address == -1) continue; - fdc_state[current_fdc].rawcmd = 2; - if (user_reset_fdc(-1, FD_RESET_ALWAYS, false)) { + fdc_state[i].rawcmd = 2; + if (user_reset_fdc(REVDRIVE(i, 0), FD_RESET_ALWAYS, false)) { /* free ioports reserved by floppy_grab_irq_and_dma() */ - floppy_release_regions(current_fdc); - fdc_state[current_fdc].address = -1; - fdc_state[current_fdc].version = FDC_NONE; + floppy_release_regions(i); + fdc_state[i].address = -1; + fdc_state[i].version = FDC_NONE; continue; } /* Try to determine the floppy controller type */ - fdc_state[current_fdc].version = get_fdc_version(current_fdc); - if (fdc_state[current_fdc].version == FDC_NONE) { + fdc_state[i].version = get_fdc_version(i); + if (fdc_state[i].version == FDC_NONE) { /* free ioports reserved by floppy_grab_irq_and_dma() */ - floppy_release_regions(current_fdc); - fdc_state[current_fdc].address = -1; + floppy_release_regions(i); + fdc_state[i].address = -1; continue; } if (can_use_virtual_dma == 2 && - fdc_state[current_fdc].version < FDC_82072A) + fdc_state[i].version < FDC_82072A) can_use_virtual_dma = 0; have_no_fdc = 0; @@ -4739,7 +4737,7 @@ static int __init do_floppy_init(void) * properly, so force a reset for the standard FDC clones, * to avoid interrupt garbage. */ - user_reset_fdc(-1, FD_RESET_ALWAYS, false); + user_reset_fdc(REVDRIVE(i, 0), FD_RESET_ALWAYS, false); } current_fdc = 0; cancel_delayed_work(&fd_timeout); From 6111a4f9bb189e76cda6a306074c9746ddeef04b Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Fri, 10 Apr 2020 12:19:02 +0200 Subject: [PATCH 0405/1043] floppy: make sure to reset all FDCs upon resume() In floppy_resume() we don't properly reinitialize all FDCs, instead we reinitialize the current FDC once per available FDC because value -1 is passed to user_reset_fdc(). Let's simply save the current drive and properly reinitialize each FDC. Link: https://lore.kernel.org/r/20200410101904.14652-1-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 8da7921659f1..b102f55dfa5d 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -4545,11 +4545,13 @@ static void floppy_device_release(struct device *dev) static int floppy_resume(struct device *dev) { int fdc; + int saved_drive; + saved_drive = current_drive; for (fdc = 0; fdc < N_FDC; fdc++) if (fdc_state[fdc].address != -1) - user_reset_fdc(-1, FD_RESET_ALWAYS, false); - + user_reset_fdc(REVDRIVE(fdc, 0), FD_RESET_ALWAYS, false); + set_fdc(saved_drive); return 0; } From 99ba6ccc7f8f362ae52ddddda2252e753329c7ec Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Fri, 10 Apr 2020 12:19:03 +0200 Subject: [PATCH 0406/1043] floppy: cleanup: get rid of current_reqD in favor of current_drive This macro equals -1 and is used as an alternative for current_drive when calling reschedule_timeout(), which in turn needs to remap it. This only adds obfuscation, let's simply use current_drive. Link: https://lore.kernel.org/r/20200410101904.14652-2-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index b102f55dfa5d..20646d4c5437 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -668,16 +668,12 @@ static struct output_log { static int output_log_pos; -#define current_reqD -1 #define MAXTIMEOUT -2 static void __reschedule_timeout(int drive, const char *message) { unsigned long delay; - if (drive == current_reqD) - drive = current_drive; - if (drive < 0 || drive >= N_DRIVE) { delay = 20UL * HZ; drive = 0; @@ -1960,7 +1956,7 @@ static void floppy_ready(void) static void floppy_start(void) { - reschedule_timeout(current_reqD, "floppy start"); + reschedule_timeout(current_drive, "floppy start"); scandrives(); debug_dcl(drive_params[current_drive].flags, @@ -2874,7 +2870,7 @@ do_request: } drive = (long)current_req->rq_disk->private_data; set_fdc(drive); - reschedule_timeout(current_reqD, "redo fd request"); + reschedule_timeout(current_drive, "redo fd request"); set_floppy(drive); raw_cmd = &default_raw_cmd; From ca1b409a3b8a190c13bb30ed3ad91585d434d8e2 Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Fri, 10 Apr 2020 12:19:04 +0200 Subject: [PATCH 0407/1043] floppy: cleanup: make set_fdc() always set current_drive and current_fd When called with a negative drive value, set_fdc() would stick to the current fdc (which was assumed to reflect the current_drive's FDC). We do not need this anymore as the last call place with a negative value was just addressed. Let's make this function always set both current_fdc and current_drive so that there's no more ambiguity. A few comments stating this were added to a few non-obvious places. Link: https://lore.kernel.org/r/20200410101904.14652-3-w@1wt.eu Signed-off-by: Willy Tarreau Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 43 ++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 20646d4c5437..2817170dd403 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -851,31 +851,42 @@ static void reset_fdc_info(int fdc, int mode) drive_state[drive].track = NEED_2_RECAL; } -/* selects the fdc and drive, and enables the fdc's input/dma. */ +/* + * selects the fdc and drive, and enables the fdc's input/dma. + * Both current_drive and current_fdc are changed to match the new drive. + */ static void set_fdc(int drive) { - unsigned int new_fdc = current_fdc; + unsigned int fdc; - if (drive >= 0 && drive < N_DRIVE) { - new_fdc = FDC(drive); - current_drive = drive; + if (drive < 0 || drive >= N_DRIVE) { + pr_info("bad drive value %d\n", drive); + return; } - if (new_fdc >= N_FDC) { + + fdc = FDC(drive); + if (fdc >= N_FDC) { pr_info("bad fdc value\n"); return; } - current_fdc = new_fdc; - set_dor(current_fdc, ~0, 8); + + set_dor(fdc, ~0, 8); #if N_FDC > 1 - set_dor(1 - current_fdc, ~8, 0); + set_dor(1 - fdc, ~8, 0); #endif - if (fdc_state[current_fdc].rawcmd == 2) - reset_fdc_info(current_fdc, 1); - if (fdc_inb(current_fdc, FD_STATUS) != STATUS_READY) - fdc_state[current_fdc].reset = 1; + if (fdc_state[fdc].rawcmd == 2) + reset_fdc_info(fdc, 1); + if (fdc_inb(fdc, FD_STATUS) != STATUS_READY) + fdc_state[fdc].reset = 1; + + current_drive = drive; + current_fdc = fdc; } -/* locks the driver */ +/* + * locks the driver. + * Both current_drive and current_fdc are changed to match the new drive. + */ static int lock_fdc(int drive) { if (WARN(atomic_read(&usage_count) == 0, @@ -3000,6 +3011,10 @@ static const struct cont_t reset_cont = { .done = generic_done }; +/* + * Resets the FDC connected to drive . + * Both current_drive and current_fdc are changed to match the new drive. + */ static int user_reset_fdc(int drive, int arg, bool interruptible) { int ret; From 29ac67633c893dec0024fb7597860fde52fdc819 Mon Sep 17 00:00:00 2001 From: Denis Efremov Date: Fri, 1 May 2020 16:44:13 +0300 Subject: [PATCH 0408/1043] floppy: use print_hex_dump() in setup_DMA() Remove pr_cont() and use print_hex_dump() in setup_DMA() to print the contents of the cmd buffer. Link: https://lore.kernel.org/r/20200501134416.72248-2-efremov@linux.com Suggested-by: Joe Perches Reviewed-by: Christoph Hellwig Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 2817170dd403..3ab6e804b5ec 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -1069,12 +1069,9 @@ static void setup_DMA(void) unsigned long f; if (raw_cmd->length == 0) { - int i; - - pr_info("zero dma transfer size:"); - for (i = 0; i < raw_cmd->cmd_count; i++) - pr_cont("%x,", raw_cmd->cmd[i]); - pr_cont("\n"); + print_hex_dump(KERN_INFO, "zero dma transfer size: ", + DUMP_PREFIX_NONE, 16, 1, + raw_cmd->cmd, raw_cmd->cmd_count, false); cont->done(0); fdc_state[current_fdc].reset = 1; return; From 9c4c5a24c85585fb8904bd2872501cd8181b3854 Mon Sep 17 00:00:00 2001 From: Denis Efremov Date: Fri, 1 May 2020 16:44:14 +0300 Subject: [PATCH 0409/1043] floppy: add FD_AUTODETECT_SIZE define for struct floppy_drive_params Use FD_AUTODETECT_SIZE for autodetect buffer size in struct floppy_drive_params instead of a magic number. Link: https://lore.kernel.org/r/20200501134416.72248-3-efremov@linux.com Reviewed-by: Christoph Hellwig Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 9 +++++---- include/uapi/linux/fd.h | 5 ++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 3ab6e804b5ec..b82b3d38b834 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -2073,7 +2073,8 @@ static int next_valid_format(int drive) probed_format = drive_state[drive].probed_format; while (1) { - if (probed_format >= 8 || !drive_params[drive].autodetect[probed_format]) { + if (probed_format >= FD_AUTODETECT_SIZE || + !drive_params[drive].autodetect[probed_format]) { drive_state[drive].probed_format = 0; return 1; } @@ -3442,13 +3443,13 @@ static int fd_getgeo(struct block_device *bdev, struct hd_geometry *geo) return 0; } -static bool valid_floppy_drive_params(const short autodetect[8], +static bool valid_floppy_drive_params(const short autodetect[FD_AUTODETECT_SIZE], int native_format) { size_t floppy_type_size = ARRAY_SIZE(floppy_type); size_t i = 0; - for (i = 0; i < 8; ++i) { + for (i = 0; i < FD_AUTODETECT_SIZE; ++i) { if (autodetect[i] < 0 || autodetect[i] >= floppy_type_size) return false; @@ -3673,7 +3674,7 @@ struct compat_floppy_drive_params { struct floppy_max_errors max_errors; char flags; char read_track; - short autodetect[8]; + short autodetect[FD_AUTODETECT_SIZE]; compat_int_t checkfreq; compat_int_t native_format; }; diff --git a/include/uapi/linux/fd.h b/include/uapi/linux/fd.h index 90fb94712c41..3f6b7be4c096 100644 --- a/include/uapi/linux/fd.h +++ b/include/uapi/linux/fd.h @@ -172,7 +172,10 @@ struct floppy_drive_params { * used in succession to try to read the disk. If the FDC cannot lock onto * the disk, the next format is tried. This uses the variable 'probing'. */ - short autodetect[8]; /* autodetected formats */ + +#define FD_AUTODETECT_SIZE 8 + + short autodetect[FD_AUTODETECT_SIZE]; /* autodetected formats */ int checkfreq; /* how often should the drive be checked for disk * changes */ From bd10a5f3e21b1cb8e2133c1f08b3e8207cee12dd Mon Sep 17 00:00:00 2001 From: Denis Efremov Date: Fri, 1 May 2020 16:44:15 +0300 Subject: [PATCH 0410/1043] floppy: add defines for sizes of cmd & reply buffers of floppy_raw_cmd Use FD_RAW_CMD_SIZE, FD_RAW_REPLY_SIZE defines instead of magic numbers for cmd & reply buffers of struct floppy_raw_cmd. Remove local to floppy.c MAX_REPLIES define, as it is now FD_RAW_REPLY_SIZE. FD_RAW_CMD_FULLSIZE added as we allow command to also fill reply_count and reply fields. Link: https://lore.kernel.org/r/20200501134416.72248-4-efremov@linux.com Reviewed-by: Christoph Hellwig Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 19 +++++-------------- include/uapi/linux/fd.h | 14 ++++++++++++-- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index b82b3d38b834..9e098d53b046 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -337,8 +337,7 @@ static bool initialized; /* * globals used by 'result()' */ -#define MAX_REPLIES 16 -static unsigned char reply_buffer[MAX_REPLIES]; +static unsigned char reply_buffer[FD_RAW_REPLY_SIZE]; static int inr; /* size of reply buffer, when called from interrupt */ #define ST0 0 #define ST1 1 @@ -1162,7 +1161,7 @@ static int result(int fdc) int i; int status = 0; - for (i = 0; i < MAX_REPLIES; i++) { + for (i = 0; i < FD_RAW_REPLY_SIZE; i++) { status = wait_til_ready(fdc); if (status < 0) break; @@ -3079,7 +3078,7 @@ static void raw_cmd_done(int flag) raw_cmd->flags |= FD_RAW_HARDFAILURE; } else { raw_cmd->reply_count = inr; - if (raw_cmd->reply_count > MAX_REPLIES) + if (raw_cmd->reply_count > FD_RAW_REPLY_SIZE) raw_cmd->reply_count = 0; for (i = 0; i < raw_cmd->reply_count; i++) raw_cmd->reply[i] = reply_buffer[i]; @@ -3190,18 +3189,10 @@ loop: if (ret) return -EFAULT; param += sizeof(struct floppy_raw_cmd); - if (ptr->cmd_count > 33) - /* the command may now also take up the space - * initially intended for the reply & the - * reply count. Needed for long 82078 commands - * such as RESTORE, which takes ... 17 command - * bytes. Murphy's law #137: When you reserve - * 16 bytes for a structure, you'll one day - * discover that you really need 17... - */ + if (ptr->cmd_count > FD_RAW_CMD_FULLSIZE) return -EINVAL; - for (i = 0; i < 16; i++) + for (i = 0; i < FD_RAW_REPLY_SIZE; i++) ptr->reply[i] = 0; ptr->resultcode = 0; diff --git a/include/uapi/linux/fd.h b/include/uapi/linux/fd.h index 3f6b7be4c096..2e9c2c1c18e6 100644 --- a/include/uapi/linux/fd.h +++ b/include/uapi/linux/fd.h @@ -360,10 +360,20 @@ struct floppy_raw_cmd { int buffer_length; /* length of allocated buffer */ unsigned char rate; + +#define FD_RAW_CMD_SIZE 16 +#define FD_RAW_REPLY_SIZE 16 +#define FD_RAW_CMD_FULLSIZE (FD_RAW_CMD_SIZE + 1 + FD_RAW_REPLY_SIZE) + + /* The command may take up the space initially intended for the reply + * and the reply count. Needed for long 82078 commands such as RESTORE, + * which takes 17 command bytes. + */ + unsigned char cmd_count; - unsigned char cmd[16]; + unsigned char cmd[FD_RAW_CMD_SIZE]; unsigned char reply_count; - unsigned char reply[16]; + unsigned char reply[FD_RAW_REPLY_SIZE]; int track; int resultcode; From 0836275df4db20daf040fff5d9a1da89c4c08a85 Mon Sep 17 00:00:00 2001 From: Denis Efremov Date: Fri, 1 May 2020 16:44:16 +0300 Subject: [PATCH 0411/1043] floppy: suppress UBSAN warning in setup_rw_floppy() UBSAN: array-index-out-of-bounds in drivers/block/floppy.c:1521:45 index 16 is out of range for type 'unsigned char [16]' Call Trace: ... setup_rw_floppy+0x5c3/0x7f0 floppy_ready+0x2be/0x13b0 process_one_work+0x2c1/0x5d0 worker_thread+0x56/0x5e0 kthread+0x122/0x170 ret_from_fork+0x35/0x40 From include/uapi/linux/fd.h: struct floppy_raw_cmd { ... unsigned char cmd_count; unsigned char cmd[16]; unsigned char reply_count; unsigned char reply[16]; ... } This out-of-bounds access is intentional. The command in struct floppy_raw_cmd may take up the space initially intended for the reply and the reply count. It is needed for long 82078 commands such as RESTORE, which takes 17 command bytes. Initial cmd size is not enough and since struct setup_rw_floppy is a part of uapi we check that cmd_count is in [0:16+1+16] in raw_cmd_copyin(). The patch adds union with original cmd,reply_count,reply fields and fullcmd field of equivalent size. The cmd accesses are turned to fullcmd where appropriate to suppress UBSAN warning. Link: https://lore.kernel.org/r/20200501134416.72248-5-efremov@linux.com Reviewed-by: Christoph Hellwig Signed-off-by: Denis Efremov --- drivers/block/floppy.c | 4 ++-- include/uapi/linux/fd.h | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 9e098d53b046..064c1acb9f00 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -1070,7 +1070,7 @@ static void setup_DMA(void) if (raw_cmd->length == 0) { print_hex_dump(KERN_INFO, "zero dma transfer size: ", DUMP_PREFIX_NONE, 16, 1, - raw_cmd->cmd, raw_cmd->cmd_count, false); + raw_cmd->fullcmd, raw_cmd->cmd_count, false); cont->done(0); fdc_state[current_fdc].reset = 1; return; @@ -1515,7 +1515,7 @@ static void setup_rw_floppy(void) r = 0; for (i = 0; i < raw_cmd->cmd_count; i++) - r |= output_byte(current_fdc, raw_cmd->cmd[i]); + r |= output_byte(current_fdc, raw_cmd->fullcmd[i]); debugt(__func__, "rw_command"); diff --git a/include/uapi/linux/fd.h b/include/uapi/linux/fd.h index 2e9c2c1c18e6..8b80c63b971c 100644 --- a/include/uapi/linux/fd.h +++ b/include/uapi/linux/fd.h @@ -371,9 +371,14 @@ struct floppy_raw_cmd { */ unsigned char cmd_count; - unsigned char cmd[FD_RAW_CMD_SIZE]; - unsigned char reply_count; - unsigned char reply[FD_RAW_REPLY_SIZE]; + union { + struct { + unsigned char cmd[FD_RAW_CMD_SIZE]; + unsigned char reply_count; + unsigned char reply[FD_RAW_REPLY_SIZE]; + }; + unsigned char fullcmd[FD_RAW_CMD_FULLSIZE]; + }; int track; int resultcode; From 26bff9eb49201aeb4e1b32d698c191831a39f5d4 Mon Sep 17 00:00:00 2001 From: Thomas Bogendoerfer Date: Wed, 6 May 2020 18:24:02 +0200 Subject: [PATCH 0412/1043] MIPS: Only include the platform file needed Instead of including all Platform files, we simply include the needed one and avoid clashes with makefile variables. Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kbuild.platforms | 73 +++++++++++++++++--------------- arch/mips/alchemy/Platform | 3 -- arch/mips/ar7/Platform | 1 - arch/mips/ath25/Platform | 1 - arch/mips/ath79/Platform | 1 - arch/mips/bcm47xx/Platform | 1 - arch/mips/bcm63xx/Platform | 1 - arch/mips/bmips/Platform | 1 - arch/mips/cavium-octeon/Platform | 1 - arch/mips/cobalt/Platform | 1 - arch/mips/dec/Platform | 1 - arch/mips/generic/Platform | 1 - arch/mips/jazz/Platform | 1 - arch/mips/jz4740/Platform | 1 - arch/mips/lantiq/Platform | 1 - arch/mips/loongson2ef/Platform | 1 - arch/mips/loongson32/Platform | 1 - arch/mips/loongson64/Platform | 1 - arch/mips/mti-malta/Platform | 1 - arch/mips/netlogic/Platform | 1 - arch/mips/paravirt/Platform | 1 - arch/mips/pic32/Platform | 1 - arch/mips/pistachio/Platform | 1 - arch/mips/pnx833x/Platform | 1 - arch/mips/rb532/Platform | 1 - arch/mips/sgi-ip22/Platform | 2 - arch/mips/sgi-ip27/Platform | 3 -- arch/mips/sgi-ip30/Platform | 3 -- arch/mips/sgi-ip32/Platform | 1 - arch/mips/sibyte/Platform | 4 -- arch/mips/sni/Platform | 1 - arch/mips/txx9/Platform | 3 -- 32 files changed, 39 insertions(+), 77 deletions(-) diff --git a/arch/mips/Kbuild.platforms b/arch/mips/Kbuild.platforms index 292b59afb4ba..5e3f6ed96292 100644 --- a/arch/mips/Kbuild.platforms +++ b/arch/mips/Kbuild.platforms @@ -1,39 +1,44 @@ # SPDX-License-Identifier: GPL-2.0 # All platforms listed in alphabetic order -platforms += alchemy -platforms += ar7 -platforms += ath25 -platforms += ath79 -platforms += bcm47xx -platforms += bcm63xx -platforms += bmips -platforms += cavium-octeon -platforms += cobalt -platforms += dec -platforms += generic -platforms += jazz -platforms += jz4740 -platforms += lantiq -platforms += loongson2ef -platforms += loongson32 -platforms += loongson64 -platforms += mti-malta -platforms += netlogic -platforms += paravirt -platforms += pic32 -platforms += pistachio -platforms += pnx833x -platforms += ralink -platforms += rb532 -platforms += sgi-ip22 -platforms += sgi-ip27 -platforms += sgi-ip30 -platforms += sgi-ip32 -platforms += sibyte -platforms += sni -platforms += txx9 -platforms += vr41xx +platform-$(CONFIG_MIPS_ALCHEMY) += alchemy/ +platform-$(CONFIG_AR7) += ar7/ +platform-$(CONFIG_ATH25) += ath25/ +platform-$(CONFIG_ATH79) += ath79/ +platform-$(CONFIG_BCM47XX) += bcm47xx/ +platform-$(CONFIG_BCM63XX) += bcm63xx/ +platform-$(CONFIG_BMIPS_GENERIC) += bmips/ +platform-$(CONFIG_CAVIUM_OCTEON_SOC) += cavium-octeon/ +platform-$(CONFIG_MIPS_COBALT) += cobalt/ +platform-$(CONFIG_MACH_DECSTATION) += dec/ +platform-$(CONFIG_MIPS_GENERIC) += generic/ +platform-$(CONFIG_MACH_JAZZ) += jazz/ +platform-$(CONFIG_MACH_INGENIC) += jz4740/ +platform-$(CONFIG_LANTIQ) += lantiq/ +platform-$(CONFIG_MACH_LOONGSON2EF) += loongson2ef/ +platform-$(CONFIG_MACH_LOONGSON32) += loongson32/ +platform-$(CONFIG_MACH_LOONGSON64) += loongson64/ +platform-$(CONFIG_MIPS_MALTA) += mti-malta/ +platform-$(CONFIG_NLM_COMMON) += netlogic/ +platform-$(CONFIG_MIPS_PARAVIRT) += paravirt/ +platform-$(CONFIG_PIC32MZDA) += pic32/ +platform-$(CONFIG_MACH_PISTACHIO) += pistachio/ +platform-$(CONFIG_SOC_PNX833X) += pnx833x/ +platform-$(CONFIG_RALINK) += ralink/ +platform-$(CONFIG_MIKROTIK_RB532) += rb532/ +platform-$(CONFIG_SGI_IP22) += sgi-ip22/ +platform-$(CONFIG_SGI_IP27) += sgi-ip27/ +platform-$(CONFIG_SGI_IP28) += sgi-ip22/ +platform-$(CONFIG_SGI_IP30) += sgi-ip30/ +platform-$(CONFIG_SGI_IP32) += sgi-ip32/ +platform-$(CONFIG_SIBYTE_BCM112X) += sibyte/ +platform-$(CONFIG_SIBYTE_SB1250) += sibyte/ +platform-$(CONFIG_SIBYTE_BCM1x55) += sibyte/ +platform-$(CONFIG_SIBYTE_BCM1x80) += sibyte/ +platform-$(CONFIG_SNI_RM) += sni/ +platform-$(CONFIG_MACH_TX39XX) += tx99/ +platform-$(CONFIG_MACH_TX49XX) += tx99/ +platform-$(CONFIG_MACH_VR41XX) += vr41xx/ # include the platform specific files -include $(patsubst %, $(srctree)/arch/mips/%/Platform, $(platforms)) +include $(patsubst %, $(srctree)/arch/mips/%/Platform, $(platform-y)) diff --git a/arch/mips/alchemy/Platform b/arch/mips/alchemy/Platform index 33c9da3b077b..c8cff50b0eda 100644 --- a/arch/mips/alchemy/Platform +++ b/arch/mips/alchemy/Platform @@ -15,19 +15,16 @@ load-$(CONFIG_MIPS_DB1XXX) += 0xffffffff80100000 # # 4G-Systems MTX-1 "MeshCube" wireless router # -platform-$(CONFIG_MIPS_MTX1) += alchemy/ load-$(CONFIG_MIPS_MTX1) += 0xffffffff80100000 # # MyCable eval board # -platform-$(CONFIG_MIPS_XXS1500) += alchemy/ load-$(CONFIG_MIPS_XXS1500) += 0xffffffff80100000 # # Trapeze ITS GRP board # -platform-$(CONFIG_MIPS_GPR) += alchemy/ load-$(CONFIG_MIPS_GPR) += 0xffffffff80100000 # boards can specify their own in one of their include dirs. diff --git a/arch/mips/ar7/Platform b/arch/mips/ar7/Platform index 21f9102d533c..a9257cc01c3c 100644 --- a/arch/mips/ar7/Platform +++ b/arch/mips/ar7/Platform @@ -1,6 +1,5 @@ # # Texas Instruments AR7 # -platform-$(CONFIG_AR7) += ar7/ cflags-$(CONFIG_AR7) += -I$(srctree)/arch/mips/include/asm/mach-ar7 load-$(CONFIG_AR7) += 0xffffffff94100000 diff --git a/arch/mips/ath25/Platform b/arch/mips/ath25/Platform index ef3f81fa080b..aef098b6f405 100644 --- a/arch/mips/ath25/Platform +++ b/arch/mips/ath25/Platform @@ -1,6 +1,5 @@ # # Atheros AR531X/AR231X WiSoC # -platform-$(CONFIG_ATH25) += ath25/ cflags-$(CONFIG_ATH25) += -I$(srctree)/arch/mips/include/asm/mach-ath25 load-$(CONFIG_ATH25) += 0xffffffff80041000 diff --git a/arch/mips/ath79/Platform b/arch/mips/ath79/Platform index 2bd663647d27..57744472ed2e 100644 --- a/arch/mips/ath79/Platform +++ b/arch/mips/ath79/Platform @@ -2,6 +2,5 @@ # Atheros AR71xx/AR724x/AR913x # -platform-$(CONFIG_ATH79) += ath79/ cflags-$(CONFIG_ATH79) += -I$(srctree)/arch/mips/include/asm/mach-ath79 load-$(CONFIG_ATH79) = 0xffffffff80060000 diff --git a/arch/mips/bcm47xx/Platform b/arch/mips/bcm47xx/Platform index 70783b75fd9d..833b204fe5da 100644 --- a/arch/mips/bcm47xx/Platform +++ b/arch/mips/bcm47xx/Platform @@ -1,7 +1,6 @@ # # Broadcom BCM47XX boards # -platform-$(CONFIG_BCM47XX) += bcm47xx/ cflags-$(CONFIG_BCM47XX) += \ -I$(srctree)/arch/mips/include/asm/mach-bcm47xx load-$(CONFIG_BCM47XX) := 0xffffffff80001000 diff --git a/arch/mips/bcm63xx/Platform b/arch/mips/bcm63xx/Platform index 5f86b2fff6de..882dc40f49a2 100644 --- a/arch/mips/bcm63xx/Platform +++ b/arch/mips/bcm63xx/Platform @@ -1,7 +1,6 @@ # # Broadcom BCM63XX boards # -platform-$(CONFIG_BCM63XX) += bcm63xx/ cflags-$(CONFIG_BCM63XX) += \ -I$(srctree)/arch/mips/include/asm/mach-bcm63xx/ load-$(CONFIG_BCM63XX) := 0xffffffff80010000 diff --git a/arch/mips/bmips/Platform b/arch/mips/bmips/Platform index 5f127fd7f4b5..1434ea31ce85 100644 --- a/arch/mips/bmips/Platform +++ b/arch/mips/bmips/Platform @@ -1,7 +1,6 @@ # # Broadcom Generic BMIPS kernel # -platform-$(CONFIG_BMIPS_GENERIC) += bmips/ cflags-$(CONFIG_BMIPS_GENERIC) += \ -I$(srctree)/arch/mips/include/asm/mach-bmips/ load-$(CONFIG_BMIPS_GENERIC) := 0xffffffff80010000 diff --git a/arch/mips/cavium-octeon/Platform b/arch/mips/cavium-octeon/Platform index 45be853700e6..4adef38dea9d 100644 --- a/arch/mips/cavium-octeon/Platform +++ b/arch/mips/cavium-octeon/Platform @@ -1,7 +1,6 @@ # # Cavium Octeon # -platform-$(CONFIG_CAVIUM_OCTEON_SOC) += cavium-octeon/ cflags-$(CONFIG_CAVIUM_OCTEON_SOC) += \ -I$(srctree)/arch/mips/include/asm/mach-cavium-octeon load-$(CONFIG_CAVIUM_OCTEON_SOC) += 0xffffffff81100000 diff --git a/arch/mips/cobalt/Platform b/arch/mips/cobalt/Platform index 34123efd6dfe..4254895ad6f4 100644 --- a/arch/mips/cobalt/Platform +++ b/arch/mips/cobalt/Platform @@ -1,6 +1,5 @@ # # Cobalt Server # -platform-$(CONFIG_MIPS_COBALT) += cobalt/ cflags-$(CONFIG_MIPS_COBALT) += -I$(srctree)/arch/mips/include/asm/mach-cobalt load-$(CONFIG_MIPS_COBALT) += 0xffffffff80080000 diff --git a/arch/mips/dec/Platform b/arch/mips/dec/Platform index cf55a6f4e720..c82391e832f9 100644 --- a/arch/mips/dec/Platform +++ b/arch/mips/dec/Platform @@ -1,7 +1,6 @@ # # DECstation family # -platform-$(CONFIG_MACH_DECSTATION) += dec/ cflags-$(CONFIG_MACH_DECSTATION) += \ -I$(srctree)/arch/mips/include/asm/mach-dec libs-$(CONFIG_MACH_DECSTATION) += arch/mips/dec/prom/ diff --git a/arch/mips/generic/Platform b/arch/mips/generic/Platform index eaa19d189324..53c33cb72974 100644 --- a/arch/mips/generic/Platform +++ b/arch/mips/generic/Platform @@ -8,7 +8,6 @@ # option) any later version. # -platform-$(CONFIG_MIPS_GENERIC) += generic/ cflags-$(CONFIG_MIPS_GENERIC) += -I$(srctree)/arch/mips/include/asm/mach-generic load-$(CONFIG_MIPS_GENERIC) += 0xffffffff80100000 all-$(CONFIG_MIPS_GENERIC) := vmlinux.gz.itb diff --git a/arch/mips/jazz/Platform b/arch/mips/jazz/Platform index 3373788acca1..eb0490ae8b09 100644 --- a/arch/mips/jazz/Platform +++ b/arch/mips/jazz/Platform @@ -1,6 +1,5 @@ # # Acer PICA 61, Mips Magnum 4000 and Olivetti M700. # -platform-$(CONFIG_MACH_JAZZ) += jazz/ cflags-$(CONFIG_MACH_JAZZ) += -I$(srctree)/arch/mips/include/asm/mach-jazz load-$(CONFIG_MACH_JAZZ) += 0xffffffff80080000 diff --git a/arch/mips/jz4740/Platform b/arch/mips/jz4740/Platform index a2a5a85ea1f9..bd35d0621b13 100644 --- a/arch/mips/jz4740/Platform +++ b/arch/mips/jz4740/Platform @@ -1,4 +1,3 @@ -platform-$(CONFIG_MACH_INGENIC) += jz4740/ cflags-$(CONFIG_MACH_INGENIC) += -I$(srctree)/arch/mips/include/asm/mach-jz4740 load-$(CONFIG_MACH_INGENIC) += 0xffffffff80010000 zload-$(CONFIG_MACH_INGENIC) += 0xffffffff81000000 diff --git a/arch/mips/lantiq/Platform b/arch/mips/lantiq/Platform index b3ec49838fd7..0bc9c0fbd431 100644 --- a/arch/mips/lantiq/Platform +++ b/arch/mips/lantiq/Platform @@ -2,7 +2,6 @@ # Lantiq # -platform-$(CONFIG_LANTIQ) += lantiq/ cflags-$(CONFIG_LANTIQ) += -I$(srctree)/arch/mips/include/asm/mach-lantiq load-$(CONFIG_LANTIQ) = 0xffffffff80002000 cflags-$(CONFIG_SOC_TYPE_XWAY) += -I$(srctree)/arch/mips/include/asm/mach-lantiq/xway diff --git a/arch/mips/loongson2ef/Platform b/arch/mips/loongson2ef/Platform index 3aca42963f35..cdad3c1a9a18 100644 --- a/arch/mips/loongson2ef/Platform +++ b/arch/mips/loongson2ef/Platform @@ -26,7 +26,6 @@ endif # Loongson Machines' Support # -platform-$(CONFIG_MACH_LOONGSON2EF) += loongson2ef/ cflags-$(CONFIG_MACH_LOONGSON2EF) += -I$(srctree)/arch/mips/include/asm/mach-loongson2ef -mno-branch-likely load-$(CONFIG_LEMOTE_FULOONG2E) += 0xffffffff80100000 load-$(CONFIG_LEMOTE_MACH2F) += 0xffffffff80200000 diff --git a/arch/mips/loongson32/Platform b/arch/mips/loongson32/Platform index 7f8e342f1ef5..3b9673e7a2fa 100644 --- a/arch/mips/loongson32/Platform +++ b/arch/mips/loongson32/Platform @@ -1,4 +1,3 @@ cflags-$(CONFIG_CPU_LOONGSON32) += -march=mips32r2 -Wa,--trap -platform-$(CONFIG_MACH_LOONGSON32) += loongson32/ cflags-$(CONFIG_MACH_LOONGSON32) += -I$(srctree)/arch/mips/include/asm/mach-loongson32 load-$(CONFIG_CPU_LOONGSON32) += 0xffffffff80200000 diff --git a/arch/mips/loongson64/Platform b/arch/mips/loongson64/Platform index d5eb94c9edb4..ec42c5085905 100644 --- a/arch/mips/loongson64/Platform +++ b/arch/mips/loongson64/Platform @@ -55,6 +55,5 @@ cflags-y += $(call cc-option,-mno-loongson-mmi) # Loongson Machines' Support # -platform-$(CONFIG_MACH_LOONGSON64) += loongson64/ cflags-$(CONFIG_MACH_LOONGSON64) += -I$(srctree)/arch/mips/include/asm/mach-loongson64 -mno-branch-likely load-$(CONFIG_CPU_LOONGSON64) += 0xffffffff80200000 diff --git a/arch/mips/mti-malta/Platform b/arch/mips/mti-malta/Platform index 2cc72c9b38e3..41e0d2a2d325 100644 --- a/arch/mips/mti-malta/Platform +++ b/arch/mips/mti-malta/Platform @@ -1,7 +1,6 @@ # # MIPS Malta board # -platform-$(CONFIG_MIPS_MALTA) += mti-malta/ cflags-$(CONFIG_MIPS_MALTA) += -I$(srctree)/arch/mips/include/asm/mach-malta ifdef CONFIG_KVM_GUEST load-$(CONFIG_MIPS_MALTA) += 0x0000000040100000 diff --git a/arch/mips/netlogic/Platform b/arch/mips/netlogic/Platform index fb8eb4c0c6ec..4195a097f5f2 100644 --- a/arch/mips/netlogic/Platform +++ b/arch/mips/netlogic/Platform @@ -13,5 +13,4 @@ cflags-$(CONFIG_CPU_XLP) += $(call cc-option,-march=xlp,-march=mips64r2) # # NETLOGIC processor support # -platform-$(CONFIG_NLM_COMMON) += netlogic/ load-$(CONFIG_NLM_COMMON) += 0xffffffff80100000 diff --git a/arch/mips/paravirt/Platform b/arch/mips/paravirt/Platform index 7e76ef25ea17..0b857580dfdd 100644 --- a/arch/mips/paravirt/Platform +++ b/arch/mips/paravirt/Platform @@ -1,7 +1,6 @@ # # Generic para-virtualized guest. # -platform-$(CONFIG_MIPS_PARAVIRT) += paravirt/ cflags-$(CONFIG_MIPS_PARAVIRT) += \ -I$(srctree)/arch/mips/include/asm/mach-paravirt diff --git a/arch/mips/pic32/Platform b/arch/mips/pic32/Platform index cd2084f44507..1e92e52a137b 100644 --- a/arch/mips/pic32/Platform +++ b/arch/mips/pic32/Platform @@ -1,7 +1,6 @@ # # PIC32MZDA # -platform-$(CONFIG_PIC32MZDA) += pic32/ cflags-$(CONFIG_PIC32MZDA) += -I$(srctree)/arch/mips/include/asm/mach-pic32 load-$(CONFIG_PIC32MZDA) += 0xffffffff88000000 all-$(CONFIG_PIC32MZDA) := $(COMPRESSION_FNAME).bin diff --git a/arch/mips/pistachio/Platform b/arch/mips/pistachio/Platform index c3592b374ad2..f73a1a929965 100644 --- a/arch/mips/pistachio/Platform +++ b/arch/mips/pistachio/Platform @@ -1,7 +1,6 @@ # # IMG Pistachio SoC # -platform-$(CONFIG_MACH_PISTACHIO) += pistachio/ cflags-$(CONFIG_MACH_PISTACHIO) += \ -I$(srctree)/arch/mips/include/asm/mach-pistachio load-$(CONFIG_MACH_PISTACHIO) += 0xffffffff80400000 diff --git a/arch/mips/pnx833x/Platform b/arch/mips/pnx833x/Platform index 287260669551..e5286a49fc3e 100644 --- a/arch/mips/pnx833x/Platform +++ b/arch/mips/pnx833x/Platform @@ -1,5 +1,4 @@ # NXP STB225 -platform-$(CONFIG_SOC_PNX833X) += pnx833x/ cflags-$(CONFIG_SOC_PNX833X) += -I$(srctree)/arch/mips/include/asm/mach-pnx833x load-$(CONFIG_NXP_STB220) += 0xffffffff80001000 load-$(CONFIG_NXP_STB225) += 0xffffffff80001000 diff --git a/arch/mips/rb532/Platform b/arch/mips/rb532/Platform index aeec45a7cbb3..12eaa8790b3e 100644 --- a/arch/mips/rb532/Platform +++ b/arch/mips/rb532/Platform @@ -1,7 +1,6 @@ # # Routerboard 532 # -platform-$(CONFIG_MIKROTIK_RB532) += rb532/ cflags-$(CONFIG_MIKROTIK_RB532) += \ -I$(srctree)/arch/mips/include/asm/mach-rc32434 load-$(CONFIG_MIKROTIK_RB532) += 0xffffffff80101000 diff --git a/arch/mips/sgi-ip22/Platform b/arch/mips/sgi-ip22/Platform index e8f6b3a42a48..62fa30bb959e 100644 --- a/arch/mips/sgi-ip22/Platform +++ b/arch/mips/sgi-ip22/Platform @@ -7,7 +7,6 @@ # current variable will break so for 64-bit kernels we have to raise the start # address by 8kb. # -platform-$(CONFIG_SGI_IP22) += sgi-ip22/ cflags-$(CONFIG_SGI_IP22) += -I$(srctree)/arch/mips/include/asm/mach-ip22 ifdef CONFIG_32BIT load-$(CONFIG_SGI_IP22) += 0xffffffff88002000 @@ -29,6 +28,5 @@ ifdef CONFIG_SGI_IP28 $(error gcc doesn't support needed option -mr10k-cache-barrier=store) endif endif -platform-$(CONFIG_SGI_IP28) += sgi-ip22/ cflags-$(CONFIG_SGI_IP28) += -mr10k-cache-barrier=store -I$(srctree)/arch/mips/include/asm/mach-ip28 load-$(CONFIG_SGI_IP28) += 0xa800000020004000 diff --git a/arch/mips/sgi-ip27/Platform b/arch/mips/sgi-ip27/Platform index 1fb9c2ea7c8f..e734ee6abd44 100644 --- a/arch/mips/sgi-ip27/Platform +++ b/arch/mips/sgi-ip27/Platform @@ -5,8 +5,6 @@ # symmon, 0xc00000000001c000 for production kernels. Note that the value must # be 16kb aligned or the handling of the current variable will break. # -ifdef CONFIG_SGI_IP27 -platform-$(CONFIG_SGI_IP27) += sgi-ip27/ cflags-$(CONFIG_SGI_IP27) += -I$(srctree)/arch/mips/include/asm/mach-ip27 ifdef CONFIG_MAPPED_KERNEL load-$(CONFIG_SGI_IP27) += 0xc00000004001c000 @@ -16,4 +14,3 @@ else load-$(CONFIG_SGI_IP27) += 0xa80000000001c000 OBJCOPYFLAGS := --change-addresses=0x57ffffff80000000 endif -endif diff --git a/arch/mips/sgi-ip30/Platform b/arch/mips/sgi-ip30/Platform index 2b5695c2049a..f6f11517e091 100644 --- a/arch/mips/sgi-ip30/Platform +++ b/arch/mips/sgi-ip30/Platform @@ -1,8 +1,5 @@ # # SGI-IP30 (Octane/Octane2) # -ifdef CONFIG_SGI_IP30 -platform-$(CONFIG_SGI_IP30) += sgi-ip30/ cflags-$(CONFIG_SGI_IP30) += -I$(srctree)/arch/mips/include/asm/mach-ip30 load-$(CONFIG_SGI_IP30) += 0xa800000020004000 -endif diff --git a/arch/mips/sgi-ip32/Platform b/arch/mips/sgi-ip32/Platform index 0fea556f3641..f58a7a02b4ca 100644 --- a/arch/mips/sgi-ip32/Platform +++ b/arch/mips/sgi-ip32/Platform @@ -6,6 +6,5 @@ # a multiple of the kernel stack size or the handling of the current variable # will break. # -platform-$(CONFIG_SGI_IP32) += sgi-ip32/ cflags-$(CONFIG_SGI_IP32) += -I$(srctree)/arch/mips/include/asm/mach-ip32 load-$(CONFIG_SGI_IP32) += 0xffffffff80004000 diff --git a/arch/mips/sibyte/Platform b/arch/mips/sibyte/Platform index af117330ce14..65b2225b76b2 100644 --- a/arch/mips/sibyte/Platform +++ b/arch/mips/sibyte/Platform @@ -1,10 +1,6 @@ # # These are all rather similar so we consider them a single platform # -platform-$(CONFIG_SIBYTE_BCM112X) += sibyte/ -platform-$(CONFIG_SIBYTE_SB1250) += sibyte/ -platform-$(CONFIG_SIBYTE_BCM1x55) += sibyte/ -platform-$(CONFIG_SIBYTE_BCM1x80) += sibyte/ # # Sibyte SB1250 / BCM1480 family of SOCs diff --git a/arch/mips/sni/Platform b/arch/mips/sni/Platform index 2644a9d63c0f..b0b3dde0bef8 100644 --- a/arch/mips/sni/Platform +++ b/arch/mips/sni/Platform @@ -1,7 +1,6 @@ # # SNI RM # -platform-$(CONFIG_SNI_RM) += sni/ cflags-$(CONFIG_SNI_RM) += -I$(srctree)/arch/mips/include/asm/mach-rm ifdef CONFIG_CPU_LITTLE_ENDIAN load-$(CONFIG_SNI_RM) += 0xffffffff80600000 diff --git a/arch/mips/txx9/Platform b/arch/mips/txx9/Platform index a176d1fd5799..7f4429ba22eb 100644 --- a/arch/mips/txx9/Platform +++ b/arch/mips/txx9/Platform @@ -1,6 +1,3 @@ -platform-$(CONFIG_MACH_TX39XX) += txx9/ -platform-$(CONFIG_MACH_TX49XX) += txx9/ - cflags-$(CONFIG_MACH_TX39XX) += \ -I$(srctree)/arch/mips/include/asm/mach-tx39xx cflags-$(CONFIG_MACH_TX49XX) += \ From fe5a90b8c14914397a3bb0c214d142103c1ba3bf Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Sat, 9 May 2020 19:18:52 +0800 Subject: [PATCH 0413/1043] selinux: netlabel: Remove unused inline function There's no callers in-tree. Signed-off-by: YueHaibing Signed-off-by: Paul Moore --- security/selinux/include/netlabel.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/security/selinux/include/netlabel.h b/security/selinux/include/netlabel.h index d30d8d7cdc9c..0c58f62dc6ab 100644 --- a/security/selinux/include/netlabel.h +++ b/security/selinux/include/netlabel.h @@ -98,12 +98,6 @@ static inline int selinux_netlbl_skbuff_setsid(struct sk_buff *skb, return 0; } -static inline int selinux_netlbl_conn_setsid(struct sock *sk, - struct sockaddr *addr) -{ - return 0; -} - static inline int selinux_netlbl_sctp_assoc_request(struct sctp_endpoint *ep, struct sk_buff *skb) { From 303cc571d107b3641d6487061b748e70ffe15ce4 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Tue, 5 May 2020 16:04:31 +0200 Subject: [PATCH 0414/1043] nsproxy: attach to namespaces via pidfds For quite a while we have been thinking about using pidfds to attach to namespaces. This patchset has existed for about a year already but we've wanted to wait to see how the general api would be received and adopted. Now that more and more programs in userspace have started using pidfds for process management it's time to send this one out. This patch makes it possible to use pidfds to attach to the namespaces of another process, i.e. they can be passed as the first argument to the setns() syscall. When only a single namespace type is specified the semantics are equivalent to passing an nsfd. That means setns(nsfd, CLONE_NEWNET) equals setns(pidfd, CLONE_NEWNET). However, when a pidfd is passed, multiple namespace flags can be specified in the second setns() argument and setns() will attach the caller to all the specified namespaces all at once or to none of them. Specifying 0 is not valid together with a pidfd. Here are just two obvious examples: setns(pidfd, CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWNET); setns(pidfd, CLONE_NEWUSER); Allowing to also attach subsets of namespaces supports various use-cases where callers setns to a subset of namespaces to retain privilege, perform an action and then re-attach another subset of namespaces. If the need arises, as Eric suggested, we can extend this patchset to assume even more context than just attaching all namespaces. His suggestion specifically was about assuming the process' root directory when setns(pidfd, 0) or setns(pidfd, SETNS_PIDFD) is specified. For now, just keep it flexible in terms of supporting subsets of namespaces but let's wait until we have users asking for even more context to be assumed. At that point we can add an extension. The obvious example where this is useful is a standard container manager interacting with a running container: pushing and pulling files or directories, injecting mounts, attaching/execing any kind of process, managing network devices all these operations require attaching to all or at least multiple namespaces at the same time. Given that nowadays most containers are spawned with all namespaces enabled we're currently looking at at least 14 syscalls, 7 to open the /proc//ns/ nsfds, another 7 to actually perform the namespace switch. With time namespaces we're looking at about 16 syscalls. (We could amortize the first 7 or 8 syscalls for opening the nsfds by stashing them in each container's monitor process but that would mean we need to send around those file descriptors through unix sockets everytime we want to interact with the container or keep on-disk state. Even in scenarios where a caller wants to join a particular namespace in a particular order callers still profit from batching other namespaces. That mostly applies to the user namespace but all container runtimes I found join the user namespace first no matter if it privileges or deprivileges the container similar to how unshare behaves.) With pidfds this becomes a single syscall no matter how many namespaces are supposed to be attached to. A decently designed, large-scale container manager usually isn't the parent of any of the containers it spawns so the containers don't die when it crashes or needs to update or reinitialize. This means that for the manager to interact with containers through pids is inherently racy especially on systems where the maximum pid number is not significicantly bumped. This is even more problematic since we often spawn and manage thousands or ten-thousands of containers. Interacting with a container through a pid thus can become risky quite quickly. Especially since we allow for an administrator to enable advanced features such as syscall interception where we're performing syscalls in lieu of the container. In all of those cases we use pidfds if they are available and we pass them around as stable references. Using them to setns() to the target process' namespaces is as reliable as using nsfds. Either the target process is already dead and we get ESRCH or we manage to attach to its namespaces but we can't accidently attach to another process' namespaces. So pidfds lend themselves to be used with this api. The other main advantage is that with this change the pidfd becomes the only relevant token for most container interactions and it's the only token we need to create and send around. Apart from significiantly reducing the number of syscalls from double digit to single digit which is a decent reason post-spectre/meltdown this also allows to switch to a set of namespaces atomically, i.e. either attaching to all the specified namespaces succeeds or we fail. If we fail we haven't changed a single namespace. There are currently three namespaces that can fail (other than for ENOMEM which really is not very interesting since we then have other problems anyway) for non-trivial reasons, user, mount, and pid namespaces. We can fail to attach to a pid namespace if it is not our current active pid namespace or a descendant of it. We can fail to attach to a user namespace because we are multi-threaded or because our current mount namespace shares filesystem state with other tasks, or because we're trying to setns() to the same user namespace, i.e. the target task has the same user namespace as we do. We can fail to attach to a mount namespace because it shares filesystem state with other tasks or because we fail to lookup the new root for the new mount namespace. In most non-pathological scenarios these issues can be somewhat mitigated. But there are cases where we're half-attached to some namespace and failing to attach to another one. I've talked about some of these problem during the hallway track (something only the pre-COVID-19 generation will remember) of Plumbers in Los Angeles in 2018(?). Even if all these issues could be avoided with super careful userspace coding it would be nicer to have this done in-kernel. Pidfds seem to lend themselves nicely for this. The other neat thing about this is that setns() becomes an actual counterpart to the namespace bits of unshare(). Signed-off-by: Christian Brauner Reviewed-by: Serge Hallyn Cc: Eric W. Biederman Cc: Serge Hallyn Cc: Jann Horn Cc: Michael Kerrisk Cc: Aleksa Sarai Link: https://lore.kernel.org/r/20200505140432.181565-3-christian.brauner@ubuntu.com --- fs/namespace.c | 5 + fs/nsfs.c | 5 + include/linux/mnt_namespace.h | 1 + include/linux/proc_fs.h | 2 + kernel/nsproxy.c | 231 +++++++++++++++++++++++++++++++--- 5 files changed, 227 insertions(+), 17 deletions(-) diff --git a/fs/namespace.c b/fs/namespace.c index 62899fad4a04..be99e80e3c7c 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -1733,6 +1733,11 @@ static struct mnt_namespace *to_mnt_ns(struct ns_common *ns) return container_of(ns, struct mnt_namespace, ns); } +struct ns_common *from_mnt_ns(struct mnt_namespace *mnt) +{ + return &mnt->ns; +} + static bool mnt_ns_loop(struct dentry *dentry) { /* Could bind mounting the mount namespace inode cause a diff --git a/fs/nsfs.c b/fs/nsfs.c index 4f1205725cfe..800c1d0eb0d0 100644 --- a/fs/nsfs.c +++ b/fs/nsfs.c @@ -229,6 +229,11 @@ int ns_get_name(char *buf, size_t size, struct task_struct *task, return res; } +bool proc_ns_file(const struct file *file) +{ + return file->f_op == &ns_file_operations; +} + struct file *proc_ns_fget(int fd) { struct file *file; diff --git a/include/linux/mnt_namespace.h b/include/linux/mnt_namespace.h index 007cfa52efb2..8f882f5881e8 100644 --- a/include/linux/mnt_namespace.h +++ b/include/linux/mnt_namespace.h @@ -11,6 +11,7 @@ struct ns_common; extern struct mnt_namespace *copy_mnt_ns(unsigned long, struct mnt_namespace *, struct user_namespace *, struct fs_struct *); extern void put_mnt_ns(struct mnt_namespace *ns); +extern struct ns_common *from_mnt_ns(struct mnt_namespace *); extern const struct file_operations proc_mounts_operations; extern const struct file_operations proc_mountinfo_operations; diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h index 45c05fd9c99d..0cfc44d7ac74 100644 --- a/include/linux/proc_fs.h +++ b/include/linux/proc_fs.h @@ -179,4 +179,6 @@ static inline struct pid_namespace *proc_pid_ns(const struct inode *inode) return inode->i_sb->s_fs_info; } +bool proc_ns_file(const struct file *file); + #endif /* _LINUX_PROC_FS_H */ diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c index b7954fd60475..b03df67621d0 100644 --- a/kernel/nsproxy.c +++ b/kernel/nsproxy.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -258,17 +259,58 @@ void exit_task_namespaces(struct task_struct *p) switch_task_namespaces(p, NULL); } +static int check_setns_flags(unsigned long flags) +{ + if (!flags || (flags & ~(CLONE_NEWNS | CLONE_NEWUTS | CLONE_NEWIPC | + CLONE_NEWNET | CLONE_NEWUSER | CLONE_NEWPID | + CLONE_NEWCGROUP))) + return -EINVAL; + +#ifndef CONFIG_USER_NS + if (flags & CLONE_NEWUSER) + return -EINVAL; +#endif +#ifndef CONFIG_PID_NS + if (flags & CLONE_NEWPID) + return -EINVAL; +#endif +#ifndef CONFIG_UTS_NS + if (flags & CLONE_NEWUTS) + return -EINVAL; +#endif +#ifndef CONFIG_IPC_NS + if (flags & CLONE_NEWIPC) + return -EINVAL; +#endif +#ifndef CONFIG_CGROUPS + if (flags & CLONE_NEWCGROUP) + return -EINVAL; +#endif +#ifndef CONFIG_NET_NS + if (flags & CLONE_NEWNET) + return -EINVAL; +#endif + + return 0; +} + static void put_nsset(struct nsset *nsset) { unsigned flags = nsset->flags; if (flags & CLONE_NEWUSER) put_cred(nsset_cred(nsset)); + /* + * We only created a temporary copy if we attached to more than just + * the mount namespace. + */ + if (nsset->fs && (flags & CLONE_NEWNS) && (flags & ~CLONE_NEWNS)) + free_fs_struct(nsset->fs); if (nsset->nsproxy) free_nsproxy(nsset->nsproxy); } -static int prepare_nsset(int nstype, struct nsset *nsset) +static int prepare_nsset(unsigned flags, struct nsset *nsset) { struct task_struct *me = current; @@ -276,17 +318,23 @@ static int prepare_nsset(int nstype, struct nsset *nsset) if (IS_ERR(nsset->nsproxy)) return PTR_ERR(nsset->nsproxy); - if (nstype == CLONE_NEWUSER) + if (flags & CLONE_NEWUSER) nsset->cred = prepare_creds(); else nsset->cred = current_cred(); if (!nsset->cred) goto out; - if (nstype == CLONE_NEWNS) + /* Only create a temporary copy of fs_struct if we really need to. */ + if (flags == CLONE_NEWNS) { nsset->fs = me->fs; + } else if (flags & CLONE_NEWNS) { + nsset->fs = copy_fs_struct(me->fs); + if (!nsset->fs) + goto out; + } - nsset->flags = nstype; + nsset->flags = flags; return 0; out: @@ -294,6 +342,138 @@ out: return -ENOMEM; } +static inline int validate_ns(struct nsset *nsset, struct ns_common *ns) +{ + return ns->ops->install(nsset, ns); +} + +/* + * This is the inverse operation to unshare(). + * Ordering is equivalent to the standard ordering used everywhere else + * during unshare and process creation. The switch to the new set of + * namespaces occurs at the point of no return after installation of + * all requested namespaces was successful in commit_nsset(). + */ +static int validate_nsset(struct nsset *nsset, struct pid *pid) +{ + int ret = 0; + unsigned flags = nsset->flags; + struct user_namespace *user_ns = NULL; + struct pid_namespace *pid_ns = NULL; + struct nsproxy *nsp; + struct task_struct *tsk; + + /* Take a "snapshot" of the target task's namespaces. */ + rcu_read_lock(); + tsk = pid_task(pid, PIDTYPE_PID); + if (!tsk) { + rcu_read_unlock(); + return -ESRCH; + } + + if (!ptrace_may_access(tsk, PTRACE_MODE_READ_REALCREDS)) { + rcu_read_unlock(); + return -EPERM; + } + + task_lock(tsk); + nsp = tsk->nsproxy; + if (nsp) + get_nsproxy(nsp); + task_unlock(tsk); + if (!nsp) { + rcu_read_unlock(); + return -ESRCH; + } + +#ifdef CONFIG_PID_NS + if (flags & CLONE_NEWPID) { + pid_ns = task_active_pid_ns(tsk); + if (unlikely(!pid_ns)) { + rcu_read_unlock(); + ret = -ESRCH; + goto out; + } + get_pid_ns(pid_ns); + } +#endif + +#ifdef CONFIG_USER_NS + if (flags & CLONE_NEWUSER) + user_ns = get_user_ns(__task_cred(tsk)->user_ns); +#endif + rcu_read_unlock(); + + /* + * Install requested namespaces. The caller will have + * verified earlier that the requested namespaces are + * supported on this kernel. We don't report errors here + * if a namespace is requested that isn't supported. + */ +#ifdef CONFIG_USER_NS + if (flags & CLONE_NEWUSER) { + ret = validate_ns(nsset, &user_ns->ns); + if (ret) + goto out; + } +#endif + + if (flags & CLONE_NEWNS) { + ret = validate_ns(nsset, from_mnt_ns(nsp->mnt_ns)); + if (ret) + goto out; + } + +#ifdef CONFIG_UTS_NS + if (flags & CLONE_NEWUTS) { + ret = validate_ns(nsset, &nsp->uts_ns->ns); + if (ret) + goto out; + } +#endif + +#ifdef CONFIG_IPC_NS + if (flags & CLONE_NEWIPC) { + ret = validate_ns(nsset, &nsp->ipc_ns->ns); + if (ret) + goto out; + } +#endif + +#ifdef CONFIG_PID_NS + if (flags & CLONE_NEWPID) { + ret = validate_ns(nsset, &pid_ns->ns); + if (ret) + goto out; + } +#endif + +#ifdef CONFIG_CGROUPS + if (flags & CLONE_NEWCGROUP) { + ret = validate_ns(nsset, &nsp->cgroup_ns->ns); + if (ret) + goto out; + } +#endif + +#ifdef CONFIG_NET_NS + if (flags & CLONE_NEWNET) { + ret = validate_ns(nsset, &nsp->net_ns->ns); + if (ret) + goto out; + } +#endif + +out: + if (pid_ns) + put_pid_ns(pid_ns); + if (nsp) + put_nsproxy(nsp); + put_user_ns(user_ns); + + return ret; +} + /* * This is the point of no return. There are just a few namespaces * that do some actual work here and it's sufficiently minimal that @@ -316,6 +496,12 @@ static void commit_nsset(struct nsset *nsset) } #endif + /* We only need to commit if we have used a temporary fs_struct. */ + if ((flags & CLONE_NEWNS) && (flags & ~CLONE_NEWNS)) { + set_fs_root(me->fs, &nsset->fs->root); + set_fs_pwd(me->fs, &nsset->fs->pwd); + } + #ifdef CONFIG_IPC_NS if (flags & CLONE_NEWIPC) exit_sem(me); @@ -326,27 +512,38 @@ static void commit_nsset(struct nsset *nsset) nsset->nsproxy = NULL; } -SYSCALL_DEFINE2(setns, int, fd, int, nstype) +SYSCALL_DEFINE2(setns, int, fd, int, flags) { struct file *file; - struct ns_common *ns; + struct ns_common *ns = NULL; struct nsset nsset = {}; - int err; + int err = 0; - file = proc_ns_fget(fd); - if (IS_ERR(file)) - return PTR_ERR(file); + file = fget(fd); + if (!file) + return -EBADF; - err = -EINVAL; - ns = get_proc_ns(file_inode(file)); - if (nstype && (ns->ops->type != nstype)) - goto out; - - err = prepare_nsset(ns->ops->type, &nsset); + if (proc_ns_file(file)) { + ns = get_proc_ns(file_inode(file)); + if (flags && (ns->ops->type != flags)) + err = -EINVAL; + flags = ns->ops->type; + } else if (!IS_ERR(pidfd_pid(file))) { + err = check_setns_flags(flags); + } else { + err = -EBADF; + } if (err) goto out; - err = ns->ops->install(&nsset, ns); + err = prepare_nsset(flags, &nsset); + if (err) + goto out; + + if (proc_ns_file(file)) + err = validate_ns(&nsset, ns); + else + err = validate_nsset(&nsset, file->private_data); if (!err) { commit_nsset(&nsset); perf_event_namespaces(current); From 2b40c5db73e239531ea54991087f4edc07fbb08e Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Tue, 5 May 2020 16:04:32 +0200 Subject: [PATCH 0415/1043] selftests/pidfd: add pidfd setns tests This is basically a test-suite for setns() and as of now contains: - test that we can't pass garbage flags - test that we can't attach to the namespaces of task that has already exited - test that we can incrementally setns into all namespaces of a target task using a pidfd - test that we can setns atomically into all namespaces of a target task - test that we can't cross setns into a user namespace outside of our user namespace hierarchy - test that we can't setns into namespaces owned by user namespaces over which we are not privileged Signed-off-by: Christian Brauner Link: https://lore.kernel.org/r/20200505140432.181565-4-christian.brauner@ubuntu.com --- tools/testing/selftests/pidfd/.gitignore | 1 + tools/testing/selftests/pidfd/Makefile | 3 +- tools/testing/selftests/pidfd/config | 6 + .../selftests/pidfd/pidfd_setns_test.c | 473 ++++++++++++++++++ 4 files changed, 482 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/pidfd/config create mode 100644 tools/testing/selftests/pidfd/pidfd_setns_test.c diff --git a/tools/testing/selftests/pidfd/.gitignore b/tools/testing/selftests/pidfd/.gitignore index 2d4db5afb142..973198a3ec3d 100644 --- a/tools/testing/selftests/pidfd/.gitignore +++ b/tools/testing/selftests/pidfd/.gitignore @@ -5,3 +5,4 @@ pidfd_test pidfd_wait pidfd_fdinfo_test pidfd_getfd_test +pidfd_setns_test diff --git a/tools/testing/selftests/pidfd/Makefile b/tools/testing/selftests/pidfd/Makefile index 75a545861375..f4a2f28f926b 100644 --- a/tools/testing/selftests/pidfd/Makefile +++ b/tools/testing/selftests/pidfd/Makefile @@ -1,7 +1,8 @@ # SPDX-License-Identifier: GPL-2.0-only CFLAGS += -g -I../../../../usr/include/ -pthread -TEST_GEN_PROGS := pidfd_test pidfd_fdinfo_test pidfd_open_test pidfd_poll_test pidfd_wait pidfd_getfd_test +TEST_GEN_PROGS := pidfd_test pidfd_fdinfo_test pidfd_open_test \ + pidfd_poll_test pidfd_wait pidfd_getfd_test pidfd_setns_test include ../lib.mk diff --git a/tools/testing/selftests/pidfd/config b/tools/testing/selftests/pidfd/config new file mode 100644 index 000000000000..bb11de90c0c9 --- /dev/null +++ b/tools/testing/selftests/pidfd/config @@ -0,0 +1,6 @@ +CONFIG_UTS_NS=y +CONFIG_IPC_NS=y +CONFIG_USER_NS=y +CONFIG_PID_NS=y +CONFIG_NET_NS=y +CONFIG_CGROUPS=y diff --git a/tools/testing/selftests/pidfd/pidfd_setns_test.c b/tools/testing/selftests/pidfd/pidfd_setns_test.c new file mode 100644 index 000000000000..133ec5b6cda8 --- /dev/null +++ b/tools/testing/selftests/pidfd/pidfd_setns_test.c @@ -0,0 +1,473 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pidfd.h" +#include "../clone3/clone3_selftests.h" +#include "../kselftest.h" +#include "../kselftest_harness.h" + +enum { + PIDFD_NS_USER, + PIDFD_NS_MNT, + PIDFD_NS_PID, + PIDFD_NS_UTS, + PIDFD_NS_IPC, + PIDFD_NS_NET, + PIDFD_NS_CGROUP, + PIDFD_NS_PIDCLD, + PIDFD_NS_MAX +}; + +const struct ns_info { + const char *name; + int flag; +} ns_info[] = { + [PIDFD_NS_USER] = { "user", CLONE_NEWUSER, }, + [PIDFD_NS_MNT] = { "mnt", CLONE_NEWNS, }, + [PIDFD_NS_PID] = { "pid", CLONE_NEWPID, }, + [PIDFD_NS_UTS] = { "uts", CLONE_NEWUTS, }, + [PIDFD_NS_IPC] = { "ipc", CLONE_NEWIPC, }, + [PIDFD_NS_NET] = { "net", CLONE_NEWNET, }, + [PIDFD_NS_CGROUP] = { "cgroup", CLONE_NEWCGROUP, }, + [PIDFD_NS_PIDCLD] = { "pid_for_children", 0, }, +}; + +FIXTURE(current_nsset) +{ + pid_t pid; + int pidfd; + int nsfds[PIDFD_NS_MAX]; + + pid_t child_pid_exited; + int child_pidfd_exited; + + pid_t child_pid1; + int child_pidfd1; + int child_nsfds1[PIDFD_NS_MAX]; + + pid_t child_pid2; + int child_pidfd2; + int child_nsfds2[PIDFD_NS_MAX]; +}; + +static int sys_waitid(int which, pid_t pid, int options) +{ + return syscall(__NR_waitid, which, pid, NULL, options, NULL); +} + +pid_t create_child(int *pidfd, unsigned flags) +{ + struct clone_args args = { + .flags = CLONE_PIDFD | flags, + .exit_signal = SIGCHLD, + .pidfd = ptr_to_u64(pidfd), + }; + + return sys_clone3(&args, sizeof(struct clone_args)); +} + +FIXTURE_SETUP(current_nsset) +{ + int i, proc_fd, ret; + + for (i = 0; i < PIDFD_NS_MAX; i++) { + self->nsfds[i] = -EBADF; + self->child_nsfds1[i] = -EBADF; + self->child_nsfds2[i] = -EBADF; + } + + proc_fd = open("/proc/self/ns", O_DIRECTORY | O_CLOEXEC); + ASSERT_GE(proc_fd, 0) { + TH_LOG("%m - Failed to open /proc/self/ns"); + } + + self->pid = getpid(); + for (i = 0; i < PIDFD_NS_MAX; i++) { + const struct ns_info *info = &ns_info[i]; + self->nsfds[i] = openat(proc_fd, info->name, O_RDONLY | O_CLOEXEC); + if (self->nsfds[i] < 0) { + EXPECT_EQ(errno, ENOENT) { + TH_LOG("%m - Failed to open %s namespace for process %d", + info->name, self->pid); + } + } + } + + self->pidfd = sys_pidfd_open(self->pid, 0); + EXPECT_GT(self->pidfd, 0) { + TH_LOG("%m - Failed to open pidfd for process %d", self->pid); + } + + /* Create task that exits right away. */ + self->child_pid_exited = create_child(&self->child_pidfd_exited, + CLONE_NEWUSER | CLONE_NEWNET); + EXPECT_GT(self->child_pid_exited, 0); + + if (self->child_pid_exited == 0) + _exit(EXIT_SUCCESS); + + ASSERT_EQ(sys_waitid(P_PID, self->child_pid_exited, WEXITED | WNOWAIT), 0); + + self->pidfd = sys_pidfd_open(self->pid, 0); + EXPECT_GE(self->pidfd, 0) { + TH_LOG("%m - Failed to open pidfd for process %d", self->pid); + } + + /* Create tasks that will be stopped. */ + self->child_pid1 = create_child(&self->child_pidfd1, + CLONE_NEWUSER | CLONE_NEWNS | + CLONE_NEWCGROUP | CLONE_NEWIPC | + CLONE_NEWUTS | CLONE_NEWPID | + CLONE_NEWNET); + EXPECT_GE(self->child_pid1, 0); + + if (self->child_pid1 == 0) { + pause(); + _exit(EXIT_SUCCESS); + } + + self->child_pid2 = create_child(&self->child_pidfd2, + CLONE_NEWUSER | CLONE_NEWNS | + CLONE_NEWCGROUP | CLONE_NEWIPC | + CLONE_NEWUTS | CLONE_NEWPID | + CLONE_NEWNET); + EXPECT_GE(self->child_pid2, 0); + + if (self->child_pid2 == 0) { + pause(); + _exit(EXIT_SUCCESS); + } + + for (i = 0; i < PIDFD_NS_MAX; i++) { + char p[100]; + + const struct ns_info *info = &ns_info[i]; + + self->nsfds[i] = openat(proc_fd, info->name, O_RDONLY | O_CLOEXEC); + if (self->nsfds[i] < 0) { + EXPECT_EQ(errno, ENOENT) { + TH_LOG("%m - Failed to open %s namespace for process %d", + info->name, self->pid); + } + } + + ret = snprintf(p, sizeof(p), "/proc/%d/ns/%s", + self->child_pid1, info->name); + EXPECT_GT(ret, 0); + EXPECT_LT(ret, sizeof(p)); + + self->child_nsfds1[i] = open(p, O_RDONLY | O_CLOEXEC); + if (self->child_nsfds1[i] < 0) { + EXPECT_EQ(errno, ENOENT) { + TH_LOG("%m - Failed to open %s namespace for process %d", + info->name, self->child_pid1); + } + } + + ret = snprintf(p, sizeof(p), "/proc/%d/ns/%s", + self->child_pid2, info->name); + EXPECT_GT(ret, 0); + EXPECT_LT(ret, sizeof(p)); + + self->child_nsfds2[i] = open(p, O_RDONLY | O_CLOEXEC); + if (self->child_nsfds2[i] < 0) { + EXPECT_EQ(errno, ENOENT) { + TH_LOG("%m - Failed to open %s namespace for process %d", + info->name, self->child_pid1); + } + } + } + + close(proc_fd); +} + +FIXTURE_TEARDOWN(current_nsset) +{ + int i; + + ASSERT_EQ(sys_pidfd_send_signal(self->child_pidfd1, + SIGKILL, NULL, 0), 0); + ASSERT_EQ(sys_pidfd_send_signal(self->child_pidfd2, + SIGKILL, NULL, 0), 0); + + for (i = 0; i < PIDFD_NS_MAX; i++) { + if (self->nsfds[i] >= 0) + close(self->nsfds[i]); + if (self->child_nsfds1[i] >= 0) + close(self->child_nsfds1[i]); + if (self->child_nsfds2[i] >= 0) + close(self->child_nsfds2[i]); + } + + if (self->child_pidfd1 >= 0) + EXPECT_EQ(0, close(self->child_pidfd1)); + if (self->child_pidfd2 >= 0) + EXPECT_EQ(0, close(self->child_pidfd2)); + ASSERT_EQ(sys_waitid(P_PID, self->child_pid_exited, WEXITED), 0); + ASSERT_EQ(sys_waitid(P_PID, self->child_pid1, WEXITED), 0); + ASSERT_EQ(sys_waitid(P_PID, self->child_pid2, WEXITED), 0); +} + +static int preserve_ns(const int pid, const char *ns) +{ + int ret; + char path[50]; + + ret = snprintf(path, sizeof(path), "/proc/%d/ns/%s", pid, ns); + if (ret < 0 || (size_t)ret >= sizeof(path)) + return -EIO; + + return open(path, O_RDONLY | O_CLOEXEC); +} + +static int in_same_namespace(int ns_fd1, pid_t pid2, const char *ns) +{ + int ns_fd2 = -EBADF; + int ret = -1; + struct stat ns_st1, ns_st2; + + ret = fstat(ns_fd1, &ns_st1); + if (ret < 0) + return -1; + + ns_fd2 = preserve_ns(pid2, ns); + if (ns_fd2 < 0) + return -1; + + ret = fstat(ns_fd2, &ns_st2); + close(ns_fd2); + if (ret < 0) + return -1; + + /* processes are in the same namespace */ + if ((ns_st1.st_dev == ns_st2.st_dev) && + (ns_st1.st_ino == ns_st2.st_ino)) + return 1; + + /* processes are in different namespaces */ + return 0; +} + +/* Test that we can't pass garbage to the kernel. */ +TEST_F(current_nsset, invalid_flags) +{ + ASSERT_NE(setns(self->pidfd, 0), 0); + EXPECT_EQ(errno, EINVAL); + + ASSERT_NE(setns(self->pidfd, -1), 0); + EXPECT_EQ(errno, EINVAL); + + ASSERT_NE(setns(self->pidfd, CLONE_VM), 0); + EXPECT_EQ(errno, EINVAL); + + ASSERT_NE(setns(self->pidfd, CLONE_NEWUSER | CLONE_VM), 0); + EXPECT_EQ(errno, EINVAL); +} + +/* Test that we can't attach to a task that has already exited. */ +TEST_F(current_nsset, pidfd_exited_child) +{ + int i; + pid_t pid; + + ASSERT_NE(setns(self->child_pidfd_exited, CLONE_NEWUSER | CLONE_NEWNET), + 0); + EXPECT_EQ(errno, ESRCH); + + pid = getpid(); + for (i = 0; i < PIDFD_NS_MAX; i++) { + const struct ns_info *info = &ns_info[i]; + /* Verify that we haven't changed any namespaces. */ + if (self->nsfds[i] >= 0) + ASSERT_EQ(in_same_namespace(self->nsfds[i], pid, info->name), 1); + } +} + +TEST_F(current_nsset, pidfd_incremental_setns) +{ + int i; + pid_t pid; + + pid = getpid(); + for (i = 0; i < PIDFD_NS_MAX; i++) { + const struct ns_info *info = &ns_info[i]; + int nsfd; + + if (self->child_nsfds1[i] < 0) + continue; + + if (info->flag) { + ASSERT_EQ(setns(self->child_pidfd1, info->flag), 0) { + TH_LOG("%m - Failed to setns to %s namespace of %d via pidfd %d", + info->name, self->child_pid1, + self->child_pidfd1); + } + } + + /* Verify that we have changed to the correct namespaces. */ + if (info->flag == CLONE_NEWPID) + nsfd = self->nsfds[i]; + else + nsfd = self->child_nsfds1[i]; + ASSERT_EQ(in_same_namespace(nsfd, pid, info->name), 1) { + TH_LOG("setns failed to place us correctly into %s namespace of %d via pidfd %d", + info->name, self->child_pid1, + self->child_pidfd1); + } + TH_LOG("Managed to correctly setns to %s namespace of %d via pidfd %d", + info->name, self->child_pid1, self->child_pidfd1); + } +} + +TEST_F(current_nsset, nsfd_incremental_setns) +{ + int i; + pid_t pid; + + pid = getpid(); + for (i = 0; i < PIDFD_NS_MAX; i++) { + const struct ns_info *info = &ns_info[i]; + int nsfd; + + if (self->child_nsfds1[i] < 0) + continue; + + if (info->flag) { + ASSERT_EQ(setns(self->child_nsfds1[i], info->flag), 0) { + TH_LOG("%m - Failed to setns to %s namespace of %d via nsfd %d", + info->name, self->child_pid1, + self->child_nsfds1[i]); + } + } + + /* Verify that we have changed to the correct namespaces. */ + if (info->flag == CLONE_NEWPID) + nsfd = self->nsfds[i]; + else + nsfd = self->child_nsfds1[i]; + ASSERT_EQ(in_same_namespace(nsfd, pid, info->name), 1) { + TH_LOG("setns failed to place us correctly into %s namespace of %d via nsfd %d", + info->name, self->child_pid1, + self->child_nsfds1[i]); + } + TH_LOG("Managed to correctly setns to %s namespace of %d via nsfd %d", + info->name, self->child_pid1, self->child_nsfds1[i]); + } +} + +TEST_F(current_nsset, pidfd_one_shot_setns) +{ + unsigned flags = 0; + int i; + pid_t pid; + + for (i = 0; i < PIDFD_NS_MAX; i++) { + const struct ns_info *info = &ns_info[i]; + + if (self->child_nsfds1[i] < 0) + continue; + + flags |= info->flag; + TH_LOG("Adding %s namespace of %d to list of namespaces to attach to", + info->name, self->child_pid1); + } + + ASSERT_EQ(setns(self->child_pidfd1, flags), 0) { + TH_LOG("%m - Failed to setns to namespaces of %d", + self->child_pid1); + } + + pid = getpid(); + for (i = 0; i < PIDFD_NS_MAX; i++) { + const struct ns_info *info = &ns_info[i]; + int nsfd; + + if (self->child_nsfds1[i] < 0) + continue; + + /* Verify that we have changed to the correct namespaces. */ + if (info->flag == CLONE_NEWPID) + nsfd = self->nsfds[i]; + else + nsfd = self->child_nsfds1[i]; + ASSERT_EQ(in_same_namespace(nsfd, pid, info->name), 1) { + TH_LOG("setns failed to place us correctly into %s namespace of %d", + info->name, self->child_pid1); + } + TH_LOG("Managed to correctly setns to %s namespace of %d", + info->name, self->child_pid1); + } +} + +TEST_F(current_nsset, no_foul_play) +{ + unsigned flags = 0; + int i; + + for (i = 0; i < PIDFD_NS_MAX; i++) { + const struct ns_info *info = &ns_info[i]; + + if (self->child_nsfds1[i] < 0) + continue; + + flags |= info->flag; + if (info->flag) /* No use logging pid_for_children. */ + TH_LOG("Adding %s namespace of %d to list of namespaces to attach to", + info->name, self->child_pid1); + } + + ASSERT_EQ(setns(self->child_pidfd1, flags), 0) { + TH_LOG("%m - Failed to setns to namespaces of %d vid pidfd %d", + self->child_pid1, self->child_pidfd1); + } + + /* + * Can't setns to a user namespace outside of our hierarchy since we + * don't have caps in there and didn't create it. That means that under + * no circumstances should we be able to setns to any of the other + * ones since they aren't owned by our user namespace. + */ + for (i = 0; i < PIDFD_NS_MAX; i++) { + const struct ns_info *info = &ns_info[i]; + + if (self->child_nsfds2[i] < 0 || !info->flag) + continue; + + ASSERT_NE(setns(self->child_pidfd2, info->flag), 0) { + TH_LOG("Managed to setns to %s namespace of %d via pidfd %d", + info->name, self->child_pid2, + self->child_pidfd2); + } + TH_LOG("%m - Correctly failed to setns to %s namespace of %d via pidfd %d", + info->name, self->child_pid2, + self->child_pidfd2); + + ASSERT_NE(setns(self->child_nsfds2[i], info->flag), 0) { + TH_LOG("Managed to setns to %s namespace of %d via nsfd %d", + info->name, self->child_pid2, + self->child_nsfds2[i]); + } + TH_LOG("%m - Correctly failed to setns to %s namespace of %d via nsfd %d", + info->name, self->child_pid2, + self->child_nsfds2[i]); + } +} + +TEST_HARNESS_MAIN From a7cf53b7abdfbf65540a569d5627a62ce118ffe9 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Wed, 13 May 2020 16:38:40 +0800 Subject: [PATCH 0416/1043] MIPS: Fix typos in arch/mips/Kbuild.platforms Commit 26bff9eb49201aeb ("MIPS: Only include the platform file needed") misspelled "txx9" to "tx99", so fix it. Fixes: 26bff9eb49201aeb ("MIPS: Only include the platform file needed") Signed-off-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kbuild.platforms | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/Kbuild.platforms b/arch/mips/Kbuild.platforms index 5e3f6ed96292..c7368a81fd1e 100644 --- a/arch/mips/Kbuild.platforms +++ b/arch/mips/Kbuild.platforms @@ -36,8 +36,8 @@ platform-$(CONFIG_SIBYTE_SB1250) += sibyte/ platform-$(CONFIG_SIBYTE_BCM1x55) += sibyte/ platform-$(CONFIG_SIBYTE_BCM1x80) += sibyte/ platform-$(CONFIG_SNI_RM) += sni/ -platform-$(CONFIG_MACH_TX39XX) += tx99/ -platform-$(CONFIG_MACH_TX49XX) += tx99/ +platform-$(CONFIG_MACH_TX39XX) += txx9/ +platform-$(CONFIG_MACH_TX49XX) += txx9/ platform-$(CONFIG_MACH_VR41XX) += vr41xx/ # include the platform specific files From 125be5868eaa5ceb9d0b5a21e281aca6c9c16a9b Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Wed, 13 May 2020 16:38:41 +0800 Subject: [PATCH 0417/1043] MIPS: Fix "make clean" error due to recent changes Commit 26bff9eb49201aeb ("MIPS: Only include the platformfile needed") moves platform-(CONFIG_XYZ) from arch/mips/xyz/Platform to arch/mips/ Kbuild.platforms. This change causes an error when "make clean": ./scripts/Makefile.clean:15: arch/mips/vr41xx/Makefile: No such file or directory make[3]: *** No rule to make target `arch/mips/vr41xx/Makefile'. Stop. make[2]: *** [arch/mips/vr41xx] Error 2 make[1]: *** [_clean_arch/mips] Error 2 make: *** [sub-make] Error 2 Clean-files are defined in arch/mips/Kbuild: obj- := $(platform-) Due to the movement of platform-(CONFIG_XYZ), "make clean" will enter arch/mips/vr41xx/ whether CONFIG_MACH_VR41XX is defined or not. Because there is no Makefile in arch/mips/vr41xx/, "make clean" fails. I don't know what is the best way to fix it, but it seems like we can avoid this error by changing the obj- definition: obj- := $(platform-y) Fixes: 26bff9eb49201aeb ("MIPS: Only include the platformfile needed") Signed-off-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kbuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/Kbuild b/arch/mips/Kbuild index a8d5e4fcbe53..d5d6ef9bb986 100644 --- a/arch/mips/Kbuild +++ b/arch/mips/Kbuild @@ -12,7 +12,7 @@ obj-y := $(platform-y) # make clean traverses $(obj-) without having included .config, so # everything ends up here -obj- := $(platform-) +obj- := $(platform-y) # mips object files # The object files are linked as core-y files would be linked From dae2f8ed7992e88c8d62c54e8295ffc8475b4a80 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Thu, 30 Apr 2020 07:41:37 -0700 Subject: [PATCH 0418/1043] fs: Lift XFS_IDONTCACHE to the VFS layer DAX effective mode (S_DAX) changes requires inode eviction. XFS has an advisory flag (XFS_IDONTCACHE) to prevent caching of the inode if no other additional references are taken. We lift this flag to the VFS layer and change the behavior slightly by allowing the flag to remain even if multiple references are taken. This will expedite the eviction of inodes to change S_DAX. Cc: Al Viro Reviewed-by: Dave Chinner Reviewed-by: Jan Kara Signed-off-by: Ira Weiny Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_icache.c | 4 ++-- fs/xfs/xfs_inode.h | 3 +-- fs/xfs/xfs_super.c | 2 +- include/linux/fs.h | 6 +++++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 8bf1d15be3f6..fa8554b3308d 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -479,7 +479,7 @@ xfs_iget_cache_hit( xfs_ilock(ip, lock_flags); if (!(flags & XFS_IGET_INCORE)) - xfs_iflags_clear(ip, XFS_ISTALE | XFS_IDONTCACHE); + xfs_iflags_clear(ip, XFS_ISTALE); XFS_STATS_INC(mp, xs_ig_found); return 0; @@ -561,7 +561,7 @@ xfs_iget_cache_miss( */ iflags = XFS_INEW; if (flags & XFS_IGET_DONTCACHE) - iflags |= XFS_IDONTCACHE; + VFS_I(ip)->i_state |= I_DONTCACHE; ip->i_udquot = NULL; ip->i_gdquot = NULL; ip->i_pdquot = NULL; diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index c6a63f6764a6..95209e3256a2 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -218,8 +218,7 @@ static inline bool xfs_inode_has_cow_data(struct xfs_inode *ip) #define XFS_IFLOCK (1 << __XFS_IFLOCK_BIT) #define __XFS_IPINNED_BIT 8 /* wakeup key for zero pin count */ #define XFS_IPINNED (1 << __XFS_IPINNED_BIT) -#define XFS_IDONTCACHE (1 << 9) /* don't cache the inode long term */ -#define XFS_IEOFBLOCKS (1 << 10)/* has the preallocblocks tag set */ +#define XFS_IEOFBLOCKS (1 << 9) /* has the preallocblocks tag set */ /* * If this unlinked inode is in the middle of recovery, don't let drop_inode * truncate and free the inode. This can happen if we iget the inode during diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 424bb9a2d532..a7f8e387ebfe 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -702,7 +702,7 @@ xfs_fs_drop_inode( return 0; } - return generic_drop_inode(inode) || (ip->i_flags & XFS_IDONTCACHE); + return generic_drop_inode(inode); } static void diff --git a/include/linux/fs.h b/include/linux/fs.h index a87cc5845a02..44bd45af760f 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2156,6 +2156,8 @@ static inline void kiocb_clone(struct kiocb *kiocb, struct kiocb *kiocb_src, * * I_CREATING New object's inode in the middle of setting up. * + * I_DONTCACHE Evict inode as soon as it is not used anymore. + * * Q: What is the difference between I_WILL_FREE and I_FREEING? */ #define I_DIRTY_SYNC (1 << 0) @@ -2178,6 +2180,7 @@ static inline void kiocb_clone(struct kiocb *kiocb, struct kiocb *kiocb_src, #define I_WB_SWITCH (1 << 13) #define I_OVL_INUSE (1 << 14) #define I_CREATING (1 << 15) +#define I_DONTCACHE (1 << 16) #define I_DIRTY_INODE (I_DIRTY_SYNC | I_DIRTY_DATASYNC) #define I_DIRTY (I_DIRTY_INODE | I_DIRTY_PAGES) @@ -3049,7 +3052,8 @@ extern int inode_needs_sync(struct inode *inode); extern int generic_delete_inode(struct inode *inode); static inline int generic_drop_inode(struct inode *inode) { - return !inode->i_nlink || inode_unhashed(inode); + return !inode->i_nlink || inode_unhashed(inode) || + (inode->i_state & I_DONTCACHE); } extern struct inode *ilookup5_nowait(struct super_block *sb, From 2c567af418e3f9380c2051aada58b4e5a4b5c2ad Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Thu, 30 Apr 2020 07:41:37 -0700 Subject: [PATCH 0419/1043] fs: Introduce DCACHE_DONTCACHE DCACHE_DONTCACHE indicates a dentry should not be cached on final dput(). Also add a helper function to mark DCACHE_DONTCACHE on all dentries pointing to a specific inode when that inode is being set I_DONTCACHE. This facilitates dropping dentry references to inodes sooner which require eviction to swap S_DAX mode. Cc: Al Viro Signed-off-by: Ira Weiny Reviewed-by: Jan Kara Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/dcache.c | 19 +++++++++++++++++++ fs/xfs/xfs_icache.c | 2 +- include/linux/dcache.h | 2 ++ include/linux/fs.h | 1 + 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/fs/dcache.c b/fs/dcache.c index b280e07e162b..0d07fb335b78 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -647,6 +647,10 @@ static inline bool retain_dentry(struct dentry *dentry) if (dentry->d_op->d_delete(dentry)) return false; } + + if (unlikely(dentry->d_flags & DCACHE_DONTCACHE)) + return false; + /* retain; LRU fodder */ dentry->d_lockref.count--; if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST))) @@ -656,6 +660,21 @@ static inline bool retain_dentry(struct dentry *dentry) return true; } +void d_mark_dontcache(struct inode *inode) +{ + struct dentry *de; + + spin_lock(&inode->i_lock); + hlist_for_each_entry(de, &inode->i_dentry, d_u.d_alias) { + spin_lock(&de->d_lock); + de->d_flags |= DCACHE_DONTCACHE; + spin_unlock(&de->d_lock); + } + inode->i_state |= I_DONTCACHE; + spin_unlock(&inode->i_lock); +} +EXPORT_SYMBOL(d_mark_dontcache); + /* * Finish off a dentry we've decided to kill. * dentry->d_lock must be held, returns with it unlocked. diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index fa8554b3308d..d7deab608e2c 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -561,7 +561,7 @@ xfs_iget_cache_miss( */ iflags = XFS_INEW; if (flags & XFS_IGET_DONTCACHE) - VFS_I(ip)->i_state |= I_DONTCACHE; + d_mark_dontcache(VFS_I(ip)); ip->i_udquot = NULL; ip->i_gdquot = NULL; ip->i_pdquot = NULL; diff --git a/include/linux/dcache.h b/include/linux/dcache.h index c1488cc84fd9..a81f0c3cf352 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -177,6 +177,8 @@ struct dentry_operations { #define DCACHE_REFERENCED 0x00000040 /* Recently used, don't discard. */ +#define DCACHE_DONTCACHE 0x00000080 /* Purge from memory on final dput() */ + #define DCACHE_CANT_MOUNT 0x00000100 #define DCACHE_GENOCIDE 0x00000200 #define DCACHE_SHRINK_LIST 0x00000400 diff --git a/include/linux/fs.h b/include/linux/fs.h index 44bd45af760f..7c3e8c0306e0 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3055,6 +3055,7 @@ static inline int generic_drop_inode(struct inode *inode) return !inode->i_nlink || inode_unhashed(inode) || (inode->i_state & I_DONTCACHE); } +extern void d_mark_dontcache(struct inode *inode); extern struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval, int (*test)(struct inode *, void *), From f74f94140fa50f768e61d626de4c146502b9102d Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 23 Apr 2020 13:22:27 -0400 Subject: [PATCH 0420/1043] KVM: SVM: introduce nested_run_pending We want to inject vmexits immediately from svm_check_nested_events, so that the interrupt/NMI window requests happen in inject_pending_event right after it returns. This however has the same issue as in vmx_check_nested_events, so introduce a nested_run_pending flag with the exact same purpose of delaying vmexit injection after the vmentry. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 4 +++- arch/x86/kvm/svm/svm.c | 1 + arch/x86/kvm/svm/svm.h | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 1429f506fe9e..7a724ea3d994 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -414,6 +414,7 @@ int nested_svm_vmrun(struct vcpu_svm *svm) copy_vmcb_control_area(hsave, vmcb); + svm->nested.nested_run_pending = 1; enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, &map); if (!nested_svm_vmrun_msrpm(svm)) { @@ -815,7 +816,8 @@ static int svm_check_nested_events(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); bool block_nested_events = - kvm_event_needs_reinjection(vcpu) || svm->nested.exit_required; + kvm_event_needs_reinjection(vcpu) || svm->nested.exit_required || + svm->nested.nested_run_pending; if (kvm_cpu_has_interrupt(vcpu) && nested_exit_on_intr(svm)) { if (block_nested_events) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index b627564e41f9..c2a4e2d21676 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3429,6 +3429,7 @@ static enum exit_fastpath_completion svm_vcpu_run(struct kvm_vcpu *vcpu) sync_cr8_to_lapic(vcpu); svm->next_rip = 0; + svm->nested.nested_run_pending = 0; svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 98c2890d561d..435f3328c99c 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -97,6 +97,10 @@ struct nested_state { /* A VMEXIT is required but not yet emulated */ bool exit_required; + /* A VMRUN has started but has not yet been performed, so + * we cannot inject a nested vmexit yet. */ + bool nested_run_pending; + /* cache for intercepts of the guest */ u32 intercept_cr; u32 intercept_dr; From 38c0b192bd6df089cdffd9828d19e764385f96e6 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 23 Apr 2020 13:13:09 -0400 Subject: [PATCH 0421/1043] KVM: SVM: leave halted state on vmexit Similar to VMX, we need to leave the halted state when performing a vmexit. Failure to do so will cause a hang after vmexit. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 7a724ea3d994..2a0f1b043145 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -473,6 +473,9 @@ int nested_svm_vmexit(struct vcpu_svm *svm) leave_guest_mode(&svm->vcpu); svm->nested.vmcb = 0; + /* in case we halted in L2 */ + svm->vcpu.arch.mp_state = KVM_MP_STATE_RUNNABLE; + /* Give the current vmcb to the guest */ disable_gif(svm); From 6e085cbfb0f0c7de4ca0f370adb572b0e07a818c Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 23 Apr 2020 13:15:33 -0400 Subject: [PATCH 0422/1043] KVM: SVM: immediately inject INTR vmexit We can immediately leave SVM guest mode in svm_check_nested_events now that we have the nested_run_pending mechanism. This makes things easier because we can run the rest of inject_pending_event with GIF=0, and KVM will naturally end up requesting the next interrupt window. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 2a0f1b043145..4654668798b7 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -801,13 +801,13 @@ int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, static void nested_svm_intr(struct vcpu_svm *svm) { + trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); + svm->vmcb->control.exit_code = SVM_EXIT_INTR; svm->vmcb->control.exit_info_1 = 0; svm->vmcb->control.exit_info_2 = 0; - /* nested_svm_vmexit this gets called afterwards from handle_exit */ - svm->nested.exit_required = true; - trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); + nested_svm_vmexit(svm); } static bool nested_exit_on_intr(struct vcpu_svm *svm) From 9c3d370a8efaac49b0520617c9ab4e552aefbf1e Mon Sep 17 00:00:00 2001 From: Cathy Avery Date: Tue, 14 Apr 2020 16:11:06 -0400 Subject: [PATCH 0423/1043] KVM: SVM: Implement check_nested_events for NMI Migrate nested guest NMI intercept processing to new check_nested_events. Signed-off-by: Cathy Avery Message-Id: <20200414201107.22952-2-cavery@redhat.com> [Reorder clauses as NMIs have higher priority than IRQs; inject immediate vmexit as is now done for IRQ vmexits. - Paolo] Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 21 +++++++++++++++++++++ arch/x86/kvm/svm/svm.c | 6 ++---- arch/x86/kvm/svm/svm.h | 15 --------------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 4654668798b7..3f268a3041a3 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -799,6 +799,20 @@ int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, return vmexit; } +static bool nested_exit_on_nmi(struct vcpu_svm *svm) +{ + return (svm->nested.intercept & (1ULL << INTERCEPT_NMI)); +} + +static void nested_svm_nmi(struct vcpu_svm *svm) +{ + svm->vmcb->control.exit_code = SVM_EXIT_NMI; + svm->vmcb->control.exit_info_1 = 0; + svm->vmcb->control.exit_info_2 = 0; + + nested_svm_vmexit(svm); +} + static void nested_svm_intr(struct vcpu_svm *svm) { trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip); @@ -822,6 +836,13 @@ static int svm_check_nested_events(struct kvm_vcpu *vcpu) kvm_event_needs_reinjection(vcpu) || svm->nested.exit_required || svm->nested.nested_run_pending; + if (vcpu->arch.nmi_pending && nested_exit_on_nmi(svm)) { + if (block_nested_events) + return -EBUSY; + nested_svm_nmi(svm); + return 0; + } + if (kvm_cpu_has_interrupt(vcpu) && nested_exit_on_intr(svm)) { if (block_nested_events) return -EBUSY; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index c2a4e2d21676..f97f29e1eb49 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3070,9 +3070,10 @@ static int svm_nmi_allowed(struct kvm_vcpu *vcpu) struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb = svm->vmcb; int ret; + ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) && !(svm->vcpu.arch.hflags & HF_NMI_MASK); - ret = ret && gif_set(svm) && nested_svm_nmi(svm); + ret = ret && gif_set(svm); return ret; } @@ -3150,9 +3151,6 @@ static void enable_nmi_window(struct kvm_vcpu *vcpu) return; /* STGI will cause a vm exit */ } - if (svm->nested.exit_required) - return; /* we're not going to run the guest yet */ - /* * Something prevents NMI from been injected. Single step over possible * problem (IRET or exception injection or interrupt shadow) diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 435f3328c99c..a2bc33aadb67 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -373,21 +373,6 @@ void disable_nmi_singlestep(struct vcpu_svm *svm); #define NESTED_EXIT_DONE 1 /* Exit caused nested vmexit */ #define NESTED_EXIT_CONTINUE 2 /* Further checks needed */ -/* This function returns true if it is save to enable the nmi window */ -static inline bool nested_svm_nmi(struct vcpu_svm *svm) -{ - if (!is_guest_mode(&svm->vcpu)) - return true; - - if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI))) - return true; - - svm->vmcb->control.exit_code = SVM_EXIT_NMI; - svm->nested.exit_required = true; - - return false; -} - static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu) { return is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK); From 6ce347af1499deff5890a52aedacd451330af706 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 22 Apr 2020 19:25:38 -0700 Subject: [PATCH 0424/1043] KVM: nVMX: Preserve exception priority irrespective of exiting behavior Short circuit vmx_check_nested_events() if an exception is pending and needs to be injected into L2, priority between coincident events is not dependent on exiting behavior. This fixes a bug where a single-step #DB that is not intercepted by L1 is incorrectly dropped due to servicing a VMX Preemption Timer VM-Exit. Injected exceptions also need to be blocked if nested VM-Enter is pending or an exception was already injected, otherwise injecting the exception could overwrite an existing event injection from L1. Technically, this scenario should be impossible, i.e. KVM shouldn't inject its own exception during nested VM-Enter. This will be addressed in a future patch. Note, event priority between SMI, NMI and INTR is incorrect for L2, e.g. SMI should take priority over VM-Exit on NMI/INTR, and NMI that is injected into L2 should take priority over VM-Exit INTR. This will also be addressed in a future patch. Fixes: b6b8a1451fc4 ("KVM: nVMX: Rework interception of IRQs and NMIs") Reported-by: Jim Mattson Cc: Oliver Upton Cc: Peter Shier Signed-off-by: Sean Christopherson Message-Id: <20200423022550.15113-2-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index b644bbf85460..de678be3c92d 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3716,11 +3716,11 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu) /* * Process any exceptions that are not debug traps before MTF. */ - if (vcpu->arch.exception.pending && - !vmx_pending_dbg_trap(vcpu) && - nested_vmx_check_exception(vcpu, &exit_qual)) { + if (vcpu->arch.exception.pending && !vmx_pending_dbg_trap(vcpu)) { if (block_nested_events) return -EBUSY; + if (!nested_vmx_check_exception(vcpu, &exit_qual)) + goto no_vmexit; nested_vmx_inject_exception_vmexit(vcpu, exit_qual); return 0; } @@ -3733,10 +3733,11 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu) return 0; } - if (vcpu->arch.exception.pending && - nested_vmx_check_exception(vcpu, &exit_qual)) { + if (vcpu->arch.exception.pending) { if (block_nested_events) return -EBUSY; + if (!nested_vmx_check_exception(vcpu, &exit_qual)) + goto no_vmexit; nested_vmx_inject_exception_vmexit(vcpu, exit_qual); return 0; } @@ -3771,6 +3772,7 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu) return 0; } +no_vmexit: vmx_complete_nested_posted_interrupt(vcpu); return 0; } From d2060bd42e4482b15c35f961a294ee57f369027d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 22 Apr 2020 19:25:39 -0700 Subject: [PATCH 0425/1043] KVM: nVMX: Open a window for pending nested VMX preemption timer Add a kvm_x86_ops hook to detect a nested pending "hypervisor timer" and use it to effectively open a window for servicing the expired timer. Like pending SMIs on VMX, opening a window simply means requesting an immediate exit. This fixes a bug where an expired VMX preemption timer (for L2) will be delayed and/or lost if a pending exception is injected into L2. The pending exception is rightly prioritized by vmx_check_nested_events() and injected into L2, with the preemption timer left pending. Because no window opened, L2 is free to run uninterrupted. Fixes: f4124500c2c13 ("KVM: nVMX: Fully emulate preemption timer") Reported-by: Jim Mattson Cc: Oliver Upton Cc: Peter Shier Signed-off-by: Sean Christopherson Message-Id: <20200423022550.15113-3-sean.j.christopherson@intel.com> [Check it in kvm_vcpu_has_events too, to ensure that the preemption timer is serviced promptly even if the vCPU is halted and L1 is not intercepting HLT. - Paolo] Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/vmx/nested.c | 10 ++++++++-- arch/x86/kvm/x86.c | 9 +++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index b3a5da27c2a5..e6671c61fd65 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1256,6 +1256,7 @@ struct kvm_x86_ops { struct kvm_x86_nested_ops { int (*check_events)(struct kvm_vcpu *vcpu); + bool (*hv_timer_pending)(struct kvm_vcpu *vcpu); int (*get_state)(struct kvm_vcpu *vcpu, struct kvm_nested_state __user *user_kvm_nested_state, unsigned user_data_size); diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index de678be3c92d..51220a14a633 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3687,6 +3687,12 @@ static void nested_vmx_update_pending_dbg(struct kvm_vcpu *vcpu) vcpu->arch.exception.payload); } +static bool nested_vmx_preemption_timer_pending(struct kvm_vcpu *vcpu) +{ + return nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) && + to_vmx(vcpu)->nested.preemption_timer_expired; +} + static int vmx_check_nested_events(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -3742,8 +3748,7 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu) return 0; } - if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) && - vmx->nested.preemption_timer_expired) { + if (nested_vmx_preemption_timer_pending(vcpu)) { if (block_nested_events) return -EBUSY; nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0); @@ -6448,6 +6453,7 @@ __init int nested_vmx_hardware_setup(struct kvm_x86_ops *ops, struct kvm_x86_nested_ops vmx_nested_ops = { .check_events = vmx_check_nested_events, + .hv_timer_pending = nested_vmx_preemption_timer_pending, .get_state = vmx_get_nested_state, .set_state = vmx_set_nested_state, .get_vmcs12_pages = nested_get_vmcs12_pages, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 542a00008caa..e874182d113c 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8355,6 +8355,10 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_x86_ops.enable_nmi_window(vcpu); if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win) kvm_x86_ops.enable_irq_window(vcpu); + if (is_guest_mode(vcpu) && + kvm_x86_ops.nested_ops->hv_timer_pending && + kvm_x86_ops.nested_ops->hv_timer_pending(vcpu)) + req_immediate_exit = true; WARN_ON(vcpu->arch.exception.pending); } @@ -10211,6 +10215,11 @@ static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) if (kvm_hv_has_stimer_pending(vcpu)) return true; + if (is_guest_mode(vcpu) && + kvm_x86_ops.nested_ops->hv_timer_pending && + kvm_x86_ops.nested_ops->hv_timer_pending(vcpu)) + return true; + return false; } From 8081ad06b68a728e676d3b08e9ab70ce4039747b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 22 Apr 2020 19:25:40 -0700 Subject: [PATCH 0426/1043] KVM: x86: Set KVM_REQ_EVENT if run is canceled with req_immediate_exit set Re-request KVM_REQ_EVENT if vcpu_enter_guest() bails after processing pending requests and an immediate exit was requested. This fixes a bug where a pending event, e.g. VMX preemption timer, is delayed and/or lost if the exit was deferred due to something other than a higher priority _injected_ event, e.g. due to a pending nested VM-Enter. This bug only affects the !injected case as kvm_x86_ops.cancel_injection() sets KVM_REQ_EVENT to redo the injection, but that's purely serendipitous behavior with respect to the deferred event. Note, emulated preemption timer isn't the only event that can be affected, it simply happens to be the only event where not re-requesting KVM_REQ_EVENT is blatantly visible to the guest. Fixes: f4124500c2c13 ("KVM: nVMX: Fully emulate preemption timer") Signed-off-by: Sean Christopherson Message-Id: <20200423022550.15113-4-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index e874182d113c..bdcb4e7c579d 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8519,6 +8519,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) return r; cancel_injection: + if (req_immediate_exit) + kvm_make_request(KVM_REQ_EVENT, vcpu); kvm_x86_ops.cancel_injection(vcpu); if (unlikely(vcpu->arch.apic_attention)) kvm_lapic_sync_from_vapic(vcpu); From 88c604b66eb6a74841cd40f5bdf639112ad69115 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 22 Apr 2020 19:25:41 -0700 Subject: [PATCH 0427/1043] KVM: x86: Make return for {interrupt_nmi,smi}_allowed() a bool instead of int Return an actual bool for kvm_x86_ops' {interrupt_nmi}_allowed() hook to better reflect the return semantics, and to avoid creating an even bigger mess when the related VMX code is refactored in upcoming patches. Signed-off-by: Sean Christopherson Message-Id: <20200423022550.15113-5-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 6 +++--- arch/x86/kvm/svm/svm.c | 16 ++++++++-------- arch/x86/kvm/vmx/vmx.c | 14 +++++++------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index e6671c61fd65..44268642b3c6 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1138,8 +1138,8 @@ struct kvm_x86_ops { void (*set_nmi)(struct kvm_vcpu *vcpu); void (*queue_exception)(struct kvm_vcpu *vcpu); void (*cancel_injection)(struct kvm_vcpu *vcpu); - int (*interrupt_allowed)(struct kvm_vcpu *vcpu); - int (*nmi_allowed)(struct kvm_vcpu *vcpu); + bool (*interrupt_allowed)(struct kvm_vcpu *vcpu); + bool (*nmi_allowed)(struct kvm_vcpu *vcpu); bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked); void (*enable_nmi_window)(struct kvm_vcpu *vcpu); @@ -1237,7 +1237,7 @@ struct kvm_x86_ops { void (*setup_mce)(struct kvm_vcpu *vcpu); - int (*smi_allowed)(struct kvm_vcpu *vcpu); + bool (*smi_allowed)(struct kvm_vcpu *vcpu); int (*pre_enter_smm)(struct kvm_vcpu *vcpu, char *smstate); int (*pre_leave_smm)(struct kvm_vcpu *vcpu, const char *smstate); int (*enable_smi_window)(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index f97f29e1eb49..c6e5374db825 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3065,11 +3065,11 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) set_cr_intercept(svm, INTERCEPT_CR8_WRITE); } -static int svm_nmi_allowed(struct kvm_vcpu *vcpu) +static bool svm_nmi_allowed(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb = svm->vmcb; - int ret; + bool ret; ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) && !(svm->vcpu.arch.hflags & HF_NMI_MASK); @@ -3098,14 +3098,14 @@ static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) } } -static int svm_interrupt_allowed(struct kvm_vcpu *vcpu) +static bool svm_interrupt_allowed(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb = svm->vmcb; if (!gif_set(svm) || (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)) - return 0; + return false; if (is_guest_mode(vcpu) && (svm->vcpu.arch.hflags & HF_VINTR_MASK)) return !!(svm->vcpu.arch.hflags & HF_HIF_MASK); @@ -3767,23 +3767,23 @@ static void svm_setup_mce(struct kvm_vcpu *vcpu) vcpu->arch.mcg_cap &= 0x1ff; } -static int svm_smi_allowed(struct kvm_vcpu *vcpu) +static bool svm_smi_allowed(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); /* Per APM Vol.2 15.22.2 "Response to SMI" */ if (!gif_set(svm)) - return 0; + return false; if (is_guest_mode(&svm->vcpu) && svm->nested.intercept & (1ULL << INTERCEPT_SMI)) { /* TODO: Might need to set exit_info_1 and exit_info_2 here */ svm->vmcb->control.exit_code = SVM_EXIT_SMI; svm->nested.exit_required = true; - return 0; + return false; } - return 1; + return true; } static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 46aa3ca01929..4ca1f1f4ef1a 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4510,21 +4510,21 @@ void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) } } -static int vmx_nmi_allowed(struct kvm_vcpu *vcpu) +static bool vmx_nmi_allowed(struct kvm_vcpu *vcpu) { if (to_vmx(vcpu)->nested.nested_run_pending) - return 0; + return false; if (!enable_vnmi && to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked) - return 0; + return false; return !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI | GUEST_INTR_STATE_NMI)); } -static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu) +static bool vmx_interrupt_allowed(struct kvm_vcpu *vcpu) { if (to_vmx(vcpu)->nested.nested_run_pending) return false; @@ -7675,12 +7675,12 @@ static void vmx_setup_mce(struct kvm_vcpu *vcpu) ~FEAT_CTL_LMCE_ENABLED; } -static int vmx_smi_allowed(struct kvm_vcpu *vcpu) +static bool vmx_smi_allowed(struct kvm_vcpu *vcpu) { /* we need a nested vmexit to enter SMM, postpone if run is pending */ if (to_vmx(vcpu)->nested.nested_run_pending) - return 0; - return 1; + return false; + return true; } static int vmx_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate) From a9fa7cb6aa997ba58294f1a07d402ce5855bafe1 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 23 Apr 2020 11:02:36 -0400 Subject: [PATCH 0428/1043] KVM: x86: replace is_smm checks with kvm_x86_ops.smi_allowed Do not hardcode is_smm so that all the architectural conditions for blocking SMIs are listed in a single place. Well, in two places because this introduces some code duplication between Intel and AMD. This ensures that nested SVM obeys GIF in kvm_vcpu_has_events. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/kvm/x86.c | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index c6e5374db825..739414028536 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3783,7 +3783,7 @@ static bool svm_smi_allowed(struct kvm_vcpu *vcpu) return false; } - return true; + return !is_smm(vcpu); } static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 4ca1f1f4ef1a..d0bea514d447 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7680,7 +7680,7 @@ static bool vmx_smi_allowed(struct kvm_vcpu *vcpu) /* we need a nested vmexit to enter SMM, postpone if run is pending */ if (to_vmx(vcpu)->nested.nested_run_pending) return false; - return true; + return !is_smm(vcpu); } static int vmx_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index bdcb4e7c579d..446fbdd05bd4 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7764,8 +7764,7 @@ static int inject_pending_event(struct kvm_vcpu *vcpu) if (kvm_event_needs_reinjection(vcpu)) return 0; - if (vcpu->arch.smi_pending && !is_smm(vcpu) && - kvm_x86_ops.smi_allowed(vcpu)) { + if (vcpu->arch.smi_pending && kvm_x86_ops.smi_allowed(vcpu)) { vcpu->arch.smi_pending = false; ++vcpu->arch.smi_count; enter_smm(vcpu); @@ -10206,7 +10205,8 @@ static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) return true; if (kvm_test_request(KVM_REQ_SMI, vcpu) || - (vcpu->arch.smi_pending && !is_smm(vcpu))) + (vcpu->arch.smi_pending && + kvm_x86_ops.smi_allowed(vcpu))) return true; if (kvm_arch_interrupt_allowed(vcpu) && From 429ab576f387c8fcaeb2a172901aaba3c6794310 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 22 Apr 2020 19:25:42 -0700 Subject: [PATCH 0429/1043] KVM: nVMX: Report NMIs as allowed when in L2 and Exit-on-NMI is set Report NMIs as allowed when the vCPU is in L2 and L2 is being run with Exit-on-NMI enabled, as NMIs are always unblocked from L1's perspective in this case. Signed-off-by: Sean Christopherson Message-Id: <20200423022550.15113-7-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 5 ----- arch/x86/kvm/vmx/nested.h | 5 +++++ arch/x86/kvm/vmx/vmx.c | 3 +++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 51220a14a633..643f86d7d537 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -698,11 +698,6 @@ static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu) VM_EXIT_ACK_INTR_ON_EXIT; } -static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu) -{ - return nested_cpu_has_nmi_exiting(get_vmcs12(vcpu)); -} - static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index 7ce9572c3d3a..5cc72ae0e277 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -225,6 +225,11 @@ static inline bool nested_cpu_has_save_preemption_timer(struct vmcs12 *vmcs12) VM_EXIT_SAVE_VMX_PREEMPTION_TIMER; } +static inline bool nested_exit_on_nmi(struct kvm_vcpu *vcpu) +{ + return nested_cpu_has_nmi_exiting(get_vmcs12(vcpu)); +} + /* * In nested virtualization, check if L1 asked to exit on external interrupts. * For most existing hypervisors, this will always return true. diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d0bea514d447..f688e6e876e8 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4515,6 +4515,9 @@ static bool vmx_nmi_allowed(struct kvm_vcpu *vcpu) if (to_vmx(vcpu)->nested.nested_run_pending) return false; + if (is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) + return true; + if (!enable_vnmi && to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked) return false; From bbdad0b5a708ddb37a9f051504c2133fa92df97a Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 23 Apr 2020 08:06:43 -0400 Subject: [PATCH 0430/1043] KVM: nSVM: Report NMIs as allowed when in L2 and Exit-on-NMI is set Report NMIs as allowed when the vCPU is in L2 and L2 is being run with Exit-on-NMI enabled, as NMIs are always unblocked from L1's perspective in this case. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 5 ----- arch/x86/kvm/svm/svm.c | 3 +++ arch/x86/kvm/svm/svm.h | 5 +++++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 3f268a3041a3..2828fa5b6016 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -799,11 +799,6 @@ int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, return vmexit; } -static bool nested_exit_on_nmi(struct vcpu_svm *svm) -{ - return (svm->nested.intercept & (1ULL << INTERCEPT_NMI)); -} - static void nested_svm_nmi(struct vcpu_svm *svm) { svm->vmcb->control.exit_code = SVM_EXIT_NMI; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 739414028536..c4f1846b6259 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3071,6 +3071,9 @@ static bool svm_nmi_allowed(struct kvm_vcpu *vcpu) struct vmcb *vmcb = svm->vmcb; bool ret; + if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) + return true; + ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) && !(svm->vcpu.arch.hflags & HF_NMI_MASK); ret = ret && gif_set(svm); diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index a2bc33aadb67..d8ae654340d4 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -378,6 +378,11 @@ static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu) return is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK); } +static inline bool nested_exit_on_nmi(struct vcpu_svm *svm) +{ + return (svm->nested.intercept & (1ULL << INTERCEPT_NMI)); +} + void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, struct vmcb *nested_vmcb, struct kvm_host_map *map); int nested_svm_vmrun(struct vcpu_svm *svm); From 55714cddbf1028bbfa19fd7d69182de3f135ce99 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 23 Apr 2020 08:17:28 -0400 Subject: [PATCH 0431/1043] KVM: nSVM: Move SMI vmexit handling to svm_check_nested_events() Unlike VMX, SVM allows a hypervisor to take a SMI vmexit without having any special SMM-monitor enablement sequence. Therefore, it has to be handled like interrupts and NMIs. Check for an unblocked SMI in svm_check_nested_events() so that pending SMIs are correctly prioritized over IRQs and NMIs when the latter events will trigger VM-Exit. Note that there is no need to test explicitly for SMI vmexits, because guests always runs outside SMM and therefore can never get an SMI while they are blocked. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 16 ++++++++++++++++ arch/x86/kvm/svm/svm.c | 8 -------- arch/x86/kvm/svm/svm.h | 5 +++++ 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 2828fa5b6016..aaec6d0aa701 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -799,6 +799,15 @@ int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, return vmexit; } +static void nested_svm_smi(struct vcpu_svm *svm) +{ + svm->vmcb->control.exit_code = SVM_EXIT_SMI; + svm->vmcb->control.exit_info_1 = 0; + svm->vmcb->control.exit_info_2 = 0; + + nested_svm_vmexit(svm); +} + static void nested_svm_nmi(struct vcpu_svm *svm) { svm->vmcb->control.exit_code = SVM_EXIT_NMI; @@ -831,6 +840,13 @@ static int svm_check_nested_events(struct kvm_vcpu *vcpu) kvm_event_needs_reinjection(vcpu) || svm->nested.exit_required || svm->nested.nested_run_pending; + if (vcpu->arch.smi_pending && nested_exit_on_smi(svm)) { + if (block_nested_events) + return -EBUSY; + nested_svm_smi(svm); + return 0; + } + if (vcpu->arch.nmi_pending && nested_exit_on_nmi(svm)) { if (block_nested_events) return -EBUSY; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index c4f1846b6259..83b8bc305fe1 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3778,14 +3778,6 @@ static bool svm_smi_allowed(struct kvm_vcpu *vcpu) if (!gif_set(svm)) return false; - if (is_guest_mode(&svm->vcpu) && - svm->nested.intercept & (1ULL << INTERCEPT_SMI)) { - /* TODO: Might need to set exit_info_1 and exit_info_2 here */ - svm->vmcb->control.exit_code = SVM_EXIT_SMI; - svm->nested.exit_required = true; - return false; - } - return !is_smm(vcpu); } diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index d8ae654340d4..4dc6d2b4b721 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -378,6 +378,11 @@ static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu) return is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK); } +static inline bool nested_exit_on_smi(struct vcpu_svm *svm) +{ + return (svm->nested.intercept & (1ULL << INTERCEPT_SMI)); +} + static inline bool nested_exit_on_nmi(struct vcpu_svm *svm) { return (svm->nested.intercept & (1ULL << INTERCEPT_NMI)); From 1b660b6baaafd8b9056740b83decd7fc74023627 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 22 Apr 2020 19:25:44 -0700 Subject: [PATCH 0432/1043] KVM: VMX: Split out architectural interrupt/NMI blocking checks Move the architectural (non-KVM specific) interrupt/NMI blocking checks to a separate helper so that they can be used in a future patch by vmx_check_nested_events(). No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20200423022550.15113-8-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 35 ++++++++++++++++++++++------------- arch/x86/kvm/vmx/vmx.h | 2 ++ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index f688e6e876e8..a5140ed7dbf9 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4510,21 +4510,35 @@ void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) } } +bool vmx_nmi_blocked(struct kvm_vcpu *vcpu) +{ + if (is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) + return false; + + if (!enable_vnmi && to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked) + return true; + + return (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & + (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI | + GUEST_INTR_STATE_NMI)); +} + static bool vmx_nmi_allowed(struct kvm_vcpu *vcpu) { if (to_vmx(vcpu)->nested.nested_run_pending) return false; - if (is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) - return true; + return !vmx_nmi_blocked(vcpu); +} - if (!enable_vnmi && - to_vmx(vcpu)->loaded_vmcs->soft_vnmi_blocked) +bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu) +{ + if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) return false; - return !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & - (GUEST_INTR_STATE_MOV_SS | GUEST_INTR_STATE_STI - | GUEST_INTR_STATE_NMI)); + return !(vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) || + (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & + (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)); } static bool vmx_interrupt_allowed(struct kvm_vcpu *vcpu) @@ -4532,12 +4546,7 @@ static bool vmx_interrupt_allowed(struct kvm_vcpu *vcpu) if (to_vmx(vcpu)->nested.nested_run_pending) return false; - if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) - return true; - - return (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) && - !(vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & - (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)); + return !vmx_interrupt_blocked(vcpu); } static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index edfb739e5907..b5e773267abe 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -344,6 +344,8 @@ void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa); void update_exception_bitmap(struct kvm_vcpu *vcpu); void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu); +bool vmx_nmi_blocked(struct kvm_vcpu *vcpu); +bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu); bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu); void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked); void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu); From cae96af18452336aff85c95c6892aab1ed955eed Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 23 Apr 2020 14:19:26 -0400 Subject: [PATCH 0433/1043] KVM: SVM: Split out architectural interrupt/NMI/SMI blocking checks Move the architectural (non-KVM specific) interrupt/NMI/SMI blocking checks to a separate helper so that they can be used in a future patch by svm_check_nested_events(). No functional change intended. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 51 +++++++++++++++++++++++++++++++++--------- arch/x86/kvm/svm/svm.h | 3 +++ 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 83b8bc305fe1..5738be661a1f 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3065,22 +3065,33 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr) set_cr_intercept(svm, INTERCEPT_CR8_WRITE); } -static bool svm_nmi_allowed(struct kvm_vcpu *vcpu) +bool svm_nmi_blocked(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb = svm->vmcb; bool ret; - if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) + if (!gif_set(svm)) return true; - ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) && - !(svm->vcpu.arch.hflags & HF_NMI_MASK); - ret = ret && gif_set(svm); + if (is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) + return false; + + ret = (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) || + (svm->vcpu.arch.hflags & HF_NMI_MASK); return ret; } +static bool svm_nmi_allowed(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + if (svm->nested.nested_run_pending) + return false; + + return !svm_nmi_blocked(vcpu); +} + static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); @@ -3101,19 +3112,28 @@ static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked) } } -static bool svm_interrupt_allowed(struct kvm_vcpu *vcpu) +bool svm_interrupt_blocked(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb = svm->vmcb; if (!gif_set(svm) || (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)) - return false; + return true; if (is_guest_mode(vcpu) && (svm->vcpu.arch.hflags & HF_VINTR_MASK)) - return !!(svm->vcpu.arch.hflags & HF_HIF_MASK); + return !(svm->vcpu.arch.hflags & HF_HIF_MASK); else - return !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF); + return !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF); +} + +static bool svm_interrupt_allowed(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + if (svm->nested.nested_run_pending) + return false; + + return !svm_interrupt_blocked(vcpu); } static void enable_irq_window(struct kvm_vcpu *vcpu) @@ -3770,15 +3790,24 @@ static void svm_setup_mce(struct kvm_vcpu *vcpu) vcpu->arch.mcg_cap &= 0x1ff; } -static bool svm_smi_allowed(struct kvm_vcpu *vcpu) +bool svm_smi_blocked(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); /* Per APM Vol.2 15.22.2 "Response to SMI" */ if (!gif_set(svm)) + return true; + + return is_smm(vcpu); +} + +static bool svm_smi_allowed(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + if (svm->nested.nested_run_pending) return false; - return !is_smm(vcpu); + return !svm_smi_blocked(vcpu); } static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate) diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 4dc6d2b4b721..f6d604b72a4c 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -366,6 +366,9 @@ void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4); void svm_flush_tlb(struct kvm_vcpu *vcpu); void disable_nmi_singlestep(struct vcpu_svm *svm); +bool svm_smi_blocked(struct kvm_vcpu *vcpu); +bool svm_nmi_blocked(struct kvm_vcpu *vcpu); +bool svm_interrupt_blocked(struct kvm_vcpu *vcpu); /* nested.c */ From 15ff0b450b6541120803de40af5ca96dde1b56b2 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 22 Apr 2020 19:25:45 -0700 Subject: [PATCH 0434/1043] KVM: nVMX: Preserve IRQ/NMI priority irrespective of exiting behavior Short circuit vmx_check_nested_events() if an unblocked IRQ/NMI is pending and needs to be injected into L2, priority between coincident events is not dependent on exiting behavior. Fixes: b6b8a1451fc4 ("KVM: nVMX: Rework interception of IRQs and NMIs") Signed-off-by: Sean Christopherson Message-Id: <20200423022550.15113-9-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 643f86d7d537..224887edd39e 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3750,9 +3750,12 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu) return 0; } - if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) { + if (vcpu->arch.nmi_pending && !vmx_nmi_blocked(vcpu)) { if (block_nested_events) return -EBUSY; + if (!nested_exit_on_nmi(vcpu)) + goto no_vmexit; + nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, NMI_VECTOR | INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK, 0); @@ -3765,9 +3768,11 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu) return 0; } - if (kvm_cpu_has_interrupt(vcpu) && nested_exit_on_intr(vcpu)) { + if (kvm_cpu_has_interrupt(vcpu) && !vmx_interrupt_blocked(vcpu)) { if (block_nested_events) return -EBUSY; + if (!nested_exit_on_intr(vcpu)) + goto no_vmexit; nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0); return 0; } From 1cd2f0b0dd9218fd11ced9ac97237f0c9517f49e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 22 Apr 2020 19:25:46 -0700 Subject: [PATCH 0435/1043] KVM: nVMX: Prioritize SMI over nested IRQ/NMI Check for an unblocked SMI in vmx_check_nested_events() so that pending SMIs are correctly prioritized over IRQs and NMIs when the latter events will trigger VM-Exit. This also fixes an issue where an SMI that was marked pending while processing a nested VM-Enter wouldn't trigger an immediate exit, i.e. would be incorrectly delayed until L2 happened to take a VM-Exit. Fixes: 64d6067057d96 ("KVM: x86: stubs for SMM support") Signed-off-by: Sean Christopherson Message-Id: <20200423022550.15113-10-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 224887edd39e..669445136144 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3750,6 +3750,12 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu) return 0; } + if (vcpu->arch.smi_pending && !is_smm(vcpu)) { + if (block_nested_events) + return -EBUSY; + goto no_vmexit; + } + if (vcpu->arch.nmi_pending && !vmx_nmi_blocked(vcpu)) { if (block_nested_events) return -EBUSY; From fc6f7c03ad80693c953d5cdacfad41f174289531 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 23 Apr 2020 18:02:45 -0400 Subject: [PATCH 0436/1043] KVM: nSVM: Report interrupts as allowed when in L2 and exit-on-interrupt is set Report interrupts as allowed when the vCPU is in L2 and L2 is being run with exit-on-interrupts enabled and EFLAGS.IF=1 (either on the host or on the guest according to VINTR). Interrupts are always unblocked from L1's perspective in this case. While moving nested_exit_on_intr to svm.h, use INTERCEPT_INTR properly instead of assuming it's zero (which it is of course). Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 5 ----- arch/x86/kvm/svm/svm.c | 23 +++++++++++++++++------ arch/x86/kvm/svm/svm.h | 5 +++++ 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index aaec6d0aa701..9c813e08e2cf 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -828,11 +828,6 @@ static void nested_svm_intr(struct vcpu_svm *svm) nested_svm_vmexit(svm); } -static bool nested_exit_on_intr(struct vcpu_svm *svm) -{ - return (svm->nested.intercept & 1ULL); -} - static int svm_check_nested_events(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 5738be661a1f..fba8bdcfed0e 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3117,14 +3117,25 @@ bool svm_interrupt_blocked(struct kvm_vcpu *vcpu) struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb = svm->vmcb; - if (!gif_set(svm) || - (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)) + if (!gif_set(svm)) return true; - if (is_guest_mode(vcpu) && (svm->vcpu.arch.hflags & HF_VINTR_MASK)) - return !(svm->vcpu.arch.hflags & HF_HIF_MASK); - else - return !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF); + if (is_guest_mode(vcpu)) { + /* As long as interrupts are being delivered... */ + if ((svm->vcpu.arch.hflags & HF_VINTR_MASK) + ? !(svm->vcpu.arch.hflags & HF_HIF_MASK) + : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF)) + return true; + + /* ... vmexits aren't blocked by the interrupt shadow */ + if (nested_exit_on_intr(svm)) + return false; + } else { + if (!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF)) + return true; + } + + return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK); } static bool svm_interrupt_allowed(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index f6d604b72a4c..5cc559ab862d 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -386,6 +386,11 @@ static inline bool nested_exit_on_smi(struct vcpu_svm *svm) return (svm->nested.intercept & (1ULL << INTERCEPT_SMI)); } +static inline bool nested_exit_on_intr(struct vcpu_svm *svm) +{ + return (svm->nested.intercept & (1ULL << INTERCEPT_INTR)); +} + static inline bool nested_exit_on_nmi(struct vcpu_svm *svm) { return (svm->nested.intercept & (1ULL << INTERCEPT_NMI)); From 221e761090b4ffadf41acaca1e1f6dd97d84ef4f Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 23 Apr 2020 08:13:10 -0400 Subject: [PATCH 0437/1043] KVM: nSVM: Preserve IRQ/NMI/SMI priority irrespective of exiting behavior Short circuit vmx_check_nested_events() if an unblocked IRQ/NMI/SMI is pending and needs to be injected into L2, priority between coincident events is not dependent on exiting behavior. Fixes: b518ba9fa691 ("KVM: nSVM: implement check_nested_events for interrupts") Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 9c813e08e2cf..712db507c819 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -835,23 +835,29 @@ static int svm_check_nested_events(struct kvm_vcpu *vcpu) kvm_event_needs_reinjection(vcpu) || svm->nested.exit_required || svm->nested.nested_run_pending; - if (vcpu->arch.smi_pending && nested_exit_on_smi(svm)) { + if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) { if (block_nested_events) return -EBUSY; + if (!nested_exit_on_smi(svm)) + return 0; nested_svm_smi(svm); return 0; } - if (vcpu->arch.nmi_pending && nested_exit_on_nmi(svm)) { + if (vcpu->arch.nmi_pending && !svm_nmi_blocked(vcpu)) { if (block_nested_events) return -EBUSY; + if (!nested_exit_on_nmi(svm)) + return 0; nested_svm_nmi(svm); return 0; } - if (kvm_cpu_has_interrupt(vcpu) && nested_exit_on_intr(svm)) { + if (kvm_cpu_has_interrupt(vcpu) && !svm_interrupt_blocked(vcpu)) { if (block_nested_events) return -EBUSY; + if (!nested_exit_on_intr(svm)) + return 0; nested_svm_intr(svm); return 0; } From 3b82b8d7fdf7c1c284cfa6ebe273435483bcbd9d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 22 Apr 2020 19:25:47 -0700 Subject: [PATCH 0438/1043] KVM: x86: WARN on injected+pending exception even in nested case WARN if a pending exception is coincident with an injected exception before calling check_nested_events() so that the WARN will fire even if inject_pending_event() bails early because check_nested_events() detects the conflict. Bailing early isn't problematic (quite the opposite), but suppressing the WARN is undesirable as it could mask a bug elsewhere in KVM. Signed-off-by: Sean Christopherson Message-Id: <20200423022550.15113-11-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 446fbdd05bd4..cb250f16bdf5 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7713,6 +7713,9 @@ static int inject_pending_event(struct kvm_vcpu *vcpu) kvm_x86_ops.set_irq(vcpu); } + WARN_ON_ONCE(vcpu->arch.exception.injected && + vcpu->arch.exception.pending); + /* * Call check_nested_events() even if we reinjected a previous event * in order for caller to determine if it should require immediate-exit @@ -7731,7 +7734,6 @@ static int inject_pending_event(struct kvm_vcpu *vcpu) vcpu->arch.exception.has_error_code, vcpu->arch.exception.error_code); - WARN_ON_ONCE(vcpu->arch.exception.injected); vcpu->arch.exception.pending = false; vcpu->arch.exception.injected = true; From db438592807e059e3b730f556c22ae201f4d4ebc Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 22 Apr 2020 19:25:48 -0700 Subject: [PATCH 0439/1043] KVM: VMX: Use vmx_interrupt_blocked() directly from vmx_handle_exit() Use vmx_interrupt_blocked() instead of bouncing through vmx_interrupt_allowed() when handling edge cases in vmx_handle_exit(). The nested_run_pending check in vmx_interrupt_allowed() should never evaluate true in the VM-Exit path. Hoist the WARN in handle_invalid_guest_state() up to vmx_handle_exit() to enforce the above assumption for the !enable_vnmi case, and to detect any other potential bugs with nested VM-Enter. Signed-off-by: Sean Christopherson Message-Id: <20200423022550.15113-12-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index a5140ed7dbf9..50a2e464562f 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5285,18 +5285,11 @@ static int handle_invalid_guest_state(struct kvm_vcpu *vcpu) bool intr_window_requested; unsigned count = 130; - /* - * We should never reach the point where we are emulating L2 - * due to invalid guest state as that means we incorrectly - * allowed a nested VMEntry with an invalid vmcs12. - */ - WARN_ON_ONCE(vmx->emulation_required && vmx->nested.nested_run_pending); - intr_window_requested = exec_controls_get(vmx) & CPU_BASED_INTR_WINDOW_EXITING; while (vmx->emulation_required && count-- != 0) { - if (intr_window_requested && vmx_interrupt_allowed(vcpu)) + if (intr_window_requested && !vmx_interrupt_blocked(vcpu)) return handle_interrupt_window(&vmx->vcpu); if (kvm_test_request(KVM_REQ_EVENT, vcpu)) @@ -5913,6 +5906,14 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu, if (enable_pml) vmx_flush_pml_buffer(vcpu); + /* + * We should never reach this point with a pending nested VM-Enter, and + * more specifically emulation of L2 due to invalid guest state (see + * below) should never happen as that means we incorrectly allowed a + * nested VM-Enter with an invalid vmcs12. + */ + WARN_ON_ONCE(vmx->nested.nested_run_pending); + /* If guest state is invalid, start emulating */ if (vmx->emulation_required) return handle_invalid_guest_state(vcpu); @@ -5979,7 +5980,7 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu, if (unlikely(!enable_vnmi && vmx->loaded_vmcs->soft_vnmi_blocked)) { - if (vmx_interrupt_allowed(vcpu)) { + if (!vmx_interrupt_blocked(vcpu)) { vmx->loaded_vmcs->soft_vnmi_blocked = 0; } else if (vmx->loaded_vmcs->vnmi_blocked_time > 1000000000LL && vcpu->arch.nmi_pending) { From 7ab0abdb553e62704be5b7879a0971acf9f36a06 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 22 Apr 2020 19:25:50 -0700 Subject: [PATCH 0440/1043] KVM: VMX: Use vmx_get_rflags() to query RFLAGS in vmx_interrupt_blocked() Use vmx_get_rflags() instead of manually reading vmcs.GUEST_RFLAGS when querying RFLAGS.IF so that multiple checks against interrupt blocking in a single run loop only require a single VMREAD. Signed-off-by: Sean Christopherson Message-Id: <20200423022550.15113-14-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 50a2e464562f..71cd210ec368 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4536,7 +4536,7 @@ bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu) if (is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) return false; - return !(vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) || + return !(vmx_get_rflags(vcpu) & X86_EFLAGS_IF) || (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)); } From c300ab9f08df9e4b9f39d53a0691e234330df124 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Thu, 23 Apr 2020 14:08:58 -0400 Subject: [PATCH 0441/1043] KVM: x86: Replace late check_nested_events() hack with more precise fix Add an argument to interrupt_allowed and nmi_allowed, to checking if interrupt injection is blocked. Use the hook to handle the case where an interrupt arrives between check_nested_events() and the injection logic. Drop the retry of check_nested_events() that hack-a-fixed the same condition. Blocking injection is also a bit of a hack, e.g. KVM should do exiting and non-exiting interrupt processing in a single pass, but it's a more precise hack. The old comment is also misleading, e.g. KVM_REQ_EVENT is purely an optimization, setting it on every run loop (which KVM doesn't do) should not affect functionality, only performance. Signed-off-by: Sean Christopherson Message-Id: <20200423022550.15113-13-sean.j.christopherson@intel.com> [Extend to SVM, add SMI and NMI. Even though NMI and SMI cannot come asynchronously right now, making the fix generic is easy and removes a special case. - Paolo] Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 6 +++--- arch/x86/kvm/svm/svm.c | 25 ++++++++++++++++++----- arch/x86/kvm/vmx/vmx.c | 17 +++++++++++++--- arch/x86/kvm/x86.c | 36 +++++++++++---------------------- 4 files changed, 49 insertions(+), 35 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 44268642b3c6..efe6199c596c 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1138,8 +1138,8 @@ struct kvm_x86_ops { void (*set_nmi)(struct kvm_vcpu *vcpu); void (*queue_exception)(struct kvm_vcpu *vcpu); void (*cancel_injection)(struct kvm_vcpu *vcpu); - bool (*interrupt_allowed)(struct kvm_vcpu *vcpu); - bool (*nmi_allowed)(struct kvm_vcpu *vcpu); + bool (*interrupt_allowed)(struct kvm_vcpu *vcpu, bool for_injection); + bool (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked); void (*enable_nmi_window)(struct kvm_vcpu *vcpu); @@ -1237,7 +1237,7 @@ struct kvm_x86_ops { void (*setup_mce)(struct kvm_vcpu *vcpu); - bool (*smi_allowed)(struct kvm_vcpu *vcpu); + bool (*smi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); int (*pre_enter_smm)(struct kvm_vcpu *vcpu, char *smstate); int (*pre_leave_smm)(struct kvm_vcpu *vcpu, const char *smstate); int (*enable_smi_window)(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index fba8bdcfed0e..45c6e4b87eee 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3083,13 +3083,17 @@ bool svm_nmi_blocked(struct kvm_vcpu *vcpu) return ret; } -static bool svm_nmi_allowed(struct kvm_vcpu *vcpu) +static bool svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) { struct vcpu_svm *svm = to_svm(vcpu); if (svm->nested.nested_run_pending) return false; - return !svm_nmi_blocked(vcpu); + /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ + if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) + return false; + + return !svm_nmi_blocked(vcpu); } static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu) @@ -3138,13 +3142,20 @@ bool svm_interrupt_blocked(struct kvm_vcpu *vcpu) return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK); } -static bool svm_interrupt_allowed(struct kvm_vcpu *vcpu) +static bool svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) { struct vcpu_svm *svm = to_svm(vcpu); if (svm->nested.nested_run_pending) return false; - return !svm_interrupt_blocked(vcpu); + /* + * An IRQ must not be injected into L2 if it's supposed to VM-Exit, + * e.g. if the IRQ arrived asynchronously after checking nested events. + */ + if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm)) + return false; + + return !svm_interrupt_blocked(vcpu); } static void enable_irq_window(struct kvm_vcpu *vcpu) @@ -3812,12 +3823,16 @@ bool svm_smi_blocked(struct kvm_vcpu *vcpu) return is_smm(vcpu); } -static bool svm_smi_allowed(struct kvm_vcpu *vcpu) +static bool svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) { struct vcpu_svm *svm = to_svm(vcpu); if (svm->nested.nested_run_pending) return false; + /* An SMI must not be injected into L2 if it's supposed to VM-Exit. */ + if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm)) + return false; + return !svm_smi_blocked(vcpu); } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 71cd210ec368..e1f5fc919fd9 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4523,11 +4523,15 @@ bool vmx_nmi_blocked(struct kvm_vcpu *vcpu) GUEST_INTR_STATE_NMI)); } -static bool vmx_nmi_allowed(struct kvm_vcpu *vcpu) +static bool vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) { if (to_vmx(vcpu)->nested.nested_run_pending) return false; + /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ + if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) + return false; + return !vmx_nmi_blocked(vcpu); } @@ -4541,11 +4545,18 @@ bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu) (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)); } -static bool vmx_interrupt_allowed(struct kvm_vcpu *vcpu) +static bool vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) { if (to_vmx(vcpu)->nested.nested_run_pending) return false; + /* + * An IRQ must not be injected into L2 if it's supposed to VM-Exit, + * e.g. if the IRQ arrived asynchronously after checking nested events. + */ + if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) + return false; + return !vmx_interrupt_blocked(vcpu); } @@ -7688,7 +7699,7 @@ static void vmx_setup_mce(struct kvm_vcpu *vcpu) ~FEAT_CTL_LMCE_ENABLED; } -static bool vmx_smi_allowed(struct kvm_vcpu *vcpu) +static bool vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) { /* we need a nested vmexit to enter SMM, postpone if run is pending */ if (to_vmx(vcpu)->nested.nested_run_pending) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index cb250f16bdf5..ec20368ac025 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7766,32 +7766,20 @@ static int inject_pending_event(struct kvm_vcpu *vcpu) if (kvm_event_needs_reinjection(vcpu)) return 0; - if (vcpu->arch.smi_pending && kvm_x86_ops.smi_allowed(vcpu)) { + if (vcpu->arch.smi_pending && + kvm_x86_ops.smi_allowed(vcpu, true)) { vcpu->arch.smi_pending = false; ++vcpu->arch.smi_count; enter_smm(vcpu); - } else if (vcpu->arch.nmi_pending && kvm_x86_ops.nmi_allowed(vcpu)) { + } else if (vcpu->arch.nmi_pending && + kvm_x86_ops.nmi_allowed(vcpu, true)) { --vcpu->arch.nmi_pending; vcpu->arch.nmi_injected = true; kvm_x86_ops.set_nmi(vcpu); - } else if (kvm_cpu_has_injectable_intr(vcpu)) { - /* - * Because interrupts can be injected asynchronously, we are - * calling check_nested_events again here to avoid a race condition. - * See https://lkml.org/lkml/2014/7/2/60 for discussion about this - * proposal and current concerns. Perhaps we should be setting - * KVM_REQ_EVENT only on certain events and not unconditionally? - */ - if (is_guest_mode(vcpu)) { - r = kvm_x86_ops.nested_ops->check_events(vcpu); - if (r != 0) - return r; - } - if (kvm_x86_ops.interrupt_allowed(vcpu)) { - kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), - false); - kvm_x86_ops.set_irq(vcpu); - } + } else if (kvm_cpu_has_injectable_intr(vcpu) && + kvm_x86_ops.interrupt_allowed(vcpu, true)) { + kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false); + kvm_x86_ops.set_irq(vcpu); } return 0; @@ -10203,12 +10191,12 @@ static inline bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) if (kvm_test_request(KVM_REQ_NMI, vcpu) || (vcpu->arch.nmi_pending && - kvm_x86_ops.nmi_allowed(vcpu))) + kvm_x86_ops.nmi_allowed(vcpu, false))) return true; if (kvm_test_request(KVM_REQ_SMI, vcpu) || (vcpu->arch.smi_pending && - kvm_x86_ops.smi_allowed(vcpu))) + kvm_x86_ops.smi_allowed(vcpu, false))) return true; if (kvm_arch_interrupt_allowed(vcpu) && @@ -10260,7 +10248,7 @@ int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) int kvm_arch_interrupt_allowed(struct kvm_vcpu *vcpu) { - return kvm_x86_ops.interrupt_allowed(vcpu); + return kvm_x86_ops.interrupt_allowed(vcpu, false); } unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu) @@ -10425,7 +10413,7 @@ bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu) * If interrupts are off we cannot even use an artificial * halt state. */ - return kvm_x86_ops.interrupt_allowed(vcpu); + return kvm_arch_interrupt_allowed(vcpu); } void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, From c9d64a1b2d0b9c81aef5e907bbeff549fd844d13 Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Thu, 23 Apr 2020 22:48:33 -0700 Subject: [PATCH 0442/1043] rcuwait: Fix stale wake call name in comment The 'trywake' name was renamed to simply 'wake', update the comment. Acked-by: Peter Zijlstra (Intel) Signed-off-by: Davidlohr Bueso Message-Id: <20200424054837.5138-2-dave@stgolabs.net> Signed-off-by: Paolo Bonzini --- kernel/exit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/exit.c b/kernel/exit.c index 389a88cb3081..9f9015f3f6b0 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -236,7 +236,7 @@ void rcuwait_wake_up(struct rcuwait *w) /* * Order condition vs @task, such that everything prior to the load * of @task is visible. This is the condition as to why the user called - * rcuwait_trywake() in the first place. Pairs with set_current_state() + * rcuwait_wake() in the first place. Pairs with set_current_state() * barrier (A) in rcuwait_wait_event(). * * WAIT WAKE From 9d9a6ebfea329cadeda87544dceae01e9c27aaef Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Thu, 23 Apr 2020 22:48:34 -0700 Subject: [PATCH 0443/1043] rcuwait: Let rcuwait_wake_up() return whether or not a task was awoken Propagating the return value of wake_up_process() back to the caller can come in handy for future users, such as for statistics or accounting purposes. Acked-by: Peter Zijlstra (Intel) Signed-off-by: Davidlohr Bueso Message-Id: <20200424054837.5138-3-dave@stgolabs.net> Signed-off-by: Paolo Bonzini --- include/linux/rcuwait.h | 2 +- kernel/exit.c | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/include/linux/rcuwait.h b/include/linux/rcuwait.h index 2ffe1ee6d482..6ebb23258a27 100644 --- a/include/linux/rcuwait.h +++ b/include/linux/rcuwait.h @@ -25,7 +25,7 @@ static inline void rcuwait_init(struct rcuwait *w) w->task = NULL; } -extern void rcuwait_wake_up(struct rcuwait *w); +extern int rcuwait_wake_up(struct rcuwait *w); /* * The caller is responsible for locking around rcuwait_wait_event(), diff --git a/kernel/exit.c b/kernel/exit.c index 9f9015f3f6b0..f3beb637acf7 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -227,8 +227,9 @@ repeat: goto repeat; } -void rcuwait_wake_up(struct rcuwait *w) +int rcuwait_wake_up(struct rcuwait *w) { + int ret = 0; struct task_struct *task; rcu_read_lock(); @@ -248,8 +249,10 @@ void rcuwait_wake_up(struct rcuwait *w) task = rcu_dereference(w->task); if (task) - wake_up_process(task); + ret = wake_up_process(task); rcu_read_unlock(); + + return ret; } EXPORT_SYMBOL_GPL(rcuwait_wake_up); From 5c21f7b322cb9fcb97f2e22f5976a0ae8dd71354 Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Thu, 23 Apr 2020 22:48:35 -0700 Subject: [PATCH 0444/1043] rcuwait: Introduce prepare_to and finish_rcuwait This allows further flexibility for some callers to implement ad-hoc versions of the generic rcuwait_wait_event(). For example, kvm will need this to maintain tracing semantics. The naming is of course similar to what waitqueue apis offer. Also go ahead and make use of rcu_assign_pointer() for both task writes as it will make the __rcu sparse people happy - this will be the special nil case, thus no added serialization. Acked-by: Peter Zijlstra (Intel) Signed-off-by: Davidlohr Bueso Message-Id: <20200424054837.5138-4-dave@stgolabs.net> Signed-off-by: Paolo Bonzini --- include/linux/rcuwait.h | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/include/linux/rcuwait.h b/include/linux/rcuwait.h index 6ebb23258a27..45bc6604e9b1 100644 --- a/include/linux/rcuwait.h +++ b/include/linux/rcuwait.h @@ -29,12 +29,25 @@ extern int rcuwait_wake_up(struct rcuwait *w); /* * The caller is responsible for locking around rcuwait_wait_event(), - * such that writes to @task are properly serialized. + * and [prepare_to/finish]_rcuwait() such that writes to @task are + * properly serialized. */ + +static inline void prepare_to_rcuwait(struct rcuwait *w) +{ + rcu_assign_pointer(w->task, current); +} + +static inline void finish_rcuwait(struct rcuwait *w) +{ + rcu_assign_pointer(w->task, NULL); + __set_current_state(TASK_RUNNING); +} + #define rcuwait_wait_event(w, condition, state) \ ({ \ int __ret = 0; \ - rcu_assign_pointer((w)->task, current); \ + prepare_to_rcuwait(w); \ for (;;) { \ /* \ * Implicit barrier (A) pairs with (B) in \ @@ -51,9 +64,7 @@ extern int rcuwait_wake_up(struct rcuwait *w); \ schedule(); \ } \ - \ - WRITE_ONCE((w)->task, NULL); \ - __set_current_state(TASK_RUNNING); \ + finish_rcuwait(w); \ __ret; \ }) From 191a43be61d6791fd4a9098a35c1a09e73f55228 Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Thu, 23 Apr 2020 22:48:36 -0700 Subject: [PATCH 0445/1043] rcuwait: Introduce rcuwait_active() This call is lockless and thus should not be trusted blindly. For example, the return value of rcuwait_wakeup() should be used to track whether a process was woken up. Signed-off-by: Davidlohr Bueso Message-Id: <20200424054837.5138-5-dave@stgolabs.net> Acked-by: Peter Zijlstra (Intel) Signed-off-by: Paolo Bonzini --- include/linux/rcuwait.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/include/linux/rcuwait.h b/include/linux/rcuwait.h index 45bc6604e9b1..c1414ce44abc 100644 --- a/include/linux/rcuwait.h +++ b/include/linux/rcuwait.h @@ -25,6 +25,15 @@ static inline void rcuwait_init(struct rcuwait *w) w->task = NULL; } +/* + * Note: this provides no serialization and, just as with waitqueues, + * requires care to estimate as to whether or not the wait is active. + */ +static inline int rcuwait_active(struct rcuwait *w) +{ + return !!rcu_dereference(w->task); +} + extern int rcuwait_wake_up(struct rcuwait *w); /* From da4ad88cab5867ee240dfd0585e9d115a8cc47db Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Thu, 23 Apr 2020 22:48:37 -0700 Subject: [PATCH 0446/1043] kvm: Replace vcpu->swait with rcuwait The use of any sort of waitqueue (simple or regular) for wait/waking vcpus has always been an overkill and semantically wrong. Because this is per-vcpu (which is blocked) there is only ever a single waiting vcpu, thus no need for any sort of queue. As such, make use of the rcuwait primitive, with the following considerations: - rcuwait already provides the proper barriers that serialize concurrent waiter and waker. - Task wakeup is done in rcu read critical region, with a stable task pointer. - Because there is no concurrency among waiters, we need not worry about rcuwait_wait_event() calls corrupting the wait->task. As a consequence, this saves the locking done in swait when modifying the queue. This also applies to per-vcore wait for powerpc kvm-hv. The x86 tscdeadline_latency test mentioned in 8577370fb0cb ("KVM: Use simple waitqueue for vcpu->wq") shows that, on avg, latency is reduced by around 15-20% with this change. Cc: Paul Mackerras Cc: kvmarm@lists.cs.columbia.edu Cc: linux-mips@vger.kernel.org Reviewed-by: Marc Zyngier Signed-off-by: Davidlohr Bueso Message-Id: <20200424054837.5138-6-dave@stgolabs.net> [Avoid extra logic changes. - Paolo] Signed-off-by: Paolo Bonzini --- arch/mips/kvm/mips.c | 6 ++---- arch/powerpc/include/asm/kvm_book3s.h | 2 +- arch/powerpc/include/asm/kvm_host.h | 2 +- arch/powerpc/kvm/book3s_hv.c | 23 ++++++++++------------- arch/powerpc/kvm/powerpc.c | 2 +- arch/x86/kvm/lapic.c | 2 +- include/linux/kvm_host.h | 10 +++++----- virt/kvm/arm/arch_timer.c | 3 ++- virt/kvm/arm/arm.c | 9 +++++---- virt/kvm/async_pf.c | 3 +-- virt/kvm/kvm_main.c | 19 +++++++++---------- 11 files changed, 38 insertions(+), 43 deletions(-) diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index 9f50ceef9978..9787cdec33e6 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -283,8 +283,7 @@ static enum hrtimer_restart kvm_mips_comparecount_wakeup(struct hrtimer *timer) kvm_mips_callbacks->queue_timer_int(vcpu); vcpu->arch.wait = 0; - if (swq_has_sleeper(&vcpu->wq)) - swake_up_one(&vcpu->wq); + rcuwait_wake_up(&vcpu->wait); return kvm_mips_count_timeout(vcpu); } @@ -511,8 +510,7 @@ int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, dvcpu->arch.wait = 0; - if (swq_has_sleeper(&dvcpu->wq)) - swake_up_one(&dvcpu->wq); + rcuwait_wake_up(&dvcpu->wait); return 0; } diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h index 506e4df2d730..6e5d85ba588d 100644 --- a/arch/powerpc/include/asm/kvm_book3s.h +++ b/arch/powerpc/include/asm/kvm_book3s.h @@ -78,7 +78,7 @@ struct kvmppc_vcore { struct kvm_vcpu *runnable_threads[MAX_SMT_THREADS]; struct list_head preempt_list; spinlock_t lock; - struct swait_queue_head wq; + struct rcuwait wait; spinlock_t stoltb_lock; /* protects stolen_tb and preempt_tb */ u64 stolen_tb; u64 preempt_tb; diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 1dc63101ffe1..337047ba4a56 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -751,7 +751,7 @@ struct kvm_vcpu_arch { u8 irq_pending; /* Used by XIVE to signal pending guest irqs */ u32 last_inst; - struct swait_queue_head *wqp; + struct rcuwait *waitp; struct kvmppc_vcore *vcore; int ret; int trap; diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 93493f0cbfe8..7f59c47a5b9d 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -230,13 +230,11 @@ static bool kvmppc_ipi_thread(int cpu) static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) { int cpu; - struct swait_queue_head *wqp; + struct rcuwait *waitp; - wqp = kvm_arch_vcpu_wq(vcpu); - if (swq_has_sleeper(wqp)) { - swake_up_one(wqp); + waitp = kvm_arch_vcpu_get_wait(vcpu); + if (rcuwait_wake_up(waitp)) ++vcpu->stat.halt_wakeup; - } cpu = READ_ONCE(vcpu->arch.thread_cpu); if (cpu >= 0 && kvmppc_ipi_thread(cpu)) @@ -2125,7 +2123,7 @@ static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int id) spin_lock_init(&vcore->lock); spin_lock_init(&vcore->stoltb_lock); - init_swait_queue_head(&vcore->wq); + rcuwait_init(&vcore->wait); vcore->preempt_tb = TB_NIL; vcore->lpcr = kvm->arch.lpcr; vcore->first_vcpuid = id; @@ -3784,7 +3782,6 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) ktime_t cur, start_poll, start_wait; int do_sleep = 1; u64 block_ns; - DECLARE_SWAITQUEUE(wait); /* Poll for pending exceptions and ceded state */ cur = start_poll = ktime_get(); @@ -3812,10 +3809,10 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) } } - prepare_to_swait_exclusive(&vc->wq, &wait, TASK_INTERRUPTIBLE); - + prepare_to_rcuwait(&vc->wait); + set_current_state(TASK_INTERRUPTIBLE); if (kvmppc_vcore_check_block(vc)) { - finish_swait(&vc->wq, &wait); + finish_rcuwait(&vc->wait); do_sleep = 0; /* If we polled, count this as a successful poll */ if (vc->halt_poll_ns) @@ -3829,7 +3826,7 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) trace_kvmppc_vcore_blocked(vc, 0); spin_unlock(&vc->lock); schedule(); - finish_swait(&vc->wq, &wait); + finish_rcuwait(&vc->wait); spin_lock(&vc->lock); vc->vcore_state = VCORE_INACTIVE; trace_kvmppc_vcore_blocked(vc, 1); @@ -3940,7 +3937,7 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) kvmppc_start_thread(vcpu, vc); trace_kvm_guest_enter(vcpu); } else if (vc->vcore_state == VCORE_SLEEPING) { - swake_up_one(&vc->wq); + rcuwait_wake_up(&vc->wait); } } @@ -4279,7 +4276,7 @@ static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu) } user_vrsave = mfspr(SPRN_VRSAVE); - vcpu->arch.wqp = &vcpu->arch.vcore->wq; + vcpu->arch.waitp = &vcpu->arch.vcore->wait; vcpu->arch.pgdir = kvm->mm->pgd; vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 052614e9d468..27ccff612903 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -752,7 +752,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) if (err) goto out_vcpu_uninit; - vcpu->arch.wqp = &vcpu->wq; + vcpu->arch.waitp = &vcpu->wait; kvmppc_create_vcpu_debugfs(vcpu, vcpu->vcpu_id); return 0; diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 38f7dc9c16ee..42cd2e3ec6fd 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1861,7 +1861,7 @@ void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) /* If the preempt notifier has already run, it also called apic_timer_expired */ if (!apic->lapic_timer.hv_timer_in_use) goto out; - WARN_ON(swait_active(&vcpu->wq)); + WARN_ON(rcuwait_active(&vcpu->wait)); cancel_hv_timer(apic); apic_timer_expired(apic); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index abfa71cb5d2d..161684696610 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include #include #include @@ -277,7 +277,7 @@ struct kvm_vcpu { struct mutex mutex; struct kvm_run *run; - struct swait_queue_head wq; + struct rcuwait wait; struct pid __rcu *pid; int sigset_active; sigset_t sigset; @@ -960,12 +960,12 @@ static inline bool kvm_arch_has_assigned_device(struct kvm *kvm) } #endif -static inline struct swait_queue_head *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu) +static inline struct rcuwait *kvm_arch_vcpu_get_wait(struct kvm_vcpu *vcpu) { #ifdef __KVM_HAVE_ARCH_WQP - return vcpu->arch.wqp; + return vcpu->arch.waitp; #else - return &vcpu->wq; + return &vcpu->wait; #endif } diff --git a/virt/kvm/arm/arch_timer.c b/virt/kvm/arm/arch_timer.c index 93bd59b46848..d5024416e722 100644 --- a/virt/kvm/arm/arch_timer.c +++ b/virt/kvm/arm/arch_timer.c @@ -571,6 +571,7 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) { struct arch_timer_cpu *timer = vcpu_timer(vcpu); struct timer_map map; + struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); if (unlikely(!timer->enabled)) return; @@ -593,7 +594,7 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) if (map.emul_ptimer) soft_timer_cancel(&map.emul_ptimer->hrtimer); - if (swait_active(kvm_arch_vcpu_wq(vcpu))) + if (rcuwait_active(wait)) kvm_timer_blocking(vcpu); /* diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c index f5390ac2165b..d5db0d6141ff 100644 --- a/virt/kvm/arm/arm.c +++ b/virt/kvm/arm/arm.c @@ -579,16 +579,17 @@ void kvm_arm_resume_guest(struct kvm *kvm) kvm_for_each_vcpu(i, vcpu, kvm) { vcpu->arch.pause = false; - swake_up_one(kvm_arch_vcpu_wq(vcpu)); + rcuwait_wake_up(kvm_arch_vcpu_get_wait(vcpu)); } } static void vcpu_req_sleep(struct kvm_vcpu *vcpu) { - struct swait_queue_head *wq = kvm_arch_vcpu_wq(vcpu); + struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); - swait_event_interruptible_exclusive(*wq, ((!vcpu->arch.power_off) && - (!vcpu->arch.pause))); + rcuwait_wait_event(wait, + (!vcpu->arch.power_off) &&(!vcpu->arch.pause), + TASK_INTERRUPTIBLE); if (vcpu->arch.power_off || vcpu->arch.pause) { /* Awaken to handle a signal, request we sleep again later. */ diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c index 15e5b037f92d..10b533f641a6 100644 --- a/virt/kvm/async_pf.c +++ b/virt/kvm/async_pf.c @@ -80,8 +80,7 @@ static void async_pf_execute(struct work_struct *work) trace_kvm_async_pf_completed(addr, cr2_or_gpa); - if (swq_has_sleeper(&vcpu->wq)) - swake_up_one(&vcpu->wq); + rcuwait_wake_up(&vcpu->wait); mmput(mm); kvm_put_kvm(vcpu->kvm); diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 7525f3838160..e12317f32c5e 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -349,7 +349,7 @@ static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) vcpu->kvm = kvm; vcpu->vcpu_id = id; vcpu->pid = NULL; - init_swait_queue_head(&vcpu->wq); + rcuwait_init(&vcpu->wait); kvm_async_pf_vcpu_init(vcpu); vcpu->pre_pcpu = -1; @@ -2678,7 +2678,6 @@ out: void kvm_vcpu_block(struct kvm_vcpu *vcpu) { ktime_t start, cur; - DECLARE_SWAITQUEUE(wait); bool waited = false; u64 block_ns; @@ -2704,8 +2703,9 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) } while (single_task_running() && ktime_before(cur, stop)); } + prepare_to_rcuwait(&vcpu->wait); for (;;) { - prepare_to_swait_exclusive(&vcpu->wq, &wait, TASK_INTERRUPTIBLE); + set_current_state(TASK_INTERRUPTIBLE); if (kvm_vcpu_check_block(vcpu) < 0) break; @@ -2713,8 +2713,7 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) waited = true; schedule(); } - - finish_swait(&vcpu->wq, &wait); + finish_rcuwait(&vcpu->wait); cur = ktime_get(); out: kvm_arch_vcpu_unblocking(vcpu); @@ -2746,11 +2745,10 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_block); bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu) { - struct swait_queue_head *wqp; + struct rcuwait *waitp; - wqp = kvm_arch_vcpu_wq(vcpu); - if (swq_has_sleeper(wqp)) { - swake_up_one(wqp); + waitp = kvm_arch_vcpu_get_wait(vcpu); + if (rcuwait_wake_up(waitp)) { WRITE_ONCE(vcpu->ready, true); ++vcpu->stat.halt_wakeup; return true; @@ -2892,7 +2890,8 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode) continue; if (vcpu == me) continue; - if (swait_active(&vcpu->wq) && !vcpu_dy_runnable(vcpu)) + if (rcuwait_active(&vcpu->wait) && + !vcpu_dy_runnable(vcpu)) continue; if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode && !kvm_arch_vcpu_in_kernel(vcpu)) From c4e115f08c08cb9f3b70247b42323e40b9afd1fd Mon Sep 17 00:00:00 2001 From: Jason Yan Date: Mon, 20 Apr 2020 20:38:05 +0800 Subject: [PATCH 0447/1043] kvm/eventfd: remove unneeded conversion to bool The '==' expression itself is bool, no need to convert it to bool again. This fixes the following coccicheck warning: virt/kvm/eventfd.c:724:38-43: WARNING: conversion to bool not needed here Signed-off-by: Jason Yan Message-Id: <20200420123805.4494-1-yanaijie@huawei.com> Reviewed-by: Peter Xu Signed-off-by: Paolo Bonzini --- virt/kvm/eventfd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index 67b6fc153e9c..0c4ede45e6bd 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -721,7 +721,7 @@ ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val) return false; } - return _val == p->datamatch ? true : false; + return _val == p->datamatch; } /* MMIO/PIO writes trigger an event if the addr/val match */ From fede8076aab4c2280c673492f8f7a2e87712e8b4 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 27 Apr 2020 11:55:59 -0400 Subject: [PATCH 0448/1043] KVM: x86: handle wrap around 32-bit address space KVM is not handling the case where EIP wraps around the 32-bit address space (that is, outside long mode). This is needed both in vmx.c and in emulate.c. SVM with NRIPS is okay, but it can still print an error to dmesg due to integer overflow. Reported-by: Nick Peterson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/emulate.c | 2 ++ arch/x86/kvm/svm/svm.c | 3 --- arch/x86/kvm/vmx/vmx.c | 15 ++++++++++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index bddaba9c68dd..de5476f8683e 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -5798,6 +5798,8 @@ writeback: } ctxt->eip = ctxt->_eip; + if (ctxt->mode != X86EMUL_MODE_PROT64) + ctxt->eip = (u32)ctxt->_eip; done: if (rc == X86EMUL_PROPAGATE_FAULT) { diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 45c6e4b87eee..83175933c0a1 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -319,9 +319,6 @@ static int skip_emulated_instruction(struct kvm_vcpu *vcpu) if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) return 0; } else { - if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE) - pr_err("%s: ip 0x%lx next 0x%llx\n", - __func__, kvm_rip_read(vcpu), svm->next_rip); kvm_rip_write(vcpu, svm->next_rip); } svm_set_interrupt_shadow(vcpu, 0); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index e1f5fc919fd9..f519fc7a22a3 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1555,7 +1555,7 @@ static int vmx_rtit_ctl_check(struct kvm_vcpu *vcpu, u64 data) static int skip_emulated_instruction(struct kvm_vcpu *vcpu) { - unsigned long rip; + unsigned long rip, orig_rip; /* * Using VMCS.VM_EXIT_INSTRUCTION_LEN on EPT misconfig depends on @@ -1567,8 +1567,17 @@ static int skip_emulated_instruction(struct kvm_vcpu *vcpu) */ if (!static_cpu_has(X86_FEATURE_HYPERVISOR) || to_vmx(vcpu)->exit_reason != EXIT_REASON_EPT_MISCONFIG) { - rip = kvm_rip_read(vcpu); - rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN); + orig_rip = kvm_rip_read(vcpu); + rip = orig_rip + vmcs_read32(VM_EXIT_INSTRUCTION_LEN); +#ifdef CONFIG_X86_64 + /* + * We need to mask out the high 32 bits of RIP if not in 64-bit + * mode, but just finding out that we are in 64-bit mode is + * quite expensive. Only do it if there was a carry. + */ + if (unlikely(((rip ^ orig_rip) >> 31) == 3) && !is_64_bit_mode(vcpu)) + rip = (u32)rip; +#endif kvm_rip_write(vcpu, rip); } else { if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP)) From f27ad73a6ef791731fa75ca183d5c64a930b9cc1 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 27 Apr 2020 10:18:37 -0700 Subject: [PATCH 0449/1043] KVM: VMX: Use accessor to read vmcs.INTR_INFO when handling exception Use vmx_get_intr_info() when grabbing the cached vmcs.INTR_INFO in handle_exception_nmi() to ensure the cache isn't stale. Bypassing the caching accessor doesn't cause any known issues as the cache is always refreshed by handle_exception_nmi_irqoff(), but the whole point of adding the proper caching mechanism was to avoid such dependencies. Fixes: 8791585837f6 ("KVM: VMX: Cache vmcs.EXIT_INTR_INFO using arch avail_reg flags") Signed-off-by: Sean Christopherson Message-Id: <20200427171837.22613-1-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index f519fc7a22a3..fbede089d0f7 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4705,7 +4705,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) u32 vect_info; vect_info = vmx->idt_vectoring_info; - intr_info = vmx->exit_intr_info; + intr_info = vmx_get_intr_info(vcpu); if (is_machine_check(intr_info) || is_nmi(intr_info)) return 1; /* handled by handle_exception_nmi_irqoff() */ From 5c911beff20aa8639e7a1f28988736c13e03ed54 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 1 May 2020 09:31:17 -0700 Subject: [PATCH 0450/1043] KVM: nVMX: Skip IBPB when switching between vmcs01 and vmcs02 Skip the Indirect Branch Prediction Barrier that is triggered on a VMCS switch when running with spectre_v2_user=on/auto if the switch is between two VMCSes in the same guest, i.e. between vmcs01 and vmcs02. The IBPB is intended to prevent one guest from attacking another, which is unnecessary in the nested case as it's the same guest from KVM's perspective. This all but eliminates the overhead observed for nested VMX transitions when running with CONFIG_RETPOLINE=y and spectre_v2_user=on/auto, which can be significant, e.g. roughly 3x on current systems. Reported-by: Alexander Graf Cc: KarimAllah Raslan Cc: stable@vger.kernel.org Fixes: 15d45071523d ("KVM/x86: Add IBPB support") Signed-off-by: Sean Christopherson Message-Id: <20200501163117.4655-1-sean.j.christopherson@intel.com> [Invert direction of bool argument. - Paolo] Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/vmx/vmx.c | 18 ++++++++++++++---- arch/x86/kvm/vmx/vmx.h | 3 ++- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 669445136144..7f754b3bc6dd 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -303,7 +303,7 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs) cpu = get_cpu(); prev = vmx->loaded_vmcs; vmx->loaded_vmcs = vmcs; - vmx_vcpu_load_vmcs(vcpu, cpu); + vmx_vcpu_load_vmcs(vcpu, cpu, prev); vmx_sync_vmcs_host_state(vmx, prev); put_cpu(); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index fbede089d0f7..db842ce74e5d 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1311,10 +1311,12 @@ after_clear_sn: pi_set_on(pi_desc); } -void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu) +void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, + struct loaded_vmcs *buddy) { struct vcpu_vmx *vmx = to_vmx(vcpu); bool already_loaded = vmx->loaded_vmcs->cpu == cpu; + struct vmcs *prev; if (!already_loaded) { loaded_vmcs_clear(vmx->loaded_vmcs); @@ -1333,10 +1335,18 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu) local_irq_enable(); } - if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) { + prev = per_cpu(current_vmcs, cpu); + if (prev != vmx->loaded_vmcs->vmcs) { per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs; vmcs_load(vmx->loaded_vmcs->vmcs); - indirect_branch_prediction_barrier(); + + /* + * No indirect branch prediction barrier needed when switching + * the active VMCS within a guest, e.g. on nested VM-Enter. + * The L1 VMM can protect itself with retpolines, IBPB or IBRS. + */ + if (!buddy || WARN_ON_ONCE(buddy->vmcs != prev)) + indirect_branch_prediction_barrier(); } if (!already_loaded) { @@ -1377,7 +1387,7 @@ void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - vmx_vcpu_load_vmcs(vcpu, cpu); + vmx_vcpu_load_vmcs(vcpu, cpu, NULL); vmx_vcpu_pi_load(vcpu, cpu); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index b5e773267abe..d3d48acc6bd9 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -320,7 +320,8 @@ struct kvm_vmx { }; bool nested_vmx_allowed(struct kvm_vcpu *vcpu); -void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu); +void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, + struct loaded_vmcs *buddy); void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu); int allocate_vpid(void); void free_vpid(int vpid); From 1af1bb05625bcdd09522f416b62bcc72cc2fdd3b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 6 May 2020 16:58:50 -0700 Subject: [PATCH 0451/1043] KVM: nVMX: Skip IBPB when temporarily switching between vmcs01 and vmcs02 Skip the Indirect Branch Prediction Barrier that is triggered on a VMCS switch when temporarily loading vmcs02 to synchronize it to vmcs12, i.e. give copy_vmcs02_to_vmcs12_rare() the same treatment as vmx_switch_vmcs(). Make vmx_vcpu_load() static now that it's only referenced within vmx.c. Signed-off-by: Sean Christopherson Message-Id: <20200506235850.22600-3-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 4 ++-- arch/x86/kvm/vmx/vmx.c | 2 +- arch/x86/kvm/vmx/vmx.h | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 7f754b3bc6dd..b69f5a2fa362 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3910,12 +3910,12 @@ static void copy_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu, cpu = get_cpu(); vmx->loaded_vmcs = &vmx->nested.vmcs02; - vmx_vcpu_load(&vmx->vcpu, cpu); + vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->vmcs01); sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); vmx->loaded_vmcs = &vmx->vmcs01; - vmx_vcpu_load(&vmx->vcpu, cpu); + vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->nested.vmcs02); put_cpu(); } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index db842ce74e5d..d044a05a38bd 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1383,7 +1383,7 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, * Switches to specified vcpu, until a matching vcpu_put(), but assumes * vcpu mutex is already taken. */ -void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) +static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index d3d48acc6bd9..373674d455e1 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -322,7 +322,6 @@ struct kvm_vmx { bool nested_vmx_allowed(struct kvm_vcpu *vcpu); void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, struct loaded_vmcs *buddy); -void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu); int allocate_vpid(void); void free_vpid(int vpid); void vmx_set_constant_host_state(struct vcpu_vmx *vmx); From 56ba77a459a72a7d95be74355a40a91e1f6dd7f7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 1 May 2020 21:32:25 -0700 Subject: [PATCH 0452/1043] KVM: x86: Save L1 TSC offset in 'struct kvm_vcpu_arch' Save L1's TSC offset in 'struct kvm_vcpu_arch' and drop the kvm_x86_ops hook read_l1_tsc_offset(). This avoids a retpoline (when configured) when reading L1's effective TSC, which is done at least once on every VM-Exit. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20200502043234.12481-2-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/svm/svm.c | 11 ----------- arch/x86/kvm/vmx/vmx.c | 12 ------------ arch/x86/kvm/x86.c | 9 ++++----- 4 files changed, 5 insertions(+), 29 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index efe6199c596c..3ff671e35d76 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -708,6 +708,7 @@ struct kvm_vcpu_arch { struct gfn_to_pfn_cache cache; } st; + u64 l1_tsc_offset; u64 tsc_offset; u64 last_guest_tsc; u64 last_host_tsc; @@ -1165,7 +1166,6 @@ struct kvm_x86_ops { bool (*has_wbinvd_exit)(void); - u64 (*read_l1_tsc_offset)(struct kvm_vcpu *vcpu); /* Returns actual tsc_offset set in active VMCS */ u64 (*write_l1_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 83175933c0a1..18a63641b612 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -951,16 +951,6 @@ static void init_sys_seg(struct vmcb_seg *seg, uint32_t type) seg->base = 0; } -static u64 svm_read_l1_tsc_offset(struct kvm_vcpu *vcpu) -{ - struct vcpu_svm *svm = to_svm(vcpu); - - if (is_guest_mode(vcpu)) - return svm->nested.hsave->control.tsc_offset; - - return vcpu->arch.tsc_offset; -} - static u64 svm_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) { struct vcpu_svm *svm = to_svm(vcpu); @@ -4075,7 +4065,6 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .has_wbinvd_exit = svm_has_wbinvd_exit, - .read_l1_tsc_offset = svm_read_l1_tsc_offset, .write_l1_tsc_offset = svm_write_l1_tsc_offset, .load_mmu_pgd = svm_load_mmu_pgd, diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d044a05a38bd..31a8d04a6c41 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1740,17 +1740,6 @@ static void setup_msrs(struct vcpu_vmx *vmx) vmx_update_msr_bitmap(&vmx->vcpu); } -static u64 vmx_read_l1_tsc_offset(struct kvm_vcpu *vcpu) -{ - struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - - if (is_guest_mode(vcpu) && - (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING)) - return vcpu->arch.tsc_offset - vmcs12->tsc_offset; - - return vcpu->arch.tsc_offset; -} - static u64 vmx_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) { struct vmcs12 *vmcs12 = get_vmcs12(vcpu); @@ -7881,7 +7870,6 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, - .read_l1_tsc_offset = vmx_read_l1_tsc_offset, .write_l1_tsc_offset = vmx_write_l1_tsc_offset, .load_mmu_pgd = vmx_load_mmu_pgd, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index ec20368ac025..3622586a26a7 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1910,7 +1910,7 @@ static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu) static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset) { - u64 curr_offset = kvm_x86_ops.read_l1_tsc_offset(vcpu); + u64 curr_offset = vcpu->arch.l1_tsc_offset; vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset; } @@ -1952,14 +1952,13 @@ static u64 kvm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc) u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) { - u64 tsc_offset = kvm_x86_ops.read_l1_tsc_offset(vcpu); - - return tsc_offset + kvm_scale_tsc(vcpu, host_tsc); + return vcpu->arch.l1_tsc_offset + kvm_scale_tsc(vcpu, host_tsc); } EXPORT_SYMBOL_GPL(kvm_read_l1_tsc); static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) { + vcpu->arch.l1_tsc_offset = offset; vcpu->arch.tsc_offset = kvm_x86_ops.write_l1_tsc_offset(vcpu, offset); } @@ -2084,7 +2083,7 @@ EXPORT_SYMBOL_GPL(kvm_write_tsc); static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, s64 adjustment) { - u64 tsc_offset = kvm_x86_ops.read_l1_tsc_offset(vcpu); + u64 tsc_offset = vcpu->arch.l1_tsc_offset; kvm_vcpu_write_tsc_offset(vcpu, tsc_offset + adjustment); } From 0cc69204e77275105d5b0fc4cf3c970e3579457f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 1 May 2020 21:32:26 -0700 Subject: [PATCH 0453/1043] KVM: nVMX: Unconditionally validate CR3 during nested transitions Unconditionally check the validity of the incoming CR3 during nested VM-Enter/VM-Exit to avoid invoking kvm_read_cr3() in the common case where the guest isn't using PAE paging. If vmcs.GUEST_CR3 hasn't yet been cached (common case), kvm_read_cr3() will trigger a VMREAD. The VMREAD (~30 cycles) alone is likely slower than nested_cr3_valid() (~5 cycles if vcpu->arch.maxphyaddr gets a cache hit), and the poor exchange only gets worse when retpolines are enabled as the call to kvm_x86_ops.cache_reg() will incur a retpoline (60+ cycles). Signed-off-by: Sean Christopherson Message-Id: <20200502043234.12481-3-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index b69f5a2fa362..90ba278dd572 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1119,21 +1119,20 @@ static bool nested_vmx_transition_mmu_sync(struct kvm_vcpu *vcpu) static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept, u32 *entry_failure_code) { - if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) { - if (CC(!nested_cr3_valid(vcpu, cr3))) { - *entry_failure_code = ENTRY_FAIL_DEFAULT; - return -EINVAL; - } + if (CC(!nested_cr3_valid(vcpu, cr3))) { + *entry_failure_code = ENTRY_FAIL_DEFAULT; + return -EINVAL; + } - /* - * If PAE paging and EPT are both on, CR3 is not used by the CPU and - * must not be dereferenced. - */ - if (is_pae_paging(vcpu) && !nested_ept) { - if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) { - *entry_failure_code = ENTRY_FAIL_PDPTE; - return -EINVAL; - } + /* + * If PAE paging and EPT are both on, CR3 is not used by the CPU and + * must not be dereferenced. + */ + if (!nested_ept && is_pae_paging(vcpu) && + (cr3 != kvm_read_cr3(vcpu) || pdptrs_changed(vcpu))) { + if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) { + *entry_failure_code = ENTRY_FAIL_PDPTE; + return -EINVAL; } } From f98c1e77127de7d9ff558570c25d02ef077df50f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 1 May 2020 21:32:30 -0700 Subject: [PATCH 0454/1043] KVM: VMX: Add proper cache tracking for CR4 Move CR4 caching into the standard register caching mechanism in order to take advantage of the availability checks provided by regs_avail. This avoids multiple VMREADs and retpolines (when configured) during nested VMX transitions as kvm_read_cr4_bits() is invoked multiple times on each transition, e.g. when stuffing CR0 and CR3. As an added bonus, this eliminates a kvm_x86_ops hook, saves a retpoline on SVM when reading CR4, and squashes the confusing naming discrepancy of "cache_reg" vs. "decache_cr4_guest_bits". No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20200502043234.12481-7-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/kvm_cache_regs.h | 5 +++-- arch/x86/kvm/svm/svm.c | 5 ----- arch/x86/kvm/vmx/vmx.c | 18 +++++++++--------- arch/x86/kvm/vmx/vmx.h | 1 + 5 files changed, 14 insertions(+), 17 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 3ff671e35d76..3e2f582160fe 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -168,6 +168,7 @@ enum kvm_reg { VCPU_EXREG_PDPTR = NR_VCPU_REGS, VCPU_EXREG_CR3, + VCPU_EXREG_CR4, VCPU_EXREG_RFLAGS, VCPU_EXREG_SEGMENTS, VCPU_EXREG_EXIT_INFO_1, @@ -1092,7 +1093,6 @@ struct kvm_x86_ops { struct kvm_segment *var, int seg); void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu); - void (*decache_cr4_guest_bits)(struct kvm_vcpu *vcpu); void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); int (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/kvm_cache_regs.h index 62558b9bdda7..921a539bcb96 100644 --- a/arch/x86/kvm/kvm_cache_regs.h +++ b/arch/x86/kvm/kvm_cache_regs.h @@ -129,8 +129,9 @@ static inline ulong kvm_read_cr0(struct kvm_vcpu *vcpu) static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask) { ulong tmask = mask & KVM_POSSIBLE_CR4_GUEST_BITS; - if (tmask & vcpu->arch.cr4_guest_owned_bits) - kvm_x86_ops.decache_cr4_guest_bits(vcpu); + if ((tmask & vcpu->arch.cr4_guest_owned_bits) && + !kvm_register_is_available(vcpu, VCPU_EXREG_CR4)) + kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR4); return vcpu->arch.cr4 & mask; } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 18a63641b612..805297e9a1cb 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1525,10 +1525,6 @@ static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu) { } -static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu) -{ -} - static void update_cr0_intercept(struct vcpu_svm *svm) { ulong gcr0 = svm->vcpu.arch.cr0; @@ -4007,7 +4003,6 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .get_cpl = svm_get_cpl, .get_cs_db_l_bits = kvm_get_cs_db_l_bits, .decache_cr0_guest_bits = svm_decache_cr0_guest_bits, - .decache_cr4_guest_bits = svm_decache_cr4_guest_bits, .set_cr0 = svm_set_cr0, .set_cr4 = svm_set_cr4, .set_efer = svm_set_efer, diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 31a8d04a6c41..83e3fe083679 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2204,6 +2204,8 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) { + unsigned long guest_owned_bits; + kvm_register_mark_available(vcpu, reg); switch (reg) { @@ -2221,6 +2223,12 @@ static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) if (enable_unrestricted_guest || (enable_ept && is_paging(vcpu))) vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); break; + case VCPU_EXREG_CR4: + guest_owned_bits = vcpu->arch.cr4_guest_owned_bits; + + vcpu->arch.cr4 &= ~guest_owned_bits; + vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & guest_owned_bits; + break; default: WARN_ON_ONCE(1); break; @@ -2922,14 +2930,6 @@ static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu) vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits; } -static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu) -{ - ulong cr4_guest_owned_bits = vcpu->arch.cr4_guest_owned_bits; - - vcpu->arch.cr4 &= ~cr4_guest_owned_bits; - vcpu->arch.cr4 |= vmcs_readl(GUEST_CR4) & cr4_guest_owned_bits; -} - static void ept_load_pdptrs(struct kvm_vcpu *vcpu) { struct kvm_mmu *mmu = vcpu->arch.walk_mmu; @@ -3128,6 +3128,7 @@ int vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) return 1; vcpu->arch.cr4 = cr4; + kvm_register_mark_available(vcpu, VCPU_EXREG_CR4); if (!enable_unrestricted_guest) { if (enable_ept) { @@ -7809,7 +7810,6 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .get_cpl = vmx_get_cpl, .get_cs_db_l_bits = vmx_get_cs_db_l_bits, .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits, - .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits, .set_cr0 = vmx_set_cr0, .set_cr4 = vmx_set_cr4, .set_efer = vmx_set_efer, diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 373674d455e1..04bb557acdd2 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -453,6 +453,7 @@ static inline void vmx_register_cache_reset(struct kvm_vcpu *vcpu) | (1 << VCPU_EXREG_PDPTR) | (1 << VCPU_EXREG_SEGMENTS) | (1 << VCPU_EXREG_CR3) + | (1 << VCPU_EXREG_CR4) | (1 << VCPU_EXREG_EXIT_INFO_1) | (1 << VCPU_EXREG_EXIT_INFO_2)); vcpu->arch.regs_dirty = 0; From bd31fe495d0d1a67fe6f44f06dfef637f202241d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 1 May 2020 21:32:31 -0700 Subject: [PATCH 0455/1043] KVM: VMX: Add proper cache tracking for CR0 Move CR0 caching into the standard register caching mechanism in order to take advantage of the availability checks provided by regs_avail. This avoids multiple VMREADs in the (uncommon) case where kvm_read_cr0() is called multiple times in a single VM-Exit, and more importantly eliminates a kvm_x86_ops hook, saves a retpoline on SVM when reading CR0, and squashes the confusing naming discrepancy of "cache_reg" vs. "decache_cr0_guest_bits". No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20200502043234.12481-8-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/kvm_cache_regs.h | 5 +++-- arch/x86/kvm/svm/svm.c | 5 ----- arch/x86/kvm/vmx/vmx.c | 16 +++++++--------- arch/x86/kvm/vmx/vmx.h | 1 + 5 files changed, 12 insertions(+), 17 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 3e2f582160fe..ad882127079f 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -167,6 +167,7 @@ enum kvm_reg { NR_VCPU_REGS, VCPU_EXREG_PDPTR = NR_VCPU_REGS, + VCPU_EXREG_CR0, VCPU_EXREG_CR3, VCPU_EXREG_CR4, VCPU_EXREG_RFLAGS, @@ -1092,7 +1093,6 @@ struct kvm_x86_ops { void (*set_segment)(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); void (*get_cs_db_l_bits)(struct kvm_vcpu *vcpu, int *db, int *l); - void (*decache_cr0_guest_bits)(struct kvm_vcpu *vcpu); void (*set_cr0)(struct kvm_vcpu *vcpu, unsigned long cr0); int (*set_cr4)(struct kvm_vcpu *vcpu, unsigned long cr4); void (*set_efer)(struct kvm_vcpu *vcpu, u64 efer); diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/kvm_cache_regs.h index 921a539bcb96..ff2d0e9ca3bc 100644 --- a/arch/x86/kvm/kvm_cache_regs.h +++ b/arch/x86/kvm/kvm_cache_regs.h @@ -116,8 +116,9 @@ static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index) static inline ulong kvm_read_cr0_bits(struct kvm_vcpu *vcpu, ulong mask) { ulong tmask = mask & KVM_POSSIBLE_CR0_GUEST_BITS; - if (tmask & vcpu->arch.cr0_guest_owned_bits) - kvm_x86_ops.decache_cr0_guest_bits(vcpu); + if ((tmask & vcpu->arch.cr0_guest_owned_bits) && + !kvm_register_is_available(vcpu, VCPU_EXREG_CR0)) + kvm_x86_ops.cache_reg(vcpu, VCPU_EXREG_CR0); return vcpu->arch.cr0 & mask; } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 805297e9a1cb..9e24b5b6fae1 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1521,10 +1521,6 @@ static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt) mark_dirty(svm->vmcb, VMCB_DT); } -static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu) -{ -} - static void update_cr0_intercept(struct vcpu_svm *svm) { ulong gcr0 = svm->vcpu.arch.cr0; @@ -4002,7 +3998,6 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .set_segment = svm_set_segment, .get_cpl = svm_get_cpl, .get_cs_db_l_bits = kvm_get_cs_db_l_bits, - .decache_cr0_guest_bits = svm_decache_cr0_guest_bits, .set_cr0 = svm_set_cr0, .set_cr4 = svm_set_cr4, .set_efer = svm_set_efer, diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 83e3fe083679..a7dca7e305b4 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2219,6 +2219,12 @@ static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) if (enable_ept) ept_save_pdptrs(vcpu); break; + case VCPU_EXREG_CR0: + guest_owned_bits = vcpu->arch.cr0_guest_owned_bits; + + vcpu->arch.cr0 &= ~guest_owned_bits; + vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & guest_owned_bits; + break; case VCPU_EXREG_CR3: if (enable_unrestricted_guest || (enable_ept && is_paging(vcpu))) vcpu->arch.cr3 = vmcs_readl(GUEST_CR3); @@ -2922,14 +2928,6 @@ static void vmx_flush_tlb_guest(struct kvm_vcpu *vcpu) vpid_sync_context(to_vmx(vcpu)->vpid); } -static void vmx_decache_cr0_guest_bits(struct kvm_vcpu *vcpu) -{ - ulong cr0_guest_owned_bits = vcpu->arch.cr0_guest_owned_bits; - - vcpu->arch.cr0 &= ~cr0_guest_owned_bits; - vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & cr0_guest_owned_bits; -} - static void ept_load_pdptrs(struct kvm_vcpu *vcpu) { struct kvm_mmu *mmu = vcpu->arch.walk_mmu; @@ -3019,6 +3017,7 @@ void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) vmcs_writel(CR0_READ_SHADOW, cr0); vmcs_writel(GUEST_CR0, hw_cr0); vcpu->arch.cr0 = cr0; + kvm_register_mark_available(vcpu, VCPU_EXREG_CR0); /* depends on vcpu->arch.cr0 to be set to a new value */ vmx->emulation_required = emulation_required(vcpu); @@ -7809,7 +7808,6 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .set_segment = vmx_set_segment, .get_cpl = vmx_get_cpl, .get_cs_db_l_bits = vmx_get_cs_db_l_bits, - .decache_cr0_guest_bits = vmx_decache_cr0_guest_bits, .set_cr0 = vmx_set_cr0, .set_cr4 = vmx_set_cr4, .set_efer = vmx_set_efer, diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 04bb557acdd2..298ddef79d00 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -452,6 +452,7 @@ static inline void vmx_register_cache_reset(struct kvm_vcpu *vcpu) | (1 << VCPU_EXREG_RFLAGS) | (1 << VCPU_EXREG_PDPTR) | (1 << VCPU_EXREG_SEGMENTS) + | (1 << VCPU_EXREG_CR0) | (1 << VCPU_EXREG_CR3) | (1 << VCPU_EXREG_CR4) | (1 << VCPU_EXREG_EXIT_INFO_1) From 0047fcade4cb1204cafdf62150081a39356f4dbd Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 1 May 2020 21:32:33 -0700 Subject: [PATCH 0456/1043] KVM: VMX: Move nested EPT out of kvm_x86_ops.get_tdp_level() hook Separate the "core" TDP level handling from the nested EPT path to make it clear that kvm_x86_ops.get_tdp_level() is used if and only if nested EPT is not in use (kvm_init_shadow_ept_mmu() calculates the level from the passed in vmcs12->eptp). Add a WARN_ON() to enforce that the kvm_x86_ops hook is not called for nested EPT. This sets the stage for snapshotting the non-"nested EPT" TDP page level during kvm_cpuid_update() to avoid the retpoline associated with kvm_x86_ops.get_tdp_level() when resetting the MMU, a relatively frequent operation when running a nested guest. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20200502043234.12481-10-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index a7dca7e305b4..7a43fbe05e2d 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -3023,13 +3023,21 @@ void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) vmx->emulation_required = emulation_required(vcpu); } +static int vmx_get_tdp_level(struct kvm_vcpu *vcpu) +{ + WARN_ON(is_guest_mode(vcpu) && nested_cpu_has_ept(get_vmcs12(vcpu))); + + if (cpu_has_vmx_ept_5levels() && (cpuid_maxphyaddr(vcpu) > 48)) + return 5; + return 4; +} + static int get_ept_level(struct kvm_vcpu *vcpu) { if (is_guest_mode(vcpu) && nested_cpu_has_ept(get_vmcs12(vcpu))) return vmx_eptp_page_walk_level(nested_ept_get_eptp(vcpu)); - if (cpu_has_vmx_ept_5levels() && (cpuid_maxphyaddr(vcpu) > 48)) - return 5; - return 4; + + return vmx_get_tdp_level(vcpu); } u64 construct_eptp(struct kvm_vcpu *vcpu, unsigned long root_hpa) @@ -7859,7 +7867,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .set_tss_addr = vmx_set_tss_addr, .set_identity_map_addr = vmx_set_identity_map_addr, - .get_tdp_level = get_ept_level, + .get_tdp_level = vmx_get_tdp_level, .get_mt_mask = vmx_get_mt_mask, .get_exit_info = vmx_get_exit_info, From e93fd3b3e89e9664039281fe7e56e6f764f2a909 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 1 May 2020 21:32:34 -0700 Subject: [PATCH 0457/1043] KVM: x86/mmu: Capture TDP level when updating CPUID Snapshot the TDP level now that it's invariant (SVM) or dependent only on host capabilities and guest CPUID (VMX). This avoids having to call kvm_x86_ops.get_tdp_level() when initializing a TDP MMU and/or calculating the page role, and thus avoids the associated retpoline. Drop the WARN in vmx_get_tdp_level() as updating CPUID while L2 is active is legal, if dodgy. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20200502043234.12481-11-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/cpuid.c | 3 ++- arch/x86/kvm/mmu/mmu.c | 6 +++--- arch/x86/kvm/svm/nested.c | 2 +- arch/x86/kvm/vmx/vmx.c | 2 -- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index ad882127079f..505cf19ace80 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -687,6 +687,7 @@ struct kvm_vcpu_arch { struct kvm_cpuid_entry2 cpuid_entries[KVM_MAX_CPUID_ENTRIES]; int maxphyaddr; + int tdp_level; /* emulate context */ diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 6828be99b908..44dfaefdad0e 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -124,8 +124,9 @@ int kvm_update_cpuid(struct kvm_vcpu *vcpu) MSR_IA32_MISC_ENABLE_MWAIT); } - /* Update physical-address width */ + /* Note, maxphyaddr must be updated before tdp_level. */ vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu); + vcpu->arch.tdp_level = kvm_x86_ops.get_tdp_level(vcpu); kvm_mmu_reset_context(vcpu); kvm_pmu_refresh(vcpu); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index e618472c572b..10cb8db54cd0 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4894,7 +4894,7 @@ kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only) union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, base_only); role.base.ad_disabled = (shadow_accessed_mask == 0); - role.base.level = kvm_x86_ops.get_tdp_level(vcpu); + role.base.level = vcpu->arch.tdp_level; role.base.direct = true; role.base.gpte_is_8_bytes = true; @@ -4915,7 +4915,7 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) context->sync_page = nonpaging_sync_page; context->invlpg = NULL; context->update_pte = nonpaging_update_pte; - context->shadow_root_level = kvm_x86_ops.get_tdp_level(vcpu); + context->shadow_root_level = vcpu->arch.tdp_level; context->direct_map = true; context->get_guest_pgd = get_cr3; context->get_pdptr = kvm_pdptr_read; @@ -5680,7 +5680,7 @@ static int alloc_mmu_pages(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) * SVM's 32-bit NPT support, TDP paging doesn't use PAE paging and can * skip allocating the PDP table. */ - if (tdp_enabled && kvm_x86_ops.get_tdp_level(vcpu) > PT32E_ROOT_LEVEL) + if (tdp_enabled && vcpu->arch.tdp_level > PT32E_ROOT_LEVEL) return 0; page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32); diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 712db507c819..a89a166d1cb8 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -86,7 +86,7 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; - vcpu->arch.mmu->shadow_root_level = kvm_x86_ops.get_tdp_level(vcpu); + vcpu->arch.mmu->shadow_root_level = vcpu->arch.tdp_level; reset_shadow_zero_bits_mask(vcpu, vcpu->arch.mmu); vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 7a43fbe05e2d..93b2a708b1da 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -3025,8 +3025,6 @@ void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) static int vmx_get_tdp_level(struct kvm_vcpu *vcpu) { - WARN_ON(is_guest_mode(vcpu) && nested_cpu_has_ept(get_vmcs12(vcpu))); - if (cpu_has_vmx_ept_5levels() && (cpuid_maxphyaddr(vcpu) > 48)) return 5; return 4; From 21e0958ec9684e76e32f822c5e611a7d7ea0a5ba Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Sat, 4 Apr 2020 23:57:07 +0200 Subject: [PATCH 0458/1043] md: add checkings before flush md_misc_wq Coly reported possible circular locking dependencyi with LOCKDEP enabled, quote the below info from the detailed report [1]. [ 1607.673903] Chain exists of: [ 1607.673903] kn->count#256 --> (wq_completion)md_misc --> (work_completion)(&rdev->del_work) [ 1607.673903] [ 1607.827946] Possible unsafe locking scenario: [ 1607.827946] [ 1607.898780] CPU0 CPU1 [ 1607.952980] ---- ---- [ 1608.007173] lock((work_completion)(&rdev->del_work)); [ 1608.069690] lock((wq_completion)md_misc); [ 1608.149887] lock((work_completion)(&rdev->del_work)); [ 1608.242563] lock(kn->count#256); [ 1608.283238] [ 1608.283238] *** DEADLOCK *** [ 1608.283238] [ 1608.354078] 2 locks held by kworker/5:0/843: [ 1608.405152] #0: ffff8889eecc9948 ((wq_completion)md_misc){+.+.}, at: process_one_work+0x42b/0xb30 [ 1608.512399] #1: ffff888a1d3b7e10 ((work_completion)(&rdev->del_work)){+.+.}, at: process_one_work+0x42b/0xb30 [ 1608.632130] Since works (rdev->del_work and mddev->del_work) are queued in md_misc_wq, then lockdep_map lock is held if either of them are running, then both of them try to hold kernfs lock by call kobject_del. Then if new_dev_store or array_state_store are triggered by write to the related sysfs node, so the write operation gets kernfs lock, but need the lockdep_map because all of them would trigger flush_workqueue(md_misc_wq) finally, then the same lockdep_map lock is needed. To suppress the lockdep warnning, we should flush the workqueue in case the related work is pending. And several works are attached to md_misc_wq, so we need to check which work should be checked: 1. for __md_stop_writes, the purpose of call flush workqueue is ensure sync thread is started if it was starting, so check mddev->del_work is pending or not since md_start_sync is attached to mddev->del_work. 2. __md_stop flushes md_misc_wq to ensure event_work is done, check the event_work is enough. Assume raid_{ctr,dtr} -> md_stop -> __md_stop doesn't need the kernfs lock. 3. both new_dev_store (holds kernfs lock) and ADD_NEW_DISK ioctl (holds the bdev->bd_mutex) call flush_workqueue to ensure md_delayed_delete has completed, this case will be handled in next patch. 4. md_open flushes workqueue to ensure the previous md is disappeared, but it holds bdev->bd_mutex then try to flush workqueue, so it is better to check mddev->del_work as well to avoid potential lock issue, this will be done in another patch. [1]: https://marc.info/?l=linux-raid&m=158518958031584&w=2 Cc: Coly Li Reported-by: Coly Li Signed-off-by: Guoqing Jiang Signed-off-by: Song Liu --- drivers/md/md.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 271e8a587354..76c32ca6eee8 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -6147,7 +6147,8 @@ static void md_clean(struct mddev *mddev) static void __md_stop_writes(struct mddev *mddev) { set_bit(MD_RECOVERY_FROZEN, &mddev->recovery); - flush_workqueue(md_misc_wq); + if (work_pending(&mddev->del_work)) + flush_workqueue(md_misc_wq); if (mddev->sync_thread) { set_bit(MD_RECOVERY_INTR, &mddev->recovery); md_reap_sync_thread(mddev); @@ -6200,7 +6201,8 @@ static void __md_stop(struct mddev *mddev) md_bitmap_destroy(mddev); mddev_detach(mddev); /* Ensure ->event_work is done */ - flush_workqueue(md_misc_wq); + if (mddev->event_work.func) + flush_workqueue(md_misc_wq); spin_lock(&mddev->lock); mddev->pers = NULL; spin_unlock(&mddev->lock); From cc1ffe61c026e2318024bfa8722e8f689fd2a7f4 Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Sat, 4 Apr 2020 23:57:08 +0200 Subject: [PATCH 0459/1043] md: add new workqueue for delete rdev Since the purpose of call flush_workqueue in new_dev_store is to ensure md_delayed_delete() has completed, so we should check rdev->del_work is pending or not. To suppress lockdep warning, we have to check mddev->del_work while md_delayed_delete is attached to rdev->del_work, so it is not aligned to the purpose of flush workquee. So a new workqueue is needed to avoid the awkward situation, and introduce a new func flush_rdev_wq to flush the new workqueue after check if there was pending work. Also like new_dev_store, ADD_NEW_DISK ioctl has the same purpose to flush workqueue while it holds bdev->bd_mutex, so make the same change applies to the ioctl to avoid similar lock issue. And md_delayed_delete actually wants to delete rdev, so rename the function to rdev_delayed_delete. Signed-off-by: Guoqing Jiang Signed-off-by: Song Liu --- drivers/md/md.c | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 76c32ca6eee8..a970fdd0ce8c 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -89,6 +89,7 @@ static struct module *md_cluster_mod; static DECLARE_WAIT_QUEUE_HEAD(resync_wait); static struct workqueue_struct *md_wq; static struct workqueue_struct *md_misc_wq; +static struct workqueue_struct *md_rdev_misc_wq; static int remove_and_add_spares(struct mddev *mddev, struct md_rdev *this); @@ -2454,7 +2455,7 @@ static int bind_rdev_to_array(struct md_rdev *rdev, struct mddev *mddev) return err; } -static void md_delayed_delete(struct work_struct *ws) +static void rdev_delayed_delete(struct work_struct *ws) { struct md_rdev *rdev = container_of(ws, struct md_rdev, del_work); kobject_del(&rdev->kobj); @@ -2479,9 +2480,9 @@ static void unbind_rdev_from_array(struct md_rdev *rdev) * to delay it due to rcu usage. */ synchronize_rcu(); - INIT_WORK(&rdev->del_work, md_delayed_delete); + INIT_WORK(&rdev->del_work, rdev_delayed_delete); kobject_get(&rdev->kobj); - queue_work(md_misc_wq, &rdev->del_work); + queue_work(md_rdev_misc_wq, &rdev->del_work); } /* @@ -4514,6 +4515,20 @@ null_show(struct mddev *mddev, char *page) return -EINVAL; } +/* need to ensure rdev_delayed_delete() has completed */ +static void flush_rdev_wq(struct mddev *mddev) +{ + struct md_rdev *rdev; + + rcu_read_lock(); + rdev_for_each_rcu(rdev, mddev) + if (work_pending(&rdev->del_work)) { + flush_workqueue(md_rdev_misc_wq); + break; + } + rcu_read_unlock(); +} + static ssize_t new_dev_store(struct mddev *mddev, const char *buf, size_t len) { @@ -4541,8 +4556,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) minor != MINOR(dev)) return -EOVERFLOW; - flush_workqueue(md_misc_wq); - + flush_rdev_wq(mddev); err = mddev_lock(mddev); if (err) return err; @@ -4780,7 +4794,8 @@ action_store(struct mddev *mddev, const char *page, size_t len) clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery); if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) && mddev_lock(mddev) == 0) { - flush_workqueue(md_misc_wq); + if (work_pending(&mddev->del_work)) + flush_workqueue(md_misc_wq); if (mddev->sync_thread) { set_bit(MD_RECOVERY_INTR, &mddev->recovery); md_reap_sync_thread(mddev); @@ -7498,8 +7513,7 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode, } if (cmd == ADD_NEW_DISK) - /* need to ensure md_delayed_delete() has completed */ - flush_workqueue(md_misc_wq); + flush_rdev_wq(mddev); if (cmd == HOT_REMOVE_DISK) /* need to ensure recovery thread has run */ @@ -9471,6 +9485,10 @@ static int __init md_init(void) if (!md_misc_wq) goto err_misc_wq; + md_rdev_misc_wq = alloc_workqueue("md_rdev_misc", 0, 0); + if (!md_misc_wq) + goto err_rdev_misc_wq; + if ((ret = register_blkdev(MD_MAJOR, "md")) < 0) goto err_md; @@ -9492,6 +9510,8 @@ static int __init md_init(void) err_mdp: unregister_blkdev(MD_MAJOR, "md"); err_md: + destroy_workqueue(md_rdev_misc_wq); +err_rdev_misc_wq: destroy_workqueue(md_misc_wq); err_misc_wq: destroy_workqueue(md_wq); @@ -9778,6 +9798,7 @@ static __exit void md_exit(void) * destroy_workqueue() below will wait for that to complete. */ } + destroy_workqueue(md_rdev_misc_wq); destroy_workqueue(md_misc_wq); destroy_workqueue(md_wq); } From f6766ff6afff70e2aaf39e1511e16d471de7c3ae Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Sat, 4 Apr 2020 23:57:09 +0200 Subject: [PATCH 0460/1043] md: don't flush workqueue unconditionally in md_open We need to check mddev->del_work before flush workqueu since the purpose of flush is to ensure the previous md is disappeared. Otherwise the similar deadlock appeared if LOCKDEP is enabled, it is due to md_open holds the bdev->bd_mutex before flush workqueue. kernel: [ 154.522645] ====================================================== kernel: [ 154.522647] WARNING: possible circular locking dependency detected kernel: [ 154.522650] 5.6.0-rc7-lp151.27-default #25 Tainted: G O kernel: [ 154.522651] ------------------------------------------------------ kernel: [ 154.522653] mdadm/2482 is trying to acquire lock: kernel: [ 154.522655] ffff888078529128 ((wq_completion)md_misc){+.+.}, at: flush_workqueue+0x84/0x4b0 kernel: [ 154.522673] kernel: [ 154.522673] but task is already holding lock: kernel: [ 154.522675] ffff88804efa9338 (&bdev->bd_mutex){+.+.}, at: __blkdev_get+0x79/0x590 kernel: [ 154.522691] kernel: [ 154.522691] which lock already depends on the new lock. kernel: [ 154.522691] kernel: [ 154.522694] kernel: [ 154.522694] the existing dependency chain (in reverse order) is: kernel: [ 154.522696] kernel: [ 154.522696] -> #4 (&bdev->bd_mutex){+.+.}: kernel: [ 154.522704] __mutex_lock+0x87/0x950 kernel: [ 154.522706] __blkdev_get+0x79/0x590 kernel: [ 154.522708] blkdev_get+0x65/0x140 kernel: [ 154.522709] blkdev_get_by_dev+0x2f/0x40 kernel: [ 154.522716] lock_rdev+0x3d/0x90 [md_mod] kernel: [ 154.522719] md_import_device+0xd6/0x1b0 [md_mod] kernel: [ 154.522723] new_dev_store+0x15e/0x210 [md_mod] kernel: [ 154.522728] md_attr_store+0x7a/0xc0 [md_mod] kernel: [ 154.522732] kernfs_fop_write+0x117/0x1b0 kernel: [ 154.522735] vfs_write+0xad/0x1a0 kernel: [ 154.522737] ksys_write+0xa4/0xe0 kernel: [ 154.522745] do_syscall_64+0x64/0x2b0 kernel: [ 154.522748] entry_SYSCALL_64_after_hwframe+0x49/0xbe kernel: [ 154.522749] kernel: [ 154.522749] -> #3 (&mddev->reconfig_mutex){+.+.}: kernel: [ 154.522752] __mutex_lock+0x87/0x950 kernel: [ 154.522756] new_dev_store+0xc9/0x210 [md_mod] kernel: [ 154.522759] md_attr_store+0x7a/0xc0 [md_mod] kernel: [ 154.522761] kernfs_fop_write+0x117/0x1b0 kernel: [ 154.522763] vfs_write+0xad/0x1a0 kernel: [ 154.522765] ksys_write+0xa4/0xe0 kernel: [ 154.522767] do_syscall_64+0x64/0x2b0 kernel: [ 154.522769] entry_SYSCALL_64_after_hwframe+0x49/0xbe kernel: [ 154.522770] kernel: [ 154.522770] -> #2 (kn->count#253){++++}: kernel: [ 154.522775] __kernfs_remove+0x253/0x2c0 kernel: [ 154.522778] kernfs_remove+0x1f/0x30 kernel: [ 154.522780] kobject_del+0x28/0x60 kernel: [ 154.522783] mddev_delayed_delete+0x24/0x30 [md_mod] kernel: [ 154.522786] process_one_work+0x2a7/0x5f0 kernel: [ 154.522788] worker_thread+0x2d/0x3d0 kernel: [ 154.522793] kthread+0x117/0x130 kernel: [ 154.522795] ret_from_fork+0x3a/0x50 kernel: [ 154.522796] kernel: [ 154.522796] -> #1 ((work_completion)(&mddev->del_work)){+.+.}: kernel: [ 154.522800] process_one_work+0x27e/0x5f0 kernel: [ 154.522802] worker_thread+0x2d/0x3d0 kernel: [ 154.522804] kthread+0x117/0x130 kernel: [ 154.522806] ret_from_fork+0x3a/0x50 kernel: [ 154.522807] kernel: [ 154.522807] -> #0 ((wq_completion)md_misc){+.+.}: kernel: [ 154.522813] __lock_acquire+0x1392/0x1690 kernel: [ 154.522816] lock_acquire+0xb4/0x1a0 kernel: [ 154.522818] flush_workqueue+0xab/0x4b0 kernel: [ 154.522821] md_open+0xb6/0xc0 [md_mod] kernel: [ 154.522823] __blkdev_get+0xea/0x590 kernel: [ 154.522825] blkdev_get+0x65/0x140 kernel: [ 154.522828] do_dentry_open+0x1d1/0x380 kernel: [ 154.522831] path_openat+0x567/0xcc0 kernel: [ 154.522834] do_filp_open+0x9b/0x110 kernel: [ 154.522836] do_sys_openat2+0x201/0x2a0 kernel: [ 154.522838] do_sys_open+0x57/0x80 kernel: [ 154.522840] do_syscall_64+0x64/0x2b0 kernel: [ 154.522842] entry_SYSCALL_64_after_hwframe+0x49/0xbe kernel: [ 154.522844] kernel: [ 154.522844] other info that might help us debug this: kernel: [ 154.522844] kernel: [ 154.522846] Chain exists of: kernel: [ 154.522846] (wq_completion)md_misc --> &mddev->reconfig_mutex --> &bdev->bd_mutex kernel: [ 154.522846] kernel: [ 154.522850] Possible unsafe locking scenario: kernel: [ 154.522850] kernel: [ 154.522852] CPU0 CPU1 kernel: [ 154.522853] ---- ---- kernel: [ 154.522854] lock(&bdev->bd_mutex); kernel: [ 154.522856] lock(&mddev->reconfig_mutex); kernel: [ 154.522858] lock(&bdev->bd_mutex); kernel: [ 154.522860] lock((wq_completion)md_misc); kernel: [ 154.522861] kernel: [ 154.522861] *** DEADLOCK *** kernel: [ 154.522861] kernel: [ 154.522864] 1 lock held by mdadm/2482: kernel: [ 154.522865] #0: ffff88804efa9338 (&bdev->bd_mutex){+.+.}, at: __blkdev_get+0x79/0x590 kernel: [ 154.522868] kernel: [ 154.522868] stack backtrace: kernel: [ 154.522873] CPU: 1 PID: 2482 Comm: mdadm Tainted: G O 5.6.0-rc7-lp151.27-default #25 kernel: [ 154.522875] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1ubuntu1 04/01/2014 kernel: [ 154.522878] Call Trace: kernel: [ 154.522881] dump_stack+0x8f/0xcb kernel: [ 154.522884] check_noncircular+0x194/0x1b0 kernel: [ 154.522888] ? __lock_acquire+0x1392/0x1690 kernel: [ 154.522890] __lock_acquire+0x1392/0x1690 kernel: [ 154.522893] lock_acquire+0xb4/0x1a0 kernel: [ 154.522895] ? flush_workqueue+0x84/0x4b0 kernel: [ 154.522898] flush_workqueue+0xab/0x4b0 kernel: [ 154.522900] ? flush_workqueue+0x84/0x4b0 kernel: [ 154.522905] ? md_open+0xb6/0xc0 [md_mod] kernel: [ 154.522908] md_open+0xb6/0xc0 [md_mod] kernel: [ 154.522910] __blkdev_get+0xea/0x590 kernel: [ 154.522912] ? bd_acquire+0xc0/0xc0 kernel: [ 154.522914] blkdev_get+0x65/0x140 kernel: [ 154.522916] ? bd_acquire+0xc0/0xc0 kernel: [ 154.522918] do_dentry_open+0x1d1/0x380 kernel: [ 154.522921] path_openat+0x567/0xcc0 kernel: [ 154.522923] ? __lock_acquire+0x380/0x1690 kernel: [ 154.522926] do_filp_open+0x9b/0x110 kernel: [ 154.522929] ? __alloc_fd+0xe5/0x1f0 kernel: [ 154.522935] ? kmem_cache_alloc+0x28c/0x630 kernel: [ 154.522939] ? do_sys_openat2+0x201/0x2a0 kernel: [ 154.522941] do_sys_openat2+0x201/0x2a0 kernel: [ 154.522944] do_sys_open+0x57/0x80 kernel: [ 154.522946] do_syscall_64+0x64/0x2b0 kernel: [ 154.522948] entry_SYSCALL_64_after_hwframe+0x49/0xbe kernel: [ 154.522951] RIP: 0033:0x7f98d279d9ae And md_alloc also flushed the same workqueue, but the thing is different here. Because all the paths call md_alloc don't hold bdev->bd_mutex, and the flush is necessary to avoid race condition, so leave it as it is. Signed-off-by: Guoqing Jiang Signed-off-by: Song Liu --- drivers/md/md.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index a970fdd0ce8c..f5dfa503fb6f 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -7768,7 +7768,8 @@ static int md_open(struct block_device *bdev, fmode_t mode) */ mddev_put(mddev); /* Wait until bdev->bd_disk is definitely gone */ - flush_workqueue(md_misc_wq); + if (work_pending(&mddev->del_work)) + flush_workqueue(md_misc_wq); /* Then retry the open from the top */ return -ERESTARTSYS; } From 78b990cf2822d1150a419c13be48e74aefa83f27 Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Sat, 4 Apr 2020 23:57:10 +0200 Subject: [PATCH 0461/1043] md: flush md_rdev_misc_wq for HOT_ADD_DISK case Since rdev->kobj is removed asynchronously, it is possible that the rdev->kobj still exists when try to add the rdev again after rdev is removed. But this path md_ioctl (HOT_ADD_DISK) -> hot_add_disk -> bind_rdev_to_array missed it. Signed-off-by: Guoqing Jiang Signed-off-by: Song Liu --- drivers/md/md.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index f5dfa503fb6f..95b72e35b355 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -7512,7 +7512,7 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode, } - if (cmd == ADD_NEW_DISK) + if (cmd == ADD_NEW_DISK || cmd == HOT_ADD_DISK) flush_rdev_wq(mddev); if (cmd == HOT_REMOVE_DISK) From 3f79cc22348fe7d285bd140209394c9e381e082c Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Sat, 4 Apr 2020 23:57:11 +0200 Subject: [PATCH 0462/1043] md: remove the extra line for ->hot_add_disk It is not not necessary to add a newline for them since they don't exceed 80 characters, and it is not intutive to distinguish ->hot_add_disk() from hot_add_disk() too. Signed-off-by: Guoqing Jiang Signed-off-by: Song Liu --- drivers/md/md.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 95b72e35b355..8ff690ac6247 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -3192,8 +3192,7 @@ slot_store(struct md_rdev *rdev, const char *buf, size_t len) rdev->saved_raid_disk = -1; clear_bit(In_sync, &rdev->flags); clear_bit(Bitmap_sync, &rdev->flags); - err = rdev->mddev->pers-> - hot_add_disk(rdev->mddev, rdev); + err = rdev->mddev->pers->hot_add_disk(rdev->mddev, rdev); if (err) { rdev->raid_disk = -1; return err; @@ -9057,8 +9056,7 @@ static int remove_and_add_spares(struct mddev *mddev, rdev->recovery_offset = 0; } - if (mddev->pers-> - hot_add_disk(mddev, rdev) == 0) { + if (mddev->pers->hot_add_disk(mddev, rdev) == 0) { if (sysfs_link_rdev(mddev, rdev)) /* failure here is OK */; if (!test_bit(Journal, &rdev->flags)) From 78f57ef9d50a75326da73d352d7c27828495229a Mon Sep 17 00:00:00 2001 From: Coly Li Date: Thu, 9 Apr 2020 22:17:20 +0800 Subject: [PATCH 0463/1043] md: use memalloc scope APIs in mddev_suspend()/mddev_resume() In raid5.c:resize_chunk(), scribble_alloc() is called with GFP_NOIO flag, then it is sent into kvmalloc_array() inside scribble_alloc(). The problem is kvmalloc_array() eventually calls kvmalloc_node() which does not accept non GFP_KERNEL compatible flag like GFP_NOIO, then kmalloc_node() is called indeed to allocate physically continuous pages. When system memory is under heavy pressure, and the requesting size is large, there is high probability that allocating continueous pages will fail. But simply using GFP_KERNEL flag to call kvmalloc_array() is also progblematic. In the code path where scribble_alloc() is called, the raid array is suspended, if kvmalloc_node() triggers memory reclaim I/Os and such I/Os go back to the suspend raid array, deadlock will happen. What is desired here is to allocate non-physically (a.k.a virtually) continuous pages and avoid memory reclaim I/Os. Michal Hocko suggests to use the mmealloc sceope APIs to restrict memory reclaim I/O in allocating context, specifically to call memalloc_noio_save() when suspend the raid array and to call memalloc_noio_restore() when resume the raid array. This patch adds the memalloc scope APIs in mddev_suspend() and mddev_resume(), to restrict memory reclaim I/Os during the raid array is suspended. The benifit of adding the memalloc scope API in the unified entry point mddev_suspend()/mddev_resume() is, no matter which md raid array type (personality), we are sure the deadlock by recursive memory reclaim I/O won't happen on the suspending context. Please notice that the memalloc scope APIs only take effect on the raid array suspending context, if the memory allocation is from another new created kthread after raid array suspended, the recursive memory reclaim I/Os won't be restricted. The mddev_suspend()/mddev_resume() entries are used for the critical section where the raid metadata is modifying, creating a kthread to allocate memory inside the critical section is queer and very probably being buggy. Fixes: b330e6a49dc3 ("md: convert to kvmalloc") Suggested-by: Michal Hocko Signed-off-by: Coly Li Signed-off-by: Song Liu --- drivers/md/md.c | 4 ++++ drivers/md/md.h | 1 + 2 files changed, 5 insertions(+) diff --git a/drivers/md/md.c b/drivers/md/md.c index 8ff690ac6247..1b3316c79b7b 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -528,11 +528,15 @@ void mddev_suspend(struct mddev *mddev) wait_event(mddev->sb_wait, !test_bit(MD_UPDATING_SB, &mddev->flags)); del_timer_sync(&mddev->safemode_timer); + /* restrict memory reclaim I/O during raid array is suspend */ + mddev->noio_flag = memalloc_noio_save(); } EXPORT_SYMBOL_GPL(mddev_suspend); void mddev_resume(struct mddev *mddev) { + /* entred the memalloc scope from mddev_suspend() */ + memalloc_noio_restore(mddev->noio_flag); lockdep_assert_held(&mddev->reconfig_mutex); if (--mddev->suspended) return; diff --git a/drivers/md/md.h b/drivers/md/md.h index acd681939112..612814d07d35 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -497,6 +497,7 @@ struct mddev { void (*sync_super)(struct mddev *mddev, struct md_rdev *rdev); struct md_cluster_info *cluster_info; unsigned int good_device_nr; /* good device num within cluster raid */ + unsigned int noio_flag; /* for memalloc scope API */ bool has_superblocks:1; bool fail_last_dev:1; From ba54d4d4d2844c234f1b4692bd8c9e0f833c8a54 Mon Sep 17 00:00:00 2001 From: Coly Li Date: Thu, 9 Apr 2020 22:17:21 +0800 Subject: [PATCH 0464/1043] raid5: remove gfp flags from scribble_alloc() Using GFP_NOIO flag to call scribble_alloc() from resize_chunk() does not have the expected behavior. kvmalloc_array() inside scribble_alloc() which receives the GFP_NOIO flag will eventually call kmalloc_node() to allocate physically continuous pages. Now we have memalloc scope APIs in mddev_suspend()/mddev_resume() to prevent memory reclaim I/Os during raid array suspend context, calling to kvmalloc_array() with GFP_KERNEL flag may avoid deadlock of recursive I/O as expected. This patch removes the useless gfp flags from parameters list of scribble_alloc(), and call kvmalloc_array() with GFP_KERNEL flag. The incorrect GFP_NOIO flag does not exist anymore. Fixes: b330e6a49dc3 ("md: convert to kvmalloc") Suggested-by: Michal Hocko Signed-off-by: Coly Li Signed-off-by: Song Liu --- drivers/md/raid5.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index ba00e9877f02..190dd70db514 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -2228,14 +2228,19 @@ static int grow_stripes(struct r5conf *conf, int num) * of the P and Q blocks. */ static int scribble_alloc(struct raid5_percpu *percpu, - int num, int cnt, gfp_t flags) + int num, int cnt) { size_t obj_size = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); void *scribble; - scribble = kvmalloc_array(cnt, obj_size, flags); + /* + * If here is in raid array suspend context, it is in memalloc noio + * context as well, there is no potential recursive memory reclaim + * I/Os with the GFP_KERNEL flag. + */ + scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL); if (!scribble) return -ENOMEM; @@ -2267,8 +2272,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) percpu = per_cpu_ptr(conf->percpu, cpu); err = scribble_alloc(percpu, new_disks, - new_sectors / STRIPE_SECTORS, - GFP_NOIO); + new_sectors / STRIPE_SECTORS); if (err) break; } @@ -6759,8 +6763,7 @@ static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu conf->previous_raid_disks), max(conf->chunk_sectors, conf->prev_chunk_sectors) - / STRIPE_SECTORS, - GFP_KERNEL)) { + / STRIPE_SECTORS)) { free_scratch_buffer(conf, percpu); return -ENOMEM; } From 7f8a30e5d253793d660f30b7475d7bb41efd40b8 Mon Sep 17 00:00:00 2001 From: Coly Li Date: Thu, 9 Apr 2020 22:17:22 +0800 Subject: [PATCH 0465/1043] raid5: update code comment of scribble_alloc() Code comments of scribble_alloc() is outdated for a while. This patch update the comments in function header for the new parameter list. Suggested-by: Song Liu Signed-off-by: Coly Li Signed-off-by: Song Liu --- drivers/md/raid5.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 190dd70db514..ab8067f9ce8c 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -2215,10 +2215,13 @@ static int grow_stripes(struct r5conf *conf, int num) } /** - * scribble_len - return the required size of the scribble region + * scribble_alloc - allocate percpu scribble buffer for required size + * of the scribble region + * @percpu - from for_each_present_cpu() of the caller * @num - total number of disks in the array + * @cnt - scribble objs count for required size of the scribble region * - * The size must be enough to contain: + * The scribble buffer size must be enough to contain: * 1/ a struct page pointer for each device in the array +2 * 2/ room to convert each entry in (1) to its corresponding dma * (dma_map_page()) or page (page_address()) address. From 3024ba2d6c5573b4797e62006d98f17f47f7d103 Mon Sep 17 00:00:00 2001 From: Coly Li Date: Thu, 9 Apr 2020 22:17:23 +0800 Subject: [PATCH 0466/1043] md: remove redundant memalloc scope API usage In mddev_create_serial_pool(), memalloc scope APIs memalloc_noio_save() and memalloc_noio_restore() are used when allocating memory by calling mempool_create_kmalloc_pool(). After adding the memalloc scope APIs in raid array suspend context, it is unncessary to explicitly call them around mempool_create_kmalloc_pool() any longer. This patch removes the redundant memalloc scope APIs in mddev_create_serial_pool(). Signed-off-by: Coly Li Cc: Guoqing Jiang Signed-off-by: Song Liu --- drivers/md/md.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 1b3316c79b7b..7b2ac5f23260 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -228,13 +228,13 @@ void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev, goto abort; if (mddev->serial_info_pool == NULL) { - unsigned int noio_flag; - - noio_flag = memalloc_noio_save(); + /* + * already in memalloc noio context by + * mddev_suspend() + */ mddev->serial_info_pool = mempool_create_kmalloc_pool(NR_SERIAL_INFOS, sizeof(struct serial_info)); - memalloc_noio_restore(noio_flag); if (!mddev->serial_info_pool) { rdevs_uninit_serial(mddev); pr_err("can't alloc memory pool for serialization\n"); From c91114c2b89d5bea132d34b40568053f8bfa963d Mon Sep 17 00:00:00 2001 From: David Jeffery Date: Mon, 27 Jan 2020 10:26:19 -0500 Subject: [PATCH 0467/1043] md/raid1: release pending accounting for an I/O only after write-behind is also finished When using RAID1 and write-behind, md can deadlock when errors occur. With write-behind, r1bio structs can be accounted by raid1 as queued but not counted as pending. The pending count is dropped when the original bio is returned complete but write-behind for the r1bio may still be active. This breaks the accounting used in some conditions to know when the raid1 md device has reached an idle state. It can result in calls to freeze_array deadlocking. freeze_array will never complete from a negative "unqueued" value being calculated due to a queued count larger than the pending count. To properly account for write-behind, move the call to allow_barrier from call_bio_endio to raid_end_bio_io. When using write-behind, md can call call_bio_endio before all write-behind I/O is complete. Using raid_end_bio_io for the point to call allow_barrier will release the pending count at a point where all I/O for an r1bio, even write-behind, is done. Signed-off-by: David Jeffery Signed-off-by: Song Liu --- drivers/md/raid1.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index cd810e195086..dcd27f3da84e 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -296,22 +296,17 @@ static void reschedule_retry(struct r1bio *r1_bio) static void call_bio_endio(struct r1bio *r1_bio) { struct bio *bio = r1_bio->master_bio; - struct r1conf *conf = r1_bio->mddev->private; if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) bio->bi_status = BLK_STS_IOERR; bio_endio(bio); - /* - * Wake up any possible resync thread that waits for the device - * to go idle. - */ - allow_barrier(conf, r1_bio->sector); } static void raid_end_bio_io(struct r1bio *r1_bio) { struct bio *bio = r1_bio->master_bio; + struct r1conf *conf = r1_bio->mddev->private; /* if nobody has done the final endio yet, do it now */ if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) { @@ -322,6 +317,12 @@ static void raid_end_bio_io(struct r1bio *r1_bio) call_bio_endio(r1_bio); } + /* + * Wake up any possible resync thread that waits for the device + * to go idle. All I/Os, even write-behind writes, are done. + */ + allow_barrier(conf, r1_bio->sector); + free_r1bio(r1_bio); } From e4fc5a74293fbe8a1b8598b8a3c53592a5d70398 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Fri, 8 May 2020 18:15:14 +0200 Subject: [PATCH 0468/1043] md: stop using ->queuedata Pointer to mddev is already available in private_data. Signed-off-by: Christoph Hellwig Signed-off-by: Song Liu --- drivers/md/md.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 7b2ac5f23260..79e36cb4e8b0 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -467,7 +467,7 @@ static blk_qc_t md_make_request(struct request_queue *q, struct bio *bio) { const int rw = bio_data_dir(bio); const int sgrp = op_stat_group(bio_op(bio)); - struct mddev *mddev = q->queuedata; + struct mddev *mddev = bio->bi_disk->private_data; unsigned int sectors; if (unlikely(test_bit(MD_BROKEN, &mddev->flags)) && (rw == WRITE)) { @@ -5644,7 +5644,6 @@ static int md_alloc(dev_t dev, char *name) mddev->queue = blk_alloc_queue(md_make_request, NUMA_NO_NODE); if (!mddev->queue) goto abort; - mddev->queue->queuedata = mddev; blk_set_stacking_limits(&mddev->queue->limits); From 3f99980c8f70bc56584450c0a69973eef7b65913 Mon Sep 17 00:00:00 2001 From: Xiongfeng Wang Date: Mon, 11 May 2020 16:23:25 +0800 Subject: [PATCH 0469/1043] md: add a newline when printing parameter 'start_ro' by sysfs Add a missing newline when printing module parameter 'start_ro' by sysfs. Signed-off-by: Xiongfeng Wang Signed-off-by: Song Liu --- drivers/md/md.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 79e36cb4e8b0..f567f536b529 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -9810,7 +9810,7 @@ module_exit(md_exit) static int get_ro(char *buffer, const struct kernel_param *kp) { - return sprintf(buffer, "%d", start_readonly); + return sprintf(buffer, "%d\n", start_readonly); } static int set_ro(const char *val, const struct kernel_param *kp) { From 358369f03ac94637c9fd9d8f94a2dfde86b9f25f Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 7 May 2020 14:22:10 -0500 Subject: [PATCH 0470/1043] md/raid1: Replace zero-length array with flexible-array The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] sizeof(flexible-array-member) triggers a warning because flexible array members have incomplete type[1]. There are some instances of code in which the sizeof operator is being incorrectly/erroneously applied to zero-length arrays and the result is zero. Such instances may be hiding some bugs. So, this work (flexible-array member conversions) will also help to get completely rid of those sorts of issues. This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Song Liu --- drivers/md/md-linear.h | 2 +- drivers/md/raid1.h | 2 +- drivers/md/raid10.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/md/md-linear.h b/drivers/md/md-linear.h index 8381d651d4ed..24e97db50ebb 100644 --- a/drivers/md/md-linear.h +++ b/drivers/md/md-linear.h @@ -12,6 +12,6 @@ struct linear_conf struct rcu_head rcu; sector_t array_sectors; int raid_disks; /* a copy of mddev->raid_disks */ - struct dev_info disks[0]; + struct dev_info disks[]; }; #endif diff --git a/drivers/md/raid1.h b/drivers/md/raid1.h index e7ccad898736..b7eb09e8c025 100644 --- a/drivers/md/raid1.h +++ b/drivers/md/raid1.h @@ -180,7 +180,7 @@ struct r1bio { * if the IO is in WRITE direction, then multiple bios are used. * We choose the number when they are allocated. */ - struct bio *bios[0]; + struct bio *bios[]; /* DO NOT PUT ANY NEW FIELDS HERE - bios array is contiguously alloced*/ }; diff --git a/drivers/md/raid10.h b/drivers/md/raid10.h index d3eaaf3eb1bc..79cd2b7d3128 100644 --- a/drivers/md/raid10.h +++ b/drivers/md/raid10.h @@ -153,7 +153,7 @@ struct r10bio { }; sector_t addr; int devnum; - } devs[0]; + } devs[]; }; /* bits for r10bio.state */ From d203c2d3eed5a6f1d617591335ccef7b1926461b Mon Sep 17 00:00:00 2001 From: Lars Povlsen Date: Wed, 13 May 2020 15:23:47 +0200 Subject: [PATCH 0471/1043] MIPS: dts: mscc: Updated changed name for miim pinctrl function This is an add-on patch to the main SoC Sparx5 series (Message-ID: <20200513125532.24585-1-lars.povlsen@microchip.com>). This changes the miim pinctrl function name from "miim1" to "miim" due to refactoring in the driver, obsoleting the instance number. The change in the driver was to better fit new platforms, as the instance number is redundant information. Specifically, support for the Microchip Sparx5 SoC is being submitted, where this change became necessary. Reviewed-by: Alexandre Belloni Signed-off-by: Lars Povlsen Acked-by: Alexandre Belloni Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/mscc/ocelot.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/boot/dts/mscc/ocelot.dtsi b/arch/mips/boot/dts/mscc/ocelot.dtsi index 797d336db54d..f94e8a02ed06 100644 --- a/arch/mips/boot/dts/mscc/ocelot.dtsi +++ b/arch/mips/boot/dts/mscc/ocelot.dtsi @@ -214,7 +214,7 @@ miim1: miim1 { pins = "GPIO_14", "GPIO_15"; - function = "miim1"; + function = "miim"; }; }; From 04146f22cb47b84faff62f9a1ce6165b4a1be5cb Mon Sep 17 00:00:00 2001 From: Jason Yan Date: Wed, 29 Apr 2020 22:09:35 +0800 Subject: [PATCH 0472/1043] KVM: MIPS: use true,false for bool variable Fix the following coccicheck warning: arch/mips/kvm/mips.c:82:1-28: WARNING: Assignment of 0/1 to bool variable arch/mips/kvm/mips.c:88:1-28: WARNING: Assignment of 0/1 to bool variable Signed-off-by: Jason Yan Signed-off-by: Thomas Bogendoerfer --- arch/mips/kvm/mips.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index 8f05dd0a0f4e..2261c63174c5 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -80,13 +80,13 @@ bool kvm_trace_guest_mode_change; int kvm_guest_mode_change_trace_reg(void) { - kvm_trace_guest_mode_change = 1; + kvm_trace_guest_mode_change = true; return 0; } void kvm_guest_mode_change_trace_unreg(void) { - kvm_trace_guest_mode_change = 0; + kvm_trace_guest_mode_change = false; } /* From 7ff1f6264f9ca1532fe1ac8e324efbc439052f54 Mon Sep 17 00:00:00 2001 From: Jason Yan Date: Tue, 28 Apr 2020 14:32:45 +0800 Subject: [PATCH 0473/1043] KVM: MIPS/TLB: Remove Unneeded semicolon in tlb.c Fix the following coccicheck warning: arch/mips/kvm/tlb.c:472:2-3: Unneeded semicolon arch/mips/kvm/tlb.c:489:2-3: Unneeded semicolon Signed-off-by: Jason Yan Signed-off-by: Thomas Bogendoerfer --- arch/mips/kvm/tlb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/kvm/tlb.c b/arch/mips/kvm/tlb.c index 7cd92166a0b9..5d436c5216cc 100644 --- a/arch/mips/kvm/tlb.c +++ b/arch/mips/kvm/tlb.c @@ -469,7 +469,7 @@ void kvm_vz_local_flush_guesttlb_all(void) cvmmemctl2 |= CVMMEMCTL2_INHIBITTS; write_c0_cvmmemctl2(cvmmemctl2); break; - }; + } /* Invalidate guest entries in guest TLB */ write_gc0_entrylo0(0); @@ -486,7 +486,7 @@ void kvm_vz_local_flush_guesttlb_all(void) if (cvmmemctl2) { cvmmemctl2 &= ~CVMMEMCTL2_INHIBITTS; write_c0_cvmmemctl2(cvmmemctl2); - }; + } write_gc0_index(old_index); write_gc0_entryhi(old_entryhi); From 8d34509742f9ef09c6deff8dd68f4af262215153 Mon Sep 17 00:00:00 2001 From: Zou Wei Date: Thu, 30 Apr 2020 11:14:50 +0800 Subject: [PATCH 0474/1043] KVM: MIPS/VZ: Remove unneeded semicolon Fixes coccicheck warnings: arch/mips/kvm/vz.c:1186:4-5: Unneeded semicolon arch/mips/kvm/vz.c:1195:3-4: Unneeded semicolon arch/mips/kvm/vz.c:1949:3-4: Unneeded semicolon arch/mips/kvm/vz.c:1121:2-3: Unneeded semicolon arch/mips/kvm/vz.c:2188:3-4: Unneeded semicolon Reported-by: Hulk Robot Signed-off-by: Zou Wei Reviewed-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/kvm/vz.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/mips/kvm/vz.c b/arch/mips/kvm/vz.c index dde20887a70d..389dd0fbd051 100644 --- a/arch/mips/kvm/vz.c +++ b/arch/mips/kvm/vz.c @@ -1118,7 +1118,7 @@ static enum emulation_result kvm_vz_gpsi_cache(union mips_instruction inst, break; default: break; - }; + } kvm_err("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n", curr_pc, vcpu->arch.gprs[31], cache, op, base, arch->gprs[base], @@ -1183,7 +1183,7 @@ static enum emulation_result kvm_trap_vz_handle_gpsi(u32 cause, u32 *opc, trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, KVM_TRACE_HWR(rd, sel), 0); goto unknown; - }; + } trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, KVM_TRACE_HWR(rd, sel), arch->gprs[rt]); @@ -1192,7 +1192,7 @@ static enum emulation_result kvm_trap_vz_handle_gpsi(u32 cause, u32 *opc, break; default: goto unknown; - }; + } break; unknown: @@ -1946,7 +1946,7 @@ static int kvm_vz_get_one_reg(struct kvm_vcpu *vcpu, default: *v = (long)kvm_read_c0_guest_prid(cop0); break; - }; + } break; case KVM_REG_MIPS_CP0_EBASE: *v = kvm_vz_read_gc0_ebase(); @@ -2185,7 +2185,7 @@ static int kvm_vz_set_one_reg(struct kvm_vcpu *vcpu, default: kvm_write_c0_guest_prid(cop0, v); break; - }; + } break; case KVM_REG_MIPS_CP0_EBASE: kvm_vz_write_gc0_ebase(v); From b6b6c318006b680a467451846aea65397074c59d Mon Sep 17 00:00:00 2001 From: Zou Wei Date: Thu, 30 Apr 2020 11:19:36 +0800 Subject: [PATCH 0475/1043] KVM: MIPS/Emulate: Remove unneeded semicolon Fixes coccicheck warnings: arch/mips/kvm/emulate.c:1793:3-4: Unneeded semicolon arch/mips/kvm/emulate.c:1968:3-4: Unneeded semicolon Reported-by: Hulk Robot Signed-off-by: Zou Wei Reviewed-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/kvm/emulate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/kvm/emulate.c b/arch/mips/kvm/emulate.c index 8c80333816fe..7ccf9b096783 100644 --- a/arch/mips/kvm/emulate.c +++ b/arch/mips/kvm/emulate.c @@ -1790,7 +1790,7 @@ static enum emulation_result kvm_mips_guest_cache_op(int (*fn)(unsigned long), return EMULATE_EXCEPT; default: break; - }; + } } } @@ -1965,7 +1965,7 @@ enum emulation_result kvm_mips_emulate_inst(u32 cause, u32 *opc, break; default: goto unknown; - }; + } break; unknown: #endif From 237aac4624aac5d46b903539e91c8d4567a2bab5 Mon Sep 17 00:00:00 2001 From: Zheng Bin Date: Tue, 12 May 2020 11:48:35 -0700 Subject: [PATCH 0476/1043] xfs: ensure f_bfree returned by statfs() is non-negative Construct an img like this: dd if=/dev/zero of=xfs.img bs=1M count=20 mkfs.xfs -d agcount=1 xfs.img xfs_db -x xfs.img sb 0 write fdblocks 0 agf 0 write freeblks 0 write longest 0 quit mount it, df -h /mnt(xfs mount point), will show this: Filesystem Size Used Avail Use% Mounted on /dev/loop0 17M -64Z -32K 100% /mnt Reported-by: Hulk Robot Signed-off-by: Zheng Bin Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_super.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index e80bd2c4c279..aae469f73efe 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -807,7 +807,8 @@ xfs_fs_statfs( statp->f_blocks = sbp->sb_dblocks - lsize; spin_unlock(&mp->m_sb_lock); - statp->f_bfree = fdblocks - mp->m_alloc_set_aside; + /* make sure statp->f_bfree does not underflow */ + statp->f_bfree = max_t(int64_t, fdblocks - mp->m_alloc_set_aside, 0); statp->f_bavail = statp->f_bfree; fakeinos = XFS_FSB_TO_INO(mp, statp->f_bfree); From ee4064e56cd81cd3126805159122f53cf4f12ae6 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Tue, 12 May 2020 16:36:47 -0700 Subject: [PATCH 0477/1043] xfs: Replace zero-length array with flexible-array The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] sizeof(flexible-array-member) triggers a warning because flexible array members have incomplete type[1]. There are some instances of code in which the sizeof operator is being incorrectly/erroneously applied to zero-length arrays and the result is zero. Such instances may be hiding some bugs. So, this work (flexible-array member conversions) will also help to get completely rid of those sorts of issues. This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_format.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index 045556e78ee2..592f1c12ad36 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -1681,7 +1681,7 @@ struct xfs_acl_entry { struct xfs_acl { __be32 acl_cnt; - struct xfs_acl_entry acl_entry[0]; + struct xfs_acl_entry acl_entry[]; }; /* From 508578f2f5601816ea29bec5cda00ea7d95a856d Mon Sep 17 00:00:00 2001 From: Nishad Kamdar Date: Tue, 12 May 2020 16:54:17 -0700 Subject: [PATCH 0478/1043] xfs: Use the correct style for SPDX License Identifier This patch corrects the SPDX License Identifier style in header files related to XFS File System support. For C header files Documentation/process/license-rules.rst mandates C-like comments. (opposed to C source files where C++ style should be used). Changes made by using a script provided by Joe Perches here: https://lkml.org/lkml/2019/2/7/46. Suggested-by: Joe Perches Signed-off-by: Nishad Kamdar Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/kmem.h | 2 +- fs/xfs/libxfs/xfs_ag_resv.h | 2 +- fs/xfs/libxfs/xfs_alloc.h | 2 +- fs/xfs/libxfs/xfs_alloc_btree.h | 2 +- fs/xfs/libxfs/xfs_attr.h | 2 +- fs/xfs/libxfs/xfs_attr_leaf.h | 2 +- fs/xfs/libxfs/xfs_attr_remote.h | 2 +- fs/xfs/libxfs/xfs_attr_sf.h | 2 +- fs/xfs/libxfs/xfs_bit.h | 2 +- fs/xfs/libxfs/xfs_bmap.h | 2 +- fs/xfs/libxfs/xfs_bmap_btree.h | 2 +- fs/xfs/libxfs/xfs_btree.h | 2 +- fs/xfs/libxfs/xfs_da_btree.h | 2 +- fs/xfs/libxfs/xfs_da_format.h | 2 +- fs/xfs/libxfs/xfs_defer.h | 2 +- fs/xfs/libxfs/xfs_dir2.h | 2 +- fs/xfs/libxfs/xfs_dir2_priv.h | 2 +- fs/xfs/libxfs/xfs_errortag.h | 2 +- fs/xfs/libxfs/xfs_format.h | 2 +- fs/xfs/libxfs/xfs_fs.h | 2 +- fs/xfs/libxfs/xfs_health.h | 2 +- 21 files changed, 21 insertions(+), 21 deletions(-) diff --git a/fs/xfs/kmem.h b/fs/xfs/kmem.h index 6143117770e9..fc87ea9f6843 100644 --- a/fs/xfs/kmem.h +++ b/fs/xfs/kmem.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000-2005 Silicon Graphics, Inc. * All Rights Reserved. diff --git a/fs/xfs/libxfs/xfs_ag_resv.h b/fs/xfs/libxfs/xfs_ag_resv.h index c0352edc8e41..f3fd0ee9a7f7 100644 --- a/fs/xfs/libxfs/xfs_ag_resv.h +++ b/fs/xfs/libxfs/xfs_ag_resv.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +/* SPDX-License-Identifier: GPL-2.0+ */ /* * Copyright (C) 2016 Oracle. All Rights Reserved. * Author: Darrick J. Wong diff --git a/fs/xfs/libxfs/xfs_alloc.h b/fs/xfs/libxfs/xfs_alloc.h index a851bf77f17b..6c22b12176b8 100644 --- a/fs/xfs/libxfs/xfs_alloc.h +++ b/fs/xfs/libxfs/xfs_alloc.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. * All Rights Reserved. diff --git a/fs/xfs/libxfs/xfs_alloc_btree.h b/fs/xfs/libxfs/xfs_alloc_btree.h index 047f09f0be3c..a5b998e950fe 100644 --- a/fs/xfs/libxfs/xfs_alloc_btree.h +++ b/fs/xfs/libxfs/xfs_alloc_btree.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000,2005 Silicon Graphics, Inc. * All Rights Reserved. diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h index 0d2d05908537..db4717657ca1 100644 --- a/fs/xfs/libxfs/xfs_attr.h +++ b/fs/xfs/libxfs/xfs_attr.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000,2002-2003,2005 Silicon Graphics, Inc. * All Rights Reserved. diff --git a/fs/xfs/libxfs/xfs_attr_leaf.h b/fs/xfs/libxfs/xfs_attr_leaf.h index 6dd2d937a42a..5be6be309302 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.h +++ b/fs/xfs/libxfs/xfs_attr_leaf.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000,2002-2003,2005 Silicon Graphics, Inc. * Copyright (c) 2013 Red Hat, Inc. diff --git a/fs/xfs/libxfs/xfs_attr_remote.h b/fs/xfs/libxfs/xfs_attr_remote.h index 6fb4572845ce..e1144f22b005 100644 --- a/fs/xfs/libxfs/xfs_attr_remote.h +++ b/fs/xfs/libxfs/xfs_attr_remote.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2013 Red Hat, Inc. * All Rights Reserved. diff --git a/fs/xfs/libxfs/xfs_attr_sf.h b/fs/xfs/libxfs/xfs_attr_sf.h index aafa4fe70624..bb004fb7944a 100644 --- a/fs/xfs/libxfs/xfs_attr_sf.h +++ b/fs/xfs/libxfs/xfs_attr_sf.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000,2002,2005 Silicon Graphics, Inc. * All Rights Reserved. diff --git a/fs/xfs/libxfs/xfs_bit.h b/fs/xfs/libxfs/xfs_bit.h index 99017b8df292..a04f266ae644 100644 --- a/fs/xfs/libxfs/xfs_bit.h +++ b/fs/xfs/libxfs/xfs_bit.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000,2002,2005 Silicon Graphics, Inc. * All Rights Reserved. diff --git a/fs/xfs/libxfs/xfs_bmap.h b/fs/xfs/libxfs/xfs_bmap.h index f3259ad5c22c..6028a3c825ba 100644 --- a/fs/xfs/libxfs/xfs_bmap.h +++ b/fs/xfs/libxfs/xfs_bmap.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000-2006 Silicon Graphics, Inc. * All Rights Reserved. diff --git a/fs/xfs/libxfs/xfs_bmap_btree.h b/fs/xfs/libxfs/xfs_bmap_btree.h index 29b407d053b4..72bf74c79fb9 100644 --- a/fs/xfs/libxfs/xfs_bmap_btree.h +++ b/fs/xfs/libxfs/xfs_bmap_btree.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000,2002-2005 Silicon Graphics, Inc. * All Rights Reserved. diff --git a/fs/xfs/libxfs/xfs_btree.h b/fs/xfs/libxfs/xfs_btree.h index 8626c5a81aad..10e50cbacacf 100644 --- a/fs/xfs/libxfs/xfs_btree.h +++ b/fs/xfs/libxfs/xfs_btree.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. * All Rights Reserved. diff --git a/fs/xfs/libxfs/xfs_da_btree.h b/fs/xfs/libxfs/xfs_da_btree.h index 53e503b6f186..6e25de6621e4 100644 --- a/fs/xfs/libxfs/xfs_da_btree.h +++ b/fs/xfs/libxfs/xfs_da_btree.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000,2002,2005 Silicon Graphics, Inc. * Copyright (c) 2013 Red Hat, Inc. diff --git a/fs/xfs/libxfs/xfs_da_format.h b/fs/xfs/libxfs/xfs_da_format.h index 08c0a4d98b89..059ac108b1b3 100644 --- a/fs/xfs/libxfs/xfs_da_format.h +++ b/fs/xfs/libxfs/xfs_da_format.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. * Copyright (c) 2013 Red Hat, Inc. diff --git a/fs/xfs/libxfs/xfs_defer.h b/fs/xfs/libxfs/xfs_defer.h index 3bf7c2c4d851..6b2ca580f2b0 100644 --- a/fs/xfs/libxfs/xfs_defer.h +++ b/fs/xfs/libxfs/xfs_defer.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +/* SPDX-License-Identifier: GPL-2.0+ */ /* * Copyright (C) 2016 Oracle. All Rights Reserved. * Author: Darrick J. Wong diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h index 033777e282f2..e55378640b05 100644 --- a/fs/xfs/libxfs/xfs_dir2.h +++ b/fs/xfs/libxfs/xfs_dir2.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. * All Rights Reserved. diff --git a/fs/xfs/libxfs/xfs_dir2_priv.h b/fs/xfs/libxfs/xfs_dir2_priv.h index 01ee0b926572..44c6a77cba05 100644 --- a/fs/xfs/libxfs/xfs_dir2_priv.h +++ b/fs/xfs/libxfs/xfs_dir2_priv.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. * All Rights Reserved. diff --git a/fs/xfs/libxfs/xfs_errortag.h b/fs/xfs/libxfs/xfs_errortag.h index 2486dab19023..53b305dea381 100644 --- a/fs/xfs/libxfs/xfs_errortag.h +++ b/fs/xfs/libxfs/xfs_errortag.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +/* SPDX-License-Identifier: GPL-2.0+ */ /* * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. * Copyright (C) 2017 Oracle. diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index 592f1c12ad36..f2228d9e317a 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2000-2005 Silicon Graphics, Inc. * All Rights Reserved. diff --git a/fs/xfs/libxfs/xfs_fs.h b/fs/xfs/libxfs/xfs_fs.h index 245188e4f6d3..84bcffa87753 100644 --- a/fs/xfs/libxfs/xfs_fs.h +++ b/fs/xfs/libxfs/xfs_fs.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: LGPL-2.1 +/* SPDX-License-Identifier: LGPL-2.1 */ /* * Copyright (c) 1995-2005 Silicon Graphics, Inc. * All Rights Reserved. diff --git a/fs/xfs/libxfs/xfs_health.h b/fs/xfs/libxfs/xfs_health.h index 272005ac8c88..99e796256c5d 100644 --- a/fs/xfs/libxfs/xfs_health.h +++ b/fs/xfs/libxfs/xfs_health.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +/* SPDX-License-Identifier: GPL-2.0+ */ /* * Copyright (C) 2019 Oracle. All Rights Reserved. * Author: Darrick J. Wong From 60cf7c5ed5f7087c4de87a7676b8c82d96fd166c Mon Sep 17 00:00:00 2001 From: Jeremy Cline Date: Thu, 14 May 2020 10:05:46 -0400 Subject: [PATCH 0479/1043] lockdown: Allow unprivileged users to see lockdown status A number of userspace tools, such as systemtap, need a way to see the current lockdown state so they can gracefully deal with the kernel being locked down. The state is already exposed in /sys/kernel/security/lockdown, but is only readable by root. Adjust the permissions so unprivileged users can read the state. Fixes: 000d388ed3bb ("security: Add a static lockdown policy LSM") Cc: Frank Ch. Eigler Signed-off-by: Jeremy Cline Signed-off-by: James Morris --- security/lockdown/lockdown.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security/lockdown/lockdown.c b/security/lockdown/lockdown.c index 40b790536def..ae594c0a127f 100644 --- a/security/lockdown/lockdown.c +++ b/security/lockdown/lockdown.c @@ -175,7 +175,7 @@ static int __init lockdown_secfs_init(void) { struct dentry *dentry; - dentry = securityfs_create_file("lockdown", 0600, NULL, NULL, + dentry = securityfs_create_file("lockdown", 0644, NULL, NULL, &lockdown_ops); return PTR_ERR_OR_ZERO(dentry); } From 0f158b4cf20e7983d5b33878a6aad118cfac4f05 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 14 May 2020 17:18:39 -0600 Subject: [PATCH 0480/1043] io_uring: name sq thread and ref completions We used to have three completions, now we just have two. With the two, let's not allocate them dynamically, just embed then in the ctx and name them appropriately. Signed-off-by: Jens Axboe --- fs/io_uring.c | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index d2e37215d05a..414e940323d4 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -279,8 +279,8 @@ struct io_ring_ctx { const struct cred *creds; - /* 0 is for ctx quiesce/reinit/free, 1 is for sqo_thread started */ - struct completion *completions; + struct completion ref_comp; + struct completion sq_thread_comp; /* if all else fails... */ struct io_kiocb *fallback_req; @@ -883,7 +883,7 @@ static void io_ring_ctx_ref_free(struct percpu_ref *ref) { struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); - complete(&ctx->completions[0]); + complete(&ctx->ref_comp); } static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) @@ -899,10 +899,6 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) if (!ctx->fallback_req) goto err; - ctx->completions = kmalloc(2 * sizeof(struct completion), GFP_KERNEL); - if (!ctx->completions) - goto err; - /* * Use 5 bits less than the max cq entries, that should give us around * 32 entries per hash list if totally full and uniformly spread. @@ -925,8 +921,8 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) ctx->flags = p->flags; init_waitqueue_head(&ctx->cq_wait); INIT_LIST_HEAD(&ctx->cq_overflow_list); - init_completion(&ctx->completions[0]); - init_completion(&ctx->completions[1]); + init_completion(&ctx->ref_comp); + init_completion(&ctx->sq_thread_comp); idr_init(&ctx->io_buffer_idr); idr_init(&ctx->personality_idr); mutex_init(&ctx->uring_lock); @@ -942,7 +938,6 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) err: if (ctx->fallback_req) kmem_cache_free(req_cachep, ctx->fallback_req); - kfree(ctx->completions); kfree(ctx->cancel_hash); kfree(ctx); return NULL; @@ -5933,7 +5928,7 @@ static int io_sq_thread(void *data) unsigned long timeout; int ret = 0; - complete(&ctx->completions[1]); + complete(&ctx->sq_thread_comp); old_fs = get_fs(); set_fs(USER_DS); @@ -6212,7 +6207,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) static void io_sq_thread_stop(struct io_ring_ctx *ctx) { if (ctx->sqo_thread) { - wait_for_completion(&ctx->completions[1]); + wait_for_completion(&ctx->sq_thread_comp); /* * The park is a bit of a work-around, without it we get * warning spews on shutdown with SQPOLL set and affinity @@ -7241,7 +7236,6 @@ static void io_ring_ctx_free(struct io_ring_ctx *ctx) ring_pages(ctx->sq_entries, ctx->cq_entries)); free_uid(ctx->user); put_cred(ctx->creds); - kfree(ctx->completions); kfree(ctx->cancel_hash); kmem_cache_free(req_cachep, ctx->fallback_req); kfree(ctx); @@ -7293,7 +7287,7 @@ static void io_ring_exit_work(struct work_struct *work) if (ctx->rings) io_cqring_overflow_flush(ctx, true); - wait_for_completion(&ctx->completions[0]); + wait_for_completion(&ctx->ref_comp); io_ring_ctx_free(ctx); } @@ -7992,7 +7986,7 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, * after we've killed the percpu ref. */ mutex_unlock(&ctx->uring_lock); - ret = wait_for_completion_interruptible(&ctx->completions[0]); + ret = wait_for_completion_interruptible(&ctx->ref_comp); mutex_lock(&ctx->uring_lock); if (ret) { percpu_ref_resurrect(&ctx->refs); @@ -8069,7 +8063,7 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, /* bring the ctx back to life */ percpu_ref_reinit(&ctx->refs); out: - reinit_completion(&ctx->completions[0]); + reinit_completion(&ctx->ref_comp); } return ret; } From 8a4e2779570fe29da33e99a1c9addc4205110819 Mon Sep 17 00:00:00 2001 From: Thomas Bogendoerfer Date: Thu, 14 May 2020 13:07:23 +0200 Subject: [PATCH 0481/1043] MIPS: Fix builds for VR41xx platforms Changing inclusion of Platform files, broke VR41xx platforms. Add Makefile to vr41xx directory and traverse subdirs from it. Fixes: 26bff9eb49201aeb ("MIPS: Only include the platformfile needed") Signed-off-by: Thomas Bogendoerfer --- arch/mips/vr41xx/Makefile | 5 +++++ arch/mips/vr41xx/Platform | 3 --- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 arch/mips/vr41xx/Makefile diff --git a/arch/mips/vr41xx/Makefile b/arch/mips/vr41xx/Makefile new file mode 100644 index 000000000000..765020d5ee4d --- /dev/null +++ b/arch/mips/vr41xx/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 +# +obj-$(CONFIG_MACH_VR41XX) += common/ +obj-$(CONFIG_CASIO_E55) += casio-e55/ +obj-$(CONFIG_IBM_WORKPAD) += ibm-workpad/ diff --git a/arch/mips/vr41xx/Platform b/arch/mips/vr41xx/Platform index b6c8d5c08ddb..3f593a3e5678 100644 --- a/arch/mips/vr41xx/Platform +++ b/arch/mips/vr41xx/Platform @@ -1,19 +1,16 @@ # # NEC VR4100 series based machines # -platform-$(CONFIG_MACH_VR41XX) += vr41xx/common/ cflags-$(CONFIG_MACH_VR41XX) += -I$(srctree)/arch/mips/include/asm/mach-vr41xx # # CASIO CASSIPEIA E-55/65 (VR4111) # -platform-$(CONFIG_CASIO_E55) += vr41xx/casio-e55/ load-$(CONFIG_CASIO_E55) += 0xffffffff80004000 # # IBM WorkPad z50 (VR4121) # -platform-$(CONFIG_IBM_WORKPAD) += vr41xx/ibm-workpad/ load-$(CONFIG_IBM_WORKPAD) += 0xffffffff80004000 # From 1fded93fb154a75d99823b0492dfb1688958d64a Mon Sep 17 00:00:00 2001 From: Thomas Bogendoerfer Date: Thu, 14 May 2020 16:31:22 +0200 Subject: [PATCH 0482/1043] MIPS: ralink: Don't include objects twice With the change of platform file inclusion object were included via platform-y and core-y. Remove the core-y part to fix it. Fixes: 26bff9eb49201aeb ("MIPS: Only include the platformfile needed") Signed-off-by: Thomas Bogendoerfer --- arch/mips/ralink/Platform | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/mips/ralink/Platform b/arch/mips/ralink/Platform index 6095fcc334f4..02ee0791481d 100644 --- a/arch/mips/ralink/Platform +++ b/arch/mips/ralink/Platform @@ -1,7 +1,6 @@ # # Ralink SoC common stuff # -core-$(CONFIG_RALINK) += arch/mips/ralink/ cflags-$(CONFIG_RALINK) += -I$(srctree)/arch/mips/include/asm/mach-ralink # From 83dd9a0b664397209a11849c47f9a59901af4da9 Mon Sep 17 00:00:00 2001 From: Joshua Kinard Date: Thu, 14 May 2020 23:24:22 -0400 Subject: [PATCH 0483/1043] MIPS: SGI-IP27: Use the _AC() macro in spaces.h The attached patch wraps several of the macros in IP27's spaces.h header file with the _AC() macro. This matches most of the other spaces.h files in the MIPS tree. Signed-off-by: Joshua Kinard Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mach-ip27/spaces.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/arch/mips/include/asm/mach-ip27/spaces.h b/arch/mips/include/asm/mach-ip27/spaces.h index 24d5e31bcfa6..66421e9a6aa6 100644 --- a/arch/mips/include/asm/mach-ip27/spaces.h +++ b/arch/mips/include/asm/mach-ip27/spaces.h @@ -10,17 +10,19 @@ #ifndef _ASM_MACH_IP27_SPACES_H #define _ASM_MACH_IP27_SPACES_H +#include + /* * IP27 uses the R10000's uncached attribute feature. Attribute 3 selects * uncached memory addressing. Hide the definitions on 32-bit compilation * of the compat-vdso code. */ #ifdef CONFIG_64BIT -#define HSPEC_BASE 0x9000000000000000 -#define IO_BASE 0x9200000000000000 -#define MSPEC_BASE 0x9400000000000000 -#define UNCAC_BASE 0x9600000000000000 -#define CAC_BASE 0xa800000000000000 +#define HSPEC_BASE _AC(0x9000000000000000, UL) +#define IO_BASE _AC(0x9200000000000000, UL) +#define MSPEC_BASE _AC(0x9400000000000000, UL) +#define UNCAC_BASE _AC(0x9600000000000000, UL) +#define CAC_BASE _AC(0xa800000000000000, UL) #endif #define TO_MSPEC(x) (MSPEC_BASE | ((x) & TO_PHYS_MASK)) From 24ce659dcc02c21f8d6c0a7589c3320a4dfa8152 Mon Sep 17 00:00:00 2001 From: Oleksij Rempel Date: Sun, 10 May 2020 08:03:23 +0200 Subject: [PATCH 0484/1043] MIPS: ath79: ar9331_dpt_module: update led nodes Fit led nodes to the latest naming schema. Signed-off-by: Oleksij Rempel Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/qca/ar9331_dpt_module.dts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/mips/boot/dts/qca/ar9331_dpt_module.dts b/arch/mips/boot/dts/qca/ar9331_dpt_module.dts index 0f2b20044834..7695d326df11 100644 --- a/arch/mips/boot/dts/qca/ar9331_dpt_module.dts +++ b/arch/mips/boot/dts/qca/ar9331_dpt_module.dts @@ -3,6 +3,7 @@ #include #include +#include #include "ar9331.dtsi" @@ -22,8 +23,9 @@ leds { compatible = "gpio-leds"; - system { - label = "dpt-module:green:system"; + led-0 { + function = LED_FUNCTION_STATUS; + color = ; gpios = <&gpio 27 GPIO_ACTIVE_LOW>; default-state = "off"; }; From e795688eeedfab9755727bf42e8b64c31a833ea0 Mon Sep 17 00:00:00 2001 From: Oleksij Rempel Date: Sun, 10 May 2020 08:03:24 +0200 Subject: [PATCH 0485/1043] MIPS: ath79: ar9331: rename uart to serial node schema violation was detected by the dtbs_check Signed-off-by: Oleksij Rempel Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/qca/ar9331.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/boot/dts/qca/ar9331.dtsi b/arch/mips/boot/dts/qca/ar9331.dtsi index 8f5aed760abb..83b3c0ce135a 100644 --- a/arch/mips/boot/dts/qca/ar9331.dtsi +++ b/arch/mips/boot/dts/qca/ar9331.dtsi @@ -59,7 +59,7 @@ #qca,ddr-wb-channel-cells = <1>; }; - uart: uart@18020000 { + uart: serial@18020000 { compatible = "qca,ar9330-uart"; reg = <0x18020000 0x14>; From 68cda40d9f3c4cb880108eb22f974d9e3d5dc6c5 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 11 May 2020 15:05:29 -0700 Subject: [PATCH 0486/1043] KVM: nVMX: Tweak handling of failure code for nested VM-Enter failure Use an enum for passing around the failure code for a failed VM-Enter that results in VM-Exit to provide a level of indirection from the final resting place of the failure code, vmcs.EXIT_QUALIFICATION. The exit qualification field is an unsigned long, e.g. passing around 'u32 exit_qual' throws up red flags as it suggests KVM may be dropping bits when reporting errors to L1. This is a red herring because the only defined failure codes are 0, 2, 3, and 4, i.e. don't come remotely close to overflowing a u32. Setting vmcs.EXIT_QUALIFICATION on entry failure is further complicated by the MSR load list, which returns the (1-based) entry that failed, and the number of MSRs to load is a 32-bit VMCS field. At first blush, it would appear that overflowing a u32 is possible, but the number of MSRs that can be loaded is hardcapped at 4096 (limited by MSR_IA32_VMX_MISC). In other words, there are two completely disparate types of data that eventually get stuffed into vmcs.EXIT_QUALIFICATION, neither of which is an 'unsigned long' in nature. This was presumably the reasoning for switching to 'u32' when the related code was refactored in commit ca0bde28f2ed6 ("kvm: nVMX: Split VMCS checks from nested_vmx_run()"). Using an enum for the failure code addresses the technically-possible- but-will-never-happen scenario where Intel defines a failure code that doesn't fit in a 32-bit integer. The enum variables and values will either be automatically sized (gcc 5.4 behavior) or be subjected to some combination of truncation. The former case will simply work, while the latter will trigger a compile-time warning unless the compiler is being particularly unhelpful. Separating the failure code from the failed MSR entry allows for disassociating both from vmcs.EXIT_QUALIFICATION, which avoids the conundrum where KVM has to choose between 'u32 exit_qual' and tracking values as 'unsigned long' that have no business being tracked as such. To cement the split, set vmcs12->exit_qualification directly from the entry error code or failed MSR index instead of bouncing through a local variable. Opportunistically rename the variables in load_vmcs12_host_state() and vmx_set_nested_state() to call out that they're ignored, set exit_reason on demand on nested VM-Enter failure, and add a comment in nested_vmx_load_msr() to call out that returning 'i + 1' can't wrap. No functional change intended. Reported-by: Vitaly Kuznetsov Cc: Jim Mattson Signed-off-by: Sean Christopherson Message-Id: <20200511220529.11402-1-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/vmx.h | 10 ++++---- arch/x86/kvm/vmx/nested.c | 47 +++++++++++++++++++++++--------------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 5e090d1f03f8..cd7de4b401fe 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -527,10 +527,12 @@ struct vmx_msr_entry { /* * Exit Qualifications for entry failure during or after loading guest state */ -#define ENTRY_FAIL_DEFAULT 0 -#define ENTRY_FAIL_PDPTE 2 -#define ENTRY_FAIL_NMI 3 -#define ENTRY_FAIL_VMCS_LINK_PTR 4 +enum vm_entry_failure_code { + ENTRY_FAIL_DEFAULT = 0, + ENTRY_FAIL_PDPTE = 2, + ENTRY_FAIL_NMI = 3, + ENTRY_FAIL_VMCS_LINK_PTR = 4, +}; /* * Exit Qualifications for EPT Violations diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 90ba278dd572..b2cd8d399644 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -922,6 +922,7 @@ static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count) } return 0; fail: + /* Note, max_msr_list_size is at most 4096, i.e. this can't wrap. */ return i + 1; } @@ -1117,7 +1118,7 @@ static bool nested_vmx_transition_mmu_sync(struct kvm_vcpu *vcpu) * @entry_failure_code. */ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept, - u32 *entry_failure_code) + enum vm_entry_failure_code *entry_failure_code) { if (CC(!nested_cr3_valid(vcpu, cr3))) { *entry_failure_code = ENTRY_FAIL_DEFAULT; @@ -2469,7 +2470,7 @@ static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) * is assigned to entry_failure_code on failure. */ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, - u32 *entry_failure_code) + enum vm_entry_failure_code *entry_failure_code) { struct vcpu_vmx *vmx = to_vmx(vcpu); struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; @@ -2929,11 +2930,11 @@ static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12) static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, - u32 *exit_qual) + enum vm_entry_failure_code *entry_failure_code) { bool ia32e; - *exit_qual = ENTRY_FAIL_DEFAULT; + *entry_failure_code = ENTRY_FAIL_DEFAULT; if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) || CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4))) @@ -2948,7 +2949,7 @@ static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu, return -EINVAL; if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) { - *exit_qual = ENTRY_FAIL_VMCS_LINK_PTR; + *entry_failure_code = ENTRY_FAIL_VMCS_LINK_PTR; return -EINVAL; } @@ -3240,9 +3241,9 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, { struct vcpu_vmx *vmx = to_vmx(vcpu); struct vmcs12 *vmcs12 = get_vmcs12(vcpu); + enum vm_entry_failure_code entry_failure_code; bool evaluate_pending_interrupts; - u32 exit_reason = EXIT_REASON_INVALID_STATE; - u32 exit_qual; + u32 exit_reason, failed_index; if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu)) kvm_vcpu_flush_tlb_current(vcpu); @@ -3290,24 +3291,33 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, return NVMX_VMENTRY_VMFAIL; } - if (nested_vmx_check_guest_state(vcpu, vmcs12, &exit_qual)) + if (nested_vmx_check_guest_state(vcpu, vmcs12, + &entry_failure_code)) { + exit_reason = EXIT_REASON_INVALID_STATE; + vmcs12->exit_qualification = entry_failure_code; goto vmentry_fail_vmexit; + } } enter_guest_mode(vcpu); if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) vcpu->arch.tsc_offset += vmcs12->tsc_offset; - if (prepare_vmcs02(vcpu, vmcs12, &exit_qual)) + if (prepare_vmcs02(vcpu, vmcs12, &entry_failure_code)) { + exit_reason = EXIT_REASON_INVALID_STATE; + vmcs12->exit_qualification = entry_failure_code; goto vmentry_fail_vmexit_guest_mode; + } if (from_vmentry) { - exit_reason = EXIT_REASON_MSR_LOAD_FAIL; - exit_qual = nested_vmx_load_msr(vcpu, - vmcs12->vm_entry_msr_load_addr, - vmcs12->vm_entry_msr_load_count); - if (exit_qual) + failed_index = nested_vmx_load_msr(vcpu, + vmcs12->vm_entry_msr_load_addr, + vmcs12->vm_entry_msr_load_count); + if (failed_index) { + exit_reason = EXIT_REASON_MSR_LOAD_FAIL; + vmcs12->exit_qualification = failed_index; goto vmentry_fail_vmexit_guest_mode; + } } else { /* * The MMU is not initialized to point at the right entities yet and @@ -3371,7 +3381,6 @@ vmentry_fail_vmexit: load_vmcs12_host_state(vcpu, vmcs12); vmcs12->vm_exit_reason = exit_reason | VMX_EXIT_REASONS_FAILED_VMENTRY; - vmcs12->exit_qualification = exit_qual; if (enable_shadow_vmcs || vmx->nested.hv_evmcs) vmx->nested.need_vmcs12_to_shadow_sync = true; return NVMX_VMENTRY_VMEXIT; @@ -4065,8 +4074,8 @@ static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { + enum vm_entry_failure_code ignored; struct kvm_segment seg; - u32 entry_failure_code; if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) vcpu->arch.efer = vmcs12->host_ia32_efer; @@ -4101,7 +4110,7 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, * Only PDPTE load can fail as the value of cr3 was checked on entry and * couldn't have changed. */ - if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code)) + if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &ignored)) nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); if (!enable_ept) @@ -6001,7 +6010,7 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu, { struct vcpu_vmx *vmx = to_vmx(vcpu); struct vmcs12 *vmcs12; - u32 exit_qual; + enum vm_entry_failure_code ignored; struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = &user_kvm_nested_state->data.vmx[0]; int ret; @@ -6142,7 +6151,7 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu, if (nested_vmx_check_controls(vcpu, vmcs12) || nested_vmx_check_host_state(vcpu, vmcs12) || - nested_vmx_check_guest_state(vcpu, vmcs12, &exit_qual)) + nested_vmx_check_guest_state(vcpu, vmcs12, &ignored)) goto error_guest_mode; vmx->nested.dirty_vmcs12 = true; From a71936ab46f1da1539d97a98dfb2f94ee383d687 Mon Sep 17 00:00:00 2001 From: Xiaoyao Li Date: Wed, 29 Apr 2020 23:43:12 +0800 Subject: [PATCH 0487/1043] kvm: x86: Cleanup vcpu->arch.guest_xstate_size vcpu->arch.guest_xstate_size lost its only user since commit df1daba7d1cb ("KVM: x86: support XSAVES usage in the host"), so clean it up. Signed-off-by: Xiaoyao Li Message-Id: <20200429154312.1411-1-xiaoyao.li@intel.com> Reviewed-by: Vitaly Kuznetsov Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 - arch/x86/kvm/cpuid.c | 4 +--- arch/x86/kvm/x86.c | 2 -- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 505cf19ace80..86295a01f5ca 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -657,7 +657,6 @@ struct kvm_vcpu_arch { u64 xcr0; u64 guest_supported_xcr0; - u32 guest_xstate_size; struct kvm_pio_request pio; void *pio_data; diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 44dfaefdad0e..cd708b0b460a 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -86,12 +86,10 @@ int kvm_update_cpuid(struct kvm_vcpu *vcpu) best = kvm_find_cpuid_entry(vcpu, 0xD, 0); if (!best) { vcpu->arch.guest_supported_xcr0 = 0; - vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET; } else { vcpu->arch.guest_supported_xcr0 = (best->eax | ((u64)best->edx << 32)) & supported_xcr0; - vcpu->arch.guest_xstate_size = best->ebx = - xstate_required_size(vcpu->arch.xcr0, false); + best->ebx = xstate_required_size(vcpu->arch.xcr0, false); } best = kvm_find_cpuid_entry(vcpu, 0xD, 1); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 3622586a26a7..6b958d6c9427 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -9387,8 +9387,6 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) } fx_init(vcpu); - vcpu->arch.guest_xstate_size = XSAVE_HDR_SIZE + XSAVE_HDR_OFFSET; - vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu); vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT; From b2f432f872d9b4eb07d35f7bd5aa68c48a756f1a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 27 Apr 2020 17:54:20 -0700 Subject: [PATCH 0488/1043] KVM: x86/mmu: Tweak PSE hugepage handling to avoid 2M vs 4M conundrum Change the PSE hugepage handling in walk_addr_generic() to fire on any page level greater than PT_PAGE_TABLE_LEVEL, a.k.a. PG_LEVEL_4K. PSE paging only has two levels, so "== 2" and "> 1" are functionally the same, i.e. this is a nop. A future patch will drop KVM's PT_*_LEVEL enums in favor of the kernel's PG_LEVEL_* enums, at which point "walker->level == PG_LEVEL_2M" is semantically incorrect (though still functionally ok). No functional change intended. Suggested-by: Paolo Bonzini Signed-off-by: Sean Christopherson Message-Id: <20200428005422.4235-2-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/paging_tmpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index efec7d27b8c5..ca39bd315f70 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -436,7 +436,7 @@ retry_walk: gfn = gpte_to_gfn_lvl(pte, walker->level); gfn += (addr & PT_LVL_OFFSET_MASK(walker->level)) >> PAGE_SHIFT; - if (PTTYPE == 32 && walker->level == PT_DIRECTORY_LEVEL && is_cpuid_PSE36()) + if (PTTYPE == 32 && walker->level > PT_PAGE_TABLE_LEVEL && is_cpuid_PSE36()) gfn += pse36_gfn_delta(pte); real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(gfn), access, &walker->fault); From e662ec3e0705cfee5b36aecd62adbc36df85eab3 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 27 Apr 2020 17:54:21 -0700 Subject: [PATCH 0489/1043] KVM: x86/mmu: Move max hugepage level to a separate #define Rename PT_MAX_HUGEPAGE_LEVEL to KVM_MAX_HUGEPAGE_LEVEL and make it a separate define in anticipation of dropping KVM's PT_*_LEVEL enums in favor of the kernel's PG_LEVEL_* enums. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20200428005422.4235-3-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 5 ++--- arch/x86/kvm/mmu/mmu.c | 17 +++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 86295a01f5ca..77a97a72bb3e 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -114,10 +114,9 @@ enum { PT_PAGE_TABLE_LEVEL = 1, PT_DIRECTORY_LEVEL = 2, PT_PDPE_LEVEL = 3, - /* set max level to the biggest one */ - PT_MAX_HUGEPAGE_LEVEL = PT_PDPE_LEVEL, }; -#define KVM_NR_PAGE_SIZES (PT_MAX_HUGEPAGE_LEVEL - \ +#define KVM_MAX_HUGEPAGE_LEVEL PT_PDPE_LEVEL +#define KVM_NR_PAGE_SIZES (KVM_MAX_HUGEPAGE_LEVEL - \ PT_PAGE_TABLE_LEVEL + 1) #define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9) #define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x)) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 10cb8db54cd0..7a712e903e93 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1199,7 +1199,7 @@ static void update_gfn_disallow_lpage_count(struct kvm_memory_slot *slot, struct kvm_lpage_info *linfo; int i; - for (i = PT_DIRECTORY_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) { + for (i = PT_DIRECTORY_LEVEL; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { linfo = lpage_info_slot(gfn, slot, i); linfo->disallow_lpage += count; WARN_ON(linfo->disallow_lpage < 0); @@ -1763,7 +1763,7 @@ bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, int i; bool write_protected = false; - for (i = PT_PAGE_TABLE_LEVEL; i <= PT_MAX_HUGEPAGE_LEVEL; ++i) { + for (i = PT_PAGE_TABLE_LEVEL; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { rmap_head = __gfn_to_rmap(gfn, i, slot); write_protected |= __rmap_write_protect(kvm, rmap_head, true); } @@ -1952,7 +1952,7 @@ static int kvm_handle_hva_range(struct kvm *kvm, gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot); for_each_slot_rmap_range(memslot, PT_PAGE_TABLE_LEVEL, - PT_MAX_HUGEPAGE_LEVEL, + KVM_MAX_HUGEPAGE_LEVEL, gfn_start, gfn_end - 1, &iterator) ret |= handler(kvm, iterator.rmap, memslot, @@ -4214,7 +4214,7 @@ int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, { int max_level; - for (max_level = PT_MAX_HUGEPAGE_LEVEL; + for (max_level = KVM_MAX_HUGEPAGE_LEVEL; max_level > PT_PAGE_TABLE_LEVEL; max_level--) { int page_num = KVM_PAGES_PER_HPAGE(max_level); @@ -5641,7 +5641,7 @@ slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot, slot_level_handler fn, bool lock_flush_tlb) { return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL, - PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); + KVM_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); } static __always_inline bool @@ -5649,7 +5649,7 @@ slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot, slot_level_handler fn, bool lock_flush_tlb) { return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL + 1, - PT_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); + KVM_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); } static __always_inline bool @@ -5867,7 +5867,8 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) continue; slot_handle_level_range(kvm, memslot, kvm_zap_rmapp, - PT_PAGE_TABLE_LEVEL, PT_MAX_HUGEPAGE_LEVEL, + PT_PAGE_TABLE_LEVEL, + KVM_MAX_HUGEPAGE_LEVEL, start, end - 1, true); } } @@ -5889,7 +5890,7 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, spin_lock(&kvm->mmu_lock); flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect, - start_level, PT_MAX_HUGEPAGE_LEVEL, false); + start_level, KVM_MAX_HUGEPAGE_LEVEL, false); spin_unlock(&kvm->mmu_lock); /* From 3bae0459bcd559506a2ca5807040ff722de5b136 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 27 Apr 2020 17:54:22 -0700 Subject: [PATCH 0490/1043] KVM: x86/mmu: Drop KVM's hugepage enums in favor of the kernel's enums Replace KVM's PT_PAGE_TABLE_LEVEL, PT_DIRECTORY_LEVEL and PT_PDPE_LEVEL with the kernel's PG_LEVEL_4K, PG_LEVEL_2M and PG_LEVEL_1G. KVM's enums are borderline impossible to remember and result in code that is visually difficult to audit, e.g. if (!enable_ept) ept_lpage_level = 0; else if (cpu_has_vmx_ept_1g_page()) ept_lpage_level = PT_PDPE_LEVEL; else if (cpu_has_vmx_ept_2m_page()) ept_lpage_level = PT_DIRECTORY_LEVEL; else ept_lpage_level = PT_PAGE_TABLE_LEVEL; versus if (!enable_ept) ept_lpage_level = 0; else if (cpu_has_vmx_ept_1g_page()) ept_lpage_level = PG_LEVEL_1G; else if (cpu_has_vmx_ept_2m_page()) ept_lpage_level = PG_LEVEL_2M; else ept_lpage_level = PG_LEVEL_4K; No functional change intended. Suggested-by: Barret Rhoden Signed-off-by: Sean Christopherson Message-Id: <20200428005422.4235-4-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 12 +--- arch/x86/kvm/mmu/mmu.c | 107 +++++++++++++++----------------- arch/x86/kvm/mmu/page_track.c | 4 +- arch/x86/kvm/mmu/paging_tmpl.h | 18 +++--- arch/x86/kvm/mmu_audit.c | 6 +- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/vmx/vmx.c | 6 +- arch/x86/kvm/x86.c | 4 +- 8 files changed, 73 insertions(+), 86 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 77a97a72bb3e..b8d035694b87 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -110,14 +110,8 @@ #define UNMAPPED_GVA (~(gpa_t)0) /* KVM Hugepage definitions for x86 */ -enum { - PT_PAGE_TABLE_LEVEL = 1, - PT_DIRECTORY_LEVEL = 2, - PT_PDPE_LEVEL = 3, -}; -#define KVM_MAX_HUGEPAGE_LEVEL PT_PDPE_LEVEL -#define KVM_NR_PAGE_SIZES (KVM_MAX_HUGEPAGE_LEVEL - \ - PT_PAGE_TABLE_LEVEL + 1) +#define KVM_MAX_HUGEPAGE_LEVEL PG_LEVEL_1G +#define KVM_NR_PAGE_SIZES (KVM_MAX_HUGEPAGE_LEVEL - PG_LEVEL_4K + 1) #define KVM_HPAGE_GFN_SHIFT(x) (((x) - 1) * 9) #define KVM_HPAGE_SHIFT(x) (PAGE_SHIFT + KVM_HPAGE_GFN_SHIFT(x)) #define KVM_HPAGE_SIZE(x) (1UL << KVM_HPAGE_SHIFT(x)) @@ -126,7 +120,7 @@ enum { static inline gfn_t gfn_to_index(gfn_t gfn, gfn_t base_gfn, int level) { - /* KVM_HPAGE_GFN_SHIFT(PT_PAGE_TABLE_LEVEL) must be 0. */ + /* KVM_HPAGE_GFN_SHIFT(PG_LEVEL_4K) must be 0. */ return (gfn >> KVM_HPAGE_GFN_SHIFT(level)) - (base_gfn >> KVM_HPAGE_GFN_SHIFT(level)); } diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 7a712e903e93..fd94668c0f0f 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -623,7 +623,7 @@ static int is_large_pte(u64 pte) static int is_last_spte(u64 pte, int level) { - if (level == PT_PAGE_TABLE_LEVEL) + if (level == PG_LEVEL_4K) return 1; if (is_large_pte(pte)) return 1; @@ -1199,7 +1199,7 @@ static void update_gfn_disallow_lpage_count(struct kvm_memory_slot *slot, struct kvm_lpage_info *linfo; int i; - for (i = PT_DIRECTORY_LEVEL; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { + for (i = PG_LEVEL_2M; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { linfo = lpage_info_slot(gfn, slot, i); linfo->disallow_lpage += count; WARN_ON(linfo->disallow_lpage < 0); @@ -1228,7 +1228,7 @@ static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp) slot = __gfn_to_memslot(slots, gfn); /* the non-leaf shadow pages are keeping readonly. */ - if (sp->role.level > PT_PAGE_TABLE_LEVEL) + if (sp->role.level > PG_LEVEL_4K) return kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); @@ -1256,7 +1256,7 @@ static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp) gfn = sp->gfn; slots = kvm_memslots_for_spte_role(kvm, sp->role); slot = __gfn_to_memslot(slots, gfn); - if (sp->role.level > PT_PAGE_TABLE_LEVEL) + if (sp->role.level > PG_LEVEL_4K) return kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE); @@ -1401,7 +1401,7 @@ static struct kvm_rmap_head *__gfn_to_rmap(gfn_t gfn, int level, unsigned long idx; idx = gfn_to_index(gfn, slot->base_gfn, level); - return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx]; + return &slot->arch.rmap[level - PG_LEVEL_4K][idx]; } static struct kvm_rmap_head *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, @@ -1532,8 +1532,7 @@ static void drop_spte(struct kvm *kvm, u64 *sptep) static bool __drop_large_spte(struct kvm *kvm, u64 *sptep) { if (is_large_pte(*sptep)) { - WARN_ON(page_header(__pa(sptep))->role.level == - PT_PAGE_TABLE_LEVEL); + WARN_ON(page_header(__pa(sptep))->role.level == PG_LEVEL_4K); drop_spte(kvm, sptep); --kvm->stat.lpages; return true; @@ -1685,7 +1684,7 @@ static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, while (mask) { rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask), - PT_PAGE_TABLE_LEVEL, slot); + PG_LEVEL_4K, slot); __rmap_write_protect(kvm, rmap_head, false); /* clear the first set bit */ @@ -1711,7 +1710,7 @@ void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, while (mask) { rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask), - PT_PAGE_TABLE_LEVEL, slot); + PG_LEVEL_4K, slot); __rmap_clear_dirty(kvm, rmap_head); /* clear the first set bit */ @@ -1763,7 +1762,7 @@ bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, int i; bool write_protected = false; - for (i = PT_PAGE_TABLE_LEVEL; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { + for (i = PG_LEVEL_4K; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { rmap_head = __gfn_to_rmap(gfn, i, slot); write_protected |= __rmap_write_protect(kvm, rmap_head, true); } @@ -1951,7 +1950,7 @@ static int kvm_handle_hva_range(struct kvm *kvm, gfn_start = hva_to_gfn_memslot(hva_start, memslot); gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot); - for_each_slot_rmap_range(memslot, PT_PAGE_TABLE_LEVEL, + for_each_slot_rmap_range(memslot, PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL, gfn_start, gfn_end - 1, &iterator) @@ -2346,7 +2345,7 @@ static bool kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, if (!s->unsync) continue; - WARN_ON(s->role.level != PT_PAGE_TABLE_LEVEL); + WARN_ON(s->role.level != PG_LEVEL_4K); ret |= kvm_sync_page(vcpu, s, invalid_list); } @@ -2375,7 +2374,7 @@ static int mmu_pages_next(struct kvm_mmu_pages *pvec, int level = sp->role.level; parents->idx[level-1] = idx; - if (level == PT_PAGE_TABLE_LEVEL) + if (level == PG_LEVEL_4K) break; parents->parent[level-2] = sp; @@ -2397,7 +2396,7 @@ static int mmu_pages_first(struct kvm_mmu_pages *pvec, sp = pvec->page[0].sp; level = sp->role.level; - WARN_ON(level == PT_PAGE_TABLE_LEVEL); + WARN_ON(level == PG_LEVEL_4K); parents->parent[level-2] = sp; @@ -2545,11 +2544,10 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, * be inconsistent with guest page table. */ account_shadowed(vcpu->kvm, sp); - if (level == PT_PAGE_TABLE_LEVEL && - rmap_write_protect(vcpu, gfn)) + if (level == PG_LEVEL_4K && rmap_write_protect(vcpu, gfn)) kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 1); - if (level > PT_PAGE_TABLE_LEVEL && need_sync) + if (level > PG_LEVEL_4K && need_sync) flush |= kvm_sync_pages(vcpu, gfn, &invalid_list); } clear_page(sp->spt); @@ -2600,7 +2598,7 @@ static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator, static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator) { - if (iterator->level < PT_PAGE_TABLE_LEVEL) + if (iterator->level < PG_LEVEL_4K) return false; iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level); @@ -2721,7 +2719,7 @@ static int mmu_zap_unsync_children(struct kvm *kvm, struct mmu_page_path parents; struct kvm_mmu_pages pages; - if (parent->role.level == PT_PAGE_TABLE_LEVEL) + if (parent->role.level == PG_LEVEL_4K) return 0; while (mmu_unsync_walk(parent, &pages)) { @@ -2920,7 +2918,7 @@ static bool mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn, if (sp->unsync) continue; - WARN_ON(sp->role.level != PT_PAGE_TABLE_LEVEL); + WARN_ON(sp->role.level != PG_LEVEL_4K); kvm_unsync_page(vcpu, sp); } @@ -3019,7 +3017,7 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, if (!speculative) spte |= spte_shadow_accessed_mask(spte); - if (level > PT_PAGE_TABLE_LEVEL && (pte_access & ACC_EXEC_MASK) && + if (level > PG_LEVEL_4K && (pte_access & ACC_EXEC_MASK) && is_nx_huge_page_enabled()) { pte_access &= ~ACC_EXEC_MASK; } @@ -3032,7 +3030,7 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, if (pte_access & ACC_USER_MASK) spte |= shadow_user_mask; - if (level > PT_PAGE_TABLE_LEVEL) + if (level > PG_LEVEL_4K) spte |= PT_PAGE_SIZE_MASK; if (tdp_enabled) spte |= kvm_x86_ops.get_mt_mask(vcpu, gfn, @@ -3102,8 +3100,7 @@ static int mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep, * If we overwrite a PTE page pointer with a 2MB PMD, unlink * the parent of the now unreachable PTE. */ - if (level > PT_PAGE_TABLE_LEVEL && - !is_large_pte(*sptep)) { + if (level > PG_LEVEL_4K && !is_large_pte(*sptep)) { struct kvm_mmu_page *child; u64 pte = *sptep; @@ -3227,7 +3224,7 @@ static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep) if (sp_ad_disabled(sp)) return; - if (sp->role.level > PT_PAGE_TABLE_LEVEL) + if (sp->role.level > PG_LEVEL_4K) return; __direct_pte_prefetch(vcpu, sp, sptep); @@ -3240,12 +3237,8 @@ static int host_pfn_mapping_level(struct kvm_vcpu *vcpu, gfn_t gfn, pte_t *pte; int level; - BUILD_BUG_ON(PT_PAGE_TABLE_LEVEL != (int)PG_LEVEL_4K || - PT_DIRECTORY_LEVEL != (int)PG_LEVEL_2M || - PT_PDPE_LEVEL != (int)PG_LEVEL_1G); - if (!PageCompound(pfn_to_page(pfn)) && !kvm_is_zone_device_pfn(pfn)) - return PT_PAGE_TABLE_LEVEL; + return PG_LEVEL_4K; /* * Note, using the already-retrieved memslot and __gfn_to_hva_memslot() @@ -3259,7 +3252,7 @@ static int host_pfn_mapping_level(struct kvm_vcpu *vcpu, gfn_t gfn, pte = lookup_address_in_mm(vcpu->kvm->mm, hva, &level); if (unlikely(!pte)) - return PT_PAGE_TABLE_LEVEL; + return PG_LEVEL_4K; return level; } @@ -3273,28 +3266,28 @@ static int kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t mask; int level; - if (unlikely(max_level == PT_PAGE_TABLE_LEVEL)) - return PT_PAGE_TABLE_LEVEL; + if (unlikely(max_level == PG_LEVEL_4K)) + return PG_LEVEL_4K; if (is_error_noslot_pfn(pfn) || kvm_is_reserved_pfn(pfn)) - return PT_PAGE_TABLE_LEVEL; + return PG_LEVEL_4K; slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, true); if (!slot) - return PT_PAGE_TABLE_LEVEL; + return PG_LEVEL_4K; max_level = min(max_level, max_page_level); - for ( ; max_level > PT_PAGE_TABLE_LEVEL; max_level--) { + for ( ; max_level > PG_LEVEL_4K; max_level--) { linfo = lpage_info_slot(gfn, slot, max_level); if (!linfo->disallow_lpage) break; } - if (max_level == PT_PAGE_TABLE_LEVEL) - return PT_PAGE_TABLE_LEVEL; + if (max_level == PG_LEVEL_4K) + return PG_LEVEL_4K; level = host_pfn_mapping_level(vcpu, gfn, pfn, slot); - if (level == PT_PAGE_TABLE_LEVEL) + if (level == PG_LEVEL_4K) return level; level = min(level, max_level); @@ -3316,7 +3309,7 @@ static void disallowed_hugepage_adjust(struct kvm_shadow_walk_iterator it, int level = *levelp; u64 spte = *it.sptep; - if (it.level == level && level > PT_PAGE_TABLE_LEVEL && + if (it.level == level && level > PG_LEVEL_4K && is_nx_huge_page_enabled() && is_shadow_present_pte(spte) && !is_large_pte(spte)) { @@ -3573,7 +3566,7 @@ static bool fast_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, * * See the comments in kvm_arch_commit_memory_region(). */ - if (sp->role.level > PT_PAGE_TABLE_LEVEL) + if (sp->role.level > PG_LEVEL_4K) break; } @@ -4132,7 +4125,7 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, return r; if (lpage_disallowed) - max_level = PT_PAGE_TABLE_LEVEL; + max_level = PG_LEVEL_4K; if (fast_page_fault(vcpu, gpa, error_code)) return RET_PF_RETRY; @@ -4168,7 +4161,7 @@ static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, /* This path builds a PAE pagetable, we can map 2mb pages at maximum. */ return direct_page_fault(vcpu, gpa & PAGE_MASK, error_code, prefault, - PT_DIRECTORY_LEVEL, false); + PG_LEVEL_2M, false); } int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, @@ -4215,7 +4208,7 @@ int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, int max_level; for (max_level = KVM_MAX_HUGEPAGE_LEVEL; - max_level > PT_PAGE_TABLE_LEVEL; + max_level > PG_LEVEL_4K; max_level--) { int page_num = KVM_PAGES_PER_HPAGE(max_level); gfn_t base = (gpa >> PAGE_SHIFT) & ~(page_num - 1); @@ -4376,11 +4369,11 @@ static inline bool is_last_gpte(struct kvm_mmu *mmu, gpte &= level - mmu->last_nonleaf_level; /* - * PT_PAGE_TABLE_LEVEL always terminates. The RHS has bit 7 set - * iff level <= PT_PAGE_TABLE_LEVEL, which for our purpose means - * level == PT_PAGE_TABLE_LEVEL; set PT_PAGE_SIZE_MASK in gpte then. + * PG_LEVEL_4K always terminates. The RHS has bit 7 set + * iff level <= PG_LEVEL_4K, which for our purpose means + * level == PG_LEVEL_4K; set PT_PAGE_SIZE_MASK in gpte then. */ - gpte |= level - PT_PAGE_TABLE_LEVEL - 1; + gpte |= level - PG_LEVEL_4K - 1; return gpte & PT_PAGE_SIZE_MASK; } @@ -5193,7 +5186,7 @@ static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, u64 *spte, const void *new) { - if (sp->role.level != PT_PAGE_TABLE_LEVEL) { + if (sp->role.level != PG_LEVEL_4K) { ++vcpu->kvm->stat.mmu_pde_zapped; return; } @@ -5251,7 +5244,7 @@ static bool detect_write_flooding(struct kvm_mmu_page *sp) * Skip write-flooding detected for the sp whose level is 1, because * it can become unsync, then the guest page is not write-protected. */ - if (sp->role.level == PT_PAGE_TABLE_LEVEL) + if (sp->role.level == PG_LEVEL_4K) return false; atomic_inc(&sp->write_flooding_count); @@ -5582,9 +5575,9 @@ void kvm_configure_mmu(bool enable_tdp, int tdp_page_level) if (tdp_enabled) max_page_level = tdp_page_level; else if (boot_cpu_has(X86_FEATURE_GBPAGES)) - max_page_level = PT_PDPE_LEVEL; + max_page_level = PG_LEVEL_1G; else - max_page_level = PT_DIRECTORY_LEVEL; + max_page_level = PG_LEVEL_2M; } EXPORT_SYMBOL_GPL(kvm_configure_mmu); @@ -5640,7 +5633,7 @@ static __always_inline bool slot_handle_all_level(struct kvm *kvm, struct kvm_memory_slot *memslot, slot_level_handler fn, bool lock_flush_tlb) { - return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL, + return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); } @@ -5648,7 +5641,7 @@ static __always_inline bool slot_handle_large_level(struct kvm *kvm, struct kvm_memory_slot *memslot, slot_level_handler fn, bool lock_flush_tlb) { - return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL + 1, + return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K + 1, KVM_MAX_HUGEPAGE_LEVEL, lock_flush_tlb); } @@ -5656,8 +5649,8 @@ static __always_inline bool slot_handle_leaf(struct kvm *kvm, struct kvm_memory_slot *memslot, slot_level_handler fn, bool lock_flush_tlb) { - return slot_handle_level(kvm, memslot, fn, PT_PAGE_TABLE_LEVEL, - PT_PAGE_TABLE_LEVEL, lock_flush_tlb); + return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K, + PG_LEVEL_4K, lock_flush_tlb); } static void free_mmu_pages(struct kvm_mmu *mmu) @@ -5867,7 +5860,7 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) continue; slot_handle_level_range(kvm, memslot, kvm_zap_rmapp, - PT_PAGE_TABLE_LEVEL, + PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL, start, end - 1, true); } diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index ddc1ec3bdacd..a7bcde34d1f2 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -61,7 +61,7 @@ static void update_gfn_track(struct kvm_memory_slot *slot, gfn_t gfn, { int index, val; - index = gfn_to_index(gfn, slot->base_gfn, PT_PAGE_TABLE_LEVEL); + index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); val = slot->arch.gfn_track[mode][index]; @@ -151,7 +151,7 @@ bool kvm_page_track_is_active(struct kvm_vcpu *vcpu, gfn_t gfn, if (!slot) return false; - index = gfn_to_index(gfn, slot->base_gfn, PT_PAGE_TABLE_LEVEL); + index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); return !!READ_ONCE(slot->arch.gfn_track[mode][index]); } diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index ca39bd315f70..38c576495048 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -75,7 +75,7 @@ #define PT_GUEST_ACCESSED_MASK (1 << PT_GUEST_ACCESSED_SHIFT) #define gpte_to_gfn_lvl FNAME(gpte_to_gfn_lvl) -#define gpte_to_gfn(pte) gpte_to_gfn_lvl((pte), PT_PAGE_TABLE_LEVEL) +#define gpte_to_gfn(pte) gpte_to_gfn_lvl((pte), PG_LEVEL_4K) /* * The guest_walker structure emulates the behavior of the hardware page @@ -198,7 +198,7 @@ static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu, !(gpte & PT_GUEST_ACCESSED_MASK)) goto no_present; - if (FNAME(is_rsvd_bits_set)(vcpu->arch.mmu, gpte, PT_PAGE_TABLE_LEVEL)) + if (FNAME(is_rsvd_bits_set)(vcpu->arch.mmu, gpte, PG_LEVEL_4K)) goto no_present; return false; @@ -436,7 +436,7 @@ retry_walk: gfn = gpte_to_gfn_lvl(pte, walker->level); gfn += (addr & PT_LVL_OFFSET_MASK(walker->level)) >> PAGE_SHIFT; - if (PTTYPE == 32 && walker->level > PT_PAGE_TABLE_LEVEL && is_cpuid_PSE36()) + if (PTTYPE == 32 && walker->level > PG_LEVEL_4K && is_cpuid_PSE36()) gfn += pse36_gfn_delta(pte); real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(gfn), access, &walker->fault); @@ -552,7 +552,7 @@ FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, * we call mmu_set_spte() with host_writable = true because * pte_prefetch_gfn_to_pfn always gets a writable pfn. */ - mmu_set_spte(vcpu, spte, pte_access, 0, PT_PAGE_TABLE_LEVEL, gfn, pfn, + mmu_set_spte(vcpu, spte, pte_access, 0, PG_LEVEL_4K, gfn, pfn, true, true); kvm_release_pfn_clean(pfn); @@ -575,7 +575,7 @@ static bool FNAME(gpte_changed)(struct kvm_vcpu *vcpu, u64 mask; int r, index; - if (level == PT_PAGE_TABLE_LEVEL) { + if (level == PG_LEVEL_4K) { mask = PTE_PREFETCH_NUM * sizeof(pt_element_t) - 1; base_gpa = pte_gpa & ~mask; index = (pte_gpa - base_gpa) / sizeof(pt_element_t); @@ -600,7 +600,7 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw, sp = page_header(__pa(sptep)); - if (sp->role.level > PT_PAGE_TABLE_LEVEL) + if (sp->role.level > PG_LEVEL_4K) return; if (sp->role.direct) @@ -828,7 +828,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, &walker, user_fault, &vcpu->arch.write_fault_to_shadow_pgtable); if (lpage_disallowed || is_self_change_mapping) - max_level = PT_PAGE_TABLE_LEVEL; + max_level = PG_LEVEL_4K; else max_level = walker.level; @@ -884,7 +884,7 @@ static gpa_t FNAME(get_level1_sp_gpa)(struct kvm_mmu_page *sp) { int offset = 0; - WARN_ON(sp->role.level != PT_PAGE_TABLE_LEVEL); + WARN_ON(sp->role.level != PG_LEVEL_4K); if (PTTYPE == 32) offset = sp->role.quadrant << PT64_LEVEL_BITS; @@ -1070,7 +1070,7 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) host_writable = sp->spt[i] & SPTE_HOST_WRITEABLE; set_spte_ret |= set_spte(vcpu, &sp->spt[i], - pte_access, PT_PAGE_TABLE_LEVEL, + pte_access, PG_LEVEL_4K, gfn, spte_to_pfn(sp->spt[i]), true, false, host_writable); } diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c index ca39f62aabc6..9d2844f87f6d 100644 --- a/arch/x86/kvm/mmu_audit.c +++ b/arch/x86/kvm/mmu_audit.c @@ -100,7 +100,7 @@ static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level) sp = page_header(__pa(sptep)); if (sp->unsync) { - if (level != PT_PAGE_TABLE_LEVEL) { + if (level != PG_LEVEL_4K) { audit_printk(vcpu->kvm, "unsync sp: %p " "level = %d\n", sp, level); return; @@ -176,7 +176,7 @@ static void check_mappings_rmap(struct kvm *kvm, struct kvm_mmu_page *sp) { int i; - if (sp->role.level != PT_PAGE_TABLE_LEVEL) + if (sp->role.level != PG_LEVEL_4K) return; for (i = 0; i < PT64_ENT_PER_PAGE; ++i) { @@ -200,7 +200,7 @@ static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp) slots = kvm_memslots_for_spte_role(kvm, sp->role); slot = __gfn_to_memslot(slots, sp->gfn); - rmap_head = __gfn_to_rmap(sp->gfn, PT_PAGE_TABLE_LEVEL, slot); + rmap_head = __gfn_to_rmap(sp->gfn, PG_LEVEL_4K, slot); for_each_rmap_spte(rmap_head, &iter, sptep) { if (is_writable_pte(*sptep)) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 9e24b5b6fae1..4c808cc059f1 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -888,7 +888,7 @@ static __init int svm_hardware_setup(void) if (npt_enabled && !npt) npt_enabled = false; - kvm_configure_mmu(npt_enabled, PT_PDPE_LEVEL); + kvm_configure_mmu(npt_enabled, PG_LEVEL_1G); pr_info("kvm: Nested Paging %sabled\n", npt_enabled ? "en" : "dis"); if (nrips) { diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 93b2a708b1da..30faae51573d 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8011,11 +8011,11 @@ static __init int hardware_setup(void) if (!enable_ept) ept_lpage_level = 0; else if (cpu_has_vmx_ept_1g_page()) - ept_lpage_level = PT_PDPE_LEVEL; + ept_lpage_level = PG_LEVEL_1G; else if (cpu_has_vmx_ept_2m_page()) - ept_lpage_level = PT_DIRECTORY_LEVEL; + ept_lpage_level = PG_LEVEL_2M; else - ept_lpage_level = PT_PAGE_TABLE_LEVEL; + ept_lpage_level = PG_LEVEL_4K; kvm_configure_mmu(enable_ept, ept_lpage_level); /* diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 6b958d6c9427..a52ef81fb87a 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10046,7 +10046,7 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, { /* Still write protect RO slot */ if (new->flags & KVM_MEM_READONLY) { - kvm_mmu_slot_remove_write_access(kvm, new, PT_PAGE_TABLE_LEVEL); + kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_4K); return; } @@ -10086,7 +10086,7 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, } else { int level = kvm_dirty_log_manual_protect_and_init_set(kvm) ? - PT_DIRECTORY_LEVEL : PT_PAGE_TABLE_LEVEL; + PG_LEVEL_2M : PG_LEVEL_4K; /* * If we're with initial-all-set, we don't need From 8123f265248c85603d55f4e97c68576f45eb1e4d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 27 Apr 2020 19:37:14 -0700 Subject: [PATCH 0491/1043] KVM: x86/mmu: Add a helper to consolidate root sp allocation Add a helper, mmu_alloc_root(), to consolidate the allocation of a root shadow page, which has the same basic mechanics for all flavors of TDP and shadow paging. Note, __pa(sp->spt) doesn't need to be protected by mmu_lock, sp->spt points at a kernel page. No functional change intended. Signed-off-by: Sean Christopherson Message-Id: <20200428023714.31923-1-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 88 +++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 49 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index fd94668c0f0f..907625fea7b3 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -3678,37 +3678,43 @@ static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn) return ret; } -static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu) +static hpa_t mmu_alloc_root(struct kvm_vcpu *vcpu, gfn_t gfn, gva_t gva, + u8 level, bool direct) { struct kvm_mmu_page *sp; + + spin_lock(&vcpu->kvm->mmu_lock); + + if (make_mmu_pages_available(vcpu)) { + spin_unlock(&vcpu->kvm->mmu_lock); + return INVALID_PAGE; + } + sp = kvm_mmu_get_page(vcpu, gfn, gva, level, direct, ACC_ALL); + ++sp->root_count; + + spin_unlock(&vcpu->kvm->mmu_lock); + return __pa(sp->spt); +} + +static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu) +{ + u8 shadow_root_level = vcpu->arch.mmu->shadow_root_level; + hpa_t root; unsigned i; - if (vcpu->arch.mmu->shadow_root_level >= PT64_ROOT_4LEVEL) { - spin_lock(&vcpu->kvm->mmu_lock); - if(make_mmu_pages_available(vcpu) < 0) { - spin_unlock(&vcpu->kvm->mmu_lock); + if (shadow_root_level >= PT64_ROOT_4LEVEL) { + root = mmu_alloc_root(vcpu, 0, 0, shadow_root_level, true); + if (!VALID_PAGE(root)) return -ENOSPC; - } - sp = kvm_mmu_get_page(vcpu, 0, 0, - vcpu->arch.mmu->shadow_root_level, 1, ACC_ALL); - ++sp->root_count; - spin_unlock(&vcpu->kvm->mmu_lock); - vcpu->arch.mmu->root_hpa = __pa(sp->spt); - } else if (vcpu->arch.mmu->shadow_root_level == PT32E_ROOT_LEVEL) { + vcpu->arch.mmu->root_hpa = root; + } else if (shadow_root_level == PT32E_ROOT_LEVEL) { for (i = 0; i < 4; ++i) { - hpa_t root = vcpu->arch.mmu->pae_root[i]; + MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu->pae_root[i])); - MMU_WARN_ON(VALID_PAGE(root)); - spin_lock(&vcpu->kvm->mmu_lock); - if (make_mmu_pages_available(vcpu) < 0) { - spin_unlock(&vcpu->kvm->mmu_lock); + root = mmu_alloc_root(vcpu, i << (30 - PAGE_SHIFT), + i << 30, PT32_ROOT_LEVEL, true); + if (!VALID_PAGE(root)) return -ENOSPC; - } - sp = kvm_mmu_get_page(vcpu, i << (30 - PAGE_SHIFT), - i << 30, PT32_ROOT_LEVEL, 1, ACC_ALL); - root = __pa(sp->spt); - ++sp->root_count; - spin_unlock(&vcpu->kvm->mmu_lock); vcpu->arch.mmu->pae_root[i] = root | PT_PRESENT_MASK; } vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->pae_root); @@ -3723,9 +3729,9 @@ static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu) static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) { - struct kvm_mmu_page *sp; u64 pdptr, pm_mask; gfn_t root_gfn, root_pgd; + hpa_t root; int i; root_pgd = vcpu->arch.mmu->get_guest_pgd(vcpu); @@ -3739,20 +3745,12 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) * write-protect the guests page table root. */ if (vcpu->arch.mmu->root_level >= PT64_ROOT_4LEVEL) { - hpa_t root = vcpu->arch.mmu->root_hpa; + MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu->root_hpa)); - MMU_WARN_ON(VALID_PAGE(root)); - - spin_lock(&vcpu->kvm->mmu_lock); - if (make_mmu_pages_available(vcpu) < 0) { - spin_unlock(&vcpu->kvm->mmu_lock); + root = mmu_alloc_root(vcpu, root_gfn, 0, + vcpu->arch.mmu->shadow_root_level, false); + if (!VALID_PAGE(root)) return -ENOSPC; - } - sp = kvm_mmu_get_page(vcpu, root_gfn, 0, - vcpu->arch.mmu->shadow_root_level, 0, ACC_ALL); - root = __pa(sp->spt); - ++sp->root_count; - spin_unlock(&vcpu->kvm->mmu_lock); vcpu->arch.mmu->root_hpa = root; goto set_root_pgd; } @@ -3767,9 +3765,7 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK; for (i = 0; i < 4; ++i) { - hpa_t root = vcpu->arch.mmu->pae_root[i]; - - MMU_WARN_ON(VALID_PAGE(root)); + MMU_WARN_ON(VALID_PAGE(vcpu->arch.mmu->pae_root[i])); if (vcpu->arch.mmu->root_level == PT32E_ROOT_LEVEL) { pdptr = vcpu->arch.mmu->get_pdptr(vcpu, i); if (!(pdptr & PT_PRESENT_MASK)) { @@ -3780,17 +3776,11 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) if (mmu_check_root(vcpu, root_gfn)) return 1; } - spin_lock(&vcpu->kvm->mmu_lock); - if (make_mmu_pages_available(vcpu) < 0) { - spin_unlock(&vcpu->kvm->mmu_lock); - return -ENOSPC; - } - sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, PT32_ROOT_LEVEL, - 0, ACC_ALL); - root = __pa(sp->spt); - ++sp->root_count; - spin_unlock(&vcpu->kvm->mmu_lock); + root = mmu_alloc_root(vcpu, root_gfn, i << 30, + PT32_ROOT_LEVEL, false); + if (!VALID_PAGE(root)) + return -ENOSPC; vcpu->arch.mmu->pae_root[i] = root | pm_mask; } vcpu->arch.mmu->root_hpa = __pa(vcpu->arch.mmu->pae_root); From c16312f4fa75e786f5748a7b721a05e6b0d761fe Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Mon, 27 Apr 2020 22:50:35 +0200 Subject: [PATCH 0492/1043] KVM: VMX: Remove unneeded __ASM_SIZE usage with POP instruction POP [mem] defaults to the word size, and the only legal non-default size is 16 bits, e.g. a 32-bit POP will #UD in 64-bit mode and vice versa, no need to use __ASM_SIZE macro to force operating mode. Changes since v1: - Fix commit message. Cc: Paolo Bonzini Cc: Sean Christopherson Reviewed-by: Sean Christopherson Signed-off-by: Uros Bizjak Message-Id: <20200427205035.1594232-1-ubizjak@gmail.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmenter.S | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S index 51d1a82742fd..e0a182cb3cdd 100644 --- a/arch/x86/kvm/vmx/vmenter.S +++ b/arch/x86/kvm/vmx/vmenter.S @@ -166,13 +166,13 @@ SYM_FUNC_START(__vmx_vcpu_run) mov WORD_SIZE(%_ASM_SP), %_ASM_AX /* Save all guest registers, including RAX from the stack */ - __ASM_SIZE(pop) VCPU_RAX(%_ASM_AX) - mov %_ASM_CX, VCPU_RCX(%_ASM_AX) - mov %_ASM_DX, VCPU_RDX(%_ASM_AX) - mov %_ASM_BX, VCPU_RBX(%_ASM_AX) - mov %_ASM_BP, VCPU_RBP(%_ASM_AX) - mov %_ASM_SI, VCPU_RSI(%_ASM_AX) - mov %_ASM_DI, VCPU_RDI(%_ASM_AX) + pop VCPU_RAX(%_ASM_AX) + mov %_ASM_CX, VCPU_RCX(%_ASM_AX) + mov %_ASM_DX, VCPU_RDX(%_ASM_AX) + mov %_ASM_BX, VCPU_RBX(%_ASM_AX) + mov %_ASM_BP, VCPU_RBP(%_ASM_AX) + mov %_ASM_SI, VCPU_RSI(%_ASM_AX) + mov %_ASM_DI, VCPU_RDI(%_ASM_AX) #ifdef CONFIG_X86_64 mov %r8, VCPU_R8 (%_ASM_AX) mov %r9, VCPU_R9 (%_ASM_AX) From dd03bcaad0b1a62c8ea6297e6f2a5993c1c5cd30 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Thu, 16 Apr 2020 11:58:59 -0400 Subject: [PATCH 0493/1043] KVM: X86: Force ASYNC_PF_PER_VCPU to be power of two Forcing the ASYNC_PF_PER_VCPU to be power of two is much easier to be used rather than calling roundup_pow_of_two() from time to time. Do this by adding a BUILD_BUG_ON() inside the hash function. Another point is that generally async pf does not allow concurrency over ASYNC_PF_PER_VCPU after all (see kvm_setup_async_pf()), so it does not make much sense either to have it not a power of two or some of the entries will definitely be wasted. Signed-off-by: Peter Xu Message-Id: <20200416155859.267366-1-peterx@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/x86.c | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index b8d035694b87..04f44961858b 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -763,7 +763,7 @@ struct kvm_vcpu_arch { struct { bool halted; - gfn_t gfns[roundup_pow_of_two(ASYNC_PF_PER_VCPU)]; + gfn_t gfns[ASYNC_PF_PER_VCPU]; struct gfn_to_hva_cache data; u64 msr_val; u32 id; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index a52ef81fb87a..8e28f21fb39b 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -257,7 +257,7 @@ static int emulator_fix_hypercall(struct x86_emulate_ctxt *ctxt); static inline void kvm_async_pf_hash_reset(struct kvm_vcpu *vcpu) { int i; - for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU); i++) + for (i = 0; i < ASYNC_PF_PER_VCPU; i++) vcpu->arch.apf.gfns[i] = ~0; } @@ -10310,12 +10310,14 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) static inline u32 kvm_async_pf_hash_fn(gfn_t gfn) { + BUILD_BUG_ON(!is_power_of_2(ASYNC_PF_PER_VCPU)); + return hash_32(gfn & 0xffffffff, order_base_2(ASYNC_PF_PER_VCPU)); } static inline u32 kvm_async_pf_next_probe(u32 key) { - return (key + 1) & (roundup_pow_of_two(ASYNC_PF_PER_VCPU) - 1); + return (key + 1) & (ASYNC_PF_PER_VCPU - 1); } static void kvm_add_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) @@ -10333,7 +10335,7 @@ static u32 kvm_async_pf_gfn_slot(struct kvm_vcpu *vcpu, gfn_t gfn) int i; u32 key = kvm_async_pf_hash_fn(gfn); - for (i = 0; i < roundup_pow_of_two(ASYNC_PF_PER_VCPU) && + for (i = 0; i < ASYNC_PF_PER_VCPU && (vcpu->arch.apf.gfns[key] != gfn && vcpu->arch.apf.gfns[key] != ~0); i++) key = kvm_async_pf_next_probe(key); From 5b494aea13fe9ec67365510c0d75835428cbb303 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Thu, 16 Apr 2020 11:59:06 -0400 Subject: [PATCH 0494/1043] KVM: No need to retry for hva_to_pfn_remapped() hva_to_pfn_remapped() calls fixup_user_fault(), which has already handled the retry gracefully. Even if "unlocked" is set to true, it means that we've got a VM_FAULT_RETRY inside fixup_user_fault(), however the page fault has already retried and we should have the pfn set correctly. No need to do that again. Signed-off-by: Peter Xu Message-Id: <20200416155906.267462-1-peterx@redhat.com> Signed-off-by: Paolo Bonzini --- virt/kvm/kvm_main.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index e12317f32c5e..bef3d8d40685 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1831,8 +1831,6 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, r = fixup_user_fault(current, current->mm, addr, (write_fault ? FAULT_FLAG_WRITE : 0), &unlocked); - if (unlocked) - return -EAGAIN; if (r) return r; @@ -1903,15 +1901,12 @@ static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, goto exit; } -retry: vma = find_vma_intersection(current->mm, addr, addr + 1); if (vma == NULL) pfn = KVM_PFN_ERR_FAULT; else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) { r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable, &pfn); - if (r == -EAGAIN) - goto retry; if (r < 0) pfn = KVM_PFN_ERR_FAULT; } else { From 0fd460446912dd96fd1e08995f57b14806c10478 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Thu, 16 Apr 2020 11:59:10 -0400 Subject: [PATCH 0495/1043] KVM: X86: Sanity check on gfn before removal The index returned by kvm_async_pf_gfn_slot() will be removed when an async pf gfn is going to be removed. However kvm_async_pf_gfn_slot() is not reliable in that it can return the last key it loops over even if the gfn is not found in the async gfn array. It should never happen, but it's still better to sanity check against that to make sure no unexpected gfn will be removed. Signed-off-by: Peter Xu Message-Id: <20200416155910.267514-1-peterx@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 8e28f21fb39b..370288fdedba 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10353,6 +10353,10 @@ static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) u32 i, j, k; i = j = kvm_async_pf_gfn_slot(vcpu, gfn); + + if (WARN_ON_ONCE(vcpu->arch.apf.gfns[i] != gfn)) + return; + while (true) { vcpu->arch.apf.gfns[i] = ~0; do { From 62315b639302098c177ad6e3e125fa1809398bee Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Thu, 16 Apr 2020 11:59:13 -0400 Subject: [PATCH 0496/1043] KVM: Documentation: Fix up cpuid page 0x4b564d00 and 0x4b564d01 belong to KVM_FEATURE_CLOCKSOURCE2. Signed-off-by: Peter Xu Message-Id: <20200416155913.267562-1-peterx@redhat.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/cpuid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/virt/kvm/cpuid.rst b/Documentation/virt/kvm/cpuid.rst index 01b081f6e7ea..f721c89327ec 100644 --- a/Documentation/virt/kvm/cpuid.rst +++ b/Documentation/virt/kvm/cpuid.rst @@ -50,8 +50,8 @@ KVM_FEATURE_NOP_IO_DELAY 1 not necessary to perform delays KVM_FEATURE_MMU_OP 2 deprecated KVM_FEATURE_CLOCKSOURCE2 3 kvmclock available at msrs - 0x4b564d00 and 0x4b564d01 + KVM_FEATURE_ASYNC_PF 4 async pf can be enabled by writing to msr 0x4b564d02 From 551896e0e060ca32947621e353b4918e90b88822 Mon Sep 17 00:00:00 2001 From: Uros Bizjak Date: Mon, 4 May 2020 17:57:06 +0200 Subject: [PATCH 0497/1043] KVM: VMX: Improve handle_external_interrupt_irqoff inline assembly Improve handle_external_interrupt_irqoff inline assembly in several ways: - remove unneeded %c operand modifiers and "$" prefixes - use %rsp instead of _ASM_SP, since we are in CONFIG_X86_64 part - use $-16 immediate to align %rsp - remove unneeded use of __ASM_SIZE macro - define "ss" named operand only for X86_64 The patch introduces no functional changes. Cc: Paolo Bonzini Cc: Sean Christopherson Signed-off-by: Uros Bizjak Message-Id: <20200504155706.2516956-1-ubizjak@gmail.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 30faae51573d..9ce182db7326 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6382,13 +6382,13 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu) asm volatile( #ifdef CONFIG_X86_64 - "mov %%" _ASM_SP ", %[sp]\n\t" - "and $0xfffffffffffffff0, %%" _ASM_SP "\n\t" - "push $%c[ss]\n\t" + "mov %%rsp, %[sp]\n\t" + "and $-16, %%rsp\n\t" + "push %[ss]\n\t" "push %[sp]\n\t" #endif "pushf\n\t" - __ASM_SIZE(push) " $%c[cs]\n\t" + "push %[cs]\n\t" CALL_NOSPEC : #ifdef CONFIG_X86_64 @@ -6397,7 +6397,9 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu) ASM_CALL_CONSTRAINT : [thunk_target]"r"(entry), +#ifdef CONFIG_X86_64 [ss]"i"(__KERNEL_DS), +#endif [cs]"i"(__KERNEL_CS) ); From 2408500dfc8f275f9057586b81466989f0fafc59 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 28 Apr 2020 16:10:24 -0700 Subject: [PATCH 0498/1043] KVM: nVMX: Truncate writes to vmcs.SYSENTER_EIP/ESP for 32-bit vCPU Explicitly truncate the data written to vmcs.SYSENTER_EIP/ESP on WRMSR if the virtual CPU doesn't support 64-bit mode. The SYSENTER address fields in the VMCS are natural width, i.e. bits 63:32 are dropped if the CPU doesn't support Intel 64 architectures. This behavior is visible to the guest after a VM-Exit/VM-Exit roundtrip, e.g. if the guest sets bits 63:32 in the actual MSR. Signed-off-by: Sean Christopherson Message-Id: <20200428231025.12766-2-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 9ce182db7326..fc2b660aea7d 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1943,6 +1943,16 @@ static int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return 0; } +static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu, + u64 data) +{ +#ifdef CONFIG_X86_64 + if (!guest_cpuid_has(vcpu, X86_FEATURE_LM)) + return (u32)data; +#endif + return (unsigned long)data; +} + /* * Writes msr value into the appropriate "register". * Returns 0 on success, non-0 otherwise. @@ -1980,13 +1990,17 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) vmcs_write32(GUEST_SYSENTER_CS, data); break; case MSR_IA32_SYSENTER_EIP: - if (is_guest_mode(vcpu)) + if (is_guest_mode(vcpu)) { + data = nested_vmx_truncate_sysenter_addr(vcpu, data); get_vmcs12(vcpu)->guest_sysenter_eip = data; + } vmcs_writel(GUEST_SYSENTER_EIP, data); break; case MSR_IA32_SYSENTER_ESP: - if (is_guest_mode(vcpu)) + if (is_guest_mode(vcpu)) { + data = nested_vmx_truncate_sysenter_addr(vcpu, data); get_vmcs12(vcpu)->guest_sysenter_esp = data; + } vmcs_writel(GUEST_SYSENTER_ESP, data); break; case MSR_IA32_DEBUGCTLMSR: From 9e826feb8f114964cbdce026340b6cb9bde68a18 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 28 Apr 2020 16:10:25 -0700 Subject: [PATCH 0499/1043] KVM: nVMX: Drop superfluous VMREAD of vmcs02.GUEST_SYSENTER_* Don't propagate GUEST_SYSENTER_* from vmcs02 to vmcs12 on nested VM-Exit as the vmcs12 fields are updated in vmx_set_msr(), and writes to the corresponding MSRs are always intercepted by KVM when running L2. Dropping the propagation was intended to be done in the same commit that added vmcs12 writes in vmx_set_msr()[1], but for reasons unknown was only shuffled around[2][3]. [1] https://patchwork.kernel.org/patch/10933215 [2] https://patchwork.kernel.org/patch/10933215/#22682289 [3] https://lore.kernel.org/patchwork/patch/1088643 Signed-off-by: Sean Christopherson Message-Id: <20200428231025.12766-3-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index b2cd8d399644..135adc624592 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3952,10 +3952,6 @@ static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES); vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES); - vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS); - vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP); - vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP); - vmcs12->guest_interruptibility_info = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO); From dcf068da7eb29362adf13f20e5c44a18d98ed9a3 Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Tue, 28 Apr 2020 14:23:23 +0800 Subject: [PATCH 0500/1043] KVM: VMX: Introduce generic fastpath handler Introduce generic fastpath handler to handle MSR fastpath, VMX-preemption timer fastpath etc; move it after vmx_complete_interrupts() in order to catch events delivered to the guest, and abort the fast path in later patches. While at it, move the kvm_exit tracepoint so that it is printed for fastpath vmexits as well. There is no observed performance effect for the IPI fastpath after this patch. Tested-by: Haiwei Li Cc: Haiwei Li Signed-off-by: Wanpeng Li Suggested-by: Sean Christopherson Message-Id: <1588055009-12677-2-git-send-email-wanpengli@tencent.com> Reviewed-by: Vitaly Kuznetsov Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index fc2b660aea7d..d883fbb63566 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5933,8 +5933,6 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu, u32 exit_reason = vmx->exit_reason; u32 vectoring_info = vmx->idt_vectoring_info; - trace_kvm_exit(exit_reason, vcpu, KVM_ISA_VMX); - /* * Flush logged GPAs PML buffer, this will make dirty_bitmap more * updated. Another good is, in kvm_vm_ioctl_get_dirty_log, before @@ -6630,6 +6628,16 @@ void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) } } +static enum exit_fastpath_completion vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) +{ + switch (to_vmx(vcpu)->exit_reason) { + case EXIT_REASON_MSR_WRITE: + return handle_fastpath_set_msr_irqoff(vcpu); + default: + return EXIT_FASTPATH_NONE; + } +} + bool __vmx_vcpu_run(struct vcpu_vmx *vmx, unsigned long *regs, bool launched); static enum exit_fastpath_completion vmx_vcpu_run(struct kvm_vcpu *vcpu) @@ -6784,20 +6792,21 @@ static enum exit_fastpath_completion vmx_vcpu_run(struct kvm_vcpu *vcpu) if (unlikely((u16)vmx->exit_reason == EXIT_REASON_MCE_DURING_VMENTRY)) kvm_machine_check(); + trace_kvm_exit(vmx->exit_reason, vcpu, KVM_ISA_VMX); + if (unlikely(vmx->exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) return EXIT_FASTPATH_NONE; - if (!is_guest_mode(vcpu) && vmx->exit_reason == EXIT_REASON_MSR_WRITE) - exit_fastpath = handle_fastpath_set_msr_irqoff(vcpu); - else - exit_fastpath = EXIT_FASTPATH_NONE; - vmx->loaded_vmcs->launched = 1; vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); vmx_recover_nmi_blocking(vmx); vmx_complete_interrupts(vmx); + if (is_guest_mode(vcpu)) + return EXIT_FASTPATH_NONE; + + exit_fastpath = vmx_exit_handlers_fastpath(vcpu); return exit_fastpath; } From 2c4c41325540cf3abb12aef142c0e550f6afeffc Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 8 May 2020 16:53:48 -0700 Subject: [PATCH 0501/1043] KVM: x86: Print symbolic names of VMX VM-Exit flags in traces Use __print_flags() to display the names of VMX flags in VM-Exit traces and strip the flags when printing the basic exit reason, e.g. so that a failed VM-Entry due to invalid guest state gets recorded as "INVALID_STATE FAILED_VMENTRY" instead of "0x80000021". Opportunstically fix misaligned variables in the kvm_exit and kvm_nested_vmexit_inject tracepoints. Reviewed-by: Vitaly Kuznetsov Signed-off-by: Sean Christopherson Message-Id: <20200508235348.19427-3-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/uapi/asm/vmx.h | 3 +++ arch/x86/kvm/trace.h | 32 +++++++++++++++++--------------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/arch/x86/include/uapi/asm/vmx.h b/arch/x86/include/uapi/asm/vmx.h index e95b72ec19bc..b8ff9e8ac0d5 100644 --- a/arch/x86/include/uapi/asm/vmx.h +++ b/arch/x86/include/uapi/asm/vmx.h @@ -150,6 +150,9 @@ { EXIT_REASON_UMWAIT, "UMWAIT" }, \ { EXIT_REASON_TPAUSE, "TPAUSE" } +#define VMX_EXIT_REASON_FLAGS \ + { VMX_EXIT_REASONS_FAILED_VMENTRY, "FAILED_VMENTRY" } + #define VMX_ABORT_SAVE_GUEST_MSR_FAIL 1 #define VMX_ABORT_LOAD_HOST_PDPTE_FAIL 2 #define VMX_ABORT_LOAD_HOST_MSR_FAIL 4 diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index 249062f24b94..54a10c98d746 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -225,6 +225,14 @@ TRACE_EVENT(kvm_apic, #define KVM_ISA_VMX 1 #define KVM_ISA_SVM 2 +#define kvm_print_exit_reason(exit_reason, isa) \ + (isa == KVM_ISA_VMX) ? \ + __print_symbolic(exit_reason & 0xffff, VMX_EXIT_REASONS) : \ + __print_symbolic(exit_reason, SVM_EXIT_REASONS), \ + (isa == KVM_ISA_VMX && exit_reason & ~0xffff) ? " " : "", \ + (isa == KVM_ISA_VMX) ? \ + __print_flags(exit_reason & ~0xffff, " ", VMX_EXIT_REASON_FLAGS) : "" + /* * Tracepoint for kvm guest exit: */ @@ -250,12 +258,10 @@ TRACE_EVENT(kvm_exit, &__entry->info2); ), - TP_printk("vcpu %u reason %s rip 0x%lx info %llx %llx", + TP_printk("vcpu %u reason %s%s%s rip 0x%lx info %llx %llx", __entry->vcpu_id, - (__entry->isa == KVM_ISA_VMX) ? - __print_symbolic(__entry->exit_reason, VMX_EXIT_REASONS) : - __print_symbolic(__entry->exit_reason, SVM_EXIT_REASONS), - __entry->guest_rip, __entry->info1, __entry->info2) + kvm_print_exit_reason(__entry->exit_reason, __entry->isa), + __entry->guest_rip, __entry->info1, __entry->info2) ); /* @@ -588,12 +594,10 @@ TRACE_EVENT(kvm_nested_vmexit, __entry->exit_int_info_err = exit_int_info_err; __entry->isa = isa; ), - TP_printk("rip: 0x%016llx reason: %s ext_inf1: 0x%016llx " + TP_printk("rip: 0x%016llx reason: %s%s%s ext_inf1: 0x%016llx " "ext_inf2: 0x%016llx ext_int: 0x%08x ext_int_err: 0x%08x", __entry->rip, - (__entry->isa == KVM_ISA_VMX) ? - __print_symbolic(__entry->exit_code, VMX_EXIT_REASONS) : - __print_symbolic(__entry->exit_code, SVM_EXIT_REASONS), + kvm_print_exit_reason(__entry->exit_code, __entry->isa), __entry->exit_info1, __entry->exit_info2, __entry->exit_int_info, __entry->exit_int_info_err) ); @@ -626,13 +630,11 @@ TRACE_EVENT(kvm_nested_vmexit_inject, __entry->isa = isa; ), - TP_printk("reason: %s ext_inf1: 0x%016llx " + TP_printk("reason: %s%s%s ext_inf1: 0x%016llx " "ext_inf2: 0x%016llx ext_int: 0x%08x ext_int_err: 0x%08x", - (__entry->isa == KVM_ISA_VMX) ? - __print_symbolic(__entry->exit_code, VMX_EXIT_REASONS) : - __print_symbolic(__entry->exit_code, SVM_EXIT_REASONS), - __entry->exit_info1, __entry->exit_info2, - __entry->exit_int_info, __entry->exit_int_info_err) + kvm_print_exit_reason(__entry->exit_code, __entry->isa), + __entry->exit_info1, __entry->exit_info2, + __entry->exit_int_info, __entry->exit_int_info_err) ); /* From 5a9f54435a488f8a1153efd36cccee3e7e0fc28b Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Tue, 28 Apr 2020 14:23:26 +0800 Subject: [PATCH 0502/1043] KVM: X86: Introduce kvm_vcpu_exit_request() helper Introduce kvm_vcpu_exit_request() helper, we need to check some conditions before enter guest again immediately, we skip invoking the exit handler and go through full run loop if complete fastpath but there is stuff preventing we enter guest again immediately. Tested-by: Haiwei Li Cc: Haiwei Li Signed-off-by: Wanpeng Li Message-Id: <1588055009-12677-5-git-send-email-wanpengli@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 10 ++++++++-- arch/x86/kvm/x86.h | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 370288fdedba..29a41aa98929 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1573,6 +1573,13 @@ int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr); +bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu) +{ + return vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) || + need_resched() || signal_pending(current); +} +EXPORT_SYMBOL_GPL(kvm_vcpu_exit_request); + /* * The fast path for frequent and performance sensitive wrmsr emulation, * i.e. the sending of IPI, sending IPI early in the VM-Exit flow reduces @@ -8396,8 +8403,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_lapic_enabled(vcpu) && vcpu->arch.apicv_active) kvm_x86_ops.sync_pir_to_irr(vcpu); - if (vcpu->mode == EXITING_GUEST_MODE || kvm_request_pending(vcpu) - || need_resched() || signal_pending(current)) { + if (kvm_vcpu_exit_request(vcpu)) { vcpu->mode = OUTSIDE_GUEST_MODE; smp_wmb(); local_irq_enable(); diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 7b5ed8ed628e..e02fe28254b6 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -364,5 +364,6 @@ static inline bool kvm_dr7_valid(u64 data) void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu); void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu); u64 kvm_spec_ctrl_valid_bits(struct kvm_vcpu *vcpu); +bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu); #endif From 404d5d7bff0d419fe11c7eaebca9ec8f25258f95 Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Tue, 28 Apr 2020 14:23:25 +0800 Subject: [PATCH 0503/1043] KVM: X86: Introduce more exit_fastpath_completion enum values Adds a fastpath_t typedef since enum lines are a bit long, and replace EXIT_FASTPATH_SKIP_EMUL_INS with two new exit_fastpath_completion enum values. - EXIT_FASTPATH_EXIT_HANDLED kvm will still go through it's full run loop, but it would skip invoking the exit handler. - EXIT_FASTPATH_REENTER_GUEST complete fastpath, guest can be re-entered without invoking the exit handler or going back to vcpu_run Tested-by: Haiwei Li Cc: Haiwei Li Signed-off-by: Wanpeng Li Message-Id: <1588055009-12677-4-git-send-email-wanpengli@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 4 +++- arch/x86/kvm/svm/svm.c | 15 +++++++-------- arch/x86/kvm/vmx/vmx.c | 26 ++++++++++++++++++-------- arch/x86/kvm/x86.c | 19 ++++++++++--------- arch/x86/kvm/x86.h | 2 +- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 04f44961858b..c3906fb2b93f 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -182,8 +182,10 @@ enum { enum exit_fastpath_completion { EXIT_FASTPATH_NONE, - EXIT_FASTPATH_SKIP_EMUL_INS, + EXIT_FASTPATH_REENTER_GUEST, + EXIT_FASTPATH_EXIT_HANDLED, }; +typedef enum exit_fastpath_completion fastpath_t; struct x86_emulate_ctxt; struct x86_exception; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 4c808cc059f1..29e7f7bc284d 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2893,8 +2893,7 @@ static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2) *info2 = control->exit_info_2; } -static int handle_exit(struct kvm_vcpu *vcpu, - enum exit_fastpath_completion exit_fastpath) +static int handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) { struct vcpu_svm *svm = to_svm(vcpu); struct kvm_run *kvm_run = vcpu->run; @@ -2952,10 +2951,10 @@ static int handle_exit(struct kvm_vcpu *vcpu, __func__, svm->vmcb->control.exit_int_info, exit_code); - if (exit_fastpath == EXIT_FASTPATH_SKIP_EMUL_INS) { - kvm_skip_emulated_instruction(vcpu); + if (exit_fastpath != EXIT_FASTPATH_NONE) return 1; - } else if (exit_code >= ARRAY_SIZE(svm_exit_handlers) + + if (exit_code >= ARRAY_SIZE(svm_exit_handlers) || !svm_exit_handlers[exit_code]) { vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%x\n", exit_code); dump_vmcb(vcpu); @@ -3324,7 +3323,7 @@ static void svm_cancel_injection(struct kvm_vcpu *vcpu) svm_complete_interrupts(svm); } -static enum exit_fastpath_completion svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu) +static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu) { if (!is_guest_mode(vcpu) && to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_MSR && @@ -3336,9 +3335,9 @@ static enum exit_fastpath_completion svm_exit_handlers_fastpath(struct kvm_vcpu void __svm_vcpu_run(unsigned long vmcb_pa, unsigned long *regs); -static enum exit_fastpath_completion svm_vcpu_run(struct kvm_vcpu *vcpu) +static fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) { - enum exit_fastpath_completion exit_fastpath; + fastpath_t exit_fastpath; struct vcpu_svm *svm = to_svm(vcpu); svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d883fbb63566..c7730d3aa706 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5926,8 +5926,7 @@ void dump_vmcs(void) * The guest has exited. See if we can fix it or if we need userspace * assistance. */ -static int vmx_handle_exit(struct kvm_vcpu *vcpu, - enum exit_fastpath_completion exit_fastpath) +static int vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) { struct vcpu_vmx *vmx = to_vmx(vcpu); u32 exit_reason = vmx->exit_reason; @@ -6034,10 +6033,8 @@ static int vmx_handle_exit(struct kvm_vcpu *vcpu, } } - if (exit_fastpath == EXIT_FASTPATH_SKIP_EMUL_INS) { - kvm_skip_emulated_instruction(vcpu); + if (exit_fastpath != EXIT_FASTPATH_NONE) return 1; - } if (exit_reason >= kvm_vmx_max_exit_handlers) goto unexpected_vmexit; @@ -6628,7 +6625,7 @@ void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp) } } -static enum exit_fastpath_completion vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) +static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) { switch (to_vmx(vcpu)->exit_reason) { case EXIT_REASON_MSR_WRITE: @@ -6640,12 +6637,13 @@ static enum exit_fastpath_completion vmx_exit_handlers_fastpath(struct kvm_vcpu bool __vmx_vcpu_run(struct vcpu_vmx *vmx, unsigned long *regs, bool launched); -static enum exit_fastpath_completion vmx_vcpu_run(struct kvm_vcpu *vcpu) +static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) { - enum exit_fastpath_completion exit_fastpath; + fastpath_t exit_fastpath; struct vcpu_vmx *vmx = to_vmx(vcpu); unsigned long cr3, cr4; +reenter_guest: /* Record the guest's net vcpu time for enforced NMI injections. */ if (unlikely(!enable_vnmi && vmx->loaded_vmcs->soft_vnmi_blocked)) @@ -6807,6 +6805,18 @@ static enum exit_fastpath_completion vmx_vcpu_run(struct kvm_vcpu *vcpu) return EXIT_FASTPATH_NONE; exit_fastpath = vmx_exit_handlers_fastpath(vcpu); + if (exit_fastpath == EXIT_FASTPATH_REENTER_GUEST) { + if (!kvm_vcpu_exit_request(vcpu)) { + /* + * FIXME: this goto should be a loop in vcpu_enter_guest, + * but it would incur the cost of a retpoline for now. + * Revisit once static calls are available. + */ + goto reenter_guest; + } + exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED; + } + return exit_fastpath; } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 29a41aa98929..71749fcb229e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1608,27 +1608,28 @@ static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data return 1; } -enum exit_fastpath_completion handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu) +fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu) { u32 msr = kvm_rcx_read(vcpu); u64 data; - int ret = 0; + fastpath_t ret = EXIT_FASTPATH_NONE; switch (msr) { case APIC_BASE_MSR + (APIC_ICR >> 4): data = kvm_read_edx_eax(vcpu); - ret = handle_fastpath_set_x2apic_icr_irqoff(vcpu, data); + if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) { + kvm_skip_emulated_instruction(vcpu); + ret = EXIT_FASTPATH_EXIT_HANDLED; + } break; default: - return EXIT_FASTPATH_NONE; + break; } - if (!ret) { + if (ret != EXIT_FASTPATH_NONE) trace_kvm_msr_write(msr, data); - return EXIT_FASTPATH_SKIP_EMUL_INS; - } - return EXIT_FASTPATH_NONE; + return ret; } EXPORT_SYMBOL_GPL(handle_fastpath_set_msr_irqoff); @@ -8205,7 +8206,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) bool req_int_win = dm_request_for_irq_injection(vcpu) && kvm_cpu_accept_dm_intr(vcpu); - enum exit_fastpath_completion exit_fastpath; + fastpath_t exit_fastpath; bool req_immediate_exit = false; diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index e02fe28254b6..6eb62e97e59f 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -274,7 +274,7 @@ bool kvm_mtrr_check_gfn_range_consistency(struct kvm_vcpu *vcpu, gfn_t gfn, bool kvm_vector_hashing_enabled(void); int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, int emulation_type, void *insn, int insn_len); -enum exit_fastpath_completion handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu); +fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu); extern u64 host_xcr0; extern u64 supported_xcr0; From 379a3c8ee44440d5afa505230ed8cb5b0d0e314b Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Tue, 28 Apr 2020 14:23:27 +0800 Subject: [PATCH 0504/1043] KVM: VMX: Optimize posted-interrupt delivery for timer fastpath While optimizing posted-interrupt delivery especially for the timer fastpath scenario, I measured kvm_x86_ops.deliver_posted_interrupt() to introduce substantial latency because the processor has to perform all vmentry tasks, ack the posted interrupt notification vector, read the posted-interrupt descriptor etc. This is not only slow, it is also unnecessary when delivering an interrupt to the current CPU (as is the case for the LAPIC timer) because PIR->IRR and IRR->RVI synchronization is already performed on vmentry Therefore skip kvm_vcpu_trigger_posted_interrupt in this case, and instead do vmx_sync_pir_to_irr() on the EXIT_FASTPATH_REENTER_GUEST fastpath as well. Tested-by: Haiwei Li Cc: Haiwei Li Suggested-by: Paolo Bonzini Signed-off-by: Wanpeng Li Message-Id: <1588055009-12677-6-git-send-email-wanpengli@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 5 ++++- virt/kvm/kvm_main.c | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index c7730d3aa706..8d881fcf648e 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -3936,7 +3936,8 @@ static int vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector) if (pi_test_and_set_on(&vmx->pi_desc)) return 0; - if (!kvm_vcpu_trigger_posted_interrupt(vcpu, false)) + if (vcpu != kvm_get_running_vcpu() && + !kvm_vcpu_trigger_posted_interrupt(vcpu, false)) kvm_vcpu_kick(vcpu); return 0; @@ -6812,6 +6813,8 @@ reenter_guest: * but it would incur the cost of a retpoline for now. * Revisit once static calls are available. */ + if (vcpu->arch.apicv_active) + vmx_sync_pir_to_irr(vcpu); goto reenter_guest; } exit_fastpath = EXIT_FASTPATH_EXIT_HANDLED; diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index bef3d8d40685..11844fad60fd 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4646,6 +4646,7 @@ struct kvm_vcpu *kvm_get_running_vcpu(void) return vcpu; } +EXPORT_SYMBOL_GPL(kvm_get_running_vcpu); /** * kvm_get_running_vcpus - get the per-CPU array of currently running vcpus. From 199a8b84c455cfeb81bc060c6824dea473e54dc3 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 5 May 2020 06:45:35 -0400 Subject: [PATCH 0505/1043] KVM: x86: introduce kvm_can_use_hv_timer Replace the ad hoc test in vmx_set_hv_timer with a test in the caller, start_hv_timer. This test is not Intel-specific and would be duplicated when introducing the fast path for the TSC deadline MSR. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/lapic.c | 13 ++++++++++--- arch/x86/kvm/lapic.h | 2 +- arch/x86/kvm/vmx/vmx.c | 4 ---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 42cd2e3ec6fd..73e51abca21d 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -110,11 +110,18 @@ static inline u32 kvm_x2apic_id(struct kvm_lapic *apic) return apic->vcpu->vcpu_id; } -bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu) +static bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu) { return pi_inject_timer && kvm_vcpu_apicv_active(vcpu); } -EXPORT_SYMBOL_GPL(kvm_can_post_timer_interrupt); + +bool kvm_can_use_hv_timer(struct kvm_vcpu *vcpu) +{ + return kvm_x86_ops.set_hv_timer + && !(kvm_mwait_in_guest(vcpu->kvm) || + kvm_can_post_timer_interrupt(vcpu)); +} +EXPORT_SYMBOL_GPL(kvm_can_use_hv_timer); static bool kvm_use_posted_timer_interrupt(struct kvm_vcpu *vcpu) { @@ -1788,7 +1795,7 @@ static bool start_hv_timer(struct kvm_lapic *apic) bool expired; WARN_ON(preemptible()); - if (!kvm_x86_ops.set_hv_timer) + if (!kvm_can_use_hv_timer(vcpu)) return false; if (!ktimer->tscdeadline) diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 7f15f9e69efe..754f29beb83e 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -250,7 +250,7 @@ void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu); void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu); bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu); void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu); -bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu); +bool kvm_can_use_hv_timer(struct kvm_vcpu *vcpu); static inline enum lapic_mode kvm_apic_mode(u64 apic_base) { diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 8d881fcf648e..d2678078eee8 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7405,10 +7405,6 @@ static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, u64 tscl, guest_tscl, delta_tsc, lapic_timer_advance_cycles; struct kvm_timer *ktimer = &vcpu->arch.apic->lapic_timer; - if (kvm_mwait_in_guest(vcpu->kvm) || - kvm_can_post_timer_interrupt(vcpu)) - return -EOPNOTSUPP; - vmx = to_vmx(vcpu); tscl = rdtsc(); guest_tscl = kvm_read_l1_tsc(vcpu, tscl); From ae95f566b3d22ade75c67827f1171594dacc9a03 Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Tue, 28 Apr 2020 14:23:28 +0800 Subject: [PATCH 0506/1043] KVM: X86: TSCDEADLINE MSR emulation fastpath This patch implements a fast path for emulation of writes to the TSCDEADLINE MSR. Besides shortcutting various housekeeping tasks in the vCPU loop, the fast path can also deliver the timer interrupt directly without going through KVM_REQ_PENDING_TIMER because it runs in vCPU context. Tested-by: Haiwei Li Cc: Haiwei Li Signed-off-by: Wanpeng Li Message-Id: <1588055009-12677-7-git-send-email-wanpengli@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/lapic.c | 18 ++++++++++++------ arch/x86/kvm/x86.c | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 73e51abca21d..2a3b57401a68 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1600,7 +1600,7 @@ static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic) } } -static void apic_timer_expired(struct kvm_lapic *apic) +static void apic_timer_expired(struct kvm_lapic *apic, bool from_timer_fn) { struct kvm_vcpu *vcpu = apic->vcpu; struct kvm_timer *ktimer = &apic->lapic_timer; @@ -1611,6 +1611,12 @@ static void apic_timer_expired(struct kvm_lapic *apic) if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use) ktimer->expired_tscdeadline = ktimer->tscdeadline; + if (!from_timer_fn && vcpu->arch.apicv_active) { + WARN_ON(kvm_get_running_vcpu() != vcpu); + kvm_apic_inject_pending_timer_irqs(apic); + return; + } + if (kvm_use_posted_timer_interrupt(apic->vcpu)) { if (apic->lapic_timer.timer_advance_ns) __kvm_wait_lapic_expire(vcpu); @@ -1650,7 +1656,7 @@ static void start_sw_tscdeadline(struct kvm_lapic *apic) expire = ktime_sub_ns(expire, ktimer->timer_advance_ns); hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD); } else - apic_timer_expired(apic); + apic_timer_expired(apic, false); local_irq_restore(flags); } @@ -1758,7 +1764,7 @@ static void start_sw_period(struct kvm_lapic *apic) if (ktime_after(ktime_get(), apic->lapic_timer.target_expiration)) { - apic_timer_expired(apic); + apic_timer_expired(apic, false); if (apic_lvtt_oneshot(apic)) return; @@ -1820,7 +1826,7 @@ static bool start_hv_timer(struct kvm_lapic *apic) if (atomic_read(&ktimer->pending)) { cancel_hv_timer(apic); } else if (expired) { - apic_timer_expired(apic); + apic_timer_expired(apic, false); cancel_hv_timer(apic); } } @@ -1870,7 +1876,7 @@ void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) goto out; WARN_ON(rcuwait_active(&vcpu->wait)); cancel_hv_timer(apic); - apic_timer_expired(apic); + apic_timer_expired(apic, false); if (apic_lvtt_period(apic) && apic->lapic_timer.period) { advance_periodic_target_expiration(apic); @@ -2376,7 +2382,7 @@ static enum hrtimer_restart apic_timer_fn(struct hrtimer *data) struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer); - apic_timer_expired(apic); + apic_timer_expired(apic, true); if (lapic_is_periodic(apic)) { advance_periodic_target_expiration(apic); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 71749fcb229e..7fb7c1a8f1a5 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1608,6 +1608,15 @@ static int handle_fastpath_set_x2apic_icr_irqoff(struct kvm_vcpu *vcpu, u64 data return 1; } +static int handle_fastpath_set_tscdeadline(struct kvm_vcpu *vcpu, u64 data) +{ + if (!kvm_can_use_hv_timer(vcpu)) + return 1; + + kvm_set_lapic_tscdeadline_msr(vcpu, data); + return 0; +} + fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu) { u32 msr = kvm_rcx_read(vcpu); @@ -1622,6 +1631,13 @@ fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu) ret = EXIT_FASTPATH_EXIT_HANDLED; } break; + case MSR_IA32_TSCDEADLINE: + data = kvm_read_edx_eax(vcpu); + if (!handle_fastpath_set_tscdeadline(vcpu, data)) { + kvm_skip_emulated_instruction(vcpu); + ret = EXIT_FASTPATH_REENTER_GUEST; + } + break; default: break; } From 26efe2fd92e50822674acce1dbc4f2ac6fc1788f Mon Sep 17 00:00:00 2001 From: Wanpeng Li Date: Wed, 6 May 2020 11:44:01 -0400 Subject: [PATCH 0507/1043] KVM: VMX: Handle preemption timer fastpath This patch implements a fastpath for the preemption timer vmexit. The vmexit can be handled quickly so it can be performed with interrupts off and going back directly to the guest. Testing on SKX Server. cyclictest in guest(w/o mwait exposed, adaptive advance lapic timer is default -1): 5540.5ns -> 4602ns 17% kvm-unit-test/vmexit.flat: w/o avanced timer: tscdeadline_immed: 3028.5 -> 2494.75 17.6% tscdeadline: 5765.7 -> 5285 8.3% w/ adaptive advance timer default -1: tscdeadline_immed: 3123.75 -> 2583 17.3% tscdeadline: 4663.75 -> 4537 2.7% Tested-by: Haiwei Li Cc: Haiwei Li Signed-off-by: Wanpeng Li Message-Id: <1588055009-12677-8-git-send-email-wanpengli@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index d2678078eee8..5fa162f6e1dc 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5601,14 +5601,22 @@ static int handle_pml_full(struct kvm_vcpu *vcpu) return 1; } -static int handle_preemption_timer(struct kvm_vcpu *vcpu) +static fastpath_t handle_fastpath_preemption_timer(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); if (!vmx->req_immediate_exit && - !unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled)) + !unlikely(vmx->loaded_vmcs->hv_timer_soft_disabled)) { kvm_lapic_expired_hv_timer(vcpu); + return EXIT_FASTPATH_REENTER_GUEST; + } + return EXIT_FASTPATH_NONE; +} + +static int handle_preemption_timer(struct kvm_vcpu *vcpu) +{ + handle_fastpath_preemption_timer(vcpu); return 1; } @@ -6631,6 +6639,8 @@ static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu) switch (to_vmx(vcpu)->exit_reason) { case EXIT_REASON_MSR_WRITE: return handle_fastpath_set_msr_irqoff(vcpu); + case EXIT_REASON_PREEMPTION_TIMER: + return handle_fastpath_preemption_timer(vcpu); default: return EXIT_FASTPATH_NONE; } From e14b7786cb1cf39a12e79bb04e2f43afc2eabc0f Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 May 2020 08:17:55 -0500 Subject: [PATCH 0508/1043] KVM: SVM: Merge svm_enable_vintr into svm_set_vintr Code clean up and remove unnecessary intercept check for INTERCEPT_VINTR. Signed-off-by: Suravee Suthikulpanit Message-Id: <1588771076-73790-4-git-send-email-suravee.suthikulpanit@amd.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 29e7f7bc284d..801886142cc4 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1352,12 +1352,13 @@ static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) } } -static inline void svm_enable_vintr(struct vcpu_svm *svm) +static void svm_set_vintr(struct vcpu_svm *svm) { struct vmcb_control_area *control; /* The following fields are ignored when AVIC is enabled */ WARN_ON(kvm_vcpu_apicv_active(&svm->vcpu)); + set_intercept(svm, INTERCEPT_VINTR); /* * This is just a dummy VINTR to actually cause a vmexit to happen. @@ -1371,13 +1372,6 @@ static inline void svm_enable_vintr(struct vcpu_svm *svm) mark_dirty(svm->vmcb, VMCB_INTR); } -static void svm_set_vintr(struct vcpu_svm *svm) -{ - set_intercept(svm, INTERCEPT_VINTR); - if (is_intercept(svm, INTERCEPT_VINTR)) - svm_enable_vintr(svm); -} - static void svm_clear_vintr(struct vcpu_svm *svm) { clr_intercept(svm, INTERCEPT_VINTR); From de182481629c2dc248adbb4ec5df83cd9d633dd4 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 May 2020 08:17:56 -0500 Subject: [PATCH 0509/1043] KVM: SVM: Remove unnecessary V_IRQ unsetting This has already been handled in the prior call to svm_clear_vintr(). Signed-off-by: Suravee Suthikulpanit Message-Id: <1588771076-73790-5-git-send-email-suravee.suthikulpanit@amd.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 801886142cc4..4e9cd2a73ad0 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2660,8 +2660,6 @@ static int interrupt_window_interception(struct vcpu_svm *svm) */ svm_toggle_avic_for_irq_window(&svm->vcpu, true); - svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; - mark_dirty(svm->vmcb, VMCB_INTR); ++svm->vcpu.stat.irq_window_exits; return 1; } From 6c1c6e58356b87017839f71120d1c250b4e0f7ff Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 6 May 2020 13:46:53 -0700 Subject: [PATCH 0510/1043] KVM: nVMX: Remove unused 'ops' param from nested_vmx_hardware_setup() Remove a 'struct kvm_x86_ops' param that got left behind when the nested ops were moved to their own struct. Fixes: 33b22172452f0 ("KVM: x86: move nested-related kvm_x86_ops to a separate struct") Signed-off-by: Sean Christopherson Message-Id: <20200506204653.14683-1-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 3 +-- arch/x86/kvm/vmx/nested.h | 3 +-- arch/x86/kvm/vmx/vmx.c | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 135adc624592..25f130d3158b 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -6421,8 +6421,7 @@ void nested_vmx_hardware_unsetup(void) } } -__init int nested_vmx_hardware_setup(struct kvm_x86_ops *ops, - int (*exit_handlers[])(struct kvm_vcpu *)) +__init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)) { int i; diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index 5cc72ae0e277..758bccc26cf9 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -19,8 +19,7 @@ enum nvmx_vmentry_status { void vmx_leave_nested(struct kvm_vcpu *vcpu); void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps); void nested_vmx_hardware_unsetup(void); -__init int nested_vmx_hardware_setup(struct kvm_x86_ops *ops, - int (*exit_handlers[])(struct kvm_vcpu *)); +__init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *)); void nested_vmx_set_vmcs_shadowing_bitmap(void); void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu); enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 5fa162f6e1dc..68f4efb2b9c2 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8119,8 +8119,7 @@ static __init int hardware_setup(void) nested_vmx_setup_ctls_msrs(&vmcs_config.nested, vmx_capability.ept); - r = nested_vmx_hardware_setup(&vmx_x86_ops, - kvm_vmx_exit_handlers); + r = nested_vmx_hardware_setup(kvm_vmx_exit_handlers); if (r) return r; } From 1739f3d56d24a121535b45267d85b7fe6f6cc4ab Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Fri, 8 May 2020 13:36:41 -0700 Subject: [PATCH 0511/1043] KVM: nVMX: Really make emulated nested preemption timer pinned The PINNED bit is ignored by hrtimer_init. It is only considered when starting the timer. When the hrtimer isn't pinned to the same logical processor as the vCPU thread to be interrupted, the emulated VMX-preemption timer often fails to adhere to the architectural specification. Fixes: f15a75eedc18e ("KVM: nVMX: make emulated nested preemption timer pinned") Signed-off-by: Jim Mattson Reviewed-by: Peter Shier Reviewed-by: Oliver Upton Message-Id: <20200508203643.85477-2-jmattson@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 25f130d3158b..3d50e50de626 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -2113,7 +2113,7 @@ static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu) preemption_timeout *= 1000000; do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); hrtimer_start(&vmx->nested.preemption_timer, - ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL); + ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL_PINNED); } static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) From ada0098df6569b7f9e9495beb0f1e35718895aa0 Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Fri, 8 May 2020 13:36:42 -0700 Subject: [PATCH 0512/1043] KVM: nVMX: Change emulated VMX-preemption timer hrtimer to absolute Prepare for migration of this hrtimer, by changing it from relative to absolute. (I couldn't get migration to work with a relative timer.) Signed-off-by: Jim Mattson Reviewed-by: Peter Shier Reviewed-by: Oliver Upton Message-Id: <20200508203643.85477-3-jmattson@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 3d50e50de626..51ebb60e1533 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -2113,7 +2113,8 @@ static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu) preemption_timeout *= 1000000; do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz); hrtimer_start(&vmx->nested.preemption_timer, - ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL_PINNED); + ktime_add_ns(ktime_get(), preemption_timeout), + HRTIMER_MODE_ABS_PINNED); } static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) @@ -4673,7 +4674,7 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu) goto out_shadow_vmcs; hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC, - HRTIMER_MODE_REL_PINNED); + HRTIMER_MODE_ABS_PINNED); vmx->nested.preemption_timer.function = vmx_preemption_timer_fn; vmx->nested.vpid02 = allocate_vpid(); From 93dff2fed2fb4a513196b7df05742c6fcdfd5178 Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Fri, 8 May 2020 13:36:43 -0700 Subject: [PATCH 0513/1043] KVM: nVMX: Migrate the VMX-preemption timer The hrtimer used to emulate the VMX-preemption timer must be pinned to the same logical processor as the vCPU thread to be interrupted if we want to have any hope of adhering to the architectural specification of the VMX-preemption timer. Even with this change, the emulated VMX-preemption timer VM-exit occasionally arrives too late. Signed-off-by: Jim Mattson Reviewed-by: Peter Shier Reviewed-by: Oliver Upton Message-Id: <20200508203643.85477-4-jmattson@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/irq.c | 2 ++ arch/x86/kvm/vmx/vmx.c | 11 +++++++++++ 3 files changed, 15 insertions(+) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index c3906fb2b93f..a3ce391c9e7a 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1247,6 +1247,8 @@ struct kvm_x86_ops { bool (*apic_init_signal_blocked)(struct kvm_vcpu *vcpu); int (*enable_direct_tlbflush)(struct kvm_vcpu *vcpu); + + void (*migrate_timers)(struct kvm_vcpu *vcpu); }; struct kvm_x86_nested_ops { diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c index e330e7d125f7..54f7ea68083b 100644 --- a/arch/x86/kvm/irq.c +++ b/arch/x86/kvm/irq.c @@ -159,6 +159,8 @@ void __kvm_migrate_timers(struct kvm_vcpu *vcpu) { __kvm_migrate_apic_timer(vcpu); __kvm_migrate_pit_timer(vcpu); + if (kvm_x86_ops.migrate_timers) + kvm_x86_ops.migrate_timers(vcpu); } bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 68f4efb2b9c2..6a03c27ff314 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7814,6 +7814,16 @@ static bool vmx_apic_init_signal_blocked(struct kvm_vcpu *vcpu) return to_vmx(vcpu)->nested.vmxon; } +static void vmx_migrate_timers(struct kvm_vcpu *vcpu) +{ + if (is_guest_mode(vcpu)) { + struct hrtimer *timer = &to_vmx(vcpu)->nested.preemption_timer; + + if (hrtimer_try_to_cancel(timer) == 1) + hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); + } +} + static void hardware_unsetup(void) { if (nested) @@ -7957,6 +7967,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .need_emulation_on_page_fault = vmx_need_emulation_on_page_fault, .apic_init_signal_blocked = vmx_apic_init_signal_blocked, + .migrate_timers = vmx_migrate_timers, }; static __init int hardware_setup(void) From cb953129bfe5c0f2da835a0469930873fb7e71df Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 8 May 2020 11:22:40 -0700 Subject: [PATCH 0514/1043] kvm: add halt-polling cpu usage stats Two new stats for exposing halt-polling cpu usage: halt_poll_success_ns halt_poll_fail_ns Thus sum of these 2 stats is the total cpu time spent polling. "success" means the VCPU polled until a virtual interrupt was delivered. "fail" means the VCPU had to schedule out (either because the maximum poll time was reached or it needed to yield the CPU). To avoid touching every arch's kvm_vcpu_stat struct, only update and export halt-polling cpu usage stats if we're on x86. Exporting cpu usage as a u64 and in nanoseconds means we will overflow at ~500 years, which seems reasonably large. Signed-off-by: David Matlack Signed-off-by: Jon Cargille Reviewed-by: Jim Mattson Message-Id: <20200508182240.68440-1-jcargill@google.com> Signed-off-by: Paolo Bonzini --- arch/arm64/include/asm/kvm_host.h | 2 ++ arch/arm64/kvm/guest.c | 2 ++ arch/mips/include/asm/kvm_host.h | 2 ++ arch/mips/kvm/mips.c | 2 ++ arch/powerpc/kvm/booke.c | 2 ++ arch/s390/include/asm/kvm_host.h | 2 ++ arch/s390/kvm/kvm-s390.c | 2 ++ arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/x86.c | 2 ++ virt/kvm/kvm_main.c | 18 +++++++++++++++--- 10 files changed, 33 insertions(+), 3 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 32c8a675e5a4..3833736dd064 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -415,6 +415,8 @@ struct kvm_vm_stat { struct kvm_vcpu_stat { u64 halt_successful_poll; u64 halt_attempted_poll; + u64 halt_poll_success_ns; + u64 halt_poll_fail_ns; u64 halt_poll_invalid; u64 halt_wakeup; u64 hvc_exit_stat; diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 863a0d158fb8..55ebb9ea74f6 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -40,6 +40,8 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { VCPU_STAT("mmio_exit_user", mmio_exit_user), VCPU_STAT("mmio_exit_kernel", mmio_exit_kernel), VCPU_STAT("exits", exits), + VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), + VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), { NULL } }; diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h index 2c343c346b79..e28b5a946e26 100644 --- a/arch/mips/include/asm/kvm_host.h +++ b/arch/mips/include/asm/kvm_host.h @@ -174,6 +174,8 @@ struct kvm_vcpu_stat { #endif u64 halt_successful_poll; u64 halt_attempted_poll; + u64 halt_poll_success_ns; + u64 halt_poll_fail_ns; u64 halt_poll_invalid; u64 halt_wakeup; }; diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index 9787cdec33e6..99ed08aff31e 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -72,6 +72,8 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { VCPU_STAT("halt_attempted_poll", halt_attempted_poll), VCPU_STAT("halt_poll_invalid", halt_poll_invalid), VCPU_STAT("halt_wakeup", halt_wakeup), + VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), + VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), {NULL} }; diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c index c2984cb6dfa7..888afe8d35cc 100644 --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -54,6 +54,8 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { VCPU_STAT("halt_wakeup", halt_wakeup), VCPU_STAT("doorbell", dbell_exits), VCPU_STAT("guest doorbell", gdbell_exits), + VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), + VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), VM_STAT("remote_tlb_flush", remote_tlb_flush), { NULL } }; diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index d6bcd34f3ec3..176f74cc2257 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -375,6 +375,8 @@ struct kvm_vcpu_stat { u64 halt_poll_invalid; u64 halt_no_poll_steal; u64 halt_wakeup; + u64 halt_poll_success_ns; + u64 halt_poll_fail_ns; u64 instruction_lctl; u64 instruction_lctlg; u64 instruction_stctl; diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 389ff1b7cd43..a560a368f92c 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -75,6 +75,8 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { VCPU_STAT("halt_poll_invalid", halt_poll_invalid), VCPU_STAT("halt_no_poll_steal", halt_no_poll_steal), VCPU_STAT("halt_wakeup", halt_wakeup), + VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), + VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), VCPU_STAT("instruction_lctlg", instruction_lctlg), VCPU_STAT("instruction_lctl", instruction_lctl), VCPU_STAT("instruction_stctl", instruction_stctl), diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index a3ce391c9e7a..fd78bd44b2d6 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1031,6 +1031,8 @@ struct kvm_vcpu_stat { u64 irq_injections; u64 nmi_injections; u64 req_event; + u64 halt_poll_success_ns; + u64 halt_poll_fail_ns; }; struct x86_instruction_info; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 7fb7c1a8f1a5..471fccf7f850 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -217,6 +217,8 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { VCPU_STAT("nmi_injections", nmi_injections), VCPU_STAT("req_event", req_event), VCPU_STAT("l1d_flush", l1d_flush), + VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), + VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), VM_STAT("mmu_shadow_zapped", mmu_shadow_zapped), VM_STAT("mmu_pte_write", mmu_pte_write), VM_STAT("mmu_pte_updated", mmu_pte_updated), diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 11844fad60fd..da6da386b591 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2667,18 +2667,27 @@ out: return ret; } +static inline void +update_halt_poll_stats(struct kvm_vcpu *vcpu, u64 poll_ns, bool waited) +{ + if (waited) + vcpu->stat.halt_poll_fail_ns += poll_ns; + else + vcpu->stat.halt_poll_success_ns += poll_ns; +} + /* * The vCPU has executed a HLT instruction with in-kernel mode enabled. */ void kvm_vcpu_block(struct kvm_vcpu *vcpu) { - ktime_t start, cur; + ktime_t start, cur, poll_end; bool waited = false; u64 block_ns; kvm_arch_vcpu_blocking(vcpu); - start = cur = ktime_get(); + start = cur = poll_end = ktime_get(); if (vcpu->halt_poll_ns && !kvm_arch_no_poll(vcpu)) { ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns); @@ -2694,7 +2703,7 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) ++vcpu->stat.halt_poll_invalid; goto out; } - cur = ktime_get(); + poll_end = cur = ktime_get(); } while (single_task_running() && ktime_before(cur, stop)); } @@ -2714,6 +2723,9 @@ out: kvm_arch_vcpu_unblocking(vcpu); block_ns = ktime_to_ns(cur) - ktime_to_ns(start); + update_halt_poll_stats( + vcpu, ktime_to_ns(ktime_sub(poll_end, start)), waited); + if (!kvm_arch_no_poll(vcpu)) { if (!vcpu_valid_wakeup(vcpu)) { shrink_halt_poll_ns(vcpu); From 4a38aed2a0a729ccecd84dca5b76d827b9e1294d Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 14 May 2020 17:21:15 -0600 Subject: [PATCH 0515/1043] io_uring: batch reap of dead file registrations We currently embed and queue a work item per fixed_file_ref_node that we update, but if the workload does a lot of these, then the associated kworker-events overhead can become quite noticeable. Since we rarely need to wait on these, batch them at 1 second intervals instead. If we do need to wait for them, we just flush the pending delayed work. Signed-off-by: Jens Axboe --- fs/io_uring.c | 54 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 414e940323d4..99dbd43442f2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -191,7 +191,7 @@ struct fixed_file_ref_node { struct list_head node; struct list_head file_list; struct fixed_file_data *file_data; - struct work_struct work; + struct llist_node llist; }; struct fixed_file_data { @@ -327,6 +327,9 @@ struct io_ring_ctx { struct list_head inflight_list; } ____cacheline_aligned_in_smp; + struct delayed_work file_put_work; + struct llist_head file_put_llist; + struct work_struct exit_work; }; @@ -879,6 +882,8 @@ struct sock *io_uring_get_socket(struct file *file) } EXPORT_SYMBOL(io_uring_get_socket); +static void io_file_put_work(struct work_struct *work); + static void io_ring_ctx_ref_free(struct percpu_ref *ref) { struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); @@ -934,6 +939,8 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p) init_waitqueue_head(&ctx->inflight_wait); spin_lock_init(&ctx->inflight_lock); INIT_LIST_HEAD(&ctx->inflight_list); + INIT_DELAYED_WORK(&ctx->file_put_work, io_file_put_work); + init_llist_head(&ctx->file_put_llist); return ctx; err: if (ctx->fallback_req) @@ -6190,6 +6197,7 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) percpu_ref_kill(&data->refs); /* wait for all refs nodes to complete */ + flush_delayed_work(&ctx->file_put_work); wait_for_completion(&data->done); __io_sqe_files_unregister(ctx); @@ -6420,18 +6428,13 @@ struct io_file_put { struct file *file; }; -static void io_file_put_work(struct work_struct *work) +static void __io_file_put_work(struct fixed_file_ref_node *ref_node) { - struct fixed_file_ref_node *ref_node; - struct fixed_file_data *file_data; - struct io_ring_ctx *ctx; + struct fixed_file_data *file_data = ref_node->file_data; + struct io_ring_ctx *ctx = file_data->ctx; struct io_file_put *pfile, *tmp; unsigned long flags; - ref_node = container_of(work, struct fixed_file_ref_node, work); - file_data = ref_node->file_data; - ctx = file_data->ctx; - list_for_each_entry_safe(pfile, tmp, &ref_node->file_list, list) { list_del_init(&pfile->list); io_ring_file_put(ctx, pfile->file); @@ -6447,13 +6450,42 @@ static void io_file_put_work(struct work_struct *work) percpu_ref_put(&file_data->refs); } +static void io_file_put_work(struct work_struct *work) +{ + struct io_ring_ctx *ctx; + struct llist_node *node; + + ctx = container_of(work, struct io_ring_ctx, file_put_work.work); + node = llist_del_all(&ctx->file_put_llist); + + while (node) { + struct fixed_file_ref_node *ref_node; + struct llist_node *next = node->next; + + ref_node = llist_entry(node, struct fixed_file_ref_node, llist); + __io_file_put_work(ref_node); + node = next; + } +} + static void io_file_data_ref_zero(struct percpu_ref *ref) { struct fixed_file_ref_node *ref_node; + struct io_ring_ctx *ctx; + bool first_add; + int delay = HZ; ref_node = container_of(ref, struct fixed_file_ref_node, refs); + ctx = ref_node->file_data->ctx; - queue_work(system_wq, &ref_node->work); + if (percpu_ref_is_dying(&ctx->file_data->refs)) + delay = 0; + + first_add = llist_add(&ref_node->llist, &ctx->file_put_llist); + if (!delay) + mod_delayed_work(system_wq, &ctx->file_put_work, 0); + else if (first_add) + queue_delayed_work(system_wq, &ctx->file_put_work, delay); } static struct fixed_file_ref_node *alloc_fixed_file_ref_node( @@ -6472,10 +6504,8 @@ static struct fixed_file_ref_node *alloc_fixed_file_ref_node( } INIT_LIST_HEAD(&ref_node->node); INIT_LIST_HEAD(&ref_node->file_list); - INIT_WORK(&ref_node->work, io_file_put_work); ref_node->file_data = ctx->file_data; return ref_node; - } static void destroy_fixed_file_ref_node(struct fixed_file_ref_node *ref_node) From 18bceab101adde8f38de76016bc77f3f25cf22f4 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 15 May 2020 11:56:54 -0600 Subject: [PATCH 0516/1043] io_uring: allow POLL_ADD with double poll_wait() users Some file descriptors use separate waitqueues for their f_ops->poll() handler, most commonly one for read and one for write. The io_uring poll implementation doesn't work with that, as the 2nd poll_wait() call will cause the io_uring poll request to -EINVAL. This affects (at least) tty devices and /dev/random as well. This is a big problem for event loops where some file descriptors work, and others don't. With this fix, io_uring handles multiple waitqueues. Signed-off-by: Jens Axboe --- fs/io_uring.c | 218 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 146 insertions(+), 72 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 99dbd43442f2..982066844c5a 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4099,27 +4099,6 @@ struct io_poll_table { int error; }; -static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, - struct wait_queue_head *head) -{ - if (unlikely(poll->head)) { - pt->error = -EINVAL; - return; - } - - pt->error = 0; - poll->head = head; - add_wait_queue(head, &poll->wait); -} - -static void io_async_queue_proc(struct file *file, struct wait_queue_head *head, - struct poll_table_struct *p) -{ - struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); - - __io_queue_proc(&pt->req->apoll->poll, pt, head); -} - static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll, __poll_t mask, task_work_func_t func) { @@ -4171,6 +4150,144 @@ static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll) return false; } +static void io_poll_remove_double(struct io_kiocb *req) +{ + struct io_poll_iocb *poll = (struct io_poll_iocb *) req->io; + + lockdep_assert_held(&req->ctx->completion_lock); + + if (poll && poll->head) { + struct wait_queue_head *head = poll->head; + + spin_lock(&head->lock); + list_del_init(&poll->wait.entry); + if (poll->wait.private) + refcount_dec(&req->refs); + poll->head = NULL; + spin_unlock(&head->lock); + } +} + +static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error) +{ + struct io_ring_ctx *ctx = req->ctx; + + io_poll_remove_double(req); + req->poll.done = true; + io_cqring_fill_event(req, error ? error : mangle_poll(mask)); + io_commit_cqring(ctx); +} + +static void io_poll_task_handler(struct io_kiocb *req, struct io_kiocb **nxt) +{ + struct io_ring_ctx *ctx = req->ctx; + + if (io_poll_rewait(req, &req->poll)) { + spin_unlock_irq(&ctx->completion_lock); + return; + } + + hash_del(&req->hash_node); + io_poll_complete(req, req->result, 0); + req->flags |= REQ_F_COMP_LOCKED; + io_put_req_find_next(req, nxt); + spin_unlock_irq(&ctx->completion_lock); + + io_cqring_ev_posted(ctx); +} + +static void io_poll_task_func(struct callback_head *cb) +{ + struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); + struct io_kiocb *nxt = NULL; + + io_poll_task_handler(req, &nxt); + if (nxt) { + struct io_ring_ctx *ctx = nxt->ctx; + + mutex_lock(&ctx->uring_lock); + __io_queue_sqe(nxt, NULL); + mutex_unlock(&ctx->uring_lock); + } +} + +static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode, + int sync, void *key) +{ + struct io_kiocb *req = wait->private; + struct io_poll_iocb *poll = (struct io_poll_iocb *) req->io; + __poll_t mask = key_to_poll(key); + + /* for instances that support it check for an event match first: */ + if (mask && !(mask & poll->events)) + return 0; + + if (req->poll.head) { + bool done; + + spin_lock(&req->poll.head->lock); + done = list_empty(&req->poll.wait.entry); + if (!done) + list_del_init(&req->poll.wait.entry); + spin_unlock(&req->poll.head->lock); + if (!done) + __io_async_wake(req, poll, mask, io_poll_task_func); + } + refcount_dec(&req->refs); + return 1; +} + +static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events, + wait_queue_func_t wake_func) +{ + poll->head = NULL; + poll->done = false; + poll->canceled = false; + poll->events = events; + INIT_LIST_HEAD(&poll->wait.entry); + init_waitqueue_func_entry(&poll->wait, wake_func); +} + +static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt, + struct wait_queue_head *head) +{ + struct io_kiocb *req = pt->req; + + /* + * If poll->head is already set, it's because the file being polled + * uses multiple waitqueues for poll handling (eg one for read, one + * for write). Setup a separate io_poll_iocb if this happens. + */ + if (unlikely(poll->head)) { + /* already have a 2nd entry, fail a third attempt */ + if (req->io) { + pt->error = -EINVAL; + return; + } + poll = kmalloc(sizeof(*poll), GFP_ATOMIC); + if (!poll) { + pt->error = -ENOMEM; + return; + } + io_init_poll_iocb(poll, req->poll.events, io_poll_double_wake); + refcount_inc(&req->refs); + poll->wait.private = req; + req->io = (void *) poll; + } + + pt->error = 0; + poll->head = head; + add_wait_queue(head, &poll->wait); +} + +static void io_async_queue_proc(struct file *file, struct wait_queue_head *head, + struct poll_table_struct *p) +{ + struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); + + __io_queue_proc(&pt->req->apoll->poll, pt, head); +} + static void io_async_task_func(struct callback_head *cb) { struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); @@ -4246,18 +4363,13 @@ static __poll_t __io_arm_poll_handler(struct io_kiocb *req, bool cancel = false; poll->file = req->file; - poll->head = NULL; - poll->done = poll->canceled = false; - poll->events = mask; + io_init_poll_iocb(poll, mask, wake_func); + poll->wait.private = req; ipt->pt._key = mask; ipt->req = req; ipt->error = -EINVAL; - INIT_LIST_HEAD(&poll->wait.entry); - init_waitqueue_func_entry(&poll->wait, wake_func); - poll->wait.private = req; - mask = vfs_poll(req->file, &ipt->pt) & poll->events; spin_lock_irq(&ctx->completion_lock); @@ -4288,6 +4400,7 @@ static bool io_arm_poll_handler(struct io_kiocb *req) struct async_poll *apoll; struct io_poll_table ipt; __poll_t mask, ret; + bool had_io; if (!req->file || !file_can_poll(req->file)) return false; @@ -4302,6 +4415,7 @@ static bool io_arm_poll_handler(struct io_kiocb *req) req->flags |= REQ_F_POLLED; memcpy(&apoll->work, &req->work, sizeof(req->work)); + had_io = req->io != NULL; get_task_struct(current); req->task = current; @@ -4321,7 +4435,9 @@ static bool io_arm_poll_handler(struct io_kiocb *req) io_async_wake); if (ret) { ipt.error = 0; - apoll->poll.done = true; + /* only remove double add if we did it here */ + if (!had_io) + io_poll_remove_double(req); spin_unlock_irq(&ctx->completion_lock); memcpy(&req->work, &apoll->work, sizeof(req->work)); kfree(apoll); @@ -4354,6 +4470,7 @@ static bool io_poll_remove_one(struct io_kiocb *req) bool do_complete; if (req->opcode == IORING_OP_POLL_ADD) { + io_poll_remove_double(req); do_complete = __io_poll_remove_one(req, &req->poll); } else { apoll = req->apoll; @@ -4455,49 +4572,6 @@ static int io_poll_remove(struct io_kiocb *req) return 0; } -static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error) -{ - struct io_ring_ctx *ctx = req->ctx; - - req->poll.done = true; - io_cqring_fill_event(req, error ? error : mangle_poll(mask)); - io_commit_cqring(ctx); -} - -static void io_poll_task_handler(struct io_kiocb *req, struct io_kiocb **nxt) -{ - struct io_ring_ctx *ctx = req->ctx; - struct io_poll_iocb *poll = &req->poll; - - if (io_poll_rewait(req, poll)) { - spin_unlock_irq(&ctx->completion_lock); - return; - } - - hash_del(&req->hash_node); - io_poll_complete(req, req->result, 0); - req->flags |= REQ_F_COMP_LOCKED; - io_put_req_find_next(req, nxt); - spin_unlock_irq(&ctx->completion_lock); - - io_cqring_ev_posted(ctx); -} - -static void io_poll_task_func(struct callback_head *cb) -{ - struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); - struct io_kiocb *nxt = NULL; - - io_poll_task_handler(req, &nxt); - if (nxt) { - struct io_ring_ctx *ctx = nxt->ctx; - - mutex_lock(&ctx->uring_lock); - __io_queue_sqe(nxt, NULL); - mutex_unlock(&ctx->uring_lock); - } -} - static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, void *key) { From 0d9b5b3af134cddfdc1dd31d41946a0ad389bbf2 Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Fri, 15 May 2020 18:38:04 +0200 Subject: [PATCH 0517/1043] io_uring: add 'cq_flags' field for the CQ ring This patch adds the new 'cq_flags' field that should be written by the application and read by the kernel. This new field is available to the userspace application through 'cq_off.flags'. We are using 4-bytes previously reserved and set to zero. This means that if the application finds this field to zero, then the new functionality is not supported. In the next patch we will introduce the first flag available. Signed-off-by: Stefano Garzarella Signed-off-by: Jens Axboe --- fs/io_uring.c | 10 +++++++++- include/uapi/linux/io_uring.h | 4 +++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 982066844c5a..02250693a406 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -142,7 +142,7 @@ struct io_rings { */ u32 sq_dropped; /* - * Runtime flags + * Runtime SQ flags * * Written by the kernel, shouldn't be modified by the * application. @@ -151,6 +151,13 @@ struct io_rings { * for IORING_SQ_NEED_WAKEUP after updating the sq tail. */ u32 sq_flags; + /* + * Runtime CQ flags + * + * Written by the application, shouldn't be modified by the + * kernel. + */ + u32 cq_flags; /* * Number of completion events lost because the queue was full; * this should be avoided by the application by making sure @@ -7930,6 +7937,7 @@ static int io_uring_create(unsigned entries, struct io_uring_params *p, p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries); p->cq_off.overflow = offsetof(struct io_rings, cq_overflow); p->cq_off.cqes = offsetof(struct io_rings, cqes); + p->cq_off.flags = offsetof(struct io_rings, cq_flags); p->features = IORING_FEAT_SINGLE_MMAP | IORING_FEAT_NODROP | IORING_FEAT_SUBMIT_STABLE | IORING_FEAT_RW_CUR_POS | diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index e48d746b8e2a..602bb0ece607 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -204,7 +204,9 @@ struct io_cqring_offsets { __u32 ring_entries; __u32 overflow; __u32 cqes; - __u64 resv[2]; + __u32 flags; + __u32 resv1; + __u64 resv2; }; /* From 7e55a19cf6e70ce08964b46dbbfbdb07fbc995fc Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Fri, 15 May 2020 18:38:05 +0200 Subject: [PATCH 0518/1043] io_uring: add IORING_CQ_EVENTFD_DISABLED to the CQ ring flags This new flag should be set/clear from the application to disable/enable eventfd notifications when a request is completed and queued to the CQ ring. Before this patch, notifications were always sent if an eventfd is registered, so IORING_CQ_EVENTFD_DISABLED is not set during the initialization. It will be up to the application to set the flag after initialization if no notifications are required at the beginning. Signed-off-by: Stefano Garzarella Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 ++ include/uapi/linux/io_uring.h | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/fs/io_uring.c b/fs/io_uring.c index 02250693a406..f800b0b4498f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1153,6 +1153,8 @@ static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx) { if (!ctx->cq_ev_fd) return false; + if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED) + return false; if (!ctx->eventfd_async) return true; return io_wq_current_is_worker(); diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 602bb0ece607..8c5775df08b8 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -209,6 +209,13 @@ struct io_cqring_offsets { __u64 resv2; }; +/* + * cq_ring->flags + */ + +/* disable eventfd notifications */ +#define IORING_CQ_EVENTFD_DISABLED (1U << 0) + /* * io_uring_enter(2) flags */ From 6a4d07cde5778174a35ffc445c1d1388479563ee Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 15 May 2020 14:30:38 -0600 Subject: [PATCH 0519/1043] io_uring: file registration list and lock optimization There's no point in using list_del_init() on entries that are going away, and the associated lock is always used in process context so let's not use the IRQ disabling+saving variant of the spinlock. Signed-off-by: Jens Axboe --- fs/io_uring.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index f800b0b4498f..c7622a5ece2d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6264,16 +6264,15 @@ static int io_sqe_files_unregister(struct io_ring_ctx *ctx) struct fixed_file_data *data = ctx->file_data; struct fixed_file_ref_node *ref_node = NULL; unsigned nr_tables, i; - unsigned long flags; if (!data) return -ENXIO; - spin_lock_irqsave(&data->lock, flags); + spin_lock(&data->lock); if (!list_empty(&data->ref_list)) ref_node = list_first_entry(&data->ref_list, struct fixed_file_ref_node, node); - spin_unlock_irqrestore(&data->lock, flags); + spin_unlock(&data->lock); if (ref_node) percpu_ref_kill(&ref_node->refs); @@ -6516,17 +6515,16 @@ static void __io_file_put_work(struct fixed_file_ref_node *ref_node) struct fixed_file_data *file_data = ref_node->file_data; struct io_ring_ctx *ctx = file_data->ctx; struct io_file_put *pfile, *tmp; - unsigned long flags; list_for_each_entry_safe(pfile, tmp, &ref_node->file_list, list) { - list_del_init(&pfile->list); + list_del(&pfile->list); io_ring_file_put(ctx, pfile->file); kfree(pfile); } - spin_lock_irqsave(&file_data->lock, flags); - list_del_init(&ref_node->node); - spin_unlock_irqrestore(&file_data->lock, flags); + spin_lock(&file_data->lock); + list_del(&ref_node->node); + spin_unlock(&file_data->lock); percpu_ref_exit(&ref_node->refs); kfree(ref_node); @@ -6606,7 +6604,6 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, int fd, ret = 0; unsigned i; struct fixed_file_ref_node *ref_node; - unsigned long flags; if (ctx->file_data) return -EBUSY; @@ -6714,9 +6711,9 @@ static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg, } ctx->file_data->cur_refs = &ref_node->refs; - spin_lock_irqsave(&ctx->file_data->lock, flags); + spin_lock(&ctx->file_data->lock); list_add(&ref_node->node, &ctx->file_data->ref_list); - spin_unlock_irqrestore(&ctx->file_data->lock, flags); + spin_unlock(&ctx->file_data->lock); percpu_ref_get(&ctx->file_data->refs); return ret; } @@ -6792,7 +6789,6 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, __s32 __user *fds; int fd, i, err; __u32 done; - unsigned long flags; bool needs_switch = false; if (check_add_overflow(up->offset, nr_args, &done)) @@ -6857,10 +6853,10 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx, if (needs_switch) { percpu_ref_kill(data->cur_refs); - spin_lock_irqsave(&data->lock, flags); + spin_lock(&data->lock); list_add(&ref_node->node, &data->ref_list); data->cur_refs = &ref_node->refs; - spin_unlock_irqrestore(&data->lock, flags); + spin_unlock(&data->lock); percpu_ref_get(&ctx->file_data->refs); } else destroy_fixed_file_ref_node(ref_node); From 9ed24f4b712b855dcf7be3025b75b051cb73a2b7 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 13 May 2020 11:40:34 +0100 Subject: [PATCH 0520/1043] KVM: arm64: Move virt/kvm/arm to arch/arm64 Now that the 32bit KVM/arm host is a distant memory, let's move the whole of the KVM/arm64 code into the arm64 tree. As they said in the song: Welcome Home (Sanitarium). Signed-off-by: Marc Zyngier Acked-by: Will Deacon Link: https://lore.kernel.org/r/20200513104034.74741-1-maz@kernel.org --- MAINTAINERS | 1 - arch/arm64/kvm/Makefile | 44 ++-- {virt/kvm/arm => arch/arm64/kvm}/aarch32.c | 0 {virt/kvm/arm => arch/arm64/kvm}/arch_timer.c | 0 {virt/kvm/arm => arch/arm64/kvm}/arm.c | 2 +- arch/arm64/kvm/handle_exit.c | 2 +- arch/arm64/kvm/hyp/Makefile | 9 +- .../kvm/arm => arch/arm64/kvm}/hyp/aarch32.c | 0 .../kvm/arm => arch/arm64/kvm}/hyp/timer-sr.c | 0 .../arm => arch/arm64/kvm}/hyp/vgic-v3-sr.c | 4 - {virt/kvm/arm => arch/arm64/kvm}/hypercalls.c | 0 {virt/kvm/arm => arch/arm64/kvm}/mmio.c | 0 {virt/kvm/arm => arch/arm64/kvm}/mmu.c | 0 {virt/kvm/arm => arch/arm64/kvm}/perf.c | 0 .../arm/pmu.c => arch/arm64/kvm/pmu-emul.c | 0 {virt/kvm/arm => arch/arm64/kvm}/psci.c | 0 {virt/kvm/arm => arch/arm64/kvm}/pvtime.c | 0 arch/arm64/kvm/trace.h | 216 +----------------- .../arm/trace.h => arch/arm64/kvm/trace_arm.h | 11 +- arch/arm64/kvm/trace_handle_exit.h | 215 +++++++++++++++++ arch/arm64/kvm/vgic-sys-reg-v3.c | 2 +- {virt/kvm/arm => arch/arm64/kvm}/vgic/trace.h | 2 +- .../arm => arch/arm64/kvm}/vgic/vgic-debug.c | 0 .../arm => arch/arm64/kvm}/vgic/vgic-init.c | 0 .../arm => arch/arm64/kvm}/vgic/vgic-irqfd.c | 0 .../arm => arch/arm64/kvm}/vgic/vgic-its.c | 0 .../arm64/kvm}/vgic/vgic-kvm-device.c | 0 .../arm64/kvm}/vgic/vgic-mmio-v2.c | 0 .../arm64/kvm}/vgic/vgic-mmio-v3.c | 0 .../arm => arch/arm64/kvm}/vgic/vgic-mmio.c | 0 .../arm => arch/arm64/kvm}/vgic/vgic-mmio.h | 0 .../kvm/arm => arch/arm64/kvm}/vgic/vgic-v2.c | 0 .../kvm/arm => arch/arm64/kvm}/vgic/vgic-v3.c | 2 - .../kvm/arm => arch/arm64/kvm}/vgic/vgic-v4.c | 0 {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic.c | 0 {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic.h | 0 36 files changed, 253 insertions(+), 257 deletions(-) rename {virt/kvm/arm => arch/arm64/kvm}/aarch32.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/arch_timer.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/arm.c (99%) rename {virt/kvm/arm => arch/arm64/kvm}/hyp/aarch32.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/hyp/timer-sr.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/hyp/vgic-v3-sr.c (99%) rename {virt/kvm/arm => arch/arm64/kvm}/hypercalls.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/mmio.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/mmu.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/perf.c (100%) rename virt/kvm/arm/pmu.c => arch/arm64/kvm/pmu-emul.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/psci.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/pvtime.c (100%) rename virt/kvm/arm/trace.h => arch/arm64/kvm/trace_arm.h (97%) create mode 100644 arch/arm64/kvm/trace_handle_exit.h rename {virt/kvm/arm => arch/arm64/kvm}/vgic/trace.h (93%) rename {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic-debug.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic-init.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic-irqfd.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic-its.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic-kvm-device.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic-mmio-v2.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic-mmio-v3.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic-mmio.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic-mmio.h (100%) rename {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic-v2.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic-v3.c (99%) rename {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic-v4.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic.c (100%) rename {virt/kvm/arm => arch/arm64/kvm}/vgic/vgic.h (100%) diff --git a/MAINTAINERS b/MAINTAINERS index 091ec22c1a23..6c5b928989ed 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9295,7 +9295,6 @@ F: arch/arm64/include/asm/kvm* F: arch/arm64/include/uapi/asm/kvm* F: arch/arm64/kvm/ F: include/kvm/arm_* -F: virt/kvm/arm/ KERNEL VIRTUAL MACHINE FOR MIPS (KVM/mips) L: linux-mips@vger.kernel.org diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 5ffbdc39e780..7a3768538343 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -3,37 +3,37 @@ # Makefile for Kernel-based Virtual Machine module # -ccflags-y += -I $(srctree)/$(src) -I $(srctree)/virt/kvm/arm/vgic +ccflags-y += -I $(srctree)/$(src) KVM=../../../virt/kvm obj-$(CONFIG_KVM_ARM_HOST) += kvm.o obj-$(CONFIG_KVM_ARM_HOST) += hyp/ -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o $(KVM)/vfio.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arm.o $(KVM)/arm/mmu.o $(KVM)/arm/mmio.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/psci.o $(KVM)/arm/perf.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/hypercalls.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/pvtime.o +kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o +kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/eventfd.o $(KVM)/vfio.o $(KVM)/irqchip.o +kvm-$(CONFIG_KVM_ARM_HOST) += arm.o mmu.o mmio.o +kvm-$(CONFIG_KVM_ARM_HOST) += psci.o perf.o +kvm-$(CONFIG_KVM_ARM_HOST) += hypercalls.o +kvm-$(CONFIG_KVM_ARM_HOST) += pvtime.o kvm-$(CONFIG_KVM_ARM_HOST) += inject_fault.o regmap.o va_layout.o kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o kvm-$(CONFIG_KVM_ARM_HOST) += guest.o debug.o reset.o sys_regs.o sys_regs_generic_v8.o kvm-$(CONFIG_KVM_ARM_HOST) += vgic-sys-reg-v3.o fpsimd.o pmu.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/aarch32.o +kvm-$(CONFIG_KVM_ARM_HOST) += aarch32.o +kvm-$(CONFIG_KVM_ARM_HOST) += arch_timer.o +kvm-$(CONFIG_KVM_ARM_PMU) += pmu-emul.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-init.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-irqfd.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-v2.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-v3.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-v4.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-mmio.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-mmio-v2.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-mmio-v3.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-kvm-device.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-its.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic/vgic-debug.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/irqchip.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o -kvm-$(CONFIG_KVM_ARM_PMU) += $(KVM)/arm/pmu.o +kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic.o +kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-init.o +kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-irqfd.o +kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-v2.o +kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-v3.o +kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-v4.o +kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-mmio.o +kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-mmio-v2.o +kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-mmio-v3.o +kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-kvm-device.o +kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-its.o +kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-debug.o diff --git a/virt/kvm/arm/aarch32.c b/arch/arm64/kvm/aarch32.c similarity index 100% rename from virt/kvm/arm/aarch32.c rename to arch/arm64/kvm/aarch32.c diff --git a/virt/kvm/arm/arch_timer.c b/arch/arm64/kvm/arch_timer.c similarity index 100% rename from virt/kvm/arm/arch_timer.c rename to arch/arm64/kvm/arch_timer.c diff --git a/virt/kvm/arm/arm.c b/arch/arm64/kvm/arm.c similarity index 99% rename from virt/kvm/arm/arm.c rename to arch/arm64/kvm/arm.c index 48d0ec44ad77..c958bb37b769 100644 --- a/virt/kvm/arm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -22,7 +22,7 @@ #include #define CREATE_TRACE_POINTS -#include "trace.h" +#include "trace_arm.h" #include #include diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index aacfc55de44c..eb194696ef62 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -23,7 +23,7 @@ #include #define CREATE_TRACE_POINTS -#include "trace.h" +#include "trace_handle_exit.h" typedef int (*exit_handle_fn)(struct kvm_vcpu *, struct kvm_run *); diff --git a/arch/arm64/kvm/hyp/Makefile b/arch/arm64/kvm/hyp/Makefile index ea710f674cb6..dc18274a6826 100644 --- a/arch/arm64/kvm/hyp/Makefile +++ b/arch/arm64/kvm/hyp/Makefile @@ -6,12 +6,9 @@ ccflags-y += -fno-stack-protector -DDISABLE_BRANCH_PROFILING \ $(DISABLE_STACKLEAK_PLUGIN) -KVM=../../../../virt/kvm - -obj-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/hyp/vgic-v3-sr.o -obj-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/hyp/timer-sr.o -obj-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/hyp/aarch32.o - +obj-$(CONFIG_KVM_ARM_HOST) += vgic-v3-sr.o +obj-$(CONFIG_KVM_ARM_HOST) += timer-sr.o +obj-$(CONFIG_KVM_ARM_HOST) += aarch32.o obj-$(CONFIG_KVM_ARM_HOST) += vgic-v2-cpuif-proxy.o obj-$(CONFIG_KVM_ARM_HOST) += sysreg-sr.o obj-$(CONFIG_KVM_ARM_HOST) += debug-sr.o diff --git a/virt/kvm/arm/hyp/aarch32.c b/arch/arm64/kvm/hyp/aarch32.c similarity index 100% rename from virt/kvm/arm/hyp/aarch32.c rename to arch/arm64/kvm/hyp/aarch32.c diff --git a/virt/kvm/arm/hyp/timer-sr.c b/arch/arm64/kvm/hyp/timer-sr.c similarity index 100% rename from virt/kvm/arm/hyp/timer-sr.c rename to arch/arm64/kvm/hyp/timer-sr.c diff --git a/virt/kvm/arm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c similarity index 99% rename from virt/kvm/arm/hyp/vgic-v3-sr.c rename to arch/arm64/kvm/hyp/vgic-v3-sr.c index ccf1fde9836c..49fedf6710f9 100644 --- a/virt/kvm/arm/hyp/vgic-v3-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c @@ -431,8 +431,6 @@ void __hyp_text __vgic_v3_write_vmcr(u32 vmcr) write_gicreg(vmcr, ICH_VMCR_EL2); } -#ifdef CONFIG_ARM64 - static int __hyp_text __vgic_v3_bpr_min(void) { /* See Pseudocode for VPriorityGroup */ @@ -1126,5 +1124,3 @@ int __hyp_text __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu) return 1; } - -#endif diff --git a/virt/kvm/arm/hypercalls.c b/arch/arm64/kvm/hypercalls.c similarity index 100% rename from virt/kvm/arm/hypercalls.c rename to arch/arm64/kvm/hypercalls.c diff --git a/virt/kvm/arm/mmio.c b/arch/arm64/kvm/mmio.c similarity index 100% rename from virt/kvm/arm/mmio.c rename to arch/arm64/kvm/mmio.c diff --git a/virt/kvm/arm/mmu.c b/arch/arm64/kvm/mmu.c similarity index 100% rename from virt/kvm/arm/mmu.c rename to arch/arm64/kvm/mmu.c diff --git a/virt/kvm/arm/perf.c b/arch/arm64/kvm/perf.c similarity index 100% rename from virt/kvm/arm/perf.c rename to arch/arm64/kvm/perf.c diff --git a/virt/kvm/arm/pmu.c b/arch/arm64/kvm/pmu-emul.c similarity index 100% rename from virt/kvm/arm/pmu.c rename to arch/arm64/kvm/pmu-emul.c diff --git a/virt/kvm/arm/psci.c b/arch/arm64/kvm/psci.c similarity index 100% rename from virt/kvm/arm/psci.c rename to arch/arm64/kvm/psci.c diff --git a/virt/kvm/arm/pvtime.c b/arch/arm64/kvm/pvtime.c similarity index 100% rename from virt/kvm/arm/pvtime.c rename to arch/arm64/kvm/pvtime.c diff --git a/arch/arm64/kvm/trace.h b/arch/arm64/kvm/trace.h index eab91ad0effb..86f9ea47be29 100644 --- a/arch/arm64/kvm/trace.h +++ b/arch/arm64/kvm/trace.h @@ -1,216 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#if !defined(_TRACE_ARM64_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#ifndef _TRACE_ARM64_KVM_H #define _TRACE_ARM64_KVM_H -#include -#include "sys_regs.h" +#include "trace_arm.h" +#include "trace_handle_exit.h" -#undef TRACE_SYSTEM -#define TRACE_SYSTEM kvm - -TRACE_EVENT(kvm_wfx_arm64, - TP_PROTO(unsigned long vcpu_pc, bool is_wfe), - TP_ARGS(vcpu_pc, is_wfe), - - TP_STRUCT__entry( - __field(unsigned long, vcpu_pc) - __field(bool, is_wfe) - ), - - TP_fast_assign( - __entry->vcpu_pc = vcpu_pc; - __entry->is_wfe = is_wfe; - ), - - TP_printk("guest executed wf%c at: 0x%08lx", - __entry->is_wfe ? 'e' : 'i', __entry->vcpu_pc) -); - -TRACE_EVENT(kvm_hvc_arm64, - TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm), - TP_ARGS(vcpu_pc, r0, imm), - - TP_STRUCT__entry( - __field(unsigned long, vcpu_pc) - __field(unsigned long, r0) - __field(unsigned long, imm) - ), - - TP_fast_assign( - __entry->vcpu_pc = vcpu_pc; - __entry->r0 = r0; - __entry->imm = imm; - ), - - TP_printk("HVC at 0x%08lx (r0: 0x%08lx, imm: 0x%lx)", - __entry->vcpu_pc, __entry->r0, __entry->imm) -); - -TRACE_EVENT(kvm_arm_setup_debug, - TP_PROTO(struct kvm_vcpu *vcpu, __u32 guest_debug), - TP_ARGS(vcpu, guest_debug), - - TP_STRUCT__entry( - __field(struct kvm_vcpu *, vcpu) - __field(__u32, guest_debug) - ), - - TP_fast_assign( - __entry->vcpu = vcpu; - __entry->guest_debug = guest_debug; - ), - - TP_printk("vcpu: %p, flags: 0x%08x", __entry->vcpu, __entry->guest_debug) -); - -TRACE_EVENT(kvm_arm_clear_debug, - TP_PROTO(__u32 guest_debug), - TP_ARGS(guest_debug), - - TP_STRUCT__entry( - __field(__u32, guest_debug) - ), - - TP_fast_assign( - __entry->guest_debug = guest_debug; - ), - - TP_printk("flags: 0x%08x", __entry->guest_debug) -); - -TRACE_EVENT(kvm_arm_set_dreg32, - TP_PROTO(const char *name, __u32 value), - TP_ARGS(name, value), - - TP_STRUCT__entry( - __field(const char *, name) - __field(__u32, value) - ), - - TP_fast_assign( - __entry->name = name; - __entry->value = value; - ), - - TP_printk("%s: 0x%08x", __entry->name, __entry->value) -); - -TRACE_DEFINE_SIZEOF(__u64); - -TRACE_EVENT(kvm_arm_set_regset, - TP_PROTO(const char *type, int len, __u64 *control, __u64 *value), - TP_ARGS(type, len, control, value), - TP_STRUCT__entry( - __field(const char *, name) - __field(int, len) - __array(u64, ctrls, 16) - __array(u64, values, 16) - ), - TP_fast_assign( - __entry->name = type; - __entry->len = len; - memcpy(__entry->ctrls, control, len << 3); - memcpy(__entry->values, value, len << 3); - ), - TP_printk("%d %s CTRL:%s VALUE:%s", __entry->len, __entry->name, - __print_array(__entry->ctrls, __entry->len, sizeof(__u64)), - __print_array(__entry->values, __entry->len, sizeof(__u64))) -); - -TRACE_EVENT(trap_reg, - TP_PROTO(const char *fn, int reg, bool is_write, u64 write_value), - TP_ARGS(fn, reg, is_write, write_value), - - TP_STRUCT__entry( - __field(const char *, fn) - __field(int, reg) - __field(bool, is_write) - __field(u64, write_value) - ), - - TP_fast_assign( - __entry->fn = fn; - __entry->reg = reg; - __entry->is_write = is_write; - __entry->write_value = write_value; - ), - - TP_printk("%s %s reg %d (0x%08llx)", __entry->fn, __entry->is_write?"write to":"read from", __entry->reg, __entry->write_value) -); - -TRACE_EVENT(kvm_handle_sys_reg, - TP_PROTO(unsigned long hsr), - TP_ARGS(hsr), - - TP_STRUCT__entry( - __field(unsigned long, hsr) - ), - - TP_fast_assign( - __entry->hsr = hsr; - ), - - TP_printk("HSR 0x%08lx", __entry->hsr) -); - -TRACE_EVENT(kvm_sys_access, - TP_PROTO(unsigned long vcpu_pc, struct sys_reg_params *params, const struct sys_reg_desc *reg), - TP_ARGS(vcpu_pc, params, reg), - - TP_STRUCT__entry( - __field(unsigned long, vcpu_pc) - __field(bool, is_write) - __field(const char *, name) - __field(u8, Op0) - __field(u8, Op1) - __field(u8, CRn) - __field(u8, CRm) - __field(u8, Op2) - ), - - TP_fast_assign( - __entry->vcpu_pc = vcpu_pc; - __entry->is_write = params->is_write; - __entry->name = reg->name; - __entry->Op0 = reg->Op0; - __entry->Op0 = reg->Op0; - __entry->Op1 = reg->Op1; - __entry->CRn = reg->CRn; - __entry->CRm = reg->CRm; - __entry->Op2 = reg->Op2; - ), - - TP_printk("PC: %lx %s (%d,%d,%d,%d,%d) %s", - __entry->vcpu_pc, __entry->name ?: "UNKN", - __entry->Op0, __entry->Op1, __entry->CRn, - __entry->CRm, __entry->Op2, - __entry->is_write ? "write" : "read") -); - -TRACE_EVENT(kvm_set_guest_debug, - TP_PROTO(struct kvm_vcpu *vcpu, __u32 guest_debug), - TP_ARGS(vcpu, guest_debug), - - TP_STRUCT__entry( - __field(struct kvm_vcpu *, vcpu) - __field(__u32, guest_debug) - ), - - TP_fast_assign( - __entry->vcpu = vcpu; - __entry->guest_debug = guest_debug; - ), - - TP_printk("vcpu: %p, flags: 0x%08x", __entry->vcpu, __entry->guest_debug) -); - - -#endif /* _TRACE_ARM64_KVM_H */ - -#undef TRACE_INCLUDE_PATH -#define TRACE_INCLUDE_PATH . -#undef TRACE_INCLUDE_FILE -#define TRACE_INCLUDE_FILE trace - -/* This part must be outside protection */ -#include +#endif /* _TRACE_ARM64_KVM_H */ diff --git a/virt/kvm/arm/trace.h b/arch/arm64/kvm/trace_arm.h similarity index 97% rename from virt/kvm/arm/trace.h rename to arch/arm64/kvm/trace_arm.h index cc94ccc68821..4c71270cc097 100644 --- a/virt/kvm/arm/trace.h +++ b/arch/arm64/kvm/trace_arm.h @@ -1,10 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#if !defined(_TRACE_KVM_H) || defined(TRACE_HEADER_MULTI_READ) -#define _TRACE_KVM_H +#if !defined(_TRACE_ARM_ARM64_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_ARM_ARM64_KVM_H #include #include -#include #undef TRACE_SYSTEM #define TRACE_SYSTEM kvm @@ -368,12 +367,12 @@ TRACE_EVENT(kvm_timer_emulate, __entry->timer_idx, __entry->should_fire) ); -#endif /* _TRACE_KVM_H */ +#endif /* _TRACE_ARM_ARM64_KVM_H */ #undef TRACE_INCLUDE_PATH -#define TRACE_INCLUDE_PATH ../../virt/kvm/arm +#define TRACE_INCLUDE_PATH . #undef TRACE_INCLUDE_FILE -#define TRACE_INCLUDE_FILE trace +#define TRACE_INCLUDE_FILE trace_arm /* This part must be outside protection */ #include diff --git a/arch/arm64/kvm/trace_handle_exit.h b/arch/arm64/kvm/trace_handle_exit.h new file mode 100644 index 000000000000..2c56d1e0f5bd --- /dev/null +++ b/arch/arm64/kvm/trace_handle_exit.h @@ -0,0 +1,215 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(_TRACE_HANDLE_EXIT_ARM64_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_HANDLE_EXIT_ARM64_KVM_H + +#include +#include "sys_regs.h" + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm + +TRACE_EVENT(kvm_wfx_arm64, + TP_PROTO(unsigned long vcpu_pc, bool is_wfe), + TP_ARGS(vcpu_pc, is_wfe), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_pc) + __field(bool, is_wfe) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->is_wfe = is_wfe; + ), + + TP_printk("guest executed wf%c at: 0x%08lx", + __entry->is_wfe ? 'e' : 'i', __entry->vcpu_pc) +); + +TRACE_EVENT(kvm_hvc_arm64, + TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm), + TP_ARGS(vcpu_pc, r0, imm), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_pc) + __field(unsigned long, r0) + __field(unsigned long, imm) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->r0 = r0; + __entry->imm = imm; + ), + + TP_printk("HVC at 0x%08lx (r0: 0x%08lx, imm: 0x%lx)", + __entry->vcpu_pc, __entry->r0, __entry->imm) +); + +TRACE_EVENT(kvm_arm_setup_debug, + TP_PROTO(struct kvm_vcpu *vcpu, __u32 guest_debug), + TP_ARGS(vcpu, guest_debug), + + TP_STRUCT__entry( + __field(struct kvm_vcpu *, vcpu) + __field(__u32, guest_debug) + ), + + TP_fast_assign( + __entry->vcpu = vcpu; + __entry->guest_debug = guest_debug; + ), + + TP_printk("vcpu: %p, flags: 0x%08x", __entry->vcpu, __entry->guest_debug) +); + +TRACE_EVENT(kvm_arm_clear_debug, + TP_PROTO(__u32 guest_debug), + TP_ARGS(guest_debug), + + TP_STRUCT__entry( + __field(__u32, guest_debug) + ), + + TP_fast_assign( + __entry->guest_debug = guest_debug; + ), + + TP_printk("flags: 0x%08x", __entry->guest_debug) +); + +TRACE_EVENT(kvm_arm_set_dreg32, + TP_PROTO(const char *name, __u32 value), + TP_ARGS(name, value), + + TP_STRUCT__entry( + __field(const char *, name) + __field(__u32, value) + ), + + TP_fast_assign( + __entry->name = name; + __entry->value = value; + ), + + TP_printk("%s: 0x%08x", __entry->name, __entry->value) +); + +TRACE_DEFINE_SIZEOF(__u64); + +TRACE_EVENT(kvm_arm_set_regset, + TP_PROTO(const char *type, int len, __u64 *control, __u64 *value), + TP_ARGS(type, len, control, value), + TP_STRUCT__entry( + __field(const char *, name) + __field(int, len) + __array(u64, ctrls, 16) + __array(u64, values, 16) + ), + TP_fast_assign( + __entry->name = type; + __entry->len = len; + memcpy(__entry->ctrls, control, len << 3); + memcpy(__entry->values, value, len << 3); + ), + TP_printk("%d %s CTRL:%s VALUE:%s", __entry->len, __entry->name, + __print_array(__entry->ctrls, __entry->len, sizeof(__u64)), + __print_array(__entry->values, __entry->len, sizeof(__u64))) +); + +TRACE_EVENT(trap_reg, + TP_PROTO(const char *fn, int reg, bool is_write, u64 write_value), + TP_ARGS(fn, reg, is_write, write_value), + + TP_STRUCT__entry( + __field(const char *, fn) + __field(int, reg) + __field(bool, is_write) + __field(u64, write_value) + ), + + TP_fast_assign( + __entry->fn = fn; + __entry->reg = reg; + __entry->is_write = is_write; + __entry->write_value = write_value; + ), + + TP_printk("%s %s reg %d (0x%08llx)", __entry->fn, __entry->is_write?"write to":"read from", __entry->reg, __entry->write_value) +); + +TRACE_EVENT(kvm_handle_sys_reg, + TP_PROTO(unsigned long hsr), + TP_ARGS(hsr), + + TP_STRUCT__entry( + __field(unsigned long, hsr) + ), + + TP_fast_assign( + __entry->hsr = hsr; + ), + + TP_printk("HSR 0x%08lx", __entry->hsr) +); + +TRACE_EVENT(kvm_sys_access, + TP_PROTO(unsigned long vcpu_pc, struct sys_reg_params *params, const struct sys_reg_desc *reg), + TP_ARGS(vcpu_pc, params, reg), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_pc) + __field(bool, is_write) + __field(const char *, name) + __field(u8, Op0) + __field(u8, Op1) + __field(u8, CRn) + __field(u8, CRm) + __field(u8, Op2) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->is_write = params->is_write; + __entry->name = reg->name; + __entry->Op0 = reg->Op0; + __entry->Op0 = reg->Op0; + __entry->Op1 = reg->Op1; + __entry->CRn = reg->CRn; + __entry->CRm = reg->CRm; + __entry->Op2 = reg->Op2; + ), + + TP_printk("PC: %lx %s (%d,%d,%d,%d,%d) %s", + __entry->vcpu_pc, __entry->name ?: "UNKN", + __entry->Op0, __entry->Op1, __entry->CRn, + __entry->CRm, __entry->Op2, + __entry->is_write ? "write" : "read") +); + +TRACE_EVENT(kvm_set_guest_debug, + TP_PROTO(struct kvm_vcpu *vcpu, __u32 guest_debug), + TP_ARGS(vcpu, guest_debug), + + TP_STRUCT__entry( + __field(struct kvm_vcpu *, vcpu) + __field(__u32, guest_debug) + ), + + TP_fast_assign( + __entry->vcpu = vcpu; + __entry->guest_debug = guest_debug; + ), + + TP_printk("vcpu: %p, flags: 0x%08x", __entry->vcpu, __entry->guest_debug) +); + +#endif /* _TRACE_HANDLE_EXIT_ARM64_KVM_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace_handle_exit + +/* This part must be outside protection */ +#include diff --git a/arch/arm64/kvm/vgic-sys-reg-v3.c b/arch/arm64/kvm/vgic-sys-reg-v3.c index e7d1ea92095d..2f92bdcb1188 100644 --- a/arch/arm64/kvm/vgic-sys-reg-v3.c +++ b/arch/arm64/kvm/vgic-sys-reg-v3.c @@ -7,7 +7,7 @@ #include #include #include -#include "vgic.h" +#include "vgic/vgic.h" #include "sys_regs.h" static bool access_gic_ctlr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, diff --git a/virt/kvm/arm/vgic/trace.h b/arch/arm64/kvm/vgic/trace.h similarity index 93% rename from virt/kvm/arm/vgic/trace.h rename to arch/arm64/kvm/vgic/trace.h index 4fd4f6db181b..83c64401a7fc 100644 --- a/virt/kvm/arm/vgic/trace.h +++ b/arch/arm64/kvm/vgic/trace.h @@ -30,7 +30,7 @@ TRACE_EVENT(vgic_update_irq_pending, #endif /* _TRACE_VGIC_H */ #undef TRACE_INCLUDE_PATH -#define TRACE_INCLUDE_PATH ../../virt/kvm/arm/vgic +#define TRACE_INCLUDE_PATH ../../arch/arm64/kvm/vgic #undef TRACE_INCLUDE_FILE #define TRACE_INCLUDE_FILE trace diff --git a/virt/kvm/arm/vgic/vgic-debug.c b/arch/arm64/kvm/vgic/vgic-debug.c similarity index 100% rename from virt/kvm/arm/vgic/vgic-debug.c rename to arch/arm64/kvm/vgic/vgic-debug.c diff --git a/virt/kvm/arm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c similarity index 100% rename from virt/kvm/arm/vgic/vgic-init.c rename to arch/arm64/kvm/vgic/vgic-init.c diff --git a/virt/kvm/arm/vgic/vgic-irqfd.c b/arch/arm64/kvm/vgic/vgic-irqfd.c similarity index 100% rename from virt/kvm/arm/vgic/vgic-irqfd.c rename to arch/arm64/kvm/vgic/vgic-irqfd.c diff --git a/virt/kvm/arm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c similarity index 100% rename from virt/kvm/arm/vgic/vgic-its.c rename to arch/arm64/kvm/vgic/vgic-its.c diff --git a/virt/kvm/arm/vgic/vgic-kvm-device.c b/arch/arm64/kvm/vgic/vgic-kvm-device.c similarity index 100% rename from virt/kvm/arm/vgic/vgic-kvm-device.c rename to arch/arm64/kvm/vgic/vgic-kvm-device.c diff --git a/virt/kvm/arm/vgic/vgic-mmio-v2.c b/arch/arm64/kvm/vgic/vgic-mmio-v2.c similarity index 100% rename from virt/kvm/arm/vgic/vgic-mmio-v2.c rename to arch/arm64/kvm/vgic/vgic-mmio-v2.c diff --git a/virt/kvm/arm/vgic/vgic-mmio-v3.c b/arch/arm64/kvm/vgic/vgic-mmio-v3.c similarity index 100% rename from virt/kvm/arm/vgic/vgic-mmio-v3.c rename to arch/arm64/kvm/vgic/vgic-mmio-v3.c diff --git a/virt/kvm/arm/vgic/vgic-mmio.c b/arch/arm64/kvm/vgic/vgic-mmio.c similarity index 100% rename from virt/kvm/arm/vgic/vgic-mmio.c rename to arch/arm64/kvm/vgic/vgic-mmio.c diff --git a/virt/kvm/arm/vgic/vgic-mmio.h b/arch/arm64/kvm/vgic/vgic-mmio.h similarity index 100% rename from virt/kvm/arm/vgic/vgic-mmio.h rename to arch/arm64/kvm/vgic/vgic-mmio.h diff --git a/virt/kvm/arm/vgic/vgic-v2.c b/arch/arm64/kvm/vgic/vgic-v2.c similarity index 100% rename from virt/kvm/arm/vgic/vgic-v2.c rename to arch/arm64/kvm/vgic/vgic-v2.c diff --git a/virt/kvm/arm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c similarity index 99% rename from virt/kvm/arm/vgic/vgic-v3.c rename to arch/arm64/kvm/vgic/vgic-v3.c index 2c9fc13e2c59..5bc2ab58954b 100644 --- a/virt/kvm/arm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -630,12 +630,10 @@ int vgic_v3_probe(const struct gic_kvm_info *info) if (kvm_vgic_global_state.vcpu_base == 0) kvm_info("disabling GICv2 emulation\n"); -#ifdef CONFIG_ARM64 if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_30115)) { group0_trap = true; group1_trap = true; } -#endif if (group0_trap || group1_trap || common_trap) { kvm_info("GICv3 sysreg trapping enabled ([%s%s%s], reduced performance)\n", diff --git a/virt/kvm/arm/vgic/vgic-v4.c b/arch/arm64/kvm/vgic/vgic-v4.c similarity index 100% rename from virt/kvm/arm/vgic/vgic-v4.c rename to arch/arm64/kvm/vgic/vgic-v4.c diff --git a/virt/kvm/arm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c similarity index 100% rename from virt/kvm/arm/vgic/vgic.c rename to arch/arm64/kvm/vgic/vgic.c diff --git a/virt/kvm/arm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h similarity index 100% rename from virt/kvm/arm/vgic/vgic.h rename to arch/arm64/kvm/vgic/vgic.h From d82755b2e781c8989614c82df7582f5649e265b8 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 5 May 2020 16:45:17 +0100 Subject: [PATCH 0521/1043] KVM: arm64: Kill off CONFIG_KVM_ARM_HOST CONFIG_KVM_ARM_HOST is just a proxy for CONFIG_KVM, so remove it in favour of the latter. Signed-off-by: Will Deacon Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200505154520.194120-2-tabba@google.com --- arch/arm64/kernel/asm-offsets.c | 2 +- arch/arm64/kernel/cpu_errata.c | 2 +- arch/arm64/kernel/smp.c | 2 +- arch/arm64/kvm/Kconfig | 6 ---- arch/arm64/kvm/Makefile | 52 ++++++++++++++++----------------- arch/arm64/kvm/hyp/Makefile | 22 +++++++------- 6 files changed, 40 insertions(+), 46 deletions(-) diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 9981a0a5a87f..a27e0cd731e9 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -96,7 +96,7 @@ int main(void) DEFINE(CPU_BOOT_PTRAUTH_KEY, offsetof(struct secondary_data, ptrauth_key)); #endif BLANK(); -#ifdef CONFIG_KVM_ARM_HOST +#ifdef CONFIG_KVM DEFINE(VCPU_CONTEXT, offsetof(struct kvm_vcpu, arch.ctxt)); DEFINE(VCPU_FAULT_DISR, offsetof(struct kvm_vcpu, arch.fault.disr_el1)); DEFINE(VCPU_WORKAROUND_FLAGS, offsetof(struct kvm_vcpu, arch.workaround_flags)); diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index df56d2295d16..a102321fc8a2 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -234,7 +234,7 @@ static int detect_harden_bp_fw(void) smccc_end = NULL; break; -#if IS_ENABLED(CONFIG_KVM_ARM_HOST) +#if IS_ENABLED(CONFIG_KVM) case SMCCC_CONDUIT_SMC: cb = call_smc_arch_workaround_1; smccc_start = __smccc_workaround_1_smc; diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 061f60fe452f..0a3045d9f33f 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -430,7 +430,7 @@ static void __init hyp_mode_check(void) "CPU: CPUs started in inconsistent modes"); else pr_info("CPU: All CPU(s) started at EL1\n"); - if (IS_ENABLED(CONFIG_KVM_ARM_HOST)) + if (IS_ENABLED(CONFIG_KVM)) kvm_compute_layout(); } diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 449386d76441..ce724e526689 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -28,7 +28,6 @@ config KVM select HAVE_KVM_CPU_RELAX_INTERCEPT select HAVE_KVM_ARCH_TLB_FLUSH_ALL select KVM_MMIO - select KVM_ARM_HOST select KVM_GENERIC_DIRTYLOG_READ_PROTECT select SRCU select KVM_VFIO @@ -50,11 +49,6 @@ config KVM If unsure, say N. -config KVM_ARM_HOST - bool - ---help--- - Provides host support for ARM processors. - config KVM_ARM_PMU bool ---help--- diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 7a3768538343..419696e615b3 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -7,33 +7,33 @@ ccflags-y += -I $(srctree)/$(src) KVM=../../../virt/kvm -obj-$(CONFIG_KVM_ARM_HOST) += kvm.o -obj-$(CONFIG_KVM_ARM_HOST) += hyp/ +obj-$(CONFIG_KVM) += kvm.o +obj-$(CONFIG_KVM) += hyp/ -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/eventfd.o $(KVM)/vfio.o $(KVM)/irqchip.o -kvm-$(CONFIG_KVM_ARM_HOST) += arm.o mmu.o mmio.o -kvm-$(CONFIG_KVM_ARM_HOST) += psci.o perf.o -kvm-$(CONFIG_KVM_ARM_HOST) += hypercalls.o -kvm-$(CONFIG_KVM_ARM_HOST) += pvtime.o +kvm-$(CONFIG_KVM) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o +kvm-$(CONFIG_KVM) += $(KVM)/eventfd.o $(KVM)/vfio.o $(KVM)/irqchip.o +kvm-$(CONFIG_KVM) += arm.o mmu.o mmio.o +kvm-$(CONFIG_KVM) += psci.o perf.o +kvm-$(CONFIG_KVM) += hypercalls.o +kvm-$(CONFIG_KVM) += pvtime.o -kvm-$(CONFIG_KVM_ARM_HOST) += inject_fault.o regmap.o va_layout.o -kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o -kvm-$(CONFIG_KVM_ARM_HOST) += guest.o debug.o reset.o sys_regs.o sys_regs_generic_v8.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic-sys-reg-v3.o fpsimd.o pmu.o -kvm-$(CONFIG_KVM_ARM_HOST) += aarch32.o -kvm-$(CONFIG_KVM_ARM_HOST) += arch_timer.o +kvm-$(CONFIG_KVM) += inject_fault.o regmap.o va_layout.o +kvm-$(CONFIG_KVM) += hyp.o hyp-init.o handle_exit.o +kvm-$(CONFIG_KVM) += guest.o debug.o reset.o sys_regs.o sys_regs_generic_v8.o +kvm-$(CONFIG_KVM) += vgic-sys-reg-v3.o fpsimd.o pmu.o +kvm-$(CONFIG_KVM) += aarch32.o +kvm-$(CONFIG_KVM) += arch_timer.o kvm-$(CONFIG_KVM_ARM_PMU) += pmu-emul.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-init.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-irqfd.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-v2.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-v3.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-v4.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-mmio.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-mmio-v2.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-mmio-v3.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-kvm-device.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-its.o -kvm-$(CONFIG_KVM_ARM_HOST) += vgic/vgic-debug.o +kvm-$(CONFIG_KVM) += vgic/vgic.o +kvm-$(CONFIG_KVM) += vgic/vgic-init.o +kvm-$(CONFIG_KVM) += vgic/vgic-irqfd.o +kvm-$(CONFIG_KVM) += vgic/vgic-v2.o +kvm-$(CONFIG_KVM) += vgic/vgic-v3.o +kvm-$(CONFIG_KVM) += vgic/vgic-v4.o +kvm-$(CONFIG_KVM) += vgic/vgic-mmio.o +kvm-$(CONFIG_KVM) += vgic/vgic-mmio-v2.o +kvm-$(CONFIG_KVM) += vgic/vgic-mmio-v3.o +kvm-$(CONFIG_KVM) += vgic/vgic-kvm-device.o +kvm-$(CONFIG_KVM) += vgic/vgic-its.o +kvm-$(CONFIG_KVM) += vgic/vgic-debug.o diff --git a/arch/arm64/kvm/hyp/Makefile b/arch/arm64/kvm/hyp/Makefile index dc18274a6826..8229e47ba870 100644 --- a/arch/arm64/kvm/hyp/Makefile +++ b/arch/arm64/kvm/hyp/Makefile @@ -6,17 +6,17 @@ ccflags-y += -fno-stack-protector -DDISABLE_BRANCH_PROFILING \ $(DISABLE_STACKLEAK_PLUGIN) -obj-$(CONFIG_KVM_ARM_HOST) += vgic-v3-sr.o -obj-$(CONFIG_KVM_ARM_HOST) += timer-sr.o -obj-$(CONFIG_KVM_ARM_HOST) += aarch32.o -obj-$(CONFIG_KVM_ARM_HOST) += vgic-v2-cpuif-proxy.o -obj-$(CONFIG_KVM_ARM_HOST) += sysreg-sr.o -obj-$(CONFIG_KVM_ARM_HOST) += debug-sr.o -obj-$(CONFIG_KVM_ARM_HOST) += entry.o -obj-$(CONFIG_KVM_ARM_HOST) += switch.o -obj-$(CONFIG_KVM_ARM_HOST) += fpsimd.o -obj-$(CONFIG_KVM_ARM_HOST) += tlb.o -obj-$(CONFIG_KVM_ARM_HOST) += hyp-entry.o +obj-$(CONFIG_KVM) += vgic-v3-sr.o +obj-$(CONFIG_KVM) += timer-sr.o +obj-$(CONFIG_KVM) += aarch32.o +obj-$(CONFIG_KVM) += vgic-v2-cpuif-proxy.o +obj-$(CONFIG_KVM) += sysreg-sr.o +obj-$(CONFIG_KVM) += debug-sr.o +obj-$(CONFIG_KVM) += entry.o +obj-$(CONFIG_KVM) += switch.o +obj-$(CONFIG_KVM) += fpsimd.o +obj-$(CONFIG_KVM) += tlb.o +obj-$(CONFIG_KVM) += hyp-entry.o # KVM code is run at a different exception code with a different map, so # compiler instrumentation that inserts callbacks or checks into the code may From bf7bc1df30f6c6afa34d4d1d53e1c8ad93510d3e Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 5 May 2020 16:45:18 +0100 Subject: [PATCH 0522/1043] KVM: arm64: Update help text arm64 KVM supports 16k pages since 02e0b7600f83 ("arm64: kvm: Add support for 16K pages"), so update the Kconfig help text accordingly. Signed-off-by: Will Deacon Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200505154520.194120-3-tabba@google.com --- arch/arm64/kvm/Kconfig | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index ce724e526689..d2cf4f099454 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -44,8 +44,6 @@ config KVM select TASK_DELAY_ACCT ---help--- Support hosting virtualized guest machines. - We don't support KVM with 16K page tables yet, due to the multiple - levels of fake page tables. If unsure, say N. From f26133624d602b0d984815168a2d3a1f630b02e2 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 5 May 2020 16:45:19 +0100 Subject: [PATCH 0523/1043] KVM: arm64: Change CONFIG_KVM to a menuconfig entry Changing CONFIG_KVM to be a 'menuconfig' entry in Kconfig mean that we can straightforwardly enumerate optional features, such as the virtual PMU device as dependent options. Signed-off-by: Will Deacon Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200505154520.194120-4-tabba@google.com --- arch/arm64/kvm/Kconfig | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index d2cf4f099454..f1c1f981482c 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -3,7 +3,6 @@ # KVM configuration # -source "virt/kvm/Kconfig" source "virt/lib/Kconfig" menuconfig VIRTUALIZATION @@ -18,7 +17,7 @@ menuconfig VIRTUALIZATION if VIRTUALIZATION -config KVM +menuconfig KVM bool "Kernel-based Virtual Machine (KVM) support" depends on OF # for TASKSTATS/TASK_DELAY_ACCT: @@ -33,7 +32,6 @@ config KVM select KVM_VFIO select HAVE_KVM_EVENTFD select HAVE_KVM_IRQFD - select KVM_ARM_PMU if HW_PERF_EVENTS select HAVE_KVM_MSI select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQ_ROUTING @@ -47,13 +45,21 @@ config KVM If unsure, say N. +if KVM + +source "virt/kvm/Kconfig" + config KVM_ARM_PMU - bool + bool "Virtual Performance Monitoring Unit (PMU) support" + depends on HW_PERF_EVENTS + default y ---help--- Adds support for a virtual Performance Monitoring Unit (PMU) in virtual machines. config KVM_INDIRECT_VECTORS - def_bool KVM && (HARDEN_BRANCH_PREDICTOR || HARDEN_EL2_VECTORS) + def_bool HARDEN_BRANCH_PREDICTOR || HARDEN_EL2_VECTORS + +endif # KVM endif # VIRTUALIZATION From 25357de01b95140ecacd4d9347d74df2dda789f2 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 5 May 2020 16:45:20 +0100 Subject: [PATCH 0524/1043] KVM: arm64: Clean up kvm makefiles Consolidate references to the CONFIG_KVM configuration item to encompass entire folders rather than per line. Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Acked-by: Will Deacon Link: https://lore.kernel.org/r/20200505154520.194120-5-tabba@google.com --- arch/arm64/kvm/Makefile | 38 +++++++++++++------------------------ arch/arm64/kvm/hyp/Makefile | 15 ++++----------- 2 files changed, 17 insertions(+), 36 deletions(-) diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 419696e615b3..8d3d9513cbfe 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -10,30 +10,18 @@ KVM=../../../virt/kvm obj-$(CONFIG_KVM) += kvm.o obj-$(CONFIG_KVM) += hyp/ -kvm-$(CONFIG_KVM) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o -kvm-$(CONFIG_KVM) += $(KVM)/eventfd.o $(KVM)/vfio.o $(KVM)/irqchip.o -kvm-$(CONFIG_KVM) += arm.o mmu.o mmio.o -kvm-$(CONFIG_KVM) += psci.o perf.o -kvm-$(CONFIG_KVM) += hypercalls.o -kvm-$(CONFIG_KVM) += pvtime.o +kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \ + $(KVM)/vfio.o $(KVM)/irqchip.o \ + arm.o mmu.o mmio.o psci.o perf.o hypercalls.o pvtime.o \ + inject_fault.o regmap.o va_layout.o hyp.o hyp-init.o handle_exit.o \ + guest.o debug.o reset.o sys_regs.o sys_regs_generic_v8.o \ + vgic-sys-reg-v3.o fpsimd.o pmu.o \ + aarch32.o arch_timer.o \ + vgic/vgic.o vgic/vgic-init.o \ + vgic/vgic-irqfd.o vgic/vgic-v2.o \ + vgic/vgic-v3.o vgic/vgic-v4.o \ + vgic/vgic-mmio.o vgic/vgic-mmio-v2.o \ + vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \ + vgic/vgic-its.o vgic/vgic-debug.o -kvm-$(CONFIG_KVM) += inject_fault.o regmap.o va_layout.o -kvm-$(CONFIG_KVM) += hyp.o hyp-init.o handle_exit.o -kvm-$(CONFIG_KVM) += guest.o debug.o reset.o sys_regs.o sys_regs_generic_v8.o -kvm-$(CONFIG_KVM) += vgic-sys-reg-v3.o fpsimd.o pmu.o -kvm-$(CONFIG_KVM) += aarch32.o -kvm-$(CONFIG_KVM) += arch_timer.o kvm-$(CONFIG_KVM_ARM_PMU) += pmu-emul.o - -kvm-$(CONFIG_KVM) += vgic/vgic.o -kvm-$(CONFIG_KVM) += vgic/vgic-init.o -kvm-$(CONFIG_KVM) += vgic/vgic-irqfd.o -kvm-$(CONFIG_KVM) += vgic/vgic-v2.o -kvm-$(CONFIG_KVM) += vgic/vgic-v3.o -kvm-$(CONFIG_KVM) += vgic/vgic-v4.o -kvm-$(CONFIG_KVM) += vgic/vgic-mmio.o -kvm-$(CONFIG_KVM) += vgic/vgic-mmio-v2.o -kvm-$(CONFIG_KVM) += vgic/vgic-mmio-v3.o -kvm-$(CONFIG_KVM) += vgic/vgic-kvm-device.o -kvm-$(CONFIG_KVM) += vgic/vgic-its.o -kvm-$(CONFIG_KVM) += vgic/vgic-debug.o diff --git a/arch/arm64/kvm/hyp/Makefile b/arch/arm64/kvm/hyp/Makefile index 8229e47ba870..8c9880783839 100644 --- a/arch/arm64/kvm/hyp/Makefile +++ b/arch/arm64/kvm/hyp/Makefile @@ -6,17 +6,10 @@ ccflags-y += -fno-stack-protector -DDISABLE_BRANCH_PROFILING \ $(DISABLE_STACKLEAK_PLUGIN) -obj-$(CONFIG_KVM) += vgic-v3-sr.o -obj-$(CONFIG_KVM) += timer-sr.o -obj-$(CONFIG_KVM) += aarch32.o -obj-$(CONFIG_KVM) += vgic-v2-cpuif-proxy.o -obj-$(CONFIG_KVM) += sysreg-sr.o -obj-$(CONFIG_KVM) += debug-sr.o -obj-$(CONFIG_KVM) += entry.o -obj-$(CONFIG_KVM) += switch.o -obj-$(CONFIG_KVM) += fpsimd.o -obj-$(CONFIG_KVM) += tlb.o -obj-$(CONFIG_KVM) += hyp-entry.o +obj-$(CONFIG_KVM) += hyp.o + +hyp-y := vgic-v3-sr.o timer-sr.o aarch32.o vgic-v2-cpuif-proxy.o sysreg-sr.o \ + debug-sr.o entry.o switch.o fpsimd.o tlb.o hyp-entry.o # KVM code is run at a different exception code with a different map, so # compiler instrumentation that inserts callbacks or checks into the code may From c6fe89ff8b250ad4dc4bed7bd5877bfbc35f4aba Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 13 May 2020 11:58:29 +0100 Subject: [PATCH 0525/1043] KVM: arm64: Simplify __kvm_timer_set_cntvoff implementation Now that this function isn't constrained by the 32bit PCS, let's simplify it by taking a single 64bit offset instead of two 32bit parameters. Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_asm.h | 2 +- arch/arm64/kvm/arch_timer.c | 12 +----------- arch/arm64/kvm/hyp/timer-sr.c | 3 +-- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 7c7eeeaab9fa..59e314f38e43 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -64,7 +64,7 @@ extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa); extern void __kvm_tlb_flush_vmid(struct kvm *kvm); extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu); -extern void __kvm_timer_set_cntvoff(u32 cntvoff_low, u32 cntvoff_high); +extern void __kvm_timer_set_cntvoff(u64 cntvoff); extern int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 93bd59b46848..487eba9f87cd 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -451,17 +451,7 @@ out: static void set_cntvoff(u64 cntvoff) { - u32 low = lower_32_bits(cntvoff); - u32 high = upper_32_bits(cntvoff); - - /* - * Since kvm_call_hyp doesn't fully support the ARM PCS especially on - * 32-bit systems, but rather passes register by register shifted one - * place (we put the function address in r0/x0), we cannot simply pass - * a 64-bit value as an argument, but have to split the value in two - * 32-bit halves. - */ - kvm_call_hyp(__kvm_timer_set_cntvoff, low, high); + kvm_call_hyp(__kvm_timer_set_cntvoff, cntvoff); } static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, bool active) diff --git a/arch/arm64/kvm/hyp/timer-sr.c b/arch/arm64/kvm/hyp/timer-sr.c index ff76e6845fe4..fb5c0be33223 100644 --- a/arch/arm64/kvm/hyp/timer-sr.c +++ b/arch/arm64/kvm/hyp/timer-sr.c @@ -10,9 +10,8 @@ #include -void __hyp_text __kvm_timer_set_cntvoff(u32 cntvoff_low, u32 cntvoff_high) +void __hyp_text __kvm_timer_set_cntvoff(u64 cntvoff) { - u64 cntvoff = (u64)cntvoff_high << 32 | cntvoff_low; write_sysreg(cntvoff, cntvoff_el2); } From ce6f8f02f9f6786355fa6c79d88b839639dd75d8 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 13 May 2020 11:38:28 +0100 Subject: [PATCH 0526/1043] KVM: arm64: Use cpus_have_final_cap for has_vhe() By the time we start using the has_vhe() helper, we have long discovered whether we are running VHE or not. It thus makes sense to use cpus_have_final_cap() instead of cpus_have_const_cap(), which leads to a small text size reduction. Signed-off-by: Marc Zyngier Acked-by: David Brazdil Link: https://lore.kernel.org/r/20200513103828.74580-1-maz@kernel.org --- arch/arm64/include/asm/virt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h index 61fd26752adc..5051b388c654 100644 --- a/arch/arm64/include/asm/virt.h +++ b/arch/arm64/include/asm/virt.h @@ -85,7 +85,7 @@ static inline bool is_kernel_in_hyp_mode(void) static __always_inline bool has_vhe(void) { - if (cpus_have_const_cap(ARM64_HAS_VIRT_HOST_EXTN)) + if (cpus_have_final_cap(ARM64_HAS_VIRT_HOST_EXTN)) return true; return false; From 656012c731fcfd0f770007366e2b952a613745f2 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 1 Apr 2020 15:03:10 +0100 Subject: [PATCH 0527/1043] KVM: Fix spelling in code comments Fix spelling and typos (e.g., repeated words) in comments. Signed-off-by: Fuad Tabba Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200401140310.29701-1-tabba@google.com --- arch/arm64/kvm/arm.c | 6 +++--- arch/arm64/kvm/guest.c | 4 ++-- arch/arm64/kvm/hyp/vgic-v3-sr.c | 2 +- arch/arm64/kvm/mmio.c | 2 +- arch/arm64/kvm/mmu.c | 6 +++--- arch/arm64/kvm/psci.c | 6 +++--- arch/arm64/kvm/reset.c | 6 +++--- arch/arm64/kvm/sys_regs.c | 6 +++--- arch/arm64/kvm/vgic/vgic-v3.c | 2 +- virt/kvm/coalesced_mmio.c | 2 +- virt/kvm/eventfd.c | 2 +- virt/kvm/kvm_main.c | 2 +- 12 files changed, 23 insertions(+), 23 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index c958bb37b769..ee1b5bba1d08 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -455,9 +455,9 @@ void force_vm_exit(const cpumask_t *mask) * * The hardware supports a limited set of values with the value zero reserved * for the host, so we check if an assigned value belongs to a previous - * generation, which which requires us to assign a new value. If we're the - * first to use a VMID for the new generation, we must flush necessary caches - * and TLBs on all CPUs. + * generation, which requires us to assign a new value. If we're the first to + * use a VMID for the new generation, we must flush necessary caches and TLBs + * on all CPUs. */ static bool need_new_vmid_gen(struct kvm_vmid *vmid) { diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 50a279d3ddd7..871d51729b63 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -267,7 +267,7 @@ static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) /* * Vector lengths supported by the host can't currently be * hidden from the guest individually: instead we can only set a - * maxmium via ZCR_EL2.LEN. So, make sure the available vector + * maximum via ZCR_EL2.LEN. So, make sure the available vector * lengths match the set requested exactly up to the requested * maximum: */ @@ -337,7 +337,7 @@ static int sve_reg_to_region(struct sve_state_reg_region *region, unsigned int reg_num; unsigned int reqoffset, reqlen; /* User-requested offset and length */ - unsigned int maxlen; /* Maxmimum permitted length */ + unsigned int maxlen; /* Maximum permitted length */ size_t sve_state_size; diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c index 49fedf6710f9..6b85773e15c4 100644 --- a/arch/arm64/kvm/hyp/vgic-v3-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c @@ -577,7 +577,7 @@ static u8 __hyp_text __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp) /* * The priority value is independent of any of the BPR values, so we - * normalize it using the minumal BPR value. This guarantees that no + * normalize it using the minimal BPR value. This guarantees that no * matter what the guest does with its BPR, we can always set/get the * same value of a priority. */ diff --git a/arch/arm64/kvm/mmio.c b/arch/arm64/kvm/mmio.c index aedfcff99ac5..4e0366759726 100644 --- a/arch/arm64/kvm/mmio.c +++ b/arch/arm64/kvm/mmio.c @@ -131,7 +131,7 @@ int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, /* * No valid syndrome? Ask userspace for help if it has - * voluntered to do so, and bail out otherwise. + * volunteered to do so, and bail out otherwise. */ if (!kvm_vcpu_dabt_isvalid(vcpu)) { if (vcpu->kvm->arch.return_nisv_io_abort_to_user) { diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index e3b9ee268823..29d8f24df944 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -784,7 +784,7 @@ static int __create_hyp_private_mapping(phys_addr_t phys_addr, size_t size, mutex_lock(&kvm_hyp_pgd_mutex); /* - * This assumes that we we have enough space below the idmap + * This assumes that we have enough space below the idmap * page to allocate our VAs. If not, the check below will * kick. A potential alternative would be to detect that * overflow and switch to an allocation above the idmap. @@ -964,7 +964,7 @@ static void stage2_unmap_memslot(struct kvm *kvm, * stage2_unmap_vm - Unmap Stage-2 RAM mappings * @kvm: The struct kvm pointer * - * Go through the memregions and unmap any reguler RAM + * Go through the memregions and unmap any regular RAM * backing memory already mapped to the VM. */ void stage2_unmap_vm(struct kvm *kvm) @@ -2262,7 +2262,7 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, { /* * At this point memslot has been committed and there is an - * allocated dirty_bitmap[], dirty pages will be be tracked while the + * allocated dirty_bitmap[], dirty pages will be tracked while the * memory slot is write protected. */ if (change != KVM_MR_DELETE && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index ae364716ee40..83415e96b589 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -94,7 +94,7 @@ static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) /* * NOTE: We always update r0 (or x0) because for PSCI v0.1 - * the general puspose registers are undefined upon CPU_ON. + * the general purpose registers are undefined upon CPU_ON. */ reset_state->r0 = smccc_get_arg3(source_vcpu); @@ -265,10 +265,10 @@ static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu) case PSCI_0_2_FN_SYSTEM_OFF: kvm_psci_system_off(vcpu); /* - * We should'nt be going back to guest VCPU after + * We shouldn't be going back to guest VCPU after * receiving SYSTEM_OFF request. * - * If user space accidently/deliberately resumes + * If user space accidentally/deliberately resumes * guest VCPU after SYSTEM_OFF request then guest * VCPU should see internal failure from PSCI return * value. To achieve this, we preload r0 (or x0) with diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index 30b7ea680f66..658f3a79617b 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -163,7 +163,7 @@ static int kvm_vcpu_finalize_sve(struct kvm_vcpu *vcpu) vl = vcpu->arch.sve_max_vl; /* - * Resposibility for these properties is shared between + * Responsibility for these properties is shared between * kvm_arm_init_arch_resources(), kvm_vcpu_enable_sve() and * set_sve_vls(). Double-check here just to be sure: */ @@ -249,7 +249,7 @@ static int kvm_vcpu_enable_ptrauth(struct kvm_vcpu *vcpu) * ioctl or as part of handling a request issued by another VCPU in the PSCI * handling code. In the first case, the VCPU will not be loaded, and in the * second case the VCPU will be loaded. Because this function operates purely - * on the memory-backed valus of system registers, we want to do a full put if + * on the memory-backed values of system registers, we want to do a full put if * we were loaded (handling a request) and load the values back at the end of * the function. Otherwise we leave the state alone. In both cases, we * disable preemption around the vcpu reset as we would otherwise race with @@ -357,7 +357,7 @@ void kvm_set_ipa_limit(void) * * So clamp the ipa limit further down to limit the number of levels. * Since we can concatenate upto 16 tables at entry level, we could - * go upto 4bits above the maximum VA addressible with the current + * go upto 4bits above the maximum VA addressable with the current * number of levels. */ va_max = PGDIR_SHIFT + PAGE_SHIFT - 3; diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 51db934702b6..620eaf11e672 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -34,7 +34,7 @@ #include "trace.h" /* - * All of this file is extremly similar to the ARM coproc.c, but the + * All of this file is extremely similar to the ARM coproc.c, but the * types are different. My gut feeling is that it should be pretty * easy to merge, but that would be an ABI breakage -- again. VFP * would also need to be abstracted. @@ -118,8 +118,8 @@ void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg) * entry to the guest but are only restored on vcpu_load. * * Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but - * should never be listed below, because the the MPIDR should only be - * set once, before running the VCPU, and never changed later. + * should never be listed below, because the MPIDR should only be set + * once, before running the VCPU, and never changed later. */ switch (reg) { case CSSELR_EL1: write_sysreg_s(val, SYS_CSSELR_EL1); return; diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 5bc2ab58954b..3ccd6d3cb4d3 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -587,7 +587,7 @@ int vgic_v3_probe(const struct gic_kvm_info *info) int ret; /* - * The ListRegs field is 5 bits, but there is a architectural + * The ListRegs field is 5 bits, but there is an architectural * maximum of 16 list registers. Just ignore bit 4... */ kvm_vgic_global_state.nr_lr = (ich_vtr_el2 & 0xf) + 1; diff --git a/virt/kvm/coalesced_mmio.c b/virt/kvm/coalesced_mmio.c index 00c747dbc82e..e2c197fd4f9d 100644 --- a/virt/kvm/coalesced_mmio.c +++ b/virt/kvm/coalesced_mmio.c @@ -119,7 +119,7 @@ int kvm_coalesced_mmio_init(struct kvm *kvm) /* * We're using this spinlock to sync access to the coalesced ring. - * The list doesn't need it's own lock since device registration and + * The list doesn't need its own lock since device registration and * unregistration should only happen when kvm->slots_lock is held. */ spin_lock_init(&kvm->ring_lock); diff --git a/virt/kvm/eventfd.c b/virt/kvm/eventfd.c index 67b6fc153e9c..e586d1395c28 100644 --- a/virt/kvm/eventfd.c +++ b/virt/kvm/eventfd.c @@ -116,7 +116,7 @@ irqfd_shutdown(struct work_struct *work) struct kvm *kvm = irqfd->kvm; u64 cnt; - /* Make sure irqfd has been initalized in assign path. */ + /* Make sure irqfd has been initialized in assign path. */ synchronize_srcu(&kvm->irq_srcu); /* diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 74bdb7bf3295..f57792b1541b 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2799,7 +2799,7 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to); * * (a) VCPU which has not done pl-exit or cpu relax intercepted recently * (preempted lock holder), indicated by @in_spin_loop. - * Set at the beiginning and cleared at the end of interception/PLE handler. + * Set at the beginning and cleared at the end of interception/PLE handler. * * (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get * chance last time (mostly it has become eligible now since we have probably From 892713e97ca146591515b3c115f99cdf632030fb Mon Sep 17 00:00:00 2001 From: Zenghui Yu Date: Wed, 15 Apr 2020 15:28:35 +0800 Subject: [PATCH 0528/1043] KVM: arm64: Sidestep stage2_unmap_vm() on vcpu reset when S2FWB is supported stage2_unmap_vm() was introduced to unmap user RAM region in the stage2 page table to make the caches coherent. E.g., a guest reboot with stage1 MMU disabled will access memory using non-cacheable attributes. If the RAM and caches are not coherent at this stage, some evicted dirty cache line may go and corrupt guest data in RAM. Since ARMv8.4, S2FWB feature is mandatory and KVM will take advantage of it to configure the stage2 page table and the attributes of memory access. So we ensure that guests always access memory using cacheable attributes and thus, the caches always be coherent. So on CPUs that support S2FWB, we can safely reset the vcpu without a heavy stage2 unmapping. Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200415072835.1164-1-yuzenghui@huawei.com --- arch/arm64/kvm/arm.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index ee1b5bba1d08..0ea9a0266d9a 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -983,8 +983,11 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, /* * Ensure a rebooted VM will fault in RAM pages and detect if the * guest MMU is turned off and flush the caches as needed. + * + * S2FWB enforces all memory accesses to RAM being cacheable, we + * ensure that the cache is always coherent. */ - if (vcpu->arch.has_run_once) + if (vcpu->arch.has_run_once && !cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) stage2_unmap_vm(vcpu->kvm); vcpu_reset_hcr(vcpu); From 48c963e31bc664afafd31058483ea8390da63980 Mon Sep 17 00:00:00 2001 From: Jiang Yi Date: Wed, 15 Apr 2020 10:42:29 +0200 Subject: [PATCH 0529/1043] KVM: arm/arm64: Release kvm->mmu_lock in loop to prevent starvation Do cond_resched_lock() in stage2_flush_memslot() like what is done in unmap_stage2_range() and other places holding mmu_lock while processing a possibly large range of memory. Signed-off-by: Jiang Yi Signed-off-by: Marc Zyngier Reviewed-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20200415084229.29992-1-giangyi@amazon.com --- arch/arm64/kvm/mmu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 29d8f24df944..917363375e8a 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -422,6 +422,9 @@ static void stage2_flush_memslot(struct kvm *kvm, next = stage2_pgd_addr_end(kvm, addr, end); if (!stage2_pgd_none(kvm, *pgd)) stage2_flush_puds(kvm, pgd, addr, next); + + if (next != end) + cond_resched_lock(&kvm->mmu_lock); } while (pgd++, addr = next, addr != end); } From 9f2836146b11cdf98d5c8f8f71b0fce28fbd83c8 Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Thu, 7 May 2020 20:35:45 +0800 Subject: [PATCH 0530/1043] KVM: arm64: Clean up the checking for huge mapping If we are checking whether the stage2 can map PAGE_SIZE, we don't have to do the boundary checks as both the host VMA and the guest memslots are page aligned. Bail the case easily. While we're at it, fixup a typo in the comment below. Signed-off-by: Suzuki K Poulose Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200507123546.1875-2-yuzenghui@huawei.com --- arch/arm64/kvm/mmu.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 917363375e8a..ccb44e7d30d9 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1610,6 +1610,10 @@ static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot, hva_t uaddr_start, uaddr_end; size_t size; + /* The memslot and the VMA are guaranteed to be aligned to PAGE_SIZE */ + if (map_size == PAGE_SIZE) + return true; + size = memslot->npages * PAGE_SIZE; gpa_start = memslot->base_gfn << PAGE_SHIFT; @@ -1629,7 +1633,7 @@ static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot, * |abcde|fgh Stage-1 block | Stage-1 block tv|xyz| * +-----+--------------------+--------------------+---+ * - * memslot->base_gfn << PAGE_SIZE: + * memslot->base_gfn << PAGE_SHIFT: * +---+--------------------+--------------------+-----+ * |abc|def Stage-2 block | Stage-2 block |tvxyz| * +---+--------------------+--------------------+-----+ From 0529c9021252a58b6d3808da86986a614b900b1b Mon Sep 17 00:00:00 2001 From: Suzuki K Poulose Date: Thu, 7 May 2020 20:35:46 +0800 Subject: [PATCH 0531/1043] KVM: arm64: Unify handling THP backed host memory We support mapping host memory backed by PMD transparent hugepages at stage2 as huge pages. However the checks are now spread across two different places. Let us unify the handling of the THPs to keep the code cleaner (and future proof for PUD THP support). This patch moves transparent_hugepage_adjust() closer to the caller to avoid a forward declaration for fault_supports_stage2_huge_mappings(). Also, since we already handle the case where the host VA and the guest PA may not be aligned, the explicit VM_BUG_ON() is not required. Signed-off-by: Suzuki K Poulose Signed-off-by: Zenghui Yu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200507123546.1875-3-yuzenghui@huawei.com --- arch/arm64/kvm/mmu.c | 115 ++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 55 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index ccb44e7d30d9..66eb8e3f6e8c 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -1375,47 +1375,6 @@ out: return ret; } -static bool transparent_hugepage_adjust(kvm_pfn_t *pfnp, phys_addr_t *ipap) -{ - kvm_pfn_t pfn = *pfnp; - gfn_t gfn = *ipap >> PAGE_SHIFT; - - if (kvm_is_transparent_hugepage(pfn)) { - unsigned long mask; - /* - * The address we faulted on is backed by a transparent huge - * page. However, because we map the compound huge page and - * not the individual tail page, we need to transfer the - * refcount to the head page. We have to be careful that the - * THP doesn't start to split while we are adjusting the - * refcounts. - * - * We are sure this doesn't happen, because mmu_notifier_retry - * was successful and we are holding the mmu_lock, so if this - * THP is trying to split, it will be blocked in the mmu - * notifier before touching any of the pages, specifically - * before being able to call __split_huge_page_refcount(). - * - * We can therefore safely transfer the refcount from PG_tail - * to PG_head and switch the pfn from a tail page to the head - * page accordingly. - */ - mask = PTRS_PER_PMD - 1; - VM_BUG_ON((gfn & mask) != (pfn & mask)); - if (pfn & mask) { - *ipap &= PMD_MASK; - kvm_release_pfn_clean(pfn); - pfn &= ~mask; - kvm_get_pfn(pfn); - *pfnp = pfn; - } - - return true; - } - - return false; -} - /** * stage2_wp_ptes - write protect PMD range * @pmd: pointer to pmd entry @@ -1663,6 +1622,59 @@ static bool fault_supports_stage2_huge_mapping(struct kvm_memory_slot *memslot, (hva & ~(map_size - 1)) + map_size <= uaddr_end; } +/* + * Check if the given hva is backed by a transparent huge page (THP) and + * whether it can be mapped using block mapping in stage2. If so, adjust + * the stage2 PFN and IPA accordingly. Only PMD_SIZE THPs are currently + * supported. This will need to be updated to support other THP sizes. + * + * Returns the size of the mapping. + */ +static unsigned long +transparent_hugepage_adjust(struct kvm_memory_slot *memslot, + unsigned long hva, kvm_pfn_t *pfnp, + phys_addr_t *ipap) +{ + kvm_pfn_t pfn = *pfnp; + + /* + * Make sure the adjustment is done only for THP pages. Also make + * sure that the HVA and IPA are sufficiently aligned and that the + * block map is contained within the memslot. + */ + if (kvm_is_transparent_hugepage(pfn) && + fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE)) { + /* + * The address we faulted on is backed by a transparent huge + * page. However, because we map the compound huge page and + * not the individual tail page, we need to transfer the + * refcount to the head page. We have to be careful that the + * THP doesn't start to split while we are adjusting the + * refcounts. + * + * We are sure this doesn't happen, because mmu_notifier_retry + * was successful and we are holding the mmu_lock, so if this + * THP is trying to split, it will be blocked in the mmu + * notifier before touching any of the pages, specifically + * before being able to call __split_huge_page_refcount(). + * + * We can therefore safely transfer the refcount from PG_tail + * to PG_head and switch the pfn from a tail page to the head + * page accordingly. + */ + *ipap &= PMD_MASK; + kvm_release_pfn_clean(pfn); + pfn &= ~(PTRS_PER_PMD - 1); + kvm_get_pfn(pfn); + *pfnp = pfn; + + return PMD_SIZE; + } + + /* Use page mapping if we cannot use block mapping. */ + return PAGE_SIZE; +} + static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, struct kvm_memory_slot *memslot, unsigned long hva, unsigned long fault_status) @@ -1776,20 +1788,13 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, if (mmu_notifier_retry(kvm, mmu_seq)) goto out_unlock; - if (vma_pagesize == PAGE_SIZE && !force_pte) { - /* - * Only PMD_SIZE transparent hugepages(THP) are - * currently supported. This code will need to be - * updated to support other THP sizes. - * - * Make sure the host VA and the guest IPA are sufficiently - * aligned and that the block is contained within the memslot. - */ - if (fault_supports_stage2_huge_mapping(memslot, hva, PMD_SIZE) && - transparent_hugepage_adjust(&pfn, &fault_ipa)) - vma_pagesize = PMD_SIZE; - } - + /* + * If we are not forced to use page mapping, check if we are + * backed by a THP and thus use block mapping if possible. + */ + if (vma_pagesize == PAGE_SIZE && !force_pte) + vma_pagesize = transparent_hugepage_adjust(memslot, hva, + &pfn, &fault_ipa); if (writable) kvm_set_pfn_dirty(pfn); From c862626e19efdc26b26481515470b160e8fe52f3 Mon Sep 17 00:00:00 2001 From: Keqian Zhu Date: Mon, 13 Apr 2020 20:20:23 +0800 Subject: [PATCH 0532/1043] KVM: arm64: Support enabling dirty log gradually in small chunks There is already support of enabling dirty log gradually in small chunks for x86 in commit 3c9bd4006bfc ("KVM: x86: enable dirty log gradually in small chunks"). This adds support for arm64. x86 still writes protect all huge pages when DIRTY_LOG_INITIALLY_ALL_SET is enabled. However, for arm64, both huge pages and normal pages can be write protected gradually by userspace. Under the Huawei Kunpeng 920 2.6GHz platform, I did some tests on 128G Linux VMs with different page size. The memory pressure is 127G in each case. The time taken of memory_global_dirty_log_start in QEMU is listed below: Page Size Before After Optimization 4K 650ms 1.8ms 2M 4ms 1.8ms 1G 2ms 1.8ms Besides the time reduction, the biggest improvement is that we will minimize the performance side effect (because of dissolving huge pages and marking memslots dirty) on guest after enabling dirty log. Signed-off-by: Keqian Zhu Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200413122023.52583-1-zhukeqian1@huawei.com --- Documentation/virt/kvm/api.rst | 2 +- arch/arm64/include/asm/kvm_host.h | 3 +++ arch/arm64/kvm/mmu.c | 12 ++++++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index efbbe570aa9b..0017f63fa44f 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -5777,7 +5777,7 @@ will be initialized to 1 when created. This also improves performance because dirty logging can be enabled gradually in small chunks on the first call to KVM_CLEAR_DIRTY_LOG. KVM_DIRTY_LOG_INITIALLY_SET depends on KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE (it is also only available on -x86 for now). +x86 and arm64 for now). KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 was previously available under the name KVM_CAP_MANUAL_DIRTY_LOG_PROTECT, but the implementation had bugs that make diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 32c8a675e5a4..a723f84fab83 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -46,6 +46,9 @@ #define KVM_REQ_RECORD_STEAL KVM_ARCH_REQ(3) #define KVM_REQ_RELOAD_GICv4 KVM_ARCH_REQ(4) +#define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ + KVM_DIRTY_LOG_INITIALLY_SET) + DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use); extern unsigned int kvm_sve_max_vl; diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 66eb8e3f6e8c..ddf85bf21897 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2277,8 +2277,16 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, * allocated dirty_bitmap[], dirty pages will be tracked while the * memory slot is write protected. */ - if (change != KVM_MR_DELETE && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) - kvm_mmu_wp_memory_region(kvm, mem->slot); + if (change != KVM_MR_DELETE && mem->flags & KVM_MEM_LOG_DIRTY_PAGES) { + /* + * If we're with initial-all-set, we don't need to write + * protect any pages because they're all reported as dirty. + * Huge pages and normal pages will be write protect gradually. + */ + if (!kvm_dirty_log_manual_protect_and_init_set(kvm)) { + kvm_mmu_wp_memory_region(kvm, mem->slot); + } + } } int kvm_arch_prepare_memory_region(struct kvm *kvm, From 5107000faa6e8c2b0ff7a91a6d1f010f84596cd2 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 27 Apr 2020 15:15:07 +0100 Subject: [PATCH 0533/1043] KVM: arm64: Make KVM_CAP_MAX_VCPUS compatible with the selected GIC version KVM_CAP_MAX_VCPUS always return the maximum possible number of VCPUs, irrespective of the selected interrupt controller. This is pretty misleading for userspace that selects a GICv2 on a GICv3 system that supports v2 compat: It always gets a maximum of 512 VCPUs, even if the effective limit is 8. The 9th VCPU will fail to be created, which is unexpected as far as userspace is concerned. Fortunately, we already have the right information stashed in the kvm structure, and we can return it as requested. Reported-by: Ard Biesheuvel Signed-off-by: Marc Zyngier Tested-by: Alexandru Elisei Reviewed-by: Alexandru Elisei Link: https://lore.kernel.org/r/20200427141507.284985-1-maz@kernel.org --- arch/arm64/kvm/arm.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 0ea9a0266d9a..e01d44df98df 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -95,6 +95,11 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, return r; } +static int kvm_arm_default_max_vcpus(void) +{ + return vgic_present ? kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS; +} + /** * kvm_arch_init_vm - initializes a VM data structure * @kvm: pointer to the KVM struct @@ -128,8 +133,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm->arch.vmid.vmid_gen = 0; /* The maximum number of VCPUs is limited by the host's GIC model */ - kvm->arch.max_vcpus = vgic_present ? - kvm_vgic_get_max_vcpus() : KVM_MAX_VCPUS; + kvm->arch.max_vcpus = kvm_arm_default_max_vcpus(); return ret; out_free_stage2_pgd: @@ -204,10 +208,11 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = num_online_cpus(); break; case KVM_CAP_MAX_VCPUS: - r = KVM_MAX_VCPUS; - break; case KVM_CAP_MAX_VCPU_ID: - r = KVM_MAX_VCPU_ID; + if (kvm) + r = kvm->arch.max_vcpus; + else + r = kvm_arm_default_max_vcpus(); break; case KVM_CAP_MSI_DEVID: if (!kvm) From c65165651d595fd77c38a9a25c14ade14444bc13 Mon Sep 17 00:00:00 2001 From: Xu Wang Date: Thu, 7 May 2020 15:12:11 +0800 Subject: [PATCH 0534/1043] block/swim3: use set_current_state macro Use set_current_state macro instead of current->state = TASK_RUNNING. Signed-off-by: Xu Wang Reviewed-by: Chaitanya Kulkarni Signed-off-by: Jens Axboe --- drivers/block/swim.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/block/swim.c b/drivers/block/swim.c index 4c297f69171d..dd34504382e5 100644 --- a/drivers/block/swim.c +++ b/drivers/block/swim.c @@ -327,7 +327,7 @@ static inline void swim_motor(struct swim __iomem *base, swim_select(base, RELAX); if (swim_readbit(base, MOTOR_ON)) break; - current->state = TASK_INTERRUPTIBLE; + set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(1); } } else if (action == OFF) { @@ -346,7 +346,7 @@ static inline void swim_eject(struct swim __iomem *base) swim_select(base, RELAX); if (!swim_readbit(base, DISK_IN)) break; - current->state = TASK_INTERRUPTIBLE; + set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(1); } swim_select(base, RELAX); @@ -370,7 +370,7 @@ static inline int swim_step(struct swim __iomem *base) for (wait = 0; wait < HZ; wait++) { - current->state = TASK_INTERRUPTIBLE; + set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(1); swim_select(base, RELAX); From a44de7497f91834df0b8b6d459e259788ba66794 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Sat, 16 May 2020 10:15:48 +0800 Subject: [PATCH 0535/1043] MIPS: Loongson: Build ATI Radeon GPU driver as module When ATI Radeon GPU driver has been compiled directly into the kernel instead of as a module, we should make sure the firmware for the model (check available ones in /lib/firmware/radeon) is built-in to the kernel as well, otherwise there exists the following fatal error during GPU init, change CONFIG_DRM_RADEON=y to CONFIG_DRM_RADEON=m to fix it. [ 1.900997] [drm] Loading RS780 Microcode [ 1.905077] radeon 0000:01:05.0: Direct firmware load for radeon/RS780_pfp.bin failed with error -2 [ 1.914140] r600_cp: Failed to load firmware "radeon/RS780_pfp.bin" [ 1.920405] [drm:r600_init] *ERROR* Failed to load firmware! [ 1.926069] radeon 0000:01:05.0: Fatal error during GPU init [ 1.931729] [drm] radeon: finishing device. Fixes: 024e6a8b5bb1 ("MIPS: Loongson: Add a Loongson-3 default config file") Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/configs/loongson3_defconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/configs/loongson3_defconfig b/arch/mips/configs/loongson3_defconfig index 6768c1682671..4df24343e8ba 100644 --- a/arch/mips/configs/loongson3_defconfig +++ b/arch/mips/configs/loongson3_defconfig @@ -230,7 +230,7 @@ CONFIG_MEDIA_CAMERA_SUPPORT=y CONFIG_MEDIA_USB_SUPPORT=y CONFIG_USB_VIDEO_CLASS=m CONFIG_DRM=y -CONFIG_DRM_RADEON=y +CONFIG_DRM_RADEON=m CONFIG_FB_RADEON=y CONFIG_LCD_CLASS_DEVICE=y CONFIG_LCD_PLATFORM=m From d9a51fd594d8034be49060dd8438f6315754cc8e Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Sat, 16 May 2020 10:15:49 +0800 Subject: [PATCH 0536/1043] MIPS: Remove not used 8250-platform.c When CONFIG_HAVE_STD_PC_SERIAL_PORT is set, there exists build errors of 8250-platform.c due to linux/module.h is not included. CONFIG_HAVE_STD_PC_SERIAL_PORT is not used in arch/mips for many years, 8250-platform.c is also not built and used, so it is not necessary to fix the build errors, just remove the not used file 8250-platform.c and the related code in Kconfig and Makefile. Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kconfig | 3 --- arch/mips/kernel/8250-platform.c | 46 -------------------------------- arch/mips/kernel/Makefile | 2 -- 3 files changed, 51 deletions(-) delete mode 100644 arch/mips/kernel/8250-platform.c diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index bfa9cd962b06..b2ff77f8366f 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1358,9 +1358,6 @@ config MIPS_L1_CACHE_SHIFT default "4" if MIPS_L1_CACHE_SHIFT_4 default "5" -config HAVE_STD_PC_SERIAL_PORT - bool - config ARC_CMDLINE_ONLY bool diff --git a/arch/mips/kernel/8250-platform.c b/arch/mips/kernel/8250-platform.c deleted file mode 100644 index 5c6b2ab1f56e..000000000000 --- a/arch/mips/kernel/8250-platform.c +++ /dev/null @@ -1,46 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2007 Ralf Baechle (ralf@linux-mips.org) - */ -#include -#include - -#define PORT(base, int) \ -{ \ - .iobase = base, \ - .irq = int, \ - .uartclk = 1843200, \ - .iotype = UPIO_PORT, \ - .flags = UPF_BOOT_AUTOCONF | UPF_SKIP_TEST, \ - .regshift = 0, \ -} - -static struct plat_serial8250_port uart8250_data[] = { - PORT(0x3F8, 4), - PORT(0x2F8, 3), - PORT(0x3E8, 4), - PORT(0x2E8, 3), - { }, -}; - -static struct platform_device uart8250_device = { - .name = "serial8250", - .id = PLAT8250_DEV_PLATFORM, - .dev = { - .platform_data = uart8250_data, - }, -}; - -static int __init uart8250_init(void) -{ - return platform_device_register(&uart8250_device); -} - -module_init(uart8250_init); - -MODULE_AUTHOR("Ralf Baechle "); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Generic 8250 UART probe driver"); diff --git a/arch/mips/kernel/Makefile b/arch/mips/kernel/Makefile index d6e97df51cfb..8c7a043295ed 100644 --- a/arch/mips/kernel/Makefile +++ b/arch/mips/kernel/Makefile @@ -98,8 +98,6 @@ obj-$(CONFIG_MIPSR2_TO_R6_EMULATOR) += mips-r2-to-r6-emul.o CFLAGS_cpu-bugs64.o = $(shell if $(CC) $(KBUILD_CFLAGS) -Wa,-mdaddi -c -o /dev/null -x c /dev/null >/dev/null 2>&1; then echo "-DHAVE_AS_SET_DADDI"; fi) -obj-$(CONFIG_HAVE_STD_PC_SERIAL_PORT) += 8250-platform.o - obj-$(CONFIG_PERF_EVENTS) += perf_event.o obj-$(CONFIG_HW_PERF_EVENTS) += perf_event_mipsxx.o From de541d60157a1716c0e2f04971fd2dd97ade7b7e Mon Sep 17 00:00:00 2001 From: WANG Xuerui Date: Sun, 3 May 2020 18:33:02 +0800 Subject: [PATCH 0537/1043] MIPS: Loongson64: fix typos in loongson_regs.h Fix some symbol names to align with Loongson's User Manual wording. Also correct the comment in csr_readq() suggesting the wrong instruction in use. Fixes: 6a6f9b7dafd50efc ("MIPS: Loongson: Add CFUCFG&CSR support") Signed-off-by: WANG Xuerui Cc: Huacai Chen Cc: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mach-loongson64/loongson_regs.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/mips/include/asm/mach-loongson64/loongson_regs.h b/arch/mips/include/asm/mach-loongson64/loongson_regs.h index 363a47a5d26e..07aab3c5b22c 100644 --- a/arch/mips/include/asm/mach-loongson64/loongson_regs.h +++ b/arch/mips/include/asm/mach-loongson64/loongson_regs.h @@ -77,12 +77,12 @@ static inline u32 read_cpucfg(u32 reg) #define LOONGSON_CFG2_LBT3 BIT(6) #define LOONGSON_CFG2_LBTMMU BIT(7) #define LOONGSON_CFG2_LPMP BIT(8) -#define LOONGSON_CFG2_LPMPREV GENMASK(11, 9) +#define LOONGSON_CFG2_LPMREV GENMASK(11, 9) #define LOONGSON_CFG2_LAMO BIT(12) #define LOONGSON_CFG2_LPIXU BIT(13) -#define LOONGSON_CFG2_LPIXUN BIT(14) -#define LOONGSON_CFG2_LZVP BIT(15) -#define LOONGSON_CFG2_LZVREV GENMASK(18, 16) +#define LOONGSON_CFG2_LPIXNU BIT(14) +#define LOONGSON_CFG2_LVZP BIT(15) +#define LOONGSON_CFG2_LVZREV GENMASK(18, 16) #define LOONGSON_CFG2_LGFTP BIT(19) #define LOONGSON_CFG2_LGFTPREV GENMASK(22, 20) #define LOONGSON_CFG2_LLFTP BIT(23) @@ -139,7 +139,7 @@ static inline u64 csr_readq(u32 reg) { u64 __res; - /* DWRCSR reg, val */ + /* DRDCSR reg, val */ __asm__ __volatile__( "parse_r __res,%0\n\t" "parse_r reg,%1\n\t" From fdec207e466d57e658ca18755ff990a18e85d5ae Mon Sep 17 00:00:00 2001 From: WANG Xuerui Date: Sun, 3 May 2020 18:33:03 +0800 Subject: [PATCH 0538/1043] MIPS: Loongson64: define offsets and known revisions for some CPUCFG features Add the constants for easier and maintainable composition of CPUCFG values. Signed-off-by: WANG Xuerui Cc: Huacai Chen Cc: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- .../asm/mach-loongson64/loongson_regs.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/arch/mips/include/asm/mach-loongson64/loongson_regs.h b/arch/mips/include/asm/mach-loongson64/loongson_regs.h index 07aab3c5b22c..83dbb9fdf9c2 100644 --- a/arch/mips/include/asm/mach-loongson64/loongson_regs.h +++ b/arch/mips/include/asm/mach-loongson64/loongson_regs.h @@ -67,6 +67,8 @@ static inline u32 read_cpucfg(u32 reg) #define LOONGSON_CFG1_SFBP BIT(29) #define LOONGSON_CFG1_CDMAP BIT(30) +#define LOONGSON_CFG1_FPREV_OFFSET 1 + #define LOONGSON_CFG2 0x2 #define LOONGSON_CFG2_LEXT1 BIT(0) #define LOONGSON_CFG2_LEXT2 BIT(1) @@ -90,6 +92,13 @@ static inline u32 read_cpucfg(u32 reg) #define LOONGSON_CFG2_LCSRP BIT(27) #define LOONGSON_CFG2_LDISBLIKELY BIT(28) +#define LOONGSON_CFG2_LPMREV_OFFSET 9 +#define LOONGSON_CFG2_LPM_REV1 (1 << LOONGSON_CFG2_LPMREV_OFFSET) +#define LOONGSON_CFG2_LPM_REV2 (2 << LOONGSON_CFG2_LPMREV_OFFSET) +#define LOONGSON_CFG2_LVZREV_OFFSET 16 +#define LOONGSON_CFG2_LVZ_REV1 (1 << LOONGSON_CFG2_LVZREV_OFFSET) +#define LOONGSON_CFG2_LVZ_REV2 (2 << LOONGSON_CFG2_LVZREV_OFFSET) + #define LOONGSON_CFG3 0x3 #define LOONGSON_CFG3_LCAMP BIT(0) #define LOONGSON_CFG3_LCAMREV GENMASK(3, 1) @@ -97,6 +106,16 @@ static inline u32 read_cpucfg(u32 reg) #define LOONGSON_CFG3_LCAMKW GENMASK(19, 12) #define LOONGSON_CFG3_LCAMVW GENMASK(27, 20) +#define LOONGSON_CFG3_LCAMREV_OFFSET 1 +#define LOONGSON_CFG3_LCAM_REV1 (1 << LOONGSON_CFG3_LCAMREV_OFFSET) +#define LOONGSON_CFG3_LCAM_REV2 (2 << LOONGSON_CFG3_LCAMREV_OFFSET) +#define LOONGSON_CFG3_LCAMNUM_OFFSET 4 +#define LOONGSON_CFG3_LCAMNUM_REV1 (0x3f << LOONGSON_CFG3_LCAMNUM_OFFSET) +#define LOONGSON_CFG3_LCAMKW_OFFSET 12 +#define LOONGSON_CFG3_LCAMKW_REV1 (0x27 << LOONGSON_CFG3_LCAMKW_OFFSET) +#define LOONGSON_CFG3_LCAMVW_OFFSET 20 +#define LOONGSON_CFG3_LCAMVW_REV1 (0x3f << LOONGSON_CFG3_LCAMVW_OFFSET) + #define LOONGSON_CFG4 0x4 #define LOONGSON_CFG4_CCFREQ GENMASK(31, 0) From ac44d672787f7f61880114953654e22936c9b008 Mon Sep 17 00:00:00 2001 From: WANG Xuerui Date: Sun, 3 May 2020 18:33:04 +0800 Subject: [PATCH 0539/1043] MIPS: define more Loongson CP0.Config6 and CP0.Diag feature bits These are exposed to userland alternatively via the new CPUCFG instruction on Loongson-3A R4 and above. Add definitions for readback on older cores. Signed-off-by: WANG Xuerui Cc: Huacai Chen Cc: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mipsregs.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h index 796fe47cfd17..90f843c72774 100644 --- a/arch/mips/include/asm/mipsregs.h +++ b/arch/mips/include/asm/mipsregs.h @@ -681,6 +681,10 @@ #define MIPS_CONF6_FTLBDIS (_ULCAST_(1) << 22) /* FTLB probability bits */ #define MIPS_CONF6_FTLBP_SHIFT (16) +/* Loongson-3 feature bits */ +#define MIPS_CONF6_LOONGSON_SCRAND (_ULCAST_(1) << 17) +#define MIPS_CONF6_LOONGSON_LLEXC (_ULCAST_(1) << 16) +#define MIPS_CONF6_LOONGSON_STFILL (_ULCAST_(1) << 8) #define MIPS_CONF7_WII (_ULCAST_(1) << 31) @@ -997,6 +1001,8 @@ #define LOONGSON_DIAG_ITLB (_ULCAST_(1) << 2) /* Flush DTLB */ #define LOONGSON_DIAG_DTLB (_ULCAST_(1) << 3) +/* Allow some CACHE instructions (CACHE0, 1, 3, 21 and 23) in user mode */ +#define LOONGSON_DIAG_UCAC (_ULCAST_(1) << 8) /* Flush VTLB */ #define LOONGSON_DIAG_VTLB (_ULCAST_(1) << 12) /* Flush FTLB */ From 2971317ab04a38e34be5f4d62a65000b63857686 Mon Sep 17 00:00:00 2001 From: Guoyun Sun Date: Wed, 29 Apr 2020 16:44:32 +0800 Subject: [PATCH 0540/1043] mips/mm: Add page soft dirty tracking User space checkpoint and restart tool (CRIU) needs the page's change to be soft tracked. This allows to do a pre checkpoint and then dump only touched pages. Signed-off-by: Guoyun Sun Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/pgtable-bits.h | 20 ++++++++++-- arch/mips/include/asm/pgtable.h | 48 ++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/arch/mips/include/asm/pgtable-bits.h b/arch/mips/include/asm/pgtable-bits.h index 4da79b85c179..e26dc41a8a68 100644 --- a/arch/mips/include/asm/pgtable-bits.h +++ b/arch/mips/include/asm/pgtable-bits.h @@ -55,6 +55,9 @@ enum pgtable_bits { #if defined(CONFIG_ARCH_HAS_PTE_SPECIAL) _PAGE_SPECIAL_SHIFT, #endif +#if defined(CONFIG_HAVE_ARCH_SOFT_DIRTY) + _PAGE_SOFT_DIRTY_SHIFT, +#endif }; /* @@ -84,6 +87,9 @@ enum pgtable_bits { #if defined(CONFIG_ARCH_HAS_PTE_SPECIAL) _PAGE_SPECIAL_SHIFT, #endif +#if defined(CONFIG_HAVE_ARCH_SOFT_DIRTY) + _PAGE_SOFT_DIRTY_SHIFT, +#endif }; #elif defined(CONFIG_CPU_R3K_TLB) @@ -99,6 +105,9 @@ enum pgtable_bits { #if defined(CONFIG_ARCH_HAS_PTE_SPECIAL) _PAGE_SPECIAL_SHIFT, #endif +#if defined(CONFIG_HAVE_ARCH_SOFT_DIRTY) + _PAGE_SOFT_DIRTY_SHIFT, +#endif /* Used by TLB hardware (placed in EntryLo) */ _PAGE_GLOBAL_SHIFT = 8, @@ -125,7 +134,9 @@ enum pgtable_bits { #if defined(CONFIG_ARCH_HAS_PTE_SPECIAL) _PAGE_SPECIAL_SHIFT, #endif - +#if defined(CONFIG_HAVE_ARCH_SOFT_DIRTY) + _PAGE_SOFT_DIRTY_SHIFT, +#endif /* Used by TLB hardware (placed in EntryLo*) */ #if defined(CONFIG_CPU_HAS_RIXI) _PAGE_NO_EXEC_SHIFT, @@ -152,6 +163,11 @@ enum pgtable_bits { #else # define _PAGE_SPECIAL 0 #endif +#if defined(CONFIG_HAVE_ARCH_SOFT_DIRTY) +# define _PAGE_SOFT_DIRTY (1 << _PAGE_SOFT_DIRTY_SHIFT) +#else +# define _PAGE_SOFT_DIRTY 0 +#endif /* Used by TLB hardware (placed in EntryLo*) */ #if defined(CONFIG_XPA) @@ -269,6 +285,6 @@ static inline uint64_t pte_to_entrylo(unsigned long pte_val) #define __WRITEABLE (_PAGE_SILENT_WRITE | _PAGE_WRITE | _PAGE_MODIFIED) #define _PAGE_CHG_MASK (_PAGE_ACCESSED | _PAGE_MODIFIED | \ - _PFN_MASK | _CACHE_MASK) + _PAGE_SOFT_DIRTY | _PFN_MASK | _CACHE_MASK) #endif /* _ASM_PGTABLE_BITS_H */ diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h index aab0ec174f68..9b01d2d78753 100644 --- a/arch/mips/include/asm/pgtable.h +++ b/arch/mips/include/asm/pgtable.h @@ -400,7 +400,7 @@ static inline pte_t pte_mkwrite(pte_t pte) static inline pte_t pte_mkdirty(pte_t pte) { - pte_val(pte) |= _PAGE_MODIFIED; + pte_val(pte) |= _PAGE_MODIFIED | _PAGE_SOFT_DIRTY; if (pte_val(pte) & _PAGE_WRITE) pte_val(pte) |= _PAGE_SILENT_WRITE; return pte; @@ -423,6 +423,30 @@ static inline pte_t pte_mkhuge(pte_t pte) return pte; } #endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */ + +#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY +static inline bool pte_soft_dirty(pte_t pte) +{ + return pte_val(pte) & _PAGE_SOFT_DIRTY; +} +#define pte_swp_soft_dirty pte_soft_dirty + +static inline pte_t pte_mksoft_dirty(pte_t pte) +{ + pte_val(pte) |= _PAGE_SOFT_DIRTY; + return pte; +} +#define pte_swp_mksoft_dirty pte_mksoft_dirty + +static inline pte_t pte_clear_soft_dirty(pte_t pte) +{ + pte_val(pte) &= ~(_PAGE_SOFT_DIRTY); + return pte; +} +#define pte_swp_clear_soft_dirty pte_clear_soft_dirty + +#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ + #endif /* @@ -576,7 +600,7 @@ static inline pmd_t pmd_mkclean(pmd_t pmd) static inline pmd_t pmd_mkdirty(pmd_t pmd) { - pmd_val(pmd) |= _PAGE_MODIFIED; + pmd_val(pmd) |= _PAGE_MODIFIED | _PAGE_SOFT_DIRTY; if (pmd_val(pmd) & _PAGE_WRITE) pmd_val(pmd) |= _PAGE_SILENT_WRITE; @@ -605,6 +629,26 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) return pmd; } +#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY +static inline int pmd_soft_dirty(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_SOFT_DIRTY); +} + +static inline pmd_t pmd_mksoft_dirty(pmd_t pmd) +{ + pmd_val(pmd) |= _PAGE_SOFT_DIRTY; + return pmd; +} + +static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) +{ + pmd_val(pmd) &= ~(_PAGE_SOFT_DIRTY); + return pmd; +} + +#endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */ + /* Extern to avoid header file madness */ extern pmd_t mk_pmd(struct page *page, pgprot_t prot); From 3bfa5bcb26f0b52d7ae8416aa0618fff21aceaaf Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 17 May 2020 13:54:12 -0600 Subject: [PATCH 0541/1043] io_uring: cleanup io_poll_remove_one() logic We only need apoll in the one section, do the juggling with the work restoration there. This removes a special case further down as well. No functional changes in this patch. Signed-off-by: Jens Axboe --- fs/io_uring.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index c7622a5ece2d..7b529270d0d2 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4470,33 +4470,32 @@ static bool __io_poll_remove_one(struct io_kiocb *req, do_complete = true; } spin_unlock(&poll->head->lock); + hash_del(&req->hash_node); return do_complete; } static bool io_poll_remove_one(struct io_kiocb *req) { - struct async_poll *apoll = NULL; bool do_complete; if (req->opcode == IORING_OP_POLL_ADD) { io_poll_remove_double(req); do_complete = __io_poll_remove_one(req, &req->poll); } else { - apoll = req->apoll; + struct async_poll *apoll = req->apoll; + /* non-poll requests have submit ref still */ - do_complete = __io_poll_remove_one(req, &req->apoll->poll); - if (do_complete) + do_complete = __io_poll_remove_one(req, &apoll->poll); + if (do_complete) { io_put_req(req); - } - - hash_del(&req->hash_node); - - if (do_complete && apoll) { - /* - * restore ->work because we need to call io_req_work_drop_env. - */ - memcpy(&req->work, &apoll->work, sizeof(req->work)); - kfree(apoll); + /* + * restore ->work because we will call + * io_req_work_drop_env below when dropping the + * final reference. + */ + memcpy(&req->work, &apoll->work, sizeof(req->work)); + kfree(apoll); + } } if (do_complete) { From 0cdaf760f42eb8e8a714c1cc017423e5da6d4936 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 17 May 2020 14:13:40 +0300 Subject: [PATCH 0542/1043] io_uring: remove req->needs_fixed_files A submission is "async" IIF it's done by SQPOLL thread. Instead of passing @async flag into io_submit_sqes(), deduce it from ctx->flags. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 7b529270d0d2..2d54d3365925 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -626,7 +626,6 @@ struct io_kiocb { struct io_async_ctx *io; int cflags; - bool needs_fixed_file; u8 opcode; struct io_ring_ctx *ctx; @@ -891,6 +890,11 @@ EXPORT_SYMBOL(io_uring_get_socket); static void io_file_put_work(struct work_struct *work); +static inline bool io_async_submit(struct io_ring_ctx *ctx) +{ + return ctx->flags & IORING_SETUP_SQPOLL; +} + static void io_ring_ctx_ref_free(struct percpu_ref *ref) { struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); @@ -5484,7 +5488,7 @@ static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req, bool fixed; fixed = (req->flags & REQ_F_FIXED_FILE) != 0; - if (unlikely(!fixed && req->needs_fixed_file)) + if (unlikely(!fixed && io_async_submit(req->ctx))) return -EBADF; return io_file_get(state, req, fd, &req->file, fixed); @@ -5857,7 +5861,7 @@ static inline void io_consume_sqe(struct io_ring_ctx *ctx) static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, const struct io_uring_sqe *sqe, - struct io_submit_state *state, bool async) + struct io_submit_state *state) { unsigned int sqe_flags; int id; @@ -5878,7 +5882,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, refcount_set(&req->refs, 2); req->task = NULL; req->result = 0; - req->needs_fixed_file = async; INIT_IO_WORK(&req->work, io_wq_submit_work); if (unlikely(req->opcode >= IORING_OP_LAST)) @@ -5919,7 +5922,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, } static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, - struct file *ring_file, int ring_fd, bool async) + struct file *ring_file, int ring_fd) { struct io_submit_state state, *statep = NULL; struct io_kiocb *link = NULL; @@ -5963,7 +5966,7 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, break; } - err = io_init_req(ctx, req, sqe, statep, async); + err = io_init_req(ctx, req, sqe, statep); io_consume_sqe(ctx); /* will complete beyond this point, count as submitted */ submitted++; @@ -5976,7 +5979,7 @@ fail_req: } trace_io_uring_submit_sqe(ctx, req->opcode, req->user_data, - true, async); + true, io_async_submit(ctx)); err = io_submit_sqe(req, sqe, &link); if (err) goto fail_req; @@ -6115,7 +6118,7 @@ static int io_sq_thread(void *data) } mutex_lock(&ctx->uring_lock); - ret = io_submit_sqes(ctx, to_submit, NULL, -1, true); + ret = io_submit_sqes(ctx, to_submit, NULL, -1); mutex_unlock(&ctx->uring_lock); timeout = jiffies + ctx->sq_thread_idle; } @@ -7623,7 +7626,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, submitted = to_submit; } else if (to_submit) { mutex_lock(&ctx->uring_lock); - submitted = io_submit_sqes(ctx, to_submit, f.file, fd, false); + submitted = io_submit_sqes(ctx, to_submit, f.file, fd); mutex_unlock(&ctx->uring_lock); if (submitted != to_submit) From 9f13c35b33fddb186beab9ef21c555a01e45f4d7 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 17 May 2020 14:13:41 +0300 Subject: [PATCH 0543/1043] io_uring: rename io_file_put() io_file_put() deals with flushing state's file refs, adding "state" to its name makes it a bit clearer. Also, avoid double check of state->file in __io_file_get() in some cases. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 2d54d3365925..99f39d766cfc 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1999,15 +1999,19 @@ static void io_iopoll_req_issued(struct io_kiocb *req) wake_up(&ctx->sqo_wait); } -static void io_file_put(struct io_submit_state *state) +static void __io_state_file_put(struct io_submit_state *state) { - if (state->file) { - int diff = state->has_refs - state->used_refs; + int diff = state->has_refs - state->used_refs; - if (diff) - fput_many(state->file, diff); - state->file = NULL; - } + if (diff) + fput_many(state->file, diff); + state->file = NULL; +} + +static inline void io_state_file_put(struct io_submit_state *state) +{ + if (state->file) + __io_state_file_put(state); } /* @@ -2026,7 +2030,7 @@ static struct file *__io_file_get(struct io_submit_state *state, int fd) state->ios_left--; return state->file; } - io_file_put(state); + __io_state_file_put(state); } state->file = fget_many(fd, state->ios_left); if (!state->file) @@ -5790,7 +5794,7 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, static void io_submit_state_end(struct io_submit_state *state) { blk_finish_plug(&state->plug); - io_file_put(state); + io_state_file_put(state); if (state->free_reqs) kmem_cache_free_bulk(req_cachep, state->free_reqs, state->reqs); } From c11368a57be460de889696f6ff8815fbcacf4db2 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 17 May 2020 14:13:42 +0300 Subject: [PATCH 0544/1043] io_uring: don't repeat valid flag list req->flags stores all sqe->flags. After checking that sqe->flags are valid set if IOSQE* flags, no need to double check it, just forward them all. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 99f39d766cfc..c43340396f6d 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -5915,9 +5915,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, } /* same numerical values with corresponding REQ_F_*, safe to copy */ - req->flags |= sqe_flags & (IOSQE_IO_DRAIN | IOSQE_IO_HARDLINK | - IOSQE_ASYNC | IOSQE_FIXED_FILE | - IOSQE_BUFFER_SELECT | IOSQE_IO_LINK); + req->flags |= sqe_flags; if (!io_op_defs[req->opcode].needs_file) return 0; From 9dafdfc2f0a3ae551711098de3d7b621a469f11a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 17 May 2020 14:18:05 +0300 Subject: [PATCH 0545/1043] splice: export do_tee() export do_tee() for use in io_uring Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/splice.c | 3 +-- include/linux/splice.h | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/splice.c b/fs/splice.c index fd0a1e7e5959..a1dd54de24d8 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -1754,8 +1754,7 @@ static int link_pipe(struct pipe_inode_info *ipipe, * The 'flags' used are the SPLICE_F_* variants, currently the only * applicable one is SPLICE_F_NONBLOCK. */ -static long do_tee(struct file *in, struct file *out, size_t len, - unsigned int flags) +long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags) { struct pipe_inode_info *ipipe = get_pipe_info(in); struct pipe_inode_info *opipe = get_pipe_info(out); diff --git a/include/linux/splice.h b/include/linux/splice.h index ebbbfea48aa0..5c47013f708e 100644 --- a/include/linux/splice.h +++ b/include/linux/splice.h @@ -82,6 +82,9 @@ extern long do_splice(struct file *in, loff_t __user *off_in, struct file *out, loff_t __user *off_out, size_t len, unsigned int flags); +extern long do_tee(struct file *in, struct file *out, size_t len, + unsigned int flags); + /* * for dynamic pipe sizing */ From f2a8d5c7a218b9c24befb756c4eb30aa550ce822 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sun, 17 May 2020 14:18:06 +0300 Subject: [PATCH 0546/1043] io_uring: add tee(2) support Add IORING_OP_TEE implementing tee(2) support. Almost identical to splice bits, but without offsets. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 62 +++++++++++++++++++++++++++++++++-- include/uapi/linux/io_uring.h | 1 + 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index c43340396f6d..026350b9c33f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -853,6 +853,11 @@ static const struct io_op_def io_op_defs[] = { }, [IORING_OP_PROVIDE_BUFFERS] = {}, [IORING_OP_REMOVE_BUFFERS] = {}, + [IORING_OP_TEE] = { + .needs_file = 1, + .hash_reg_file = 1, + .unbound_nonreg_file = 1, + }, }; static void io_wq_submit_work(struct io_wq_work **workptr); @@ -2748,7 +2753,8 @@ out_free: return ret; } -static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +static int __io_splice_prep(struct io_kiocb *req, + const struct io_uring_sqe *sqe) { struct io_splice* sp = &req->splice; unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL; @@ -2758,8 +2764,6 @@ static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; sp->file_in = NULL; - sp->off_in = READ_ONCE(sqe->splice_off_in); - sp->off_out = READ_ONCE(sqe->off); sp->len = READ_ONCE(sqe->len); sp->flags = READ_ONCE(sqe->splice_flags); @@ -2778,6 +2782,46 @@ static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } +static int io_tee_prep(struct io_kiocb *req, + const struct io_uring_sqe *sqe) +{ + if (READ_ONCE(sqe->splice_off_in) || READ_ONCE(sqe->off)) + return -EINVAL; + return __io_splice_prep(req, sqe); +} + +static int io_tee(struct io_kiocb *req, bool force_nonblock) +{ + struct io_splice *sp = &req->splice; + struct file *in = sp->file_in; + struct file *out = sp->file_out; + unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; + long ret = 0; + + if (force_nonblock) + return -EAGAIN; + if (sp->len) + ret = do_tee(in, out, sp->len, flags); + + io_put_file(req, in, (sp->flags & SPLICE_F_FD_IN_FIXED)); + req->flags &= ~REQ_F_NEED_CLEANUP; + + io_cqring_add_event(req, ret); + if (ret != sp->len) + req_set_fail_links(req); + io_put_req(req); + return 0; +} + +static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +{ + struct io_splice* sp = &req->splice; + + sp->off_in = READ_ONCE(sqe->splice_off_in); + sp->off_out = READ_ONCE(sqe->off); + return __io_splice_prep(req, sqe); +} + static int io_splice(struct io_kiocb *req, bool force_nonblock) { struct io_splice *sp = &req->splice; @@ -5085,6 +5129,9 @@ static int io_req_defer_prep(struct io_kiocb *req, case IORING_OP_REMOVE_BUFFERS: ret = io_remove_buffers_prep(req, sqe); break; + case IORING_OP_TEE: + ret = io_tee_prep(req, sqe); + break; default: printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n", req->opcode); @@ -5158,6 +5205,7 @@ static void io_cleanup_req(struct io_kiocb *req) putname(req->open.filename); break; case IORING_OP_SPLICE: + case IORING_OP_TEE: io_put_file(req, req->splice.file_in, (req->splice.flags & SPLICE_F_FD_IN_FIXED)); break; @@ -5388,6 +5436,14 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, } ret = io_remove_buffers(req, force_nonblock); break; + case IORING_OP_TEE: + if (sqe) { + ret = io_tee_prep(req, sqe); + if (ret < 0) + break; + } + ret = io_tee(req, force_nonblock); + break; default: ret = -EINVAL; break; diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index 8c5775df08b8..92c22699a5a7 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -129,6 +129,7 @@ enum { IORING_OP_SPLICE, IORING_OP_PROVIDE_BUFFERS, IORING_OP_REMOVE_BUFFERS, + IORING_OP_TEE, /* this goes last, obviously */ IORING_OP_LAST, From 310672552f4aea2ad50704711aa3cdd45f5441e9 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 17 May 2020 17:43:31 -0600 Subject: [PATCH 0547/1043] io_uring: async task poll trigger cleanup If the request is still hashed in io_async_task_func(), then it cannot have been canceled and it's pointless to check. So save that check. Signed-off-by: Jens Axboe --- fs/io_uring.c | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 026350b9c33f..50f079417911 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4354,7 +4354,7 @@ static void io_async_task_func(struct callback_head *cb) struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work); struct async_poll *apoll = req->apoll; struct io_ring_ctx *ctx = req->ctx; - bool canceled; + bool canceled = false; trace_io_uring_task_run(req->ctx, req->opcode, req->user_data); @@ -4363,34 +4363,33 @@ static void io_async_task_func(struct callback_head *cb) return; } - if (hash_hashed(&req->hash_node)) + /* If req is still hashed, it cannot have been canceled. Don't check. */ + if (hash_hashed(&req->hash_node)) { hash_del(&req->hash_node); - - canceled = READ_ONCE(apoll->poll.canceled); - if (canceled) { - io_cqring_fill_event(req, -ECANCELED); - io_commit_cqring(ctx); + } else { + canceled = READ_ONCE(apoll->poll.canceled); + if (canceled) { + io_cqring_fill_event(req, -ECANCELED); + io_commit_cqring(ctx); + } } spin_unlock_irq(&ctx->completion_lock); /* restore ->work in case we need to retry again */ memcpy(&req->work, &apoll->work, sizeof(req->work)); + kfree(apoll); - if (canceled) { - kfree(apoll); + if (!canceled) { + __set_current_state(TASK_RUNNING); + mutex_lock(&ctx->uring_lock); + __io_queue_sqe(req, NULL); + mutex_unlock(&ctx->uring_lock); + } else { io_cqring_ev_posted(ctx); req_set_fail_links(req); io_double_put_req(req); - return; } - - __set_current_state(TASK_RUNNING); - mutex_lock(&ctx->uring_lock); - __io_queue_sqe(req, NULL); - mutex_unlock(&ctx->uring_lock); - - kfree(apoll); } static int io_async_wake(struct wait_queue_entry *wait, unsigned mode, int sync, From 143463fd33fe0abc13fbb93236b060ee9b73d5cb Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Mon, 18 May 2020 10:11:01 +0800 Subject: [PATCH 0548/1043] MIPS: Loongson: Enable devicetree based probing for 8250 ports in defconfig After commit 87fcfa7b7fe6 ("MIPS: Loongson64: Add generic dts"), there already exists the node and property of Loongson CPU UART0 in loongson3-package.dtsi: cpu_uart0: serial@1fe001e0 { compatible = "ns16550a"; reg = <0 0x1fe001e0 0x8>; clock-frequency = <33000000>; interrupt-parent = <&liointc>; interrupts = <10 IRQ_TYPE_LEVEL_HIGH>; no-loopback-test; }; In order to support for serial console on the Loongson platform, add CONFIG_SERIAL_OF_PLATFORM=y to loongson3_defconfig. With this patch, we can see the following boot messages: [ 1.877745] printk: console [ttyS0] disabled [ 1.881979] 1fe001e0.serial: ttyS0 at MMIO 0x1fe001e0 (irq = 16, base_baud = 2062500) is a 16550A [ 1.890838] printk: console [ttyS0] enabled And also, we can login normally from the serial console. Signed-off-by: Tiezhu Yang Reviewed-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/configs/loongson3_defconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/mips/configs/loongson3_defconfig b/arch/mips/configs/loongson3_defconfig index 4df24343e8ba..3d4c7e9996c5 100644 --- a/arch/mips/configs/loongson3_defconfig +++ b/arch/mips/configs/loongson3_defconfig @@ -217,6 +217,7 @@ CONFIG_SERIAL_8250_EXTENDED=y CONFIG_SERIAL_8250_MANY_PORTS=y CONFIG_SERIAL_8250_SHARE_IRQ=y CONFIG_SERIAL_8250_RSA=y +CONFIG_SERIAL_OF_PLATFORM=y CONFIG_HW_RANDOM=y CONFIG_RAW_DRIVER=m CONFIG_I2C_CHARDEV=y From 8be26bab0ea2097ab1d7bca85fb3b2e295faf5ec Mon Sep 17 00:00:00 2001 From: Joshua Kinard Date: Sun, 17 May 2020 19:24:38 -0400 Subject: [PATCH 0549/1043] MIPS: SGI-IP30: Remove R5432_CP0_INTERRUPT_WAR from war.h Remove an old macro that no longer exists anywhere else in the tree that snuck in when IP30 support was added Signed-off-by: Joshua Kinard Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mach-ip30/war.h | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/mips/include/asm/mach-ip30/war.h b/arch/mips/include/asm/mach-ip30/war.h index a98ba204f183..ad3352d3d203 100644 --- a/arch/mips/include/asm/mach-ip30/war.h +++ b/arch/mips/include/asm/mach-ip30/war.h @@ -13,7 +13,6 @@ #define SIBYTE_1956_WAR 0 #define MIPS4K_ICACHE_REFILL_WAR 0 #define MIPS34K_MISSED_ITLB_WAR 0 -#define R5432_CP0_INTERRUPT_WAR 0 #define TX49XX_ICACHE_INDEX_INV_WAR 0 #define ICACHE_REFILLS_WORKAROUND_WAR 0 From ae0bb9fda405c881848f7f6e94d912b35f6e31d2 Mon Sep 17 00:00:00 2001 From: Shaokun Zhang Date: Mon, 18 May 2020 11:00:59 +0800 Subject: [PATCH 0550/1043] platform-msi: Fix typos in comment Fix up one typos @nev -> @nr_irqs. Signed-off-by: Shaokun Zhang Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/1589770859-19340-1-git-send-email-zhangshaokun@hisilicon.com --- drivers/base/platform-msi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c index 8da314b81eab..c4a17e5edf8b 100644 --- a/drivers/base/platform-msi.c +++ b/drivers/base/platform-msi.c @@ -387,7 +387,7 @@ void platform_msi_domain_free(struct irq_domain *domain, unsigned int virq, * * @domain: The platform-msi domain * @virq: The base irq from which to perform the allocate operation - * @nvec: How many interrupts to free from @virq + * @nr_irqs: How many interrupts to free from @virq * * Return 0 on success, or an error code on failure. Must be called * with irq_domain_mutex held (which can only be done as part of a From 8a94c1ab34d53476617f83610521cfb6674db8d4 Mon Sep 17 00:00:00 2001 From: Ingo Rohloff Date: Wed, 22 Apr 2020 13:28:57 +0200 Subject: [PATCH 0551/1043] irqchip/gic-v3: Fix missing "__init" for gic_smp_init() With an SMP configuration, gic_smp_init() calls set_smp_cross_call(). set_smp_cross_call() is marked with "__init". So gic_smp_init() should also be marked with "__init". gic_smp_init() is only called from gic_init_bases(). gic_init_bases() is also marked with "__init"; So marking gic_smp_init() with "__init" is fine. Signed-off-by: Ingo Rohloff Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200422112857.4300-1-ingo.rohloff@lauterbach.com --- drivers/irqchip/irq-gic-v3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index d7006ef18a0d..98c886dab02d 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1150,7 +1150,7 @@ static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) isb(); } -static void gic_smp_init(void) +static void __init gic_smp_init(void) { set_smp_cross_call(gic_raise_softirq); cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, From 82f2202ddc97490994fad0dbfec04a014fa5163d Mon Sep 17 00:00:00 2001 From: "Wesley W. Terpstra" Date: Tue, 12 May 2020 10:26:36 -0700 Subject: [PATCH 0552/1043] irqchip/sifive-plic: Remove incorrect requirement about number of irq contexts A PLIC may not be connected to all the cores. In that case, nr_contexts may be less than num_possible_cpus. This requirement is only valid a single PLIC is the only interrupt controller for the whole system. Signed-off-by: Atish Patra Signed-off-by: "Wesley W. Terpstra" Signed-off-by: Marc Zyngier Reviewed-by: Palmer Dabbelt Acked-by: Palmer Dabbelt Link: https://lore.kernel.org/r/20200512172636.96299-1-atish.patra@wdc.com [Atish: Modified the commit text] --- drivers/irqchip/irq-sifive-plic.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c index d0a71febdadc..822e074c0600 100644 --- a/drivers/irqchip/irq-sifive-plic.c +++ b/drivers/irqchip/irq-sifive-plic.c @@ -301,8 +301,6 @@ static int __init plic_init(struct device_node *node, nr_contexts = of_irq_count(node); if (WARN_ON(!nr_contexts)) goto out_iounmap; - if (WARN_ON(nr_contexts < num_possible_cpus())) - goto out_iounmap; error = -ENOMEM; priv->irqdomain = irq_domain_add_linear(node, nr_irqs + 1, From 5c8f77a278737a6af44a892f0700d9aadb2b0de0 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Thu, 14 May 2020 10:39:00 +0200 Subject: [PATCH 0553/1043] irqdomain: Make irq_domain_reset_irq_data() available to non-hierarchical users irq_domain_reset_irq_data() doesn't modify the parent data, so it can be made available even if irq domain hierarchy is not being built. We'll subsequently use it in irq_sim code. Signed-off-by: Bartosz Golaszewski Signed-off-by: Marc Zyngier Reviewed-by: Linus Walleij Link: https://lore.kernel.org/r/20200514083901.23445-2-brgl@bgdev.pl --- include/linux/irqdomain.h | 2 +- kernel/irq/irqdomain.c | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/linux/irqdomain.h b/include/linux/irqdomain.h index 8d062e86d954..b37350c4fe37 100644 --- a/include/linux/irqdomain.h +++ b/include/linux/irqdomain.h @@ -450,6 +450,7 @@ extern void irq_domain_set_info(struct irq_domain *domain, unsigned int virq, irq_hw_number_t hwirq, struct irq_chip *chip, void *chip_data, irq_flow_handler_t handler, void *handler_data, const char *handler_name); +extern void irq_domain_reset_irq_data(struct irq_data *irq_data); #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY extern struct irq_domain *irq_domain_create_hierarchy(struct irq_domain *parent, unsigned int flags, unsigned int size, @@ -491,7 +492,6 @@ extern int irq_domain_set_hwirq_and_chip(struct irq_domain *domain, irq_hw_number_t hwirq, struct irq_chip *chip, void *chip_data); -extern void irq_domain_reset_irq_data(struct irq_data *irq_data); extern void irq_domain_free_irqs_common(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs); diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 35b8d97c3a1d..e2aa128ea3ee 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -1047,6 +1047,18 @@ int irq_domain_alloc_descs(int virq, unsigned int cnt, irq_hw_number_t hwirq, return virq; } +/** + * irq_domain_reset_irq_data - Clear hwirq, chip and chip_data in @irq_data + * @irq_data: The pointer to irq_data + */ +void irq_domain_reset_irq_data(struct irq_data *irq_data) +{ + irq_data->hwirq = 0; + irq_data->chip = &no_irq_chip; + irq_data->chip_data = NULL; +} +EXPORT_SYMBOL_GPL(irq_domain_reset_irq_data); + #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY /** * irq_domain_create_hierarchy - Add a irqdomain into the hierarchy @@ -1247,18 +1259,6 @@ void irq_domain_set_info(struct irq_domain *domain, unsigned int virq, } EXPORT_SYMBOL(irq_domain_set_info); -/** - * irq_domain_reset_irq_data - Clear hwirq, chip and chip_data in @irq_data - * @irq_data: The pointer to irq_data - */ -void irq_domain_reset_irq_data(struct irq_data *irq_data) -{ - irq_data->hwirq = 0; - irq_data->chip = &no_irq_chip; - irq_data->chip_data = NULL; -} -EXPORT_SYMBOL_GPL(irq_domain_reset_irq_data); - /** * irq_domain_free_irqs_common - Clear irq_data and free the parent * @domain: Interrupt domain to match From 337cbeb2c13eb4cab84f576fd402d7ae4ed31ae1 Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Thu, 14 May 2020 10:39:01 +0200 Subject: [PATCH 0554/1043] genirq/irq_sim: Simplify the API The interrupt simulator API exposes a lot of custom data structures and functions and doesn't reuse the interfaces already exposed by the irq subsystem. This patch tries to address it. We hide all the simulator-related data structures from users and instead rely on the well-known irq domain. When creating the interrupt simulator the user receives a pointer to a newly created irq_domain and can use it to create mappings for simulated interrupts. It is also possible to pass a handle to fwnode when creating the simulator domain and retrieve it using irq_find_matching_fwnode(). The irq_sim_fire() function is dropped as well. Instead we implement the irq_get/set_irqchip_state interface. We modify the two modules that use the simulator at the same time as adding these changes in order to reduce the intermediate bloat that would result when trying to migrate the drivers in separate patches. Signed-off-by: Bartosz Golaszewski Signed-off-by: Marc Zyngier Reviewed-by: Linus Walleij Acked-by: Jonathan Cameron #for IIO Link: https://lore.kernel.org/r/20200514083901.23445-3-brgl@bgdev.pl --- drivers/gpio/gpio-mockup.c | 53 ++++-- drivers/iio/dummy/iio_dummy_evgen.c | 34 ++-- include/linux/irq_sim.h | 33 +--- kernel/irq/Kconfig | 1 + kernel/irq/irq_sim.c | 281 +++++++++++++++++----------- 5 files changed, 244 insertions(+), 158 deletions(-) diff --git a/drivers/gpio/gpio-mockup.c b/drivers/gpio/gpio-mockup.c index 3eb94f3740d1..bc345185db26 100644 --- a/drivers/gpio/gpio-mockup.c +++ b/drivers/gpio/gpio-mockup.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -48,7 +49,7 @@ struct gpio_mockup_line_status { struct gpio_mockup_chip { struct gpio_chip gc; struct gpio_mockup_line_status *lines; - struct irq_sim irqsim; + struct irq_domain *irq_sim_domain; struct dentry *dbg_dir; struct mutex lock; }; @@ -144,14 +145,12 @@ static void gpio_mockup_set_multiple(struct gpio_chip *gc, static int gpio_mockup_apply_pull(struct gpio_mockup_chip *chip, unsigned int offset, int value) { + int curr, irq, irq_type, ret = 0; struct gpio_desc *desc; struct gpio_chip *gc; - struct irq_sim *sim; - int curr, irq, irq_type; gc = &chip->gc; desc = &gc->gpiodev->descs[offset]; - sim = &chip->irqsim; mutex_lock(&chip->lock); @@ -161,14 +160,28 @@ static int gpio_mockup_apply_pull(struct gpio_mockup_chip *chip, if (curr == value) goto out; - irq = irq_sim_irqnum(sim, offset); + irq = irq_find_mapping(chip->irq_sim_domain, offset); + if (!irq) + /* + * This is fine - it just means, nobody is listening + * for interrupts on this line, otherwise + * irq_create_mapping() would have been called from + * the to_irq() callback. + */ + goto set_value; + irq_type = irq_get_trigger_type(irq); if ((value == 1 && (irq_type & IRQ_TYPE_EDGE_RISING)) || - (value == 0 && (irq_type & IRQ_TYPE_EDGE_FALLING))) - irq_sim_fire(sim, offset); + (value == 0 && (irq_type & IRQ_TYPE_EDGE_FALLING))) { + ret = irq_set_irqchip_state(irq, IRQCHIP_STATE_PENDING, + true); + if (ret) + goto out; + } } +set_value: /* Change the value unless we're actively driving the line. */ if (!test_bit(FLAG_REQUESTED, &desc->flags) || !test_bit(FLAG_IS_OUT, &desc->flags)) @@ -177,7 +190,7 @@ static int gpio_mockup_apply_pull(struct gpio_mockup_chip *chip, out: chip->lines[offset].pull = value; mutex_unlock(&chip->lock); - return 0; + return ret; } static int gpio_mockup_set_config(struct gpio_chip *gc, @@ -236,7 +249,7 @@ static int gpio_mockup_to_irq(struct gpio_chip *gc, unsigned int offset) { struct gpio_mockup_chip *chip = gpiochip_get_data(gc); - return irq_sim_irqnum(&chip->irqsim, offset); + return irq_create_mapping(chip->irq_sim_domain, offset); } static void gpio_mockup_free(struct gpio_chip *gc, unsigned int offset) @@ -389,6 +402,19 @@ static int gpio_mockup_name_lines(struct device *dev, return 0; } +static void gpio_mockup_dispose_mappings(void *data) +{ + struct gpio_mockup_chip *chip = data; + struct gpio_chip *gc = &chip->gc; + int i, irq; + + for (i = 0; i < gc->ngpio; i++) { + irq = irq_find_mapping(chip->irq_sim_domain, i); + if (irq) + irq_dispose_mapping(irq); + } +} + static int gpio_mockup_probe(struct platform_device *pdev) { struct gpio_mockup_chip *chip; @@ -456,8 +482,13 @@ static int gpio_mockup_probe(struct platform_device *pdev) return rv; } - rv = devm_irq_sim_init(dev, &chip->irqsim, gc->ngpio); - if (rv < 0) + chip->irq_sim_domain = devm_irq_domain_create_sim(dev, NULL, + gc->ngpio); + if (IS_ERR(chip->irq_sim_domain)) + return PTR_ERR(chip->irq_sim_domain); + + rv = devm_add_action_or_reset(dev, gpio_mockup_dispose_mappings, chip); + if (rv) return rv; rv = devm_gpiochip_add_data(dev, &chip->gc, chip); diff --git a/drivers/iio/dummy/iio_dummy_evgen.c b/drivers/iio/dummy/iio_dummy_evgen.c index a6edf30567aa..409fe0f7df1c 100644 --- a/drivers/iio/dummy/iio_dummy_evgen.c +++ b/drivers/iio/dummy/iio_dummy_evgen.c @@ -37,8 +37,7 @@ struct iio_dummy_eventgen { struct iio_dummy_regs regs[IIO_EVENTGEN_NO]; struct mutex lock; bool inuse[IIO_EVENTGEN_NO]; - struct irq_sim irq_sim; - int base; + struct irq_domain *irq_sim_domain; }; /* We can only ever have one instance of this 'device' */ @@ -46,19 +45,17 @@ static struct iio_dummy_eventgen *iio_evgen; static int iio_dummy_evgen_create(void) { - int ret; - iio_evgen = kzalloc(sizeof(*iio_evgen), GFP_KERNEL); if (!iio_evgen) return -ENOMEM; - ret = irq_sim_init(&iio_evgen->irq_sim, IIO_EVENTGEN_NO); - if (ret < 0) { + iio_evgen->irq_sim_domain = irq_domain_create_sim(NULL, + IIO_EVENTGEN_NO); + if (IS_ERR(iio_evgen->irq_sim_domain)) { kfree(iio_evgen); - return ret; + return PTR_ERR(iio_evgen->irq_sim_domain); } - iio_evgen->base = irq_sim_irqnum(&iio_evgen->irq_sim, 0); mutex_init(&iio_evgen->lock); return 0; @@ -80,7 +77,7 @@ int iio_dummy_evgen_get_irq(void) mutex_lock(&iio_evgen->lock); for (i = 0; i < IIO_EVENTGEN_NO; i++) { if (!iio_evgen->inuse[i]) { - ret = irq_sim_irqnum(&iio_evgen->irq_sim, i); + ret = irq_create_mapping(iio_evgen->irq_sim_domain, i); iio_evgen->inuse[i] = true; break; } @@ -101,21 +98,27 @@ EXPORT_SYMBOL_GPL(iio_dummy_evgen_get_irq); */ void iio_dummy_evgen_release_irq(int irq) { + struct irq_data *irqd = irq_get_irq_data(irq); + mutex_lock(&iio_evgen->lock); - iio_evgen->inuse[irq - iio_evgen->base] = false; + iio_evgen->inuse[irqd_to_hwirq(irqd)] = false; + irq_dispose_mapping(irq); mutex_unlock(&iio_evgen->lock); } EXPORT_SYMBOL_GPL(iio_dummy_evgen_release_irq); struct iio_dummy_regs *iio_dummy_evgen_get_regs(int irq) { - return &iio_evgen->regs[irq - iio_evgen->base]; + struct irq_data *irqd = irq_get_irq_data(irq); + + return &iio_evgen->regs[irqd_to_hwirq(irqd)]; + } EXPORT_SYMBOL_GPL(iio_dummy_evgen_get_regs); static void iio_dummy_evgen_free(void) { - irq_sim_fini(&iio_evgen->irq_sim); + irq_domain_remove_sim(iio_evgen->irq_sim_domain); kfree(iio_evgen); } @@ -131,7 +134,7 @@ static ssize_t iio_evgen_poke(struct device *dev, { struct iio_dev_attr *this_attr = to_iio_dev_attr(attr); unsigned long event; - int ret; + int ret, irq; ret = kstrtoul(buf, 10, &event); if (ret) @@ -140,7 +143,10 @@ static ssize_t iio_evgen_poke(struct device *dev, iio_evgen->regs[this_attr->address].reg_id = this_attr->address; iio_evgen->regs[this_attr->address].reg_data = event; - irq_sim_fire(&iio_evgen->irq_sim, this_attr->address); + irq = irq_find_mapping(iio_evgen->irq_sim_domain, this_attr->address); + ret = irq_set_irqchip_state(irq, IRQCHIP_STATE_PENDING, true); + if (ret) + return ret; return len; } diff --git a/include/linux/irq_sim.h b/include/linux/irq_sim.h index 4500d453a63e..ab831e5ae748 100644 --- a/include/linux/irq_sim.h +++ b/include/linux/irq_sim.h @@ -1,41 +1,26 @@ /* SPDX-License-Identifier: GPL-2.0+ */ /* * Copyright (C) 2017-2018 Bartosz Golaszewski + * Copyright (C) 2020 Bartosz Golaszewski */ #ifndef _LINUX_IRQ_SIM_H #define _LINUX_IRQ_SIM_H -#include #include +#include +#include /* * Provides a framework for allocating simulated interrupts which can be * requested like normal irqs and enqueued from process context. */ -struct irq_sim_work_ctx { - struct irq_work work; - unsigned long *pending; -}; - -struct irq_sim_irq_ctx { - int irqnum; - bool enabled; -}; - -struct irq_sim { - struct irq_sim_work_ctx work_ctx; - int irq_base; - unsigned int irq_count; - struct irq_sim_irq_ctx *irqs; -}; - -int irq_sim_init(struct irq_sim *sim, unsigned int num_irqs); -int devm_irq_sim_init(struct device *dev, struct irq_sim *sim, - unsigned int num_irqs); -void irq_sim_fini(struct irq_sim *sim); -void irq_sim_fire(struct irq_sim *sim, unsigned int offset); -int irq_sim_irqnum(struct irq_sim *sim, unsigned int offset); +struct irq_domain *irq_domain_create_sim(struct fwnode_handle *fwnode, + unsigned int num_irqs); +struct irq_domain *devm_irq_domain_create_sim(struct device *dev, + struct fwnode_handle *fwnode, + unsigned int num_irqs); +void irq_domain_remove_sim(struct irq_domain *domain); #endif /* _LINUX_IRQ_SIM_H */ diff --git a/kernel/irq/Kconfig b/kernel/irq/Kconfig index 20d501af4f2e..d63c324895ea 100644 --- a/kernel/irq/Kconfig +++ b/kernel/irq/Kconfig @@ -72,6 +72,7 @@ config IRQ_DOMAIN config IRQ_SIM bool select IRQ_WORK + select IRQ_DOMAIN # Support for hierarchical irq domains config IRQ_DOMAIN_HIERARCHY diff --git a/kernel/irq/irq_sim.c b/kernel/irq/irq_sim.c index b992f88c5613..48006608baf0 100644 --- a/kernel/irq/irq_sim.c +++ b/kernel/irq/irq_sim.c @@ -1,14 +1,31 @@ // SPDX-License-Identifier: GPL-2.0+ /* * Copyright (C) 2017-2018 Bartosz Golaszewski + * Copyright (C) 2020 Bartosz Golaszewski */ -#include -#include #include +#include +#include +#include +#include + +struct irq_sim_work_ctx { + struct irq_work work; + int irq_base; + unsigned int irq_count; + unsigned long *pending; + struct irq_domain *domain; +}; + +struct irq_sim_irq_ctx { + int irqnum; + bool enabled; + struct irq_sim_work_ctx *work_ctx; +}; struct irq_sim_devres { - struct irq_sim *sim; + struct irq_domain *domain; }; static void irq_sim_irqmask(struct irq_data *data) @@ -36,159 +53,205 @@ static int irq_sim_set_type(struct irq_data *data, unsigned int type) return 0; } +static int irq_sim_get_irqchip_state(struct irq_data *data, + enum irqchip_irq_state which, bool *state) +{ + struct irq_sim_irq_ctx *irq_ctx = irq_data_get_irq_chip_data(data); + irq_hw_number_t hwirq = irqd_to_hwirq(data); + + switch (which) { + case IRQCHIP_STATE_PENDING: + if (irq_ctx->enabled) + *state = test_bit(hwirq, irq_ctx->work_ctx->pending); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int irq_sim_set_irqchip_state(struct irq_data *data, + enum irqchip_irq_state which, bool state) +{ + struct irq_sim_irq_ctx *irq_ctx = irq_data_get_irq_chip_data(data); + irq_hw_number_t hwirq = irqd_to_hwirq(data); + + switch (which) { + case IRQCHIP_STATE_PENDING: + if (irq_ctx->enabled) { + assign_bit(hwirq, irq_ctx->work_ctx->pending, state); + if (state) + irq_work_queue(&irq_ctx->work_ctx->work); + } + break; + default: + return -EINVAL; + } + + return 0; +} + static struct irq_chip irq_sim_irqchip = { - .name = "irq_sim", - .irq_mask = irq_sim_irqmask, - .irq_unmask = irq_sim_irqunmask, - .irq_set_type = irq_sim_set_type, + .name = "irq_sim", + .irq_mask = irq_sim_irqmask, + .irq_unmask = irq_sim_irqunmask, + .irq_set_type = irq_sim_set_type, + .irq_get_irqchip_state = irq_sim_get_irqchip_state, + .irq_set_irqchip_state = irq_sim_set_irqchip_state, }; static void irq_sim_handle_irq(struct irq_work *work) { struct irq_sim_work_ctx *work_ctx; unsigned int offset = 0; - struct irq_sim *sim; int irqnum; work_ctx = container_of(work, struct irq_sim_work_ctx, work); - sim = container_of(work_ctx, struct irq_sim, work_ctx); - while (!bitmap_empty(work_ctx->pending, sim->irq_count)) { + while (!bitmap_empty(work_ctx->pending, work_ctx->irq_count)) { offset = find_next_bit(work_ctx->pending, - sim->irq_count, offset); + work_ctx->irq_count, offset); clear_bit(offset, work_ctx->pending); - irqnum = irq_sim_irqnum(sim, offset); + irqnum = irq_find_mapping(work_ctx->domain, offset); handle_simple_irq(irq_to_desc(irqnum)); } } -/** - * irq_sim_init - Initialize the interrupt simulator: allocate a range of - * dummy interrupts. - * - * @sim: The interrupt simulator object to initialize. - * @num_irqs: Number of interrupts to allocate - * - * On success: return the base of the allocated interrupt range. - * On failure: a negative errno. - */ -int irq_sim_init(struct irq_sim *sim, unsigned int num_irqs) +static int irq_sim_domain_map(struct irq_domain *domain, + unsigned int virq, irq_hw_number_t hw) { - int i; + struct irq_sim_work_ctx *work_ctx = domain->host_data; + struct irq_sim_irq_ctx *irq_ctx; - sim->irqs = kmalloc_array(num_irqs, sizeof(*sim->irqs), GFP_KERNEL); - if (!sim->irqs) + irq_ctx = kzalloc(sizeof(*irq_ctx), GFP_KERNEL); + if (!irq_ctx) return -ENOMEM; - sim->irq_base = irq_alloc_descs(-1, 0, num_irqs, 0); - if (sim->irq_base < 0) { - kfree(sim->irqs); - return sim->irq_base; - } + irq_set_chip(virq, &irq_sim_irqchip); + irq_set_chip_data(virq, irq_ctx); + irq_set_handler(virq, handle_simple_irq); + irq_modify_status(virq, IRQ_NOREQUEST | IRQ_NOAUTOEN, IRQ_NOPROBE); + irq_ctx->work_ctx = work_ctx; - sim->work_ctx.pending = bitmap_zalloc(num_irqs, GFP_KERNEL); - if (!sim->work_ctx.pending) { - kfree(sim->irqs); - irq_free_descs(sim->irq_base, num_irqs); - return -ENOMEM; - } - - for (i = 0; i < num_irqs; i++) { - sim->irqs[i].irqnum = sim->irq_base + i; - sim->irqs[i].enabled = false; - irq_set_chip(sim->irq_base + i, &irq_sim_irqchip); - irq_set_chip_data(sim->irq_base + i, &sim->irqs[i]); - irq_set_handler(sim->irq_base + i, &handle_simple_irq); - irq_modify_status(sim->irq_base + i, - IRQ_NOREQUEST | IRQ_NOAUTOEN, IRQ_NOPROBE); - } - - init_irq_work(&sim->work_ctx.work, irq_sim_handle_irq); - sim->irq_count = num_irqs; - - return sim->irq_base; + return 0; } -EXPORT_SYMBOL_GPL(irq_sim_init); + +static void irq_sim_domain_unmap(struct irq_domain *domain, unsigned int virq) +{ + struct irq_sim_irq_ctx *irq_ctx; + struct irq_data *irqd; + + irqd = irq_domain_get_irq_data(domain, virq); + irq_ctx = irq_data_get_irq_chip_data(irqd); + + irq_set_handler(virq, NULL); + irq_domain_reset_irq_data(irqd); + kfree(irq_ctx); +} + +static const struct irq_domain_ops irq_sim_domain_ops = { + .map = irq_sim_domain_map, + .unmap = irq_sim_domain_unmap, +}; /** - * irq_sim_fini - Deinitialize the interrupt simulator: free the interrupt - * descriptors and allocated memory. + * irq_domain_create_sim - Create a new interrupt simulator irq_domain and + * allocate a range of dummy interrupts. * - * @sim: The interrupt simulator to tear down. + * @fnode: struct fwnode_handle to be associated with this domain. + * @num_irqs: Number of interrupts to allocate. + * + * On success: return a new irq_domain object. + * On failure: a negative errno wrapped with ERR_PTR(). */ -void irq_sim_fini(struct irq_sim *sim) +struct irq_domain *irq_domain_create_sim(struct fwnode_handle *fwnode, + unsigned int num_irqs) { - irq_work_sync(&sim->work_ctx.work); - bitmap_free(sim->work_ctx.pending); - irq_free_descs(sim->irq_base, sim->irq_count); - kfree(sim->irqs); -} -EXPORT_SYMBOL_GPL(irq_sim_fini); + struct irq_sim_work_ctx *work_ctx; -static void devm_irq_sim_release(struct device *dev, void *res) + work_ctx = kmalloc(sizeof(*work_ctx), GFP_KERNEL); + if (!work_ctx) + goto err_out; + + work_ctx->pending = bitmap_zalloc(num_irqs, GFP_KERNEL); + if (!work_ctx->pending) + goto err_free_work_ctx; + + work_ctx->domain = irq_domain_create_linear(fwnode, num_irqs, + &irq_sim_domain_ops, + work_ctx); + if (!work_ctx->domain) + goto err_free_bitmap; + + work_ctx->irq_count = num_irqs; + init_irq_work(&work_ctx->work, irq_sim_handle_irq); + + return work_ctx->domain; + +err_free_bitmap: + bitmap_free(work_ctx->pending); +err_free_work_ctx: + kfree(work_ctx); +err_out: + return ERR_PTR(-ENOMEM); +} +EXPORT_SYMBOL_GPL(irq_domain_create_sim); + +/** + * irq_domain_remove_sim - Deinitialize the interrupt simulator domain: free + * the interrupt descriptors and allocated memory. + * + * @domain: The interrupt simulator domain to tear down. + */ +void irq_domain_remove_sim(struct irq_domain *domain) +{ + struct irq_sim_work_ctx *work_ctx = domain->host_data; + + irq_work_sync(&work_ctx->work); + bitmap_free(work_ctx->pending); + kfree(work_ctx); + + irq_domain_remove(domain); +} +EXPORT_SYMBOL_GPL(irq_domain_remove_sim); + +static void devm_irq_domain_release_sim(struct device *dev, void *res) { struct irq_sim_devres *this = res; - irq_sim_fini(this->sim); + irq_domain_remove_sim(this->domain); } /** - * irq_sim_init - Initialize the interrupt simulator for a managed device. + * devm_irq_domain_create_sim - Create a new interrupt simulator for + * a managed device. * * @dev: Device to initialize the simulator object for. - * @sim: The interrupt simulator object to initialize. + * @fnode: struct fwnode_handle to be associated with this domain. * @num_irqs: Number of interrupts to allocate * - * On success: return the base of the allocated interrupt range. - * On failure: a negative errno. + * On success: return a new irq_domain object. + * On failure: a negative errno wrapped with ERR_PTR(). */ -int devm_irq_sim_init(struct device *dev, struct irq_sim *sim, - unsigned int num_irqs) +struct irq_domain *devm_irq_domain_create_sim(struct device *dev, + struct fwnode_handle *fwnode, + unsigned int num_irqs) { struct irq_sim_devres *dr; - int rv; - dr = devres_alloc(devm_irq_sim_release, sizeof(*dr), GFP_KERNEL); + dr = devres_alloc(devm_irq_domain_release_sim, + sizeof(*dr), GFP_KERNEL); if (!dr) - return -ENOMEM; + return ERR_PTR(-ENOMEM); - rv = irq_sim_init(sim, num_irqs); - if (rv < 0) { + dr->domain = irq_domain_create_sim(fwnode, num_irqs); + if (IS_ERR(dr->domain)) { devres_free(dr); - return rv; + return dr->domain; } - dr->sim = sim; devres_add(dev, dr); - - return rv; + return dr->domain; } -EXPORT_SYMBOL_GPL(devm_irq_sim_init); - -/** - * irq_sim_fire - Enqueue an interrupt. - * - * @sim: The interrupt simulator object. - * @offset: Offset of the simulated interrupt which should be fired. - */ -void irq_sim_fire(struct irq_sim *sim, unsigned int offset) -{ - if (sim->irqs[offset].enabled) { - set_bit(offset, sim->work_ctx.pending); - irq_work_queue(&sim->work_ctx.work); - } -} -EXPORT_SYMBOL_GPL(irq_sim_fire); - -/** - * irq_sim_irqnum - Get the allocated number of a dummy interrupt. - * - * @sim: The interrupt simulator object. - * @offset: Offset of the simulated interrupt for which to retrieve - * the number. - */ -int irq_sim_irqnum(struct irq_sim *sim, unsigned int offset) -{ - return sim->irqs[offset].irqnum; -} -EXPORT_SYMBOL_GPL(irq_sim_irqnum); +EXPORT_SYMBOL_GPL(devm_irq_domain_create_sim); From 2f13ff1d1d5c0257c97ea76b86a2d9c99c44a4b9 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 15 May 2020 17:57:51 +0100 Subject: [PATCH 0555/1043] irqchip/gic-v3-its: Track LPI distribution on a per CPU basis In order to improve the distribution of LPIs among CPUs, let start by tracking the number of LPIs assigned to CPUs, both for managed and non-managed interrupts (as separate counters). Signed-off-by: Marc Zyngier Tested-by: John Garry Link: https://lore.kernel.org/r/20200515165752.121296-2-maz@kernel.org --- drivers/irqchip/irq-gic-v3-its.c | 49 ++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 124251b0ccba..4eb8441d0c2b 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -174,6 +174,13 @@ static struct { int next_victim; } vpe_proxy; +struct cpu_lpi_count { + atomic_t managed; + atomic_t unmanaged; +}; + +static DEFINE_PER_CPU(struct cpu_lpi_count, cpu_lpi_count); + static LIST_HEAD(its_nodes); static DEFINE_RAW_SPINLOCK(its_lock); static struct rdists *gic_rdists; @@ -1510,6 +1517,30 @@ static void its_unmask_irq(struct irq_data *d) lpi_update_config(d, 0, LPI_PROP_ENABLED); } +static __maybe_unused u32 its_read_lpi_count(struct irq_data *d, int cpu) +{ + if (irqd_affinity_is_managed(d)) + return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); + + return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); +} + +static void its_inc_lpi_count(struct irq_data *d, int cpu) +{ + if (irqd_affinity_is_managed(d)) + atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); + else + atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); +} + +static void its_dec_lpi_count(struct irq_data *d, int cpu) +{ + if (irqd_affinity_is_managed(d)) + atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); + else + atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); +} + static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, bool force) { @@ -1518,34 +1549,44 @@ static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, struct its_device *its_dev = irq_data_get_irq_chip_data(d); struct its_collection *target_col; u32 id = its_get_event_id(d); + int prev_cpu; /* A forwarded interrupt should use irq_set_vcpu_affinity */ if (irqd_is_forwarded_to_vcpu(d)) return -EINVAL; + prev_cpu = its_dev->event_map.col_map[id]; + its_dec_lpi_count(d, prev_cpu); + /* lpi cannot be routed to a redistributor that is on a foreign node */ if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) { if (its_dev->its->numa_node >= 0) { cpu_mask = cpumask_of_node(its_dev->its->numa_node); if (!cpumask_intersects(mask_val, cpu_mask)) - return -EINVAL; + goto err; } } cpu = cpumask_any_and(mask_val, cpu_mask); if (cpu >= nr_cpu_ids) - return -EINVAL; + goto err; /* don't set the affinity when the target cpu is same as current one */ - if (cpu != its_dev->event_map.col_map[id]) { + if (cpu != prev_cpu) { target_col = &its_dev->its->collections[cpu]; its_send_movi(its_dev, target_col, id); its_dev->event_map.col_map[id] = cpu; irq_data_update_effective_affinity(d, cpumask_of(cpu)); } + its_inc_lpi_count(d, cpu); + return IRQ_SET_MASK_OK_DONE; + +err: + its_inc_lpi_count(d, prev_cpu); + return -EINVAL; } static u64 its_irq_get_msi_base(struct its_device *its_dev) @@ -3448,6 +3489,7 @@ static int its_irq_domain_activate(struct irq_domain *domain, cpu = cpumask_first(cpu_online_mask); } + its_inc_lpi_count(d, cpu); its_dev->event_map.col_map[event] = cpu; irq_data_update_effective_affinity(d, cpumask_of(cpu)); @@ -3462,6 +3504,7 @@ static void its_irq_domain_deactivate(struct irq_domain *domain, struct its_device *its_dev = irq_data_get_irq_chip_data(d); u32 event = its_get_event_id(d); + its_dec_lpi_count(d, its_dev->event_map.col_map[event]); /* Stop the delivery of interrupts */ its_send_discard(its_dev, event); } From 202164fbfa2b2ffa3e66b504e0f126ba9a745006 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Thu, 7 May 2020 13:08:39 -0700 Subject: [PATCH 0556/1043] kgdb: Disable WARN_CONSOLE_UNLOCKED for all kgdb In commit 81eaadcae81b ("kgdboc: disable the console lock when in kgdb") we avoided the WARN_CONSOLE_UNLOCKED() yell when we were in kgdboc. That still works fine, but it turns out that we get a similar yell when using other I/O drivers. One example is the "I/O driver" for the kgdb test suite (kgdbts). When I enabled that I again got the same yells. Even though "kgdbts" doesn't actually interact with the user over the console, using it still causes kgdb to print to the consoles. That trips the same warning: con_is_visible+0x60/0x68 con_scroll+0x110/0x1b8 lf+0x4c/0xc8 vt_console_print+0x1b8/0x348 vkdb_printf+0x320/0x89c kdb_printf+0x68/0x90 kdb_main_loop+0x190/0x860 kdb_stub+0x2cc/0x3ec kgdb_cpu_enter+0x268/0x744 kgdb_handle_exception+0x1a4/0x200 kgdb_compiled_brk_fn+0x34/0x44 brk_handler+0x7c/0xb8 do_debug_exception+0x1b4/0x228 Let's increment/decrement the "ignore_console_lock_warning" variable all the time when we enter the debugger. This will allow us to later revert commit 81eaadcae81b ("kgdboc: disable the console lock when in kgdb"). Signed-off-by: Douglas Anderson Reviewed-by: Greg Kroah-Hartman Reviewed-by: Daniel Thompson Link: https://lore.kernel.org/r/20200507130644.v4.1.Ied2b058357152ebcc8bf68edd6f20a11d98d7d4e@changeid Signed-off-by: Daniel Thompson --- kernel/debug/debug_core.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c index 2266ba27f27d..b542f36499c6 100644 --- a/kernel/debug/debug_core.c +++ b/kernel/debug/debug_core.c @@ -666,6 +666,8 @@ return_normal: if (kgdb_skipexception(ks->ex_vector, ks->linux_regs)) goto kgdb_restore; + atomic_inc(&ignore_console_lock_warning); + /* Call the I/O driver's pre_exception routine */ if (dbg_io_ops->pre_exception) dbg_io_ops->pre_exception(); @@ -738,6 +740,8 @@ cpu_master_loop: if (dbg_io_ops->post_exception) dbg_io_ops->post_exception(); + atomic_dec(&ignore_console_lock_warning); + if (!kgdb_single_step) { raw_spin_unlock(&dbg_slave_lock); /* Wait till all the CPUs have quit from the debugger. */ From 333564add0e553da9619a6591ae3cdbd561449b0 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Thu, 7 May 2020 13:08:40 -0700 Subject: [PATCH 0557/1043] Revert "kgdboc: disable the console lock when in kgdb" This reverts commit 81eaadcae81b4c1bf01649a3053d1f54e2d81cf1. Commit 81eaadcae81b ("kgdboc: disable the console lock when in kgdb") is no longer needed now that we have the patch ("kgdb: Disable WARN_CONSOLE_UNLOCKED for all kgdb"). Revert it. Signed-off-by: Douglas Anderson Reviewed-by: Greg Kroah-Hartman Reviewed-by: Daniel Thompson Link: https://lore.kernel.org/r/20200507130644.v4.2.I02258eee1497e55bcbe8dc477de90369c7c7c2c5@changeid Signed-off-by: Daniel Thompson --- drivers/tty/serial/kgdboc.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/tty/serial/kgdboc.c b/drivers/tty/serial/kgdboc.c index c9f94fa82be4..8a1a4d1b6768 100644 --- a/drivers/tty/serial/kgdboc.c +++ b/drivers/tty/serial/kgdboc.c @@ -275,14 +275,10 @@ static void kgdboc_pre_exp_handler(void) /* Increment the module count when the debugger is active */ if (!kgdb_connected) try_module_get(THIS_MODULE); - - atomic_inc(&ignore_console_lock_warning); } static void kgdboc_post_exp_handler(void) { - atomic_dec(&ignore_console_lock_warning); - /* decrement the module count when the debugger detaches */ if (!kgdb_connected) module_put(THIS_MODULE); From 68e55f61c13842baf825958129698c5371db432c Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Thu, 7 May 2020 13:08:41 -0700 Subject: [PATCH 0558/1043] kgdboc: Use a platform device to handle tty drivers showing up late If you build CONFIG_KGDB_SERIAL_CONSOLE into the kernel then you should be able to have KGDB init itself at bootup by specifying the "kgdboc=..." kernel command line parameter. This has worked OK for me for many years, but on a new device I switched to it stopped working. The problem is that on this new device the serial driver gets its probe deferred. Now when kgdb initializes it can't find the tty driver and when it gives up it never tries again. We could try to find ways to move up the initialization of the serial driver and such a thing might be worthwhile, but it's nice to be robust against serial drivers that load late. We could move kgdb to init itself later but that penalizes our ability to debug early boot code on systems where the driver inits early. We could roll our own system of detecting when new tty drivers get loaded and then use that to figure out when kgdb can init, but that's ugly. Instead, let's jump on the -EPROBE_DEFER bandwagon. We'll create a singleton instance of a "kgdboc" platform device. If we can't find our tty device when the singleton "kgdboc" probes we'll return -EPROBE_DEFER which means that the system will call us back later to try again when the tty device might be there. We won't fully transition all of the kgdboc to a platform device because early kgdb initialization (via the "ekgdboc" kernel command line parameter) still runs before the platform device has been created. The kgdb platform device is merely used as a convenient way to hook into the system's normal probe deferral mechanisms. As part of this, we'll ever-so-slightly change how the "kgdboc=..." kernel command line parameter works. Previously if you booted up and kgdb couldn't find the tty driver then later reading '/sys/module/kgdboc/parameters/kgdboc' would return a blank string. Now kgdb will keep track of the string that came as part of the command line and give it back to you. It's expected that this should be an OK change. Signed-off-by: Douglas Anderson Reviewed-by: Greg Kroah-Hartman Reviewed-by: Daniel Thompson Link: https://lore.kernel.org/r/20200507130644.v4.3.I4a493cfb0f9f740ce8fd2ab58e62dc92d18fed30@changeid [daniel.thompson@linaro.org: Make config_mutex static] Signed-off-by: Daniel Thompson --- drivers/tty/serial/kgdboc.c | 126 +++++++++++++++++++++++++++++------- 1 file changed, 101 insertions(+), 25 deletions(-) diff --git a/drivers/tty/serial/kgdboc.c b/drivers/tty/serial/kgdboc.c index 8a1a4d1b6768..e90a6016dbcb 100644 --- a/drivers/tty/serial/kgdboc.c +++ b/drivers/tty/serial/kgdboc.c @@ -20,6 +20,7 @@ #include #include #include +#include #define MAX_CONFIG_LEN 40 @@ -27,6 +28,7 @@ static struct kgdb_io kgdboc_io_ops; /* -1 = init not run yet, 0 = unconfigured, 1 = configured. */ static int configured = -1; +static DEFINE_MUTEX(config_mutex); static char config[MAX_CONFIG_LEN]; static struct kparam_string kps = { @@ -38,6 +40,8 @@ static int kgdboc_use_kms; /* 1 if we use kernel mode switching */ static struct tty_driver *kgdb_tty_driver; static int kgdb_tty_line; +static struct platform_device *kgdboc_pdev; + #ifdef CONFIG_KDB_KEYBOARD static int kgdboc_reset_connect(struct input_handler *handler, struct input_dev *dev, @@ -133,11 +137,13 @@ static void kgdboc_unregister_kbd(void) static void cleanup_kgdboc(void) { + if (configured != 1) + return; + if (kgdb_unregister_nmi_console()) return; kgdboc_unregister_kbd(); - if (configured == 1) - kgdb_unregister_io_module(&kgdboc_io_ops); + kgdb_unregister_io_module(&kgdboc_io_ops); } static int configure_kgdboc(void) @@ -198,20 +204,79 @@ nmi_con_failed: kgdb_unregister_io_module(&kgdboc_io_ops); noconfig: kgdboc_unregister_kbd(); - config[0] = 0; configured = 0; - cleanup_kgdboc(); return err; } +static int kgdboc_probe(struct platform_device *pdev) +{ + int ret = 0; + + mutex_lock(&config_mutex); + if (configured != 1) { + ret = configure_kgdboc(); + + /* Convert "no device" to "defer" so we'll keep trying */ + if (ret == -ENODEV) + ret = -EPROBE_DEFER; + } + mutex_unlock(&config_mutex); + + return ret; +} + +static struct platform_driver kgdboc_platform_driver = { + .probe = kgdboc_probe, + .driver = { + .name = "kgdboc", + .suppress_bind_attrs = true, + }, +}; + static int __init init_kgdboc(void) { - /* Already configured? */ - if (configured == 1) + int ret; + + /* + * kgdboc is a little bit of an odd "platform_driver". It can be + * up and running long before the platform_driver object is + * created and thus doesn't actually store anything in it. There's + * only one instance of kgdb so anything is stored as global state. + * The platform_driver is only created so that we can leverage the + * kernel's mechanisms (like -EPROBE_DEFER) to call us when our + * underlying tty is ready. Here we init our platform driver and + * then create the single kgdboc instance. + */ + ret = platform_driver_register(&kgdboc_platform_driver); + if (ret) + return ret; + + kgdboc_pdev = platform_device_alloc("kgdboc", PLATFORM_DEVID_NONE); + if (!kgdboc_pdev) { + ret = -ENOMEM; + goto err_did_register; + } + + ret = platform_device_add(kgdboc_pdev); + if (!ret) return 0; - return configure_kgdboc(); + platform_device_put(kgdboc_pdev); + +err_did_register: + platform_driver_unregister(&kgdboc_platform_driver); + return ret; +} + +static void exit_kgdboc(void) +{ + mutex_lock(&config_mutex); + cleanup_kgdboc(); + mutex_unlock(&config_mutex); + + platform_device_unregister(kgdboc_pdev); + platform_driver_unregister(&kgdboc_platform_driver); } static int kgdboc_get_char(void) @@ -234,24 +299,20 @@ static int param_set_kgdboc_var(const char *kmessage, const struct kernel_param *kp) { size_t len = strlen(kmessage); + int ret = 0; if (len >= MAX_CONFIG_LEN) { pr_err("config string too long\n"); return -ENOSPC; } - /* Only copy in the string if the init function has not run yet */ - if (configured < 0) { - strcpy(config, kmessage); - return 0; - } - if (kgdb_connected) { pr_err("Cannot reconfigure while KGDB is connected.\n"); - return -EBUSY; } + mutex_lock(&config_mutex); + strcpy(config, kmessage); /* Chop out \n char as a result of echo */ if (len && config[len - 1] == '\n') @@ -260,8 +321,30 @@ static int param_set_kgdboc_var(const char *kmessage, if (configured == 1) cleanup_kgdboc(); - /* Go and configure with the new params. */ - return configure_kgdboc(); + /* + * Configure with the new params as long as init already ran. + * Note that we can get called before init if someone loads us + * with "modprobe kgdboc kgdboc=..." or if they happen to use the + * the odd syntax of "kgdboc.kgdboc=..." on the kernel command. + */ + if (configured >= 0) + ret = configure_kgdboc(); + + /* + * If we couldn't configure then clear out the config. Note that + * specifying an invalid config on the kernel command line vs. + * through sysfs have slightly different behaviors. If we fail + * to configure what was specified on the kernel command line + * we'll leave it in the 'config' and return -EPROBE_DEFER from + * our probe. When specified through sysfs userspace is + * responsible for loading the tty driver before setting up. + */ + if (ret) + config[0] = '\0'; + + mutex_unlock(&config_mutex); + + return ret; } static int dbg_restore_graphics; @@ -320,15 +403,8 @@ __setup("kgdboc=", kgdboc_option_setup); /* This is only available if kgdboc is a built in for early debugging */ static int __init kgdboc_early_init(char *opt) { - /* save the first character of the config string because the - * init routine can destroy it. - */ - char save_ch; - kgdboc_option_setup(opt); - save_ch = config[0]; - init_kgdboc(); - config[0] = save_ch; + configure_kgdboc(); return 0; } @@ -336,7 +412,7 @@ early_param("ekgdboc", kgdboc_early_init); #endif /* CONFIG_KGDB_SERIAL_CONSOLE */ module_init(init_kgdboc); -module_exit(cleanup_kgdboc); +module_exit(exit_kgdboc); module_param_call(kgdboc, param_set_kgdboc_var, param_get_string, &kps, 0644); MODULE_PARM_DESC(kgdboc, "[,baud]"); MODULE_DESCRIPTION("KGDB Console TTY Driver"); From b1a57bbfcc17c87e5cc76695ebb0565380c7501a Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Thu, 7 May 2020 13:08:42 -0700 Subject: [PATCH 0559/1043] kgdb: Delay "kgdbwait" to dbg_late_init() by default Using kgdb requires at least some level of architecture-level initialization. If nothing else, it relies on the architecture to pass breakpoints / crashes onto kgdb. On some architectures this all works super early, specifically it starts working at some point in time before Linux parses early_params's. On other architectures it doesn't. A survey of a few platforms: a) x86: Presumably it all works early since "ekgdboc" is documented to work here. b) arm64: Catching crashes works; with a simple patch breakpoints can also be made to work. c) arm: Nothing in kgdb works until paging_init() -> devicemaps_init() -> early_trap_init() Let's be conservative and, by default, process "kgdbwait" (which tells the kernel to drop into the debugger ASAP at boot) a bit later at dbg_late_init() time. If an architecture has tested it and wants to re-enable super early debugging, they can select the ARCH_HAS_EARLY_DEBUG KConfig option. We'll do this for x86 to start. It should be noted that dbg_late_init() is still called quite early in the system. Note that this patch doesn't affect when kgdb runs its init. If kgdb is set to initialize early it will still initialize when parsing early_param's. This patch _only_ inhibits the initial breakpoint from "kgdbwait". This means: * Without any extra patches arm64 platforms will at least catch crashes after kgdb inits. * arm platforms will catch crashes (and could handle a hardcoded kgdb_breakpoint()) any time after early_trap_init() runs, even before dbg_late_init(). Signed-off-by: Douglas Anderson Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20200507130644.v4.4.I3113aea1b08d8ce36dc3720209392ae8b815201b@changeid Signed-off-by: Daniel Thompson --- arch/x86/Kconfig | 1 + kernel/debug/debug_core.c | 25 +++++++++++++++---------- lib/Kconfig.kgdb | 18 ++++++++++++++++++ 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 1197b5596d5a..5f44955ee21c 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -60,6 +60,7 @@ config X86 select ARCH_HAS_ACPI_TABLE_UPGRADE if ACPI select ARCH_HAS_DEBUG_VIRTUAL select ARCH_HAS_DEVMEM_IS_ALLOWED + select ARCH_HAS_EARLY_DEBUG if KGDB select ARCH_HAS_ELF_RANDOMIZE select ARCH_HAS_FAST_MULTIPLIER select ARCH_HAS_FILTER_PGPROT diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c index b542f36499c6..1cd8e8b41115 100644 --- a/kernel/debug/debug_core.c +++ b/kernel/debug/debug_core.c @@ -948,6 +948,14 @@ void kgdb_panic(const char *msg) kgdb_breakpoint(); } +static void kgdb_initial_breakpoint(void) +{ + kgdb_break_asap = 0; + + pr_crit("Waiting for connection from remote gdb...\n"); + kgdb_breakpoint(); +} + void __weak kgdb_arch_late(void) { } @@ -958,6 +966,9 @@ void __init dbg_late_init(void) if (kgdb_io_module_registered) kgdb_arch_late(); kdb_init(KDB_INIT_FULL); + + if (kgdb_io_module_registered && kgdb_break_asap) + kgdb_initial_breakpoint(); } static int @@ -1053,14 +1064,6 @@ void kgdb_schedule_breakpoint(void) } EXPORT_SYMBOL_GPL(kgdb_schedule_breakpoint); -static void kgdb_initial_breakpoint(void) -{ - kgdb_break_asap = 0; - - pr_crit("Waiting for connection from remote gdb...\n"); - kgdb_breakpoint(); -} - /** * kgdb_register_io_module - register KGDB IO module * @new_dbg_io_ops: the io ops vector @@ -1097,7 +1100,8 @@ int kgdb_register_io_module(struct kgdb_io *new_dbg_io_ops) /* Arm KGDB now. */ kgdb_register_callbacks(); - if (kgdb_break_asap) + if (kgdb_break_asap && + (!dbg_is_early || IS_ENABLED(CONFIG_ARCH_HAS_EARLY_DEBUG))) kgdb_initial_breakpoint(); return 0; @@ -1167,7 +1171,8 @@ static int __init opt_kgdb_wait(char *str) kgdb_break_asap = 1; kdb_init(KDB_INIT_EARLY); - if (kgdb_io_module_registered) + if (kgdb_io_module_registered && + IS_ENABLED(CONFIG_ARCH_HAS_EARLY_DEBUG)) kgdb_initial_breakpoint(); return 0; diff --git a/lib/Kconfig.kgdb b/lib/Kconfig.kgdb index 933680b59e2d..ffa7a76de086 100644 --- a/lib/Kconfig.kgdb +++ b/lib/Kconfig.kgdb @@ -124,4 +124,22 @@ config KDB_CONTINUE_CATASTROPHIC CONFIG_KDB_CONTINUE_CATASTROPHIC == 2. KDB forces a reboot. If you are not sure, say 0. +config ARCH_HAS_EARLY_DEBUG + bool + default n + help + If an architecture can definitely handle entering the debugger + when early_param's are parsed then it select this config. + Otherwise, if "kgdbwait" is passed on the kernel command line it + won't actually be processed until dbg_late_init() just after the + call to kgdb_arch_late() is made. + + NOTE: Even if this isn't selected by an architecture we will + still try to register kgdb to handle breakpoints and crashes + when early_param's are parsed, we just won't act on the + "kgdbwait" parameter until dbg_late_init(). If you get a + crash and try to drop into kgdb somewhere between these two + places you might or might not end up being able to use kgdb + depending on exactly how far along the architecture has initted. + endif # KGDB From 3ca676e4ca60d1834bb77535dafe24169cadacef Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Thu, 7 May 2020 13:08:44 -0700 Subject: [PATCH 0560/1043] kgdb: Prevent infinite recursive entries to the debugger If we detect that we recursively entered the debugger we should hack our I/O ops to NULL so that the panic() in the next line won't actually cause another recursion into the debugger. The first line of kgdb_panic() will check this and return. Signed-off-by: Douglas Anderson Reviewed-by: Daniel Thompson Link: https://lore.kernel.org/r/20200507130644.v4.6.I89de39f68736c9de610e6f241e68d8dbc44bc266@changeid Signed-off-by: Daniel Thompson --- kernel/debug/debug_core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c index 1cd8e8b41115..e2d67b163fb6 100644 --- a/kernel/debug/debug_core.c +++ b/kernel/debug/debug_core.c @@ -530,6 +530,7 @@ static int kgdb_reenter_check(struct kgdb_state *ks) if (exception_level > 1) { dump_stack(); + kgdb_io_module_registered = false; panic("Recursive entry to debugger"); } From eae3e19ca930a56c726fbf4dc3adc198b8f5d61d Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Thu, 7 May 2020 13:08:45 -0700 Subject: [PATCH 0561/1043] kgdboc: Remove useless #ifdef CONFIG_KGDB_SERIAL_CONSOLE in kgdboc This file is only ever compiled if that config is on since the Makefile says: obj-$(CONFIG_KGDB_SERIAL_CONSOLE) += kgdboc.o Let's get rid of the useless #ifdef. Reported-by: Daniel Thompson Signed-off-by: Douglas Anderson Link: https://lore.kernel.org/r/20200507130644.v4.7.Icb528f03d0026d957e60f537aa711ada6fd219dc@changeid Signed-off-by: Daniel Thompson --- drivers/tty/serial/kgdboc.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/tty/serial/kgdboc.c b/drivers/tty/serial/kgdboc.c index e90a6016dbcb..a260ecd13e8f 100644 --- a/drivers/tty/serial/kgdboc.c +++ b/drivers/tty/serial/kgdboc.c @@ -380,7 +380,6 @@ static struct kgdb_io kgdboc_io_ops = { .post_exception = kgdboc_post_exp_handler, }; -#ifdef CONFIG_KGDB_SERIAL_CONSOLE static int kgdboc_option_setup(char *opt) { if (!opt) { @@ -409,7 +408,6 @@ static int __init kgdboc_early_init(char *opt) } early_param("ekgdboc", kgdboc_early_init); -#endif /* CONFIG_KGDB_SERIAL_CONSOLE */ module_init(init_kgdboc); module_exit(exit_kgdboc); From 220995622da5317714b5fe659165735f7b44b87e Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Thu, 7 May 2020 13:08:46 -0700 Subject: [PATCH 0562/1043] kgdboc: Add kgdboc_earlycon to support early kgdb using boot consoles We want to enable kgdb to debug the early parts of the kernel. Unfortunately kgdb normally is a client of the tty API in the kernel and serial drivers don't register to the tty layer until fairly late in the boot process. Serial drivers do, however, commonly register a boot console. Let's enable the kgdboc driver to work with boot consoles to provide early debugging. This change co-opts the existing read() function pointer that's part of "struct console". It's assumed that if a boot console (with the flag CON_BOOT) has implemented read() that both the read() and write() function are polling functions. That means they work without interrupts and read() will return immediately (with 0 bytes read) if there's nothing to read. This should be a safe assumption since it appears that no current boot consoles implement read() right now and there seems no reason to do so unless they wanted to support "kgdboc_earlycon". The normal/expected way to make all this work is to use "kgdboc_earlycon" and "kgdboc" together. You should point them both to the same physical serial connection. At boot time, as the system transitions from the boot console to the normal console (and registers a tty), kgdb will switch over. One awkward part of all this, though, is that there can be a window where the boot console goes away and we can't quite transtion over to the main kgdboc that uses the tty layer. There are two main problems: 1. The act of registering the tty doesn't cause any call into kgdboc so there is a window of time when the tty is there but kgdboc's init code hasn't been called so we can't transition to it. 2. On some serial drivers the normal console inits (and replaces the boot console) quite early in the system. Presumably these drivers were coded up before earlycon worked as well as it does today and probably they don't need to do this anymore, but it causes us problems nontheless. Problem #1 is not too big of a deal somewhat due to the luck of probe ordering. kgdboc is last in the tty/serial/Makefile so its probe gets right after all other tty devices. It's not fun to rely on this, but it does work for the most part. Problem #2 is a big deal, but only for some serial drivers. Other serial drivers end up registering the console (which gets rid of the boot console) and tty at nearly the same time. The way we'll deal with the window when the system has stopped using the boot console and the time when we're setup using the tty is to keep using the boot console. This may sound surprising, but it has been found to work well in practice. If it doesn't work, it shouldn't be too hard for a given serial driver to make it keep working. Specifically, it's expected that the read()/write() function provided in the boot console should be the same (or nearly the same) as the normal kgdb polling functions. That means continuing to use them should work just fine. To make things even more likely to work work we'll also trap the recently added exit() function in the boot console we're using and delay any calls to it until we're all done with the boot console. NOTE: there could be ways to use all this in weird / unexpected ways. If you do something like this, it's a bit of a buyer beware situation. Specifically: - If you specify only "kgdboc_earlycon" but not "kgdboc" then (depending on your serial driver) things will probably work OK, but you'll get a warning printed the first time you use kgdb after the boot console is gone. You'd only be able to do this, of course, if the serial driver you're running atop provided an early boot console. - If your "kgdboc_earlycon" and "kgdboc" devices are not the same device things should work OK, but it'll be your job to switch over which device you're monitoring (including figuring out how to switch over gdb in-flight if you're using it). When trying to enable "kgdboc_earlycon" it should be noted that the names that are registered through the boot console layer and the tty layer are not the same for the same port. For example when debugging on one board I'd need to pass "kgdboc_earlycon=qcom_geni kgdboc=ttyMSM0" to enable things properly. Since digging up the boot console name is a pain and there will rarely be more than one boot console enabled, you can provide the "kgdboc_earlycon" parameter without specifying the name of the boot console. In this case we'll just pick the first boot that implements read() that we find. This new "kgdboc_earlycon" parameter should be contrasted to the existing "ekgdboc" parameter. While both provide a way to debug very early, the usage and mechanisms are quite different. Specifically "kgdboc_earlycon" is meant to be used in tandem with "kgdboc" and there is a transition from one to the other. The "ekgdboc" parameter, on the other hand, replaces the "kgdboc" parameter. It runs the same logic as the "kgdboc" parameter but just relies on your TTY driver being present super early. The only known usage of the old "ekgdboc" parameter is documented as "ekgdboc=kbd earlyprintk=vga". It should be noted that "kbd" has special treatment allowing it to init early as a tty device. Signed-off-by: Douglas Anderson Reviewed-by: Greg Kroah-Hartman Tested-by: Sumit Garg Link: https://lore.kernel.org/r/20200507130644.v4.8.I8fba5961bf452ab92350654aa61957f23ecf0100@changeid Signed-off-by: Daniel Thompson --- drivers/tty/serial/kgdboc.c | 136 ++++++++++++++++++++++++++++++++++++ include/linux/kgdb.h | 4 ++ kernel/debug/debug_core.c | 22 ++++-- 3 files changed, 158 insertions(+), 4 deletions(-) diff --git a/drivers/tty/serial/kgdboc.c b/drivers/tty/serial/kgdboc.c index a260ecd13e8f..34b5e91dd245 100644 --- a/drivers/tty/serial/kgdboc.c +++ b/drivers/tty/serial/kgdboc.c @@ -21,6 +21,7 @@ #include #include #include +#include #define MAX_CONFIG_LEN 40 @@ -42,6 +43,10 @@ static int kgdb_tty_line; static struct platform_device *kgdboc_pdev; +static struct kgdb_io kgdboc_earlycon_io_ops; +static struct console *earlycon; +static int (*earlycon_orig_exit)(struct console *con); + #ifdef CONFIG_KDB_KEYBOARD static int kgdboc_reset_connect(struct input_handler *handler, struct input_dev *dev, @@ -137,6 +142,9 @@ static void kgdboc_unregister_kbd(void) static void cleanup_kgdboc(void) { + if (earlycon) + kgdb_unregister_io_module(&kgdboc_earlycon_io_ops); + if (configured != 1) return; @@ -409,6 +417,134 @@ static int __init kgdboc_early_init(char *opt) early_param("ekgdboc", kgdboc_early_init); +static int kgdboc_earlycon_get_char(void) +{ + char c; + + if (!earlycon->read(earlycon, &c, 1)) + return NO_POLL_CHAR; + + return c; +} + +static void kgdboc_earlycon_put_char(u8 chr) +{ + earlycon->write(earlycon, &chr, 1); +} + +static void kgdboc_earlycon_pre_exp_handler(void) +{ + struct console *con; + static bool already_warned; + + if (already_warned) + return; + + /* + * When the first normal console comes up the kernel will take all + * the boot consoles out of the list. Really, we should stop using + * the boot console when it does that but until a TTY is registered + * we have no other choice so we keep using it. Since not all + * serial drivers might be OK with this, print a warning once per + * boot if we detect this case. + */ + for_each_console(con) + if (con == earlycon) + return; + + already_warned = true; + pr_warn("kgdboc_earlycon is still using bootconsole\n"); +} + +static int kgdboc_earlycon_deferred_exit(struct console *con) +{ + /* + * If we get here it means the boot console is going away but we + * don't yet have a suitable replacement. Don't pass through to + * the original exit routine. We'll call it later in our deinit() + * function. For now, restore the original exit() function pointer + * as a sentinal that we've hit this point. + */ + con->exit = earlycon_orig_exit; + + return 0; +} + +static void kgdboc_earlycon_deinit(void) +{ + if (!earlycon) + return; + + if (earlycon->exit == kgdboc_earlycon_deferred_exit) + /* + * kgdboc_earlycon is exiting but original boot console exit + * was never called (AKA kgdboc_earlycon_deferred_exit() + * didn't ever run). Undo our trap. + */ + earlycon->exit = earlycon_orig_exit; + else if (earlycon->exit) + /* + * We skipped calling the exit() routine so we could try to + * keep using the boot console even after it went away. We're + * finally done so call the function now. + */ + earlycon->exit(earlycon); + + earlycon = NULL; +} + +static struct kgdb_io kgdboc_earlycon_io_ops = { + .name = "kgdboc_earlycon", + .read_char = kgdboc_earlycon_get_char, + .write_char = kgdboc_earlycon_put_char, + .pre_exception = kgdboc_earlycon_pre_exp_handler, + .deinit = kgdboc_earlycon_deinit, + .is_console = true, +}; + +static int __init kgdboc_earlycon_init(char *opt) +{ + struct console *con; + + kdb_init(KDB_INIT_EARLY); + + /* + * Look for a matching console, or if the name was left blank just + * pick the first one we find. + */ + console_lock(); + for_each_console(con) { + if (con->write && con->read && + (con->flags & (CON_BOOT | CON_ENABLED)) && + (!opt || !opt[0] || strcmp(con->name, opt) == 0)) + break; + } + + if (!con) { + pr_info("Couldn't find kgdb earlycon\n"); + goto unlock; + } + + earlycon = con; + pr_info("Going to register kgdb with earlycon '%s'\n", con->name); + if (kgdb_register_io_module(&kgdboc_earlycon_io_ops) != 0) { + earlycon = NULL; + pr_info("Failed to register kgdb with earlycon\n"); + } else { + /* Trap exit so we can keep earlycon longer if needed. */ + earlycon_orig_exit = con->exit; + con->exit = kgdboc_earlycon_deferred_exit; + } + +unlock: + console_unlock(); + + /* Non-zero means malformed option so we always return zero */ + return 0; +} + +early_param("kgdboc_earlycon", kgdboc_earlycon_init); + module_init(init_kgdboc); module_exit(exit_kgdboc); module_param_call(kgdboc, param_set_kgdboc_var, param_get_string, &kps, 0644); diff --git a/include/linux/kgdb.h b/include/linux/kgdb.h index c2caee08e418..c62d76478adc 100644 --- a/include/linux/kgdb.h +++ b/include/linux/kgdb.h @@ -269,6 +269,9 @@ struct kgdb_arch { * @write_char: Pointer to a function that will write one char. * @flush: Pointer to a function that will flush any pending writes. * @init: Pointer to a function that will initialize the device. + * @deinit: Pointer to a function that will deinit the device. Implies that + * this I/O driver is temporary and expects to be replaced. Called when + * an I/O driver is replaced or explicitly unregistered. * @pre_exception: Pointer to a function that will do any prep work for * the I/O driver. * @post_exception: Pointer to a function that will do any cleanup work @@ -282,6 +285,7 @@ struct kgdb_io { void (*write_char) (u8); void (*flush) (void); int (*init) (void); + void (*deinit) (void); void (*pre_exception) (void); void (*post_exception) (void); int is_console; diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c index e2d67b163fb6..4d59aa907fdc 100644 --- a/kernel/debug/debug_core.c +++ b/kernel/debug/debug_core.c @@ -1073,15 +1073,23 @@ EXPORT_SYMBOL_GPL(kgdb_schedule_breakpoint); */ int kgdb_register_io_module(struct kgdb_io *new_dbg_io_ops) { + struct kgdb_io *old_dbg_io_ops; int err; spin_lock(&kgdb_registration_lock); - if (dbg_io_ops) { - spin_unlock(&kgdb_registration_lock); + old_dbg_io_ops = dbg_io_ops; + if (old_dbg_io_ops) { + if (!old_dbg_io_ops->deinit) { + spin_unlock(&kgdb_registration_lock); - pr_err("Another I/O driver is already registered with KGDB\n"); - return -EBUSY; + pr_err("KGDB I/O driver %s can't replace %s.\n", + new_dbg_io_ops->name, old_dbg_io_ops->name); + return -EBUSY; + } + pr_info("Replacing I/O driver %s with %s\n", + old_dbg_io_ops->name, new_dbg_io_ops->name); + old_dbg_io_ops->deinit(); } if (new_dbg_io_ops->init) { @@ -1096,6 +1104,9 @@ int kgdb_register_io_module(struct kgdb_io *new_dbg_io_ops) spin_unlock(&kgdb_registration_lock); + if (old_dbg_io_ops) + return 0; + pr_info("Registered I/O driver %s\n", new_dbg_io_ops->name); /* Arm KGDB now. */ @@ -1132,6 +1143,9 @@ void kgdb_unregister_io_module(struct kgdb_io *old_dbg_io_ops) spin_unlock(&kgdb_registration_lock); + if (old_dbg_io_ops->deinit) + old_dbg_io_ops->deinit(); + pr_info("Unregistered I/O driver %s, debugger disabled\n", old_dbg_io_ops->name); } From d15483bb49bae0f9cbb67c54becec252545752d3 Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Thu, 7 May 2020 10:23:17 -0700 Subject: [PATCH 0563/1043] clocksource/drivers/timer-ti-32k: Add support for initializing directly Let's allow probing the 32k counter directly based on devicetree data to prepare for dropping the related legacy platform code. Let's only do this if the parent node is compatible with ti-sysc to make sure we have the related devicetree data available. Let's also show the 32k counter information before registering the clocksource, now we see it after the clocksource information which is a bit confusing. Cc: linux-kernel@vger.kernel.org Cc: linux-omap@vger.kernel.org Cc: Daniel Lezcano Cc: Grygorii Strashko Cc: Keerthy Cc: Lokesh Vutla Cc: Rob Herring Cc: Tero Kristo Cc: Thomas Gleixner Signed-off-by: Tony Lindgren Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200507172330.18679-2-tony@atomide.com --- drivers/clocksource/timer-ti-32k.c | 48 +++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/drivers/clocksource/timer-ti-32k.c b/drivers/clocksource/timer-ti-32k.c index abd5f158d6e2..ae12bbf3d68c 100644 --- a/drivers/clocksource/timer-ti-32k.c +++ b/drivers/clocksource/timer-ti-32k.c @@ -24,6 +24,7 @@ * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com */ +#include #include #include #include @@ -76,6 +77,49 @@ static u64 notrace omap_32k_read_sched_clock(void) return ti_32k_read_cycles(&ti_32k_timer.cs); } +static void __init ti_32k_timer_enable_clock(struct device_node *np, + const char *name) +{ + struct clk *clock; + int error; + + clock = of_clk_get_by_name(np->parent, name); + if (IS_ERR(clock)) { + /* Only some SoCs have a separate interface clock */ + if (PTR_ERR(clock) == -EINVAL && !strncmp("ick", name, 3)) + return; + + pr_warn("%s: could not get clock %s %li\n", + __func__, name, PTR_ERR(clock)); + return; + } + + error = clk_prepare_enable(clock); + if (error) { + pr_warn("%s: could not enable %s: %i\n", + __func__, name, error); + return; + } +} + +static void __init ti_32k_timer_module_init(struct device_node *np, + void __iomem *base) +{ + void __iomem *sysc = base + 4; + + if (!of_device_is_compatible(np->parent, "ti,sysc")) + return; + + ti_32k_timer_enable_clock(np, "fck"); + ti_32k_timer_enable_clock(np, "ick"); + + /* + * Force idle module as wkup domain is active with MPU. + * No need to tag the module disabled for ti-sysc probe. + */ + writel_relaxed(0, sysc); +} + static int __init ti_32k_timer_init(struct device_node *np) { int ret; @@ -90,6 +134,7 @@ static int __init ti_32k_timer_init(struct device_node *np) ti_32k_timer.cs.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP; ti_32k_timer.counter = ti_32k_timer.base; + ti_32k_timer_module_init(np, ti_32k_timer.base); /* * 32k sync Counter IP register offsets vary between the highlander @@ -104,6 +149,8 @@ static int __init ti_32k_timer_init(struct device_node *np) else ti_32k_timer.counter += OMAP2_32KSYNCNT_CR_OFF_LOW; + pr_info("OMAP clocksource: 32k_counter at 32768 Hz\n"); + ret = clocksource_register_hz(&ti_32k_timer.cs, 32768); if (ret) { pr_err("32k_counter: can't register clocksource\n"); @@ -111,7 +158,6 @@ static int __init ti_32k_timer_init(struct device_node *np) } sched_clock_register(omap_32k_read_sched_clock, 32, 32768); - pr_info("OMAP clocksource: 32k_counter at 32768 Hz\n"); return 0; } From aba1ad05da088944a62eb87fb0cd8391152e8985 Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Thu, 7 May 2020 10:23:18 -0700 Subject: [PATCH 0564/1043] clocksource/drivers/timer-ti-dm: Add clockevent and clocksource support We can move the TI dmtimer clockevent and clocksource to live under drivers/clocksource if we rely only on the clock framework, and handle the module configuration directly in the clocksource driver based on the device tree data. This removes the early dependency with system timers to the interconnect related code, and we can probe pretty much everything else later on at the module_init level. Let's first add a new driver for timer-ti-dm-systimer based on existing arch/arm/mach-omap2/timer.c. Then let's start moving SoCs to probe with device tree data while still keeping the old timer.c. And eventually we can just drop the old timer.c. Let's take the opportunity to switch to use readl/writel as pointed out by Daniel Lezcano . This allows further clean-up of the timer-ti-dm code the a lot of the shared helpers can just become static to the non-syster related code. Note the boards can optionally configure different timer source clocks if needed with assigned-clocks and assigned-clock-parents. Cc: linux-kernel@vger.kernel.org Cc: linux-omap@vger.kernel.org Cc: Daniel Lezcano Cc: Grygorii Strashko Cc: Keerthy Cc: Lokesh Vutla Cc: Rob Herring Cc: Tero Kristo Cc: Thomas Gleixner Signed-off-by: Tony Lindgren Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200507172330.18679-3-tony@atomide.com --- drivers/clocksource/Makefile | 1 + drivers/clocksource/timer-ti-dm-systimer.c | 731 +++++++++++++++++++++ 2 files changed, 732 insertions(+) create mode 100644 drivers/clocksource/timer-ti-dm-systimer.c diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index 641ba5383ab5..bdda1a2e4097 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -18,6 +18,7 @@ obj-$(CONFIG_CLKSRC_MMIO) += mmio.o obj-$(CONFIG_DAVINCI_TIMER) += timer-davinci.o obj-$(CONFIG_DIGICOLOR_TIMER) += timer-digicolor.o obj-$(CONFIG_OMAP_DM_TIMER) += timer-ti-dm.o +obj-$(CONFIG_OMAP_DM_TIMER) += timer-ti-dm-systimer.o obj-$(CONFIG_DW_APB_TIMER) += dw_apb_timer.o obj-$(CONFIG_DW_APB_TIMER_OF) += dw_apb_timer_of.o obj-$(CONFIG_FTTMR010_TIMER) += timer-fttmr010.o diff --git a/drivers/clocksource/timer-ti-dm-systimer.c b/drivers/clocksource/timer-ti-dm-systimer.c new file mode 100644 index 000000000000..1495618a5744 --- /dev/null +++ b/drivers/clocksource/timer-ti-dm-systimer.c @@ -0,0 +1,731 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +/* For type1, set SYSC_OMAP2_CLOCKACTIVITY for fck off on idle, l4 clock on */ +#define DMTIMER_TYPE1_ENABLE ((1 << 9) | (SYSC_IDLE_SMART << 3) | \ + SYSC_OMAP2_ENAWAKEUP | SYSC_OMAP2_AUTOIDLE) + +#define DMTIMER_TYPE2_ENABLE (SYSC_IDLE_SMART_WKUP << 2) +#define DMTIMER_RESET_WAIT 100000 + +#define DMTIMER_INST_DONT_CARE ~0U + +static int counter_32k; +static u32 clocksource; +static u32 clockevent; + +/* + * Subset of the timer registers we use. Note that the register offsets + * depend on the timer revision detected. + */ +struct dmtimer_systimer { + void __iomem *base; + u8 sysc; + u8 irq_stat; + u8 irq_ena; + u8 pend; + u8 load; + u8 counter; + u8 ctrl; + u8 wakeup; + u8 ifctrl; + unsigned long rate; +}; + +struct dmtimer_clockevent { + struct clock_event_device dev; + struct dmtimer_systimer t; + u32 period; +}; + +struct dmtimer_clocksource { + struct clocksource dev; + struct dmtimer_systimer t; + unsigned int loadval; +}; + +/* Assumes v1 ip if bits [31:16] are zero */ +static bool dmtimer_systimer_revision1(struct dmtimer_systimer *t) +{ + u32 tidr = readl_relaxed(t->base); + + return !(tidr >> 16); +} + +static int __init dmtimer_systimer_type1_reset(struct dmtimer_systimer *t) +{ + void __iomem *syss = t->base + OMAP_TIMER_V1_SYS_STAT_OFFSET; + int ret; + u32 l; + + writel_relaxed(BIT(1) | BIT(2), t->base + t->ifctrl); + ret = readl_poll_timeout_atomic(syss, l, l & BIT(0), 100, + DMTIMER_RESET_WAIT); + + return ret; +} + +/* Note we must use io_base instead of func_base for type2 OCP regs */ +static int __init dmtimer_systimer_type2_reset(struct dmtimer_systimer *t) +{ + void __iomem *sysc = t->base + t->sysc; + u32 l; + + l = readl_relaxed(sysc); + l |= BIT(0); + writel_relaxed(l, sysc); + + return readl_poll_timeout_atomic(sysc, l, !(l & BIT(0)), 100, + DMTIMER_RESET_WAIT); +} + +static int __init dmtimer_systimer_reset(struct dmtimer_systimer *t) +{ + int ret; + + if (dmtimer_systimer_revision1(t)) + ret = dmtimer_systimer_type1_reset(t); + else + ret = dmtimer_systimer_type2_reset(t); + if (ret < 0) { + pr_err("%s failed with %i\n", __func__, ret); + + return ret; + } + + return 0; +} + +static const struct of_device_id counter_match_table[] = { + { .compatible = "ti,omap-counter32k" }, + { /* Sentinel */ }, +}; + +/* + * Check if the SoC als has a usable working 32 KiHz counter. The 32 KiHz + * counter is handled by timer-ti-32k, but we need to detect it as it + * affects the preferred dmtimer system timer configuration. There is + * typically no use for a dmtimer clocksource if the 32 KiHz counter is + * present, except on am437x as described below. + */ +static void __init dmtimer_systimer_check_counter32k(void) +{ + struct device_node *np; + + if (counter_32k) + return; + + np = of_find_matching_node(NULL, counter_match_table); + if (!np) { + counter_32k = -ENODEV; + + return; + } + + if (of_device_is_available(np)) + counter_32k = 1; + else + counter_32k = -ENODEV; + + of_node_put(np); +} + +static const struct of_device_id dmtimer_match_table[] = { + { .compatible = "ti,omap2420-timer", }, + { .compatible = "ti,omap3430-timer", }, + { .compatible = "ti,omap4430-timer", }, + { .compatible = "ti,omap5430-timer", }, + { .compatible = "ti,am335x-timer", }, + { .compatible = "ti,am335x-timer-1ms", }, + { .compatible = "ti,dm814-timer", }, + { .compatible = "ti,dm816-timer", }, + { /* Sentinel */ }, +}; + +/* + * Checks that system timers are configured to not reset and idle during + * the generic timer-ti-dm device driver probe. And that the system timer + * source clocks are properly configured. Also, let's not hog any DSP and + * PWM capable timers unnecessarily as system timers. + */ +static bool __init dmtimer_is_preferred(struct device_node *np) +{ + if (!of_device_is_available(np)) + return false; + + if (!of_property_read_bool(np->parent, + "ti,no-reset-on-init")) + return false; + + if (!of_property_read_bool(np->parent, "ti,no-idle")) + return false; + + /* Secure gptimer12 is always clocked with a fixed source */ + if (!of_property_read_bool(np, "ti,timer-secure")) { + if (!of_property_read_bool(np, "assigned-clocks")) + return false; + + if (!of_property_read_bool(np, "assigned-clock-parents")) + return false; + } + + if (of_property_read_bool(np, "ti,timer-dsp")) + return false; + + if (of_property_read_bool(np, "ti,timer-pwm")) + return false; + + return true; +} + +/* + * Finds the first available usable always-on timer, and assigns it to either + * clockevent or clocksource depending if the counter_32k is available on the + * SoC or not. + * + * Some omap3 boards with unreliable oscillator must not use the counter_32k + * or dmtimer1 with 32 KiHz source. Additionally, the boards with unreliable + * oscillator should really set counter_32k as disabled, and delete dmtimer1 + * ti,always-on property, but let's not count on it. For these quirky cases, + * we prefer using the always-on secure dmtimer12 with the internal 32 KiHz + * clock as the clocksource, and any available dmtimer as clockevent. + * + * For am437x, we are using am335x style dmtimer clocksource. It is unclear + * if this quirk handling is really needed, but let's change it separately + * based on testing as it might cause side effects. + */ +static void __init dmtimer_systimer_assign_alwon(void) +{ + struct device_node *np; + u32 pa = 0; + bool quirk_unreliable_oscillator = false; + + /* Quirk unreliable 32 KiHz oscillator with incomplete dts */ + if (of_machine_is_compatible("ti,omap3-beagle") || + of_machine_is_compatible("timll,omap3-devkit8000")) { + quirk_unreliable_oscillator = true; + counter_32k = -ENODEV; + } + + /* Quirk am437x using am335x style dmtimer clocksource */ + if (of_machine_is_compatible("ti,am43")) + counter_32k = -ENODEV; + + for_each_matching_node(np, dmtimer_match_table) { + if (!dmtimer_is_preferred(np)) + continue; + + if (of_property_read_bool(np, "ti,timer-alwon")) { + const __be32 *addr; + + addr = of_get_address(np, 0, NULL, NULL); + pa = of_translate_address(np, addr); + if (pa) { + /* Quirky omap3 boards must use dmtimer12 */ + if (quirk_unreliable_oscillator && + pa == 0x48318000) + continue; + + of_node_put(np); + break; + } + } + } + + /* Usually no need for dmtimer clocksource if we have counter32 */ + if (counter_32k >= 0) { + clockevent = pa; + clocksource = 0; + } else { + clocksource = pa; + clockevent = DMTIMER_INST_DONT_CARE; + } +} + +/* Finds the first usable dmtimer, used for the don't care case */ +static u32 __init dmtimer_systimer_find_first_available(void) +{ + struct device_node *np; + const __be32 *addr; + u32 pa = 0; + + for_each_matching_node(np, dmtimer_match_table) { + if (!dmtimer_is_preferred(np)) + continue; + + addr = of_get_address(np, 0, NULL, NULL); + pa = of_translate_address(np, addr); + if (pa) { + if (pa == clocksource || pa == clockevent) { + pa = 0; + continue; + } + + of_node_put(np); + break; + } + } + + return pa; +} + +/* Selects the best clocksource and clockevent to use */ +static void __init dmtimer_systimer_select_best(void) +{ + dmtimer_systimer_check_counter32k(); + dmtimer_systimer_assign_alwon(); + + if (clockevent == DMTIMER_INST_DONT_CARE) + clockevent = dmtimer_systimer_find_first_available(); + + pr_debug("%s: counter_32k: %i clocksource: %08x clockevent: %08x\n", + __func__, counter_32k, clocksource, clockevent); +} + +/* Interface clocks are only available on some SoCs variants */ +static int __init dmtimer_systimer_init_clock(struct device_node *np, + const char *name, + unsigned long *rate) +{ + struct clk *clock; + unsigned long r; + int error; + + clock = of_clk_get_by_name(np, name); + if ((PTR_ERR(clock) == -EINVAL) && !strncmp(name, "ick", 3)) + return 0; + else if (IS_ERR(clock)) + return PTR_ERR(clock); + + error = clk_prepare_enable(clock); + if (error) + return error; + + r = clk_get_rate(clock); + if (!r) + return -ENODEV; + + *rate = r; + + return 0; +} + +static void dmtimer_systimer_enable(struct dmtimer_systimer *t) +{ + u32 val; + + if (dmtimer_systimer_revision1(t)) + val = DMTIMER_TYPE1_ENABLE; + else + val = DMTIMER_TYPE2_ENABLE; + + writel_relaxed(val, t->base + t->sysc); +} + +static void dmtimer_systimer_disable(struct dmtimer_systimer *t) +{ + writel_relaxed(0, t->base + t->sysc); +} + +static int __init dmtimer_systimer_setup(struct device_node *np, + struct dmtimer_systimer *t) +{ + unsigned long rate; + u8 regbase; + int error; + + if (!of_device_is_compatible(np->parent, "ti,sysc")) + return -EINVAL; + + t->base = of_iomap(np, 0); + if (!t->base) + return -ENXIO; + + /* + * Enable optional assigned-clock-parents configured at the timer + * node level. For regular device drivers, this is done automatically + * by bus related code such as platform_drv_probe(). + */ + error = of_clk_set_defaults(np, false); + if (error < 0) + pr_err("%s: clock source init failed: %i\n", __func__, error); + + /* For ti-sysc, we have timer clocks at the parent module level */ + error = dmtimer_systimer_init_clock(np->parent, "fck", &rate); + if (error) + goto err_unmap; + + t->rate = rate; + + error = dmtimer_systimer_init_clock(np->parent, "ick", &rate); + if (error) + goto err_unmap; + + if (dmtimer_systimer_revision1(t)) { + t->irq_stat = OMAP_TIMER_V1_STAT_OFFSET; + t->irq_ena = OMAP_TIMER_V1_INT_EN_OFFSET; + t->pend = _OMAP_TIMER_WRITE_PEND_OFFSET; + regbase = 0; + } else { + t->irq_stat = OMAP_TIMER_V2_IRQSTATUS; + t->irq_ena = OMAP_TIMER_V2_IRQENABLE_SET; + regbase = OMAP_TIMER_V2_FUNC_OFFSET; + t->pend = regbase + _OMAP_TIMER_WRITE_PEND_OFFSET; + } + + t->sysc = OMAP_TIMER_OCP_CFG_OFFSET; + t->load = regbase + _OMAP_TIMER_LOAD_OFFSET; + t->counter = regbase + _OMAP_TIMER_COUNTER_OFFSET; + t->ctrl = regbase + _OMAP_TIMER_CTRL_OFFSET; + t->wakeup = regbase + _OMAP_TIMER_WAKEUP_EN_OFFSET; + t->ifctrl = regbase + _OMAP_TIMER_IF_CTRL_OFFSET; + + dmtimer_systimer_enable(t); + dmtimer_systimer_reset(t); + pr_debug("dmtimer rev %08x sysc %08x\n", readl_relaxed(t->base), + readl_relaxed(t->base + t->sysc)); + + return 0; + +err_unmap: + iounmap(t->base); + + return error; +} + +/* Clockevent */ +static struct dmtimer_clockevent * +to_dmtimer_clockevent(struct clock_event_device *clockevent) +{ + return container_of(clockevent, struct dmtimer_clockevent, dev); +} + +static irqreturn_t dmtimer_clockevent_interrupt(int irq, void *data) +{ + struct dmtimer_clockevent *clkevt = data; + struct dmtimer_systimer *t = &clkevt->t; + + writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_stat); + clkevt->dev.event_handler(&clkevt->dev); + + return IRQ_HANDLED; +} + +static int dmtimer_set_next_event(unsigned long cycles, + struct clock_event_device *evt) +{ + struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt); + struct dmtimer_systimer *t = &clkevt->t; + void __iomem *pend = t->base + t->pend; + + writel_relaxed(0xffffffff - cycles, t->base + t->counter); + while (readl_relaxed(pend) & WP_TCRR) + cpu_relax(); + + writel_relaxed(OMAP_TIMER_CTRL_ST, t->base + t->ctrl); + while (readl_relaxed(pend) & WP_TCLR) + cpu_relax(); + + return 0; +} + +static int dmtimer_clockevent_shutdown(struct clock_event_device *evt) +{ + struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt); + struct dmtimer_systimer *t = &clkevt->t; + void __iomem *ctrl = t->base + t->ctrl; + u32 l; + + l = readl_relaxed(ctrl); + if (l & OMAP_TIMER_CTRL_ST) { + l &= ~BIT(0); + writel_relaxed(l, ctrl); + /* Flush posted write */ + l = readl_relaxed(ctrl); + /* Wait for functional clock period x 3.5 */ + udelay(3500000 / t->rate + 1); + } + writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_stat); + + return 0; +} + +static int dmtimer_set_periodic(struct clock_event_device *evt) +{ + struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt); + struct dmtimer_systimer *t = &clkevt->t; + void __iomem *pend = t->base + t->pend; + + dmtimer_clockevent_shutdown(evt); + + /* Looks like we need to first set the load value separately */ + writel_relaxed(clkevt->period, t->base + t->load); + while (readl_relaxed(pend) & WP_TLDR) + cpu_relax(); + + writel_relaxed(clkevt->period, t->base + t->counter); + while (readl_relaxed(pend) & WP_TCRR) + cpu_relax(); + + writel_relaxed(OMAP_TIMER_CTRL_AR | OMAP_TIMER_CTRL_ST, + t->base + t->ctrl); + while (readl_relaxed(pend) & WP_TCLR) + cpu_relax(); + + return 0; +} + +static void omap_clockevent_idle(struct clock_event_device *evt) +{ + struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt); + struct dmtimer_systimer *t = &clkevt->t; + + dmtimer_systimer_disable(t); +} + +static void omap_clockevent_unidle(struct clock_event_device *evt) +{ + struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt); + struct dmtimer_systimer *t = &clkevt->t; + + dmtimer_systimer_enable(t); + writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_ena); + writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->wakeup); +} + +static int __init dmtimer_clockevent_init(struct device_node *np) +{ + struct dmtimer_clockevent *clkevt; + struct clock_event_device *dev; + struct dmtimer_systimer *t; + int error; + u32 pa; + + clkevt = kzalloc(sizeof(*clkevt), GFP_KERNEL); + if (!clkevt) + return -ENOMEM; + + t = &clkevt->t; + dev = &clkevt->dev; + + /* + * We mostly use cpuidle_coupled with ARM local timers for runtime, + * so there's probably no use for CLOCK_EVT_FEAT_DYNIRQ here. + */ + dev->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; + dev->rating = 300; + dev->set_next_event = dmtimer_set_next_event; + dev->set_state_shutdown = dmtimer_clockevent_shutdown; + dev->set_state_periodic = dmtimer_set_periodic; + dev->set_state_oneshot = dmtimer_clockevent_shutdown; + dev->tick_resume = dmtimer_clockevent_shutdown; + dev->cpumask = cpu_possible_mask; + + dev->irq = irq_of_parse_and_map(np, 0); + if (!dev->irq) { + error = -ENXIO; + goto err_out_free; + } + + error = dmtimer_systimer_setup(np, &clkevt->t); + if (error) + goto err_out_free; + + clkevt->period = 0xffffffff - DIV_ROUND_CLOSEST(t->rate, HZ); + + /* + * For clock-event timers we never read the timer counter and + * so we are not impacted by errata i103 and i767. Therefore, + * we can safely ignore this errata for clock-event timers. + */ + writel_relaxed(OMAP_TIMER_CTRL_POSTED, t->base + t->ifctrl); + + error = request_irq(dev->irq, dmtimer_clockevent_interrupt, + IRQF_TIMER, "clockevent", clkevt); + if (error) + goto err_out_unmap; + + writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_ena); + writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->wakeup); + + pa = of_translate_address(np, of_get_address(np, 0, NULL, NULL)); + pr_info("TI gptimer clockevent: %s%lu Hz at %pOF\n", + of_find_property(np, "ti,timer-alwon", NULL) ? + "always-on " : "", t->rate, np->parent); + + clockevents_config_and_register(dev, t->rate, + 3, /* Timer internal resynch latency */ + 0xffffffff); + + if (of_device_is_compatible(np, "ti,am33xx") || + of_device_is_compatible(np, "ti,am43")) { + dev->suspend = omap_clockevent_idle; + dev->resume = omap_clockevent_unidle; + } + + return 0; + +err_out_unmap: + iounmap(t->base); + +err_out_free: + kfree(clkevt); + + return error; +} + +/* Clocksource */ +static struct dmtimer_clocksource * +to_dmtimer_clocksource(struct clocksource *cs) +{ + return container_of(cs, struct dmtimer_clocksource, dev); +} + +static u64 dmtimer_clocksource_read_cycles(struct clocksource *cs) +{ + struct dmtimer_clocksource *clksrc = to_dmtimer_clocksource(cs); + struct dmtimer_systimer *t = &clksrc->t; + + return (u64)readl_relaxed(t->base + t->counter); +} + +static void __iomem *dmtimer_sched_clock_counter; + +static u64 notrace dmtimer_read_sched_clock(void) +{ + return readl_relaxed(dmtimer_sched_clock_counter); +} + +static void dmtimer_clocksource_suspend(struct clocksource *cs) +{ + struct dmtimer_clocksource *clksrc = to_dmtimer_clocksource(cs); + struct dmtimer_systimer *t = &clksrc->t; + + clksrc->loadval = readl_relaxed(t->base + t->counter); + dmtimer_systimer_disable(t); +} + +static void dmtimer_clocksource_resume(struct clocksource *cs) +{ + struct dmtimer_clocksource *clksrc = to_dmtimer_clocksource(cs); + struct dmtimer_systimer *t = &clksrc->t; + + dmtimer_systimer_enable(t); + writel_relaxed(clksrc->loadval, t->base + t->counter); + writel_relaxed(OMAP_TIMER_CTRL_ST | OMAP_TIMER_CTRL_AR, + t->base + t->ctrl); +} + +static int __init dmtimer_clocksource_init(struct device_node *np) +{ + struct dmtimer_clocksource *clksrc; + struct dmtimer_systimer *t; + struct clocksource *dev; + int error; + u32 pa; + + clksrc = kzalloc(sizeof(*clksrc), GFP_KERNEL); + if (!clksrc) + return -ENOMEM; + + dev = &clksrc->dev; + t = &clksrc->t; + + error = dmtimer_systimer_setup(np, t); + if (error) + goto err_out_free; + + dev->name = "dmtimer"; + dev->rating = 300; + dev->read = dmtimer_clocksource_read_cycles; + dev->mask = CLOCKSOURCE_MASK(32); + dev->flags = CLOCK_SOURCE_IS_CONTINUOUS; + + if (of_device_is_compatible(np, "ti,am33xx") || + of_device_is_compatible(np, "ti,am43")) { + dev->suspend = dmtimer_clocksource_suspend; + dev->resume = dmtimer_clocksource_resume; + } + + writel_relaxed(0, t->base + t->counter); + writel_relaxed(OMAP_TIMER_CTRL_ST | OMAP_TIMER_CTRL_AR, + t->base + t->ctrl); + + pa = of_translate_address(np, of_get_address(np, 0, NULL, NULL)); + pr_info("TI gptimer clocksource: %s%pOF\n", + of_find_property(np, "ti,timer-alwon", NULL) ? + "always-on " : "", np->parent); + + if (!dmtimer_sched_clock_counter) { + dmtimer_sched_clock_counter = t->base + t->counter; + sched_clock_register(dmtimer_read_sched_clock, 32, t->rate); + } + + if (clocksource_register_hz(dev, t->rate)) + pr_err("Could not register clocksource %pOF\n", np); + + return 0; + +err_out_free: + kfree(clksrc); + + return -ENODEV; +} + +/* + * To detect between a clocksource and clockevent, we assume the device tree + * has no interrupts configured for a clocksource timer. + */ +static int __init dmtimer_systimer_init(struct device_node *np) +{ + const __be32 *addr; + u32 pa; + + /* One time init for the preferred timer configuration */ + if (!clocksource && !clockevent) + dmtimer_systimer_select_best(); + + if (!clocksource && !clockevent) { + pr_err("%s: unable to detectt system timers, update dtb?\n", + __func__); + + return -EINVAL; + } + + addr = of_get_address(np, 0, NULL, NULL); + pa = of_translate_address(np, addr); + if (!pa) + return -EINVAL; + + if (counter_32k <= 0 && clocksource == pa) + return dmtimer_clocksource_init(np); + + if (clockevent == pa) + return dmtimer_clockevent_init(np); + + return 0; +} + +TIMER_OF_DECLARE(systimer_omap2, "ti,omap2420-timer", dmtimer_systimer_init); +TIMER_OF_DECLARE(systimer_omap3, "ti,omap3430-timer", dmtimer_systimer_init); +TIMER_OF_DECLARE(systimer_omap4, "ti,omap4430-timer", dmtimer_systimer_init); +TIMER_OF_DECLARE(systimer_omap5, "ti,omap5430-timer", dmtimer_systimer_init); +TIMER_OF_DECLARE(systimer_am33x, "ti,am335x-timer", dmtimer_systimer_init); +TIMER_OF_DECLARE(systimer_am3ms, "ti,am335x-timer-1ms", dmtimer_systimer_init); +TIMER_OF_DECLARE(systimer_dm814, "ti,dm814-timer", dmtimer_systimer_init); +TIMER_OF_DECLARE(systimer_dm816, "ti,dm816-timer", dmtimer_systimer_init); From d43e2675e96fc6ae1a633b6a69d296394448cc32 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 19 May 2020 05:34:41 -0400 Subject: [PATCH 0565/1043] KVM: x86: only do L1TF workaround on affected processors KVM stores the gfn in MMIO SPTEs as a caching optimization. These are split in two parts, as in "[high 11111 low]", to thwart any attempt to use these bits in an L1TF attack. This works as long as there are 5 free bits between MAXPHYADDR and bit 50 (inclusive), leaving bit 51 free so that the MMIO access triggers a reserved-bit-set page fault. The bit positions however were computed wrongly for AMD processors that have encryption support. In this case, x86_phys_bits is reduced (for example from 48 to 43, to account for the C bit at position 47 and four bits used internally to store the SEV ASID and other stuff) while x86_cache_bits in would remain set to 48, and _all_ bits between the reduced MAXPHYADDR and bit 51 are set. Then low_phys_bits would also cover some of the bits that are set in the shadow_mmio_value, terribly confusing the gfn caching mechanism. To fix this, avoid splitting gfns as long as the processor does not have the L1TF bug (which includes all AMD processors). When there is no splitting, low_phys_bits can be set to the reduced MAXPHYADDR removing the overlap. This fixes "npt=0" operation on EPYC processors. Thanks to Maxim Levitsky for bisecting this bug. Cc: stable@vger.kernel.org Fixes: 52918ed5fcf0 ("KVM: SVM: Override default MMIO mask if memory encryption is enabled") Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 8071952e9cf2..86619631ff6a 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -335,6 +335,8 @@ void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value, u64 access_mask) { BUG_ON((u64)(unsigned)access_mask != access_mask); BUG_ON((mmio_mask & mmio_value) != mmio_value); + WARN_ON(mmio_value & (shadow_nonpresent_or_rsvd_mask << shadow_nonpresent_or_rsvd_mask_len)); + WARN_ON(mmio_value & shadow_nonpresent_or_rsvd_lower_gfn_mask); shadow_mmio_value = mmio_value | SPTE_MMIO_MASK; shadow_mmio_mask = mmio_mask | SPTE_SPECIAL_MASK; shadow_mmio_access_mask = access_mask; @@ -583,16 +585,15 @@ static void kvm_mmu_reset_all_pte_masks(void) * the most significant bits of legal physical address space. */ shadow_nonpresent_or_rsvd_mask = 0; - low_phys_bits = boot_cpu_data.x86_cache_bits; - if (boot_cpu_data.x86_cache_bits < - 52 - shadow_nonpresent_or_rsvd_mask_len) { + low_phys_bits = boot_cpu_data.x86_phys_bits; + if (boot_cpu_has_bug(X86_BUG_L1TF) && + !WARN_ON_ONCE(boot_cpu_data.x86_cache_bits >= + 52 - shadow_nonpresent_or_rsvd_mask_len)) { + low_phys_bits = boot_cpu_data.x86_cache_bits + - shadow_nonpresent_or_rsvd_mask_len; shadow_nonpresent_or_rsvd_mask = - rsvd_bits(boot_cpu_data.x86_cache_bits - - shadow_nonpresent_or_rsvd_mask_len, - boot_cpu_data.x86_cache_bits - 1); - low_phys_bits -= shadow_nonpresent_or_rsvd_mask_len; - } else - WARN_ON_ONCE(boot_cpu_has_bug(X86_BUG_L1TF)); + rsvd_bits(low_phys_bits, boot_cpu_data.x86_cache_bits - 1); + } shadow_nonpresent_or_rsvd_lower_gfn_mask = GENMASK_ULL(low_phys_bits - 1, PAGE_SHIFT); From 0995a5dfbe49badff78e78761fb66f46579f2f9a Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 4 Mar 2020 13:09:50 +0100 Subject: [PATCH 0566/1043] tracing: Provide lockdep less trace_hardirqs_on/off() variants trace_hardirqs_on/off() is only partially safe vs. RCU idle. The tracer core itself is safe, but the resulting tracepoints can be utilized by e.g. BPF which is unsafe. Provide variants which do not contain the lockdep invocation so the lockdep and tracer invocations can be split at the call site and placed properly. This is required because lockdep needs to be aware of the state before switching away from RCU idle and after switching to RCU idle because these transitions can take locks. As these code pathes are going to be non-instrumentable the tracer can be invoked after RCU is turned on and before the switch to RCU idle. So for these new variants there is no need to invoke the rcuidle aware tracer functions. Name them so they match the lockdep counterparts. Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Peter Zijlstra Link: https://lkml.kernel.org/r/20200505134100.270771162@linutronix.de --- include/linux/irqflags.h | 4 ++++ kernel/trace/trace_preemptirq.c | 37 +++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h index 61a9ced3aa50..f150e69ab81d 100644 --- a/include/linux/irqflags.h +++ b/include/linux/irqflags.h @@ -29,6 +29,8 @@ #endif #ifdef CONFIG_TRACE_IRQFLAGS + extern void trace_hardirqs_on_prepare(void); + extern void trace_hardirqs_off_prepare(void); extern void trace_hardirqs_on(void); extern void trace_hardirqs_off(void); # define lockdep_hardirq_context(p) ((p)->hardirq_context) @@ -96,6 +98,8 @@ do { \ } while (0) #else +# define trace_hardirqs_on_prepare() do { } while (0) +# define trace_hardirqs_off_prepare() do { } while (0) # define trace_hardirqs_on() do { } while (0) # define trace_hardirqs_off() do { } while (0) # define lockdep_hardirq_context(p) 0 diff --git a/kernel/trace/trace_preemptirq.c b/kernel/trace/trace_preemptirq.c index 4d8e99fdbbbe..c00880162b06 100644 --- a/kernel/trace/trace_preemptirq.c +++ b/kernel/trace/trace_preemptirq.c @@ -19,6 +19,24 @@ /* Per-cpu variable to prevent redundant calls when IRQs already off */ static DEFINE_PER_CPU(int, tracing_irq_cpu); +/* + * Like trace_hardirqs_on() but without the lockdep invocation. This is + * used in the low level entry code where the ordering vs. RCU is important + * and lockdep uses a staged approach which splits the lockdep hardirq + * tracking into a RCU on and a RCU off section. + */ +void trace_hardirqs_on_prepare(void) +{ + if (this_cpu_read(tracing_irq_cpu)) { + if (!in_nmi()) + trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1); + tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); + this_cpu_write(tracing_irq_cpu, 0); + } +} +EXPORT_SYMBOL(trace_hardirqs_on_prepare); +NOKPROBE_SYMBOL(trace_hardirqs_on_prepare); + void trace_hardirqs_on(void) { if (this_cpu_read(tracing_irq_cpu)) { @@ -33,6 +51,25 @@ void trace_hardirqs_on(void) EXPORT_SYMBOL(trace_hardirqs_on); NOKPROBE_SYMBOL(trace_hardirqs_on); +/* + * Like trace_hardirqs_off() but without the lockdep invocation. This is + * used in the low level entry code where the ordering vs. RCU is important + * and lockdep uses a staged approach which splits the lockdep hardirq + * tracking into a RCU on and a RCU off section. + */ +void trace_hardirqs_off_prepare(void) +{ + if (!this_cpu_read(tracing_irq_cpu)) { + this_cpu_write(tracing_irq_cpu, 1); + tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); + if (!in_nmi()) + trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1); + } + +} +EXPORT_SYMBOL(trace_hardirqs_off_prepare); +NOKPROBE_SYMBOL(trace_hardirqs_off_prepare); + void trace_hardirqs_off(void) { if (!this_cpu_read(tracing_irq_cpu)) { From c86e9b987cea3dd0209203e714553a47f5d7c6dd Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Wed, 18 Mar 2020 14:22:03 +0100 Subject: [PATCH 0567/1043] lockdep: Prepare for noinstr sections Force inlining and prevent instrumentation of all sorts by marking the functions which are invoked from low level entry code with 'noinstr'. Split the irqflags tracking into two parts. One which does the heavy lifting while RCU is watching and the final one which can be invoked after RCU is turned off. Signed-off-by: Peter Zijlstra Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Link: https://lkml.kernel.org/r/20200505134100.484532537@linutronix.de --- include/linux/irqflags.h | 2 + include/linux/sched.h | 1 + kernel/locking/lockdep.c | 86 +++++++++++++++++++++++++-------- kernel/trace/trace_preemptirq.c | 2 + lib/debug_locks.c | 2 +- 5 files changed, 71 insertions(+), 22 deletions(-) diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h index f150e69ab81d..d7f7e436c3af 100644 --- a/include/linux/irqflags.h +++ b/include/linux/irqflags.h @@ -19,11 +19,13 @@ #ifdef CONFIG_PROVE_LOCKING extern void lockdep_softirqs_on(unsigned long ip); extern void lockdep_softirqs_off(unsigned long ip); + extern void lockdep_hardirqs_on_prepare(unsigned long ip); extern void lockdep_hardirqs_on(unsigned long ip); extern void lockdep_hardirqs_off(unsigned long ip); #else static inline void lockdep_softirqs_on(unsigned long ip) { } static inline void lockdep_softirqs_off(unsigned long ip) { } + static inline void lockdep_hardirqs_on_prepare(unsigned long ip) { } static inline void lockdep_hardirqs_on(unsigned long ip) { } static inline void lockdep_hardirqs_off(unsigned long ip) { } #endif diff --git a/include/linux/sched.h b/include/linux/sched.h index 4418f5cb8324..658de6164853 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -983,6 +983,7 @@ struct task_struct { unsigned int hardirq_disable_event; int hardirqs_enabled; int hardirq_context; + u64 hardirq_chain_key; unsigned long softirq_disable_ip; unsigned long softirq_enable_ip; unsigned int softirq_disable_event; diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index ac10db66cc63..9ccd675a8b5a 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -3635,13 +3635,10 @@ mark_held_locks(struct task_struct *curr, enum lock_usage_bit base_bit) /* * Hardirqs will be enabled: */ -static void __trace_hardirqs_on_caller(unsigned long ip) +static void __trace_hardirqs_on_caller(void) { struct task_struct *curr = current; - /* we'll do an OFF -> ON transition: */ - curr->hardirqs_enabled = 1; - /* * We are going to turn hardirqs on, so set the * usage bit for all held locks: @@ -3654,15 +3651,19 @@ static void __trace_hardirqs_on_caller(unsigned long ip) * this bit from being set before) */ if (curr->softirqs_enabled) - if (!mark_held_locks(curr, LOCK_ENABLED_SOFTIRQ)) - return; - - curr->hardirq_enable_ip = ip; - curr->hardirq_enable_event = ++curr->irq_events; - debug_atomic_inc(hardirqs_on_events); + mark_held_locks(curr, LOCK_ENABLED_SOFTIRQ); } -void lockdep_hardirqs_on(unsigned long ip) +/** + * lockdep_hardirqs_on_prepare - Prepare for enabling interrupts + * @ip: Caller address + * + * Invoked before a possible transition to RCU idle from exit to user or + * guest mode. This ensures that all RCU operations are done before RCU + * stops watching. After the RCU transition lockdep_hardirqs_on() has to be + * invoked to set the final state. + */ +void lockdep_hardirqs_on_prepare(unsigned long ip) { if (unlikely(!debug_locks || current->lockdep_recursion)) return; @@ -3698,20 +3699,62 @@ void lockdep_hardirqs_on(unsigned long ip) if (DEBUG_LOCKS_WARN_ON(current->hardirq_context)) return; + current->hardirq_chain_key = current->curr_chain_key; + current->lockdep_recursion++; - __trace_hardirqs_on_caller(ip); + __trace_hardirqs_on_caller(); lockdep_recursion_finish(); } -NOKPROBE_SYMBOL(lockdep_hardirqs_on); +EXPORT_SYMBOL_GPL(lockdep_hardirqs_on_prepare); + +void noinstr lockdep_hardirqs_on(unsigned long ip) +{ + struct task_struct *curr = current; + + if (unlikely(!debug_locks || curr->lockdep_recursion)) + return; + + if (curr->hardirqs_enabled) { + /* + * Neither irq nor preemption are disabled here + * so this is racy by nature but losing one hit + * in a stat is not a big deal. + */ + __debug_atomic_inc(redundant_hardirqs_on); + return; + } + + /* + * We're enabling irqs and according to our state above irqs weren't + * already enabled, yet we find the hardware thinks they are in fact + * enabled.. someone messed up their IRQ state tracing. + */ + if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) + return; + + /* + * Ensure the lock stack remained unchanged between + * lockdep_hardirqs_on_prepare() and lockdep_hardirqs_on(). + */ + DEBUG_LOCKS_WARN_ON(current->hardirq_chain_key != + current->curr_chain_key); + + /* we'll do an OFF -> ON transition: */ + curr->hardirqs_enabled = 1; + curr->hardirq_enable_ip = ip; + curr->hardirq_enable_event = ++curr->irq_events; + debug_atomic_inc(hardirqs_on_events); +} +EXPORT_SYMBOL_GPL(lockdep_hardirqs_on); /* * Hardirqs were disabled: */ -void lockdep_hardirqs_off(unsigned long ip) +void noinstr lockdep_hardirqs_off(unsigned long ip) { struct task_struct *curr = current; - if (unlikely(!debug_locks || current->lockdep_recursion)) + if (unlikely(!debug_locks || curr->lockdep_recursion)) return; /* @@ -3729,10 +3772,11 @@ void lockdep_hardirqs_off(unsigned long ip) curr->hardirq_disable_ip = ip; curr->hardirq_disable_event = ++curr->irq_events; debug_atomic_inc(hardirqs_off_events); - } else + } else { debug_atomic_inc(redundant_hardirqs_off); + } } -NOKPROBE_SYMBOL(lockdep_hardirqs_off); +EXPORT_SYMBOL_GPL(lockdep_hardirqs_off); /* * Softirqs will be enabled: @@ -4408,8 +4452,8 @@ static void print_unlock_imbalance_bug(struct task_struct *curr, dump_stack(); } -static int match_held_lock(const struct held_lock *hlock, - const struct lockdep_map *lock) +static noinstr int match_held_lock(const struct held_lock *hlock, + const struct lockdep_map *lock) { if (hlock->instance == lock) return 1; @@ -4696,7 +4740,7 @@ __lock_release(struct lockdep_map *lock, unsigned long ip) return 0; } -static nokprobe_inline +static __always_inline int __lock_is_held(const struct lockdep_map *lock, int read) { struct task_struct *curr = current; @@ -4956,7 +5000,7 @@ void lock_release(struct lockdep_map *lock, unsigned long ip) } EXPORT_SYMBOL_GPL(lock_release); -int lock_is_held_type(const struct lockdep_map *lock, int read) +noinstr int lock_is_held_type(const struct lockdep_map *lock, int read) { unsigned long flags; int ret = 0; diff --git a/kernel/trace/trace_preemptirq.c b/kernel/trace/trace_preemptirq.c index c00880162b06..fb0691b8a88d 100644 --- a/kernel/trace/trace_preemptirq.c +++ b/kernel/trace/trace_preemptirq.c @@ -46,6 +46,7 @@ void trace_hardirqs_on(void) this_cpu_write(tracing_irq_cpu, 0); } + lockdep_hardirqs_on_prepare(CALLER_ADDR0); lockdep_hardirqs_on(CALLER_ADDR0); } EXPORT_SYMBOL(trace_hardirqs_on); @@ -93,6 +94,7 @@ __visible void trace_hardirqs_on_caller(unsigned long caller_addr) this_cpu_write(tracing_irq_cpu, 0); } + lockdep_hardirqs_on_prepare(CALLER_ADDR0); lockdep_hardirqs_on(CALLER_ADDR0); } EXPORT_SYMBOL(trace_hardirqs_on_caller); diff --git a/lib/debug_locks.c b/lib/debug_locks.c index a75ee30b77cb..06d3135bd184 100644 --- a/lib/debug_locks.c +++ b/lib/debug_locks.c @@ -36,7 +36,7 @@ EXPORT_SYMBOL_GPL(debug_locks_silent); /* * Generic 'turn off all lock debugging' function: */ -int debug_locks_off(void) +noinstr int debug_locks_off(void) { if (debug_locks && __debug_locks_off()) { if (!debug_locks_silent) { From af1e56b78534c38bb0e0c712ca70e59f816b74e9 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Thu, 19 Mar 2020 14:53:56 +0100 Subject: [PATCH 0568/1043] context_tracking: Make guest_enter/exit() .noinstr ready Force inlining of the helpers and mark the instrumentable parts accordingly. Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Peter Zijlstra Link: https://lkml.kernel.org/r/20200505134341.672545766@linutronix.de --- include/linux/context_tracking.h | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 8150f5ac176c..8cac62ee6add 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -101,12 +101,14 @@ static inline void context_tracking_init(void) { } #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN /* must be called with irqs disabled */ -static inline void guest_enter_irqoff(void) +static __always_inline void guest_enter_irqoff(void) { + instrumentation_begin(); if (vtime_accounting_enabled_this_cpu()) vtime_guest_enter(current); else current->flags |= PF_VCPU; + instrumentation_end(); if (context_tracking_enabled()) __context_tracking_enter(CONTEXT_GUEST); @@ -118,39 +120,48 @@ static inline void guest_enter_irqoff(void) * one time slice). Lets treat guest mode as quiescent state, just like * we do with user-mode execution. */ - if (!context_tracking_enabled_this_cpu()) + if (!context_tracking_enabled_this_cpu()) { + instrumentation_begin(); rcu_virt_note_context_switch(smp_processor_id()); + instrumentation_end(); + } } -static inline void guest_exit_irqoff(void) +static __always_inline void guest_exit_irqoff(void) { if (context_tracking_enabled()) __context_tracking_exit(CONTEXT_GUEST); + instrumentation_begin(); if (vtime_accounting_enabled_this_cpu()) vtime_guest_exit(current); else current->flags &= ~PF_VCPU; + instrumentation_end(); } #else -static inline void guest_enter_irqoff(void) +static __always_inline void guest_enter_irqoff(void) { /* * This is running in ioctl context so its safe * to assume that it's the stime pending cputime * to flush. */ + instrumentation_begin(); vtime_account_kernel(current); current->flags |= PF_VCPU; rcu_virt_note_context_switch(smp_processor_id()); + instrumentation_end(); } -static inline void guest_exit_irqoff(void) +static __always_inline void guest_exit_irqoff(void) { + instrumentation_begin(); /* Flush the guest cputime we spent on the guest */ vtime_account_kernel(current); current->flags &= ~PF_VCPU; + instrumentation_end(); } #endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */ From ef68017eb5704eb2b0577c3aa6619e13caf2b59f Mon Sep 17 00:00:00 2001 From: Andy Lutomirski Date: Fri, 28 Feb 2020 10:42:48 -0800 Subject: [PATCH 0569/1043] x86/kvm: Handle async page faults directly through do_page_fault() KVM overloads #PF to indicate two types of not-actually-page-fault events. Right now, the KVM guest code intercepts them by modifying the IDT and hooking the #PF vector. This makes the already fragile fault code even harder to understand, and it also pollutes call traces with async_page_fault and do_async_page_fault for normal page faults. Clean it up by moving the logic into do_page_fault() using a static branch. This gets rid of the platform trap_init override mechanism completely. [ tglx: Fixed up 32bit, removed error code from the async functions and massaged coding style ] Signed-off-by: Andy Lutomirski Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Paolo Bonzini Acked-by: Peter Zijlstra Link: https://lkml.kernel.org/r/20200505134059.169270470@linutronix.de --- arch/x86/entry/entry_32.S | 8 ------- arch/x86/entry/entry_64.S | 4 ---- arch/x86/include/asm/kvm_para.h | 19 ++++++++++++++-- arch/x86/include/asm/x86_init.h | 2 -- arch/x86/kernel/kvm.c | 39 ++++++++++++++++++--------------- arch/x86/kernel/traps.c | 2 -- arch/x86/kernel/x86_init.c | 1 - arch/x86/mm/fault.c | 19 ++++++++++++++++ 8 files changed, 57 insertions(+), 37 deletions(-) diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index b67bae7091d7..8ba0985f5016 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -1693,14 +1693,6 @@ SYM_CODE_START(general_protection) jmp common_exception SYM_CODE_END(general_protection) -#ifdef CONFIG_KVM_GUEST -SYM_CODE_START(async_page_fault) - ASM_CLAC - pushl $do_async_page_fault - jmp common_exception_read_cr2 -SYM_CODE_END(async_page_fault) -#endif - SYM_CODE_START(rewind_stack_do_exit) /* Prevent any naive code from trying to unwind to our caller. */ xorl %ebp, %ebp diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 3063aa9090f9..9ab3ea6d02fc 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -1202,10 +1202,6 @@ idtentry xendebug do_debug has_error_code=0 idtentry general_protection do_general_protection has_error_code=1 idtentry page_fault do_page_fault has_error_code=1 read_cr2=1 -#ifdef CONFIG_KVM_GUEST -idtentry async_page_fault do_async_page_fault has_error_code=1 read_cr2=1 -#endif - #ifdef CONFIG_X86_MCE idtentry machine_check do_mce has_error_code=0 paranoid=1 #endif diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h index 9b4df6eaa11a..5261363adda3 100644 --- a/arch/x86/include/asm/kvm_para.h +++ b/arch/x86/include/asm/kvm_para.h @@ -91,8 +91,18 @@ unsigned int kvm_arch_para_hints(void); void kvm_async_pf_task_wait(u32 token, int interrupt_kernel); void kvm_async_pf_task_wake(u32 token); u32 kvm_read_and_reset_pf_reason(void); -extern void kvm_disable_steal_time(void); -void do_async_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address); +void kvm_disable_steal_time(void); +bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token); + +DECLARE_STATIC_KEY_FALSE(kvm_async_pf_enabled); + +static __always_inline bool kvm_handle_async_pf(struct pt_regs *regs, u32 token) +{ + if (static_branch_unlikely(&kvm_async_pf_enabled)) + return __kvm_handle_async_pf(regs, token); + else + return false; +} #ifdef CONFIG_PARAVIRT_SPINLOCKS void __init kvm_spinlock_init(void); @@ -130,6 +140,11 @@ static inline void kvm_disable_steal_time(void) { return; } + +static inline bool kvm_handle_async_pf(struct pt_regs *regs, u32 token) +{ + return false; +} #endif #endif /* _ASM_X86_KVM_PARA_H */ diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h index 96d9cd208610..6807153c0410 100644 --- a/arch/x86/include/asm/x86_init.h +++ b/arch/x86/include/asm/x86_init.h @@ -50,14 +50,12 @@ struct x86_init_resources { * @pre_vector_init: init code to run before interrupt vectors * are set up. * @intr_init: interrupt init code - * @trap_init: platform specific trap setup * @intr_mode_select: interrupt delivery mode selection * @intr_mode_init: interrupt delivery mode setup */ struct x86_init_irqs { void (*pre_vector_init)(void); void (*intr_init)(void); - void (*trap_init)(void); void (*intr_mode_select)(void); void (*intr_mode_init)(void); }; diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 6efe0410fb72..5ad3fcca2309 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -35,6 +35,8 @@ #include #include +DEFINE_STATIC_KEY_FALSE(kvm_async_pf_enabled); + static int kvmapf = 1; static int __init parse_no_kvmapf(char *arg) @@ -242,25 +244,27 @@ u32 kvm_read_and_reset_pf_reason(void) EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason); NOKPROBE_SYMBOL(kvm_read_and_reset_pf_reason); -dotraplinkage void -do_async_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address) +bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token) { + /* + * If we get a page fault right here, the pf_reason seems likely + * to be clobbered. Bummer. + */ switch (kvm_read_and_reset_pf_reason()) { default: - do_page_fault(regs, error_code, address); - break; + return false; case KVM_PV_REASON_PAGE_NOT_PRESENT: /* page is swapped out by the host. */ - kvm_async_pf_task_wait((u32)address, !user_mode(regs)); - break; + kvm_async_pf_task_wait(token, !user_mode(regs)); + return true; case KVM_PV_REASON_PAGE_READY: rcu_irq_enter(); - kvm_async_pf_task_wake((u32)address); + kvm_async_pf_task_wake(token); rcu_irq_exit(); - break; + return true; } } -NOKPROBE_SYMBOL(do_async_page_fault); +NOKPROBE_SYMBOL(__kvm_handle_async_pf); static void __init paravirt_ops_setup(void) { @@ -306,7 +310,11 @@ static notrace void kvm_guest_apic_eoi_write(u32 reg, u32 val) static void kvm_guest_cpu_init(void) { if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF) && kvmapf) { - u64 pa = slow_virt_to_phys(this_cpu_ptr(&apf_reason)); + u64 pa; + + WARN_ON_ONCE(!static_branch_likely(&kvm_async_pf_enabled)); + + pa = slow_virt_to_phys(this_cpu_ptr(&apf_reason)); #ifdef CONFIG_PREEMPTION pa |= KVM_ASYNC_PF_SEND_ALWAYS; @@ -592,12 +600,6 @@ static int kvm_cpu_down_prepare(unsigned int cpu) } #endif -static void __init kvm_apf_trap_init(void) -{ - update_intr_gate(X86_TRAP_PF, async_page_fault); -} - - static void kvm_flush_tlb_others(const struct cpumask *cpumask, const struct flush_tlb_info *info) { @@ -632,8 +634,6 @@ static void __init kvm_guest_init(void) register_reboot_notifier(&kvm_pv_reboot_nb); for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) raw_spin_lock_init(&async_pf_sleepers[i].lock); - if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF)) - x86_init.irqs.trap_init = kvm_apf_trap_init; if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) { has_steal_clock = 1; @@ -649,6 +649,9 @@ static void __init kvm_guest_init(void) if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) apic_set_eoi_write(kvm_guest_apic_eoi_write); + if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF) && kvmapf) + static_branch_enable(&kvm_async_pf_enabled); + #ifdef CONFIG_SMP smp_ops.smp_prepare_cpus = kvm_smp_prepare_cpus; smp_ops.smp_prepare_boot_cpu = kvm_smp_prepare_boot_cpu; diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index d54cffdc7cac..821fac47eef6 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -983,7 +983,5 @@ void __init trap_init(void) idt_setup_ist_traps(); - x86_init.irqs.trap_init(); - idt_setup_debugidt_traps(); } diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c index 85f1a90c55cd..123f1c1f1788 100644 --- a/arch/x86/kernel/x86_init.c +++ b/arch/x86/kernel/x86_init.c @@ -79,7 +79,6 @@ struct x86_init_ops x86_init __initdata = { .irqs = { .pre_vector_init = init_ISA_irqs, .intr_init = native_init_IRQ, - .trap_init = x86_init_noop, .intr_mode_select = apic_intr_mode_select, .intr_mode_init = apic_intr_mode_init }, diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index a51df516b87b..6486ccec1b0e 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -30,6 +30,7 @@ #include /* store_idt(), ... */ #include /* exception stack */ #include /* VMALLOC_START, ... */ +#include /* kvm_handle_async_pf */ #define CREATE_TRACE_POINTS #include @@ -1523,6 +1524,24 @@ do_page_fault(struct pt_regs *regs, unsigned long hw_error_code, unsigned long address) { prefetchw(¤t->mm->mmap_sem); + /* + * KVM has two types of events that are, logically, interrupts, but + * are unfortunately delivered using the #PF vector. These events are + * "you just accessed valid memory, but the host doesn't have it right + * now, so I'll put you to sleep if you continue" and "that memory + * you tried to access earlier is available now." + * + * We are relying on the interrupted context being sane (valid RSP, + * relevant locks not held, etc.), which is fine as long as the + * interrupted context had IF=1. We are also relying on the KVM + * async pf type field and CR2 being read consistently instead of + * getting values from real and async page faults mixed up. + * + * Fingers crossed. + */ + if (kvm_handle_async_pf(regs, (u32)address)) + return; + trace_page_fault_entries(regs, hw_error_code, address); if (unlikely(kmmio_fault(regs, address))) From 6bca69ada4bc20fa27eb44a5e09da3363d1752af Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sat, 7 Mar 2020 00:42:06 +0100 Subject: [PATCH 0570/1043] x86/kvm: Sanitize kvm_async_pf_task_wait() While working on the entry consolidation I stumbled over the KVM async page fault handler and kvm_async_pf_task_wait() in particular. It took me a while to realize that the randomly sprinkled around rcu_irq_enter()/exit() invocations are just cargo cult programming. Several patches "fixed" RCU splats by curing the symptoms without noticing that the code is flawed from a design perspective. The main problem is that this async injection is not based on a proper handshake mechanism and only respects the minimal requirement, i.e. the guest is not in a state where it has interrupts disabled. Aside of that the actual code is a convoluted one fits it all swiss army knife. It is invoked from different places with different RCU constraints: 1) Host side: vcpu_enter_guest() kvm_x86_ops->handle_exit() kvm_handle_page_fault() kvm_async_pf_task_wait() The invocation happens from fully preemptible context. 2) Guest side: The async page fault interrupted: a) user space b) preemptible kernel code which is not in a RCU read side critical section c) non-preemtible kernel code or a RCU read side critical section or kernel code with CONFIG_PREEMPTION=n which allows not to differentiate between #2b and #2c. RCU is watching for: #1 The vCPU exited and current is definitely not the idle task #2a The #PF entry code on the guest went through enter_from_user_mode() which reactivates RCU #2b There is no preemptible, interrupts enabled code in the kernel which can run with RCU looking away. (The idle task is always non preemptible). I.e. all schedulable states (#1, #2a, #2b) do not need any of this RCU voodoo at all. In #2c RCU is eventually not watching, but as that state cannot schedule anyway there is no point to worry about it so it has to invoke rcu_irq_enter() before running that code. This can be optimized, but this will be done as an extra step in course of the entry code consolidation work. So the proper solution for this is to: - Split kvm_async_pf_task_wait() into schedule and halt based waiting interfaces which share the enqueueing code. - Add comments (condensed form of this changelog) to spare others the time waste and pain of reverse engineering all of this with the help of uncomprehensible changelogs and code history. - Invoke kvm_async_pf_task_wait_schedule() from kvm_handle_page_fault(), user mode and schedulable kernel side async page faults (#1, #2a, #2b) - Invoke kvm_async_pf_task_wait_halt() for the non schedulable kernel case (#2c). For this case also remove the rcu_irq_exit()/enter() pair around the halt as it is just a pointless exercise: - vCPUs can VMEXIT at any random point and can be scheduled out for an arbitrary amount of time by the host and this is not any different except that it voluntary triggers the exit via halt. - The interrupted context could have RCU watching already. So the rcu_irq_exit() before the halt is not gaining anything aside of confusing the reader. Claiming that this might prevent RCU stalls is just an illusion. Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Paolo Bonzini Acked-by: Peter Zijlstra Link: https://lkml.kernel.org/r/20200505134059.262701431@linutronix.de --- arch/x86/include/asm/kvm_para.h | 4 +- arch/x86/kernel/kvm.c | 209 ++++++++++++++++++++++---------- arch/x86/kvm/mmu/mmu.c | 2 +- 3 files changed, 148 insertions(+), 67 deletions(-) diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h index 5261363adda3..118e5c2379f9 100644 --- a/arch/x86/include/asm/kvm_para.h +++ b/arch/x86/include/asm/kvm_para.h @@ -88,7 +88,7 @@ static inline long kvm_hypercall4(unsigned int nr, unsigned long p1, bool kvm_para_available(void); unsigned int kvm_arch_para_features(void); unsigned int kvm_arch_para_hints(void); -void kvm_async_pf_task_wait(u32 token, int interrupt_kernel); +void kvm_async_pf_task_wait_schedule(u32 token); void kvm_async_pf_task_wake(u32 token); u32 kvm_read_and_reset_pf_reason(void); void kvm_disable_steal_time(void); @@ -113,7 +113,7 @@ static inline void kvm_spinlock_init(void) #endif /* CONFIG_PARAVIRT_SPINLOCKS */ #else /* CONFIG_KVM_GUEST */ -#define kvm_async_pf_task_wait(T, I) do {} while(0) +#define kvm_async_pf_task_wait_schedule(T) do {} while(0) #define kvm_async_pf_task_wake(T) do {} while(0) static inline bool kvm_para_available(void) diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index 5ad3fcca2309..c6a82f9f537f 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -75,7 +75,7 @@ struct kvm_task_sleep_node { struct swait_queue_head wq; u32 token; int cpu; - bool halted; + bool use_halt; }; static struct kvm_task_sleep_head { @@ -98,75 +98,145 @@ static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b, return NULL; } -/* - * @interrupt_kernel: Is this called from a routine which interrupts the kernel - * (other than user space)? - */ -void kvm_async_pf_task_wait(u32 token, int interrupt_kernel) +static bool kvm_async_pf_queue_task(u32 token, bool use_halt, + struct kvm_task_sleep_node *n) { u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS); struct kvm_task_sleep_head *b = &async_pf_sleepers[key]; - struct kvm_task_sleep_node n, *e; - DECLARE_SWAITQUEUE(wait); - - rcu_irq_enter(); + struct kvm_task_sleep_node *e; raw_spin_lock(&b->lock); e = _find_apf_task(b, token); if (e) { /* dummy entry exist -> wake up was delivered ahead of PF */ hlist_del(&e->link); - kfree(e); raw_spin_unlock(&b->lock); - - rcu_irq_exit(); - return; + kfree(e); + return false; } - n.token = token; - n.cpu = smp_processor_id(); - n.halted = is_idle_task(current) || - (IS_ENABLED(CONFIG_PREEMPT_COUNT) - ? preempt_count() > 1 || rcu_preempt_depth() - : interrupt_kernel); - init_swait_queue_head(&n.wq); - hlist_add_head(&n.link, &b->list); + n->token = token; + n->cpu = smp_processor_id(); + n->use_halt = use_halt; + init_swait_queue_head(&n->wq); + hlist_add_head(&n->link, &b->list); raw_spin_unlock(&b->lock); + return true; +} + +/* + * kvm_async_pf_task_wait_schedule - Wait for pagefault to be handled + * @token: Token to identify the sleep node entry + * + * Invoked from the async pagefault handling code or from the VM exit page + * fault handler. In both cases RCU is watching. + */ +void kvm_async_pf_task_wait_schedule(u32 token) +{ + struct kvm_task_sleep_node n; + DECLARE_SWAITQUEUE(wait); + + lockdep_assert_irqs_disabled(); + + if (!kvm_async_pf_queue_task(token, false, &n)) + return; for (;;) { - if (!n.halted) - prepare_to_swait_exclusive(&n.wq, &wait, TASK_UNINTERRUPTIBLE); + prepare_to_swait_exclusive(&n.wq, &wait, TASK_UNINTERRUPTIBLE); if (hlist_unhashed(&n.link)) break; - rcu_irq_exit(); - - if (!n.halted) { - local_irq_enable(); - schedule(); - local_irq_disable(); - } else { - /* - * We cannot reschedule. So halt. - */ - native_safe_halt(); - local_irq_disable(); - } - - rcu_irq_enter(); + local_irq_enable(); + schedule(); + local_irq_disable(); + } + finish_swait(&n.wq, &wait); +} +EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait_schedule); + +/* + * Invoked from the async page fault handler. + */ +static void kvm_async_pf_task_wait_halt(u32 token) +{ + struct kvm_task_sleep_node n; + + if (!kvm_async_pf_queue_task(token, true, &n)) + return; + + for (;;) { + if (hlist_unhashed(&n.link)) + break; + /* + * No point in doing anything about RCU here. Any RCU read + * side critical section or RCU watching section can be + * interrupted by VMEXITs and the host is free to keep the + * vCPU scheduled out as long as it sees fit. This is not + * any different just because of the halt induced voluntary + * VMEXIT. + * + * Also the async page fault could have interrupted any RCU + * watching context, so invoking rcu_irq_exit()/enter() + * around this is not gaining anything. + */ + native_safe_halt(); + local_irq_disable(); + } +} + +/* Invoked from the async page fault handler */ +static void kvm_async_pf_task_wait(u32 token, bool usermode) +{ + bool can_schedule; + + /* + * No need to check whether interrupts were disabled because the + * host will (hopefully) only inject an async page fault into + * interrupt enabled regions. + * + * If CONFIG_PREEMPTION is enabled then check whether the code + * which triggered the page fault is preemptible. This covers user + * mode as well because preempt_count() is obviously 0 there. + * + * The check for rcu_preempt_depth() is also required because + * voluntary scheduling inside a rcu read locked section is not + * allowed. + * + * The idle task is already covered by this because idle always + * has a preempt count > 0. + * + * If CONFIG_PREEMPTION is disabled only allow scheduling when + * coming from user mode as there is no indication whether the + * context which triggered the page fault could schedule or not. + */ + if (IS_ENABLED(CONFIG_PREEMPTION)) + can_schedule = preempt_count() + rcu_preempt_depth() == 0; + else + can_schedule = usermode; + + /* + * If the kernel context is allowed to schedule then RCU is + * watching because no preemptible code in the kernel is inside RCU + * idle state. So it can be treated like user mode. User mode is + * safe because the #PF entry invoked enter_from_user_mode(). + * + * For the non schedulable case invoke rcu_irq_enter() for + * now. This will be moved out to the pagefault entry code later + * and only invoked when really needed. + */ + if (can_schedule) { + kvm_async_pf_task_wait_schedule(token); + } else { + rcu_irq_enter(); + kvm_async_pf_task_wait_halt(token); + rcu_irq_exit(); } - if (!n.halted) - finish_swait(&n.wq, &wait); - - rcu_irq_exit(); - return; } -EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait); static void apf_task_wake_one(struct kvm_task_sleep_node *n) { hlist_del_init(&n->link); - if (n->halted) + if (n->use_halt) smp_send_reschedule(n->cpu); else if (swq_has_sleeper(&n->wq)) swake_up_one(&n->wq); @@ -177,12 +247,13 @@ static void apf_task_wake_all(void) int i; for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) { - struct hlist_node *p, *next; struct kvm_task_sleep_head *b = &async_pf_sleepers[i]; + struct kvm_task_sleep_node *n; + struct hlist_node *p, *next; + raw_spin_lock(&b->lock); hlist_for_each_safe(p, next, &b->list) { - struct kvm_task_sleep_node *n = - hlist_entry(p, typeof(*n), link); + n = hlist_entry(p, typeof(*n), link); if (n->cpu == smp_processor_id()) apf_task_wake_one(n); } @@ -223,8 +294,9 @@ again: n->cpu = smp_processor_id(); init_swait_queue_head(&n->wq); hlist_add_head(&n->link, &b->list); - } else + } else { apf_task_wake_one(n); + } raw_spin_unlock(&b->lock); return; } @@ -246,23 +318,33 @@ NOKPROBE_SYMBOL(kvm_read_and_reset_pf_reason); bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token) { - /* - * If we get a page fault right here, the pf_reason seems likely - * to be clobbered. Bummer. - */ - switch (kvm_read_and_reset_pf_reason()) { + u32 reason = kvm_read_and_reset_pf_reason(); + + switch (reason) { + case KVM_PV_REASON_PAGE_NOT_PRESENT: + case KVM_PV_REASON_PAGE_READY: + break; default: return false; - case KVM_PV_REASON_PAGE_NOT_PRESENT: + } + + /* + * If the host managed to inject an async #PF into an interrupt + * disabled region, then die hard as this is not going to end well + * and the host side is seriously broken. + */ + if (unlikely(!(regs->flags & X86_EFLAGS_IF))) + panic("Host injected async #PF in interrupt disabled region\n"); + + if (reason == KVM_PV_REASON_PAGE_NOT_PRESENT) { /* page is swapped out by the host. */ - kvm_async_pf_task_wait(token, !user_mode(regs)); - return true; - case KVM_PV_REASON_PAGE_READY: + kvm_async_pf_task_wait(token, user_mode(regs)); + } else { rcu_irq_enter(); kvm_async_pf_task_wake(token); rcu_irq_exit(); - return true; } + return true; } NOKPROBE_SYMBOL(__kvm_handle_async_pf); @@ -326,12 +408,12 @@ static void kvm_guest_cpu_init(void) wrmsrl(MSR_KVM_ASYNC_PF_EN, pa); __this_cpu_write(apf_reason.enabled, 1); - printk(KERN_INFO"KVM setup async PF for cpu %d\n", - smp_processor_id()); + pr_info("KVM setup async PF for cpu %d\n", smp_processor_id()); } if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) { unsigned long pa; + /* Size alignment is implied but just to make it explicit. */ BUILD_BUG_ON(__alignof__(kvm_apic_eoi) < 4); __this_cpu_write(kvm_apic_eoi, 0); @@ -352,8 +434,7 @@ static void kvm_pv_disable_apf(void) wrmsrl(MSR_KVM_ASYNC_PF_EN, 0); __this_cpu_write(apf_reason.enabled, 0); - printk(KERN_INFO"Unregister pv shared memory for cpu %d\n", - smp_processor_id()); + pr_info("Unregister pv shared memory for cpu %d\n", smp_processor_id()); } static void kvm_pv_guest_cpu_reboot(void *unused) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 8071952e9cf2..dd900a648059 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4198,7 +4198,7 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, case KVM_PV_REASON_PAGE_NOT_PRESENT: vcpu->arch.apf.host_apf_reason = 0; local_irq_disable(); - kvm_async_pf_task_wait(fault_address, 0); + kvm_async_pf_task_wait_schedule(fault_address); local_irq_enable(); break; case KVM_PV_REASON_PAGE_READY: From 3a7c8fafd1b42adea229fd204132f6a2fb3cd2d9 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 24 Apr 2020 09:57:56 +0200 Subject: [PATCH 0571/1043] x86/kvm: Restrict ASYNC_PF to user space The async page fault injection into kernel space creates more problems than it solves. The host has absolutely no knowledge about the state of the guest if the fault happens in CPL0. The only restriction for the host is interrupt disabled state. If interrupts are enabled in the guest then the exception can hit arbitrary code. The HALT based wait in non-preemotible code is a hacky replacement for a proper hypercall. For the ongoing work to restrict instrumentation and make the RCU idle interaction well defined the required extra work for supporting async pagefault in CPL0 is just not justified and creates complexity for a dubious benefit. The CPL3 injection is well defined and does not cause any issues as it is more or less the same as a regular page fault from CPL3. Suggested-by: Andy Lutomirski Signed-off-by: Thomas Gleixner Reviewed-by: Alexandre Chartre Acked-by: Paolo Bonzini Acked-by: Peter Zijlstra Link: https://lkml.kernel.org/r/20200505134059.369802541@linutronix.de --- arch/x86/kernel/kvm.c | 100 +++--------------------------------------- 1 file changed, 7 insertions(+), 93 deletions(-) diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index c6a82f9f537f..b3d9b0d7a37d 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -75,7 +75,6 @@ struct kvm_task_sleep_node { struct swait_queue_head wq; u32 token; int cpu; - bool use_halt; }; static struct kvm_task_sleep_head { @@ -98,8 +97,7 @@ static struct kvm_task_sleep_node *_find_apf_task(struct kvm_task_sleep_head *b, return NULL; } -static bool kvm_async_pf_queue_task(u32 token, bool use_halt, - struct kvm_task_sleep_node *n) +static bool kvm_async_pf_queue_task(u32 token, struct kvm_task_sleep_node *n) { u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS); struct kvm_task_sleep_head *b = &async_pf_sleepers[key]; @@ -117,7 +115,6 @@ static bool kvm_async_pf_queue_task(u32 token, bool use_halt, n->token = token; n->cpu = smp_processor_id(); - n->use_halt = use_halt; init_swait_queue_head(&n->wq); hlist_add_head(&n->link, &b->list); raw_spin_unlock(&b->lock); @@ -138,7 +135,7 @@ void kvm_async_pf_task_wait_schedule(u32 token) lockdep_assert_irqs_disabled(); - if (!kvm_async_pf_queue_task(token, false, &n)) + if (!kvm_async_pf_queue_task(token, &n)) return; for (;;) { @@ -154,91 +151,10 @@ void kvm_async_pf_task_wait_schedule(u32 token) } EXPORT_SYMBOL_GPL(kvm_async_pf_task_wait_schedule); -/* - * Invoked from the async page fault handler. - */ -static void kvm_async_pf_task_wait_halt(u32 token) -{ - struct kvm_task_sleep_node n; - - if (!kvm_async_pf_queue_task(token, true, &n)) - return; - - for (;;) { - if (hlist_unhashed(&n.link)) - break; - /* - * No point in doing anything about RCU here. Any RCU read - * side critical section or RCU watching section can be - * interrupted by VMEXITs and the host is free to keep the - * vCPU scheduled out as long as it sees fit. This is not - * any different just because of the halt induced voluntary - * VMEXIT. - * - * Also the async page fault could have interrupted any RCU - * watching context, so invoking rcu_irq_exit()/enter() - * around this is not gaining anything. - */ - native_safe_halt(); - local_irq_disable(); - } -} - -/* Invoked from the async page fault handler */ -static void kvm_async_pf_task_wait(u32 token, bool usermode) -{ - bool can_schedule; - - /* - * No need to check whether interrupts were disabled because the - * host will (hopefully) only inject an async page fault into - * interrupt enabled regions. - * - * If CONFIG_PREEMPTION is enabled then check whether the code - * which triggered the page fault is preemptible. This covers user - * mode as well because preempt_count() is obviously 0 there. - * - * The check for rcu_preempt_depth() is also required because - * voluntary scheduling inside a rcu read locked section is not - * allowed. - * - * The idle task is already covered by this because idle always - * has a preempt count > 0. - * - * If CONFIG_PREEMPTION is disabled only allow scheduling when - * coming from user mode as there is no indication whether the - * context which triggered the page fault could schedule or not. - */ - if (IS_ENABLED(CONFIG_PREEMPTION)) - can_schedule = preempt_count() + rcu_preempt_depth() == 0; - else - can_schedule = usermode; - - /* - * If the kernel context is allowed to schedule then RCU is - * watching because no preemptible code in the kernel is inside RCU - * idle state. So it can be treated like user mode. User mode is - * safe because the #PF entry invoked enter_from_user_mode(). - * - * For the non schedulable case invoke rcu_irq_enter() for - * now. This will be moved out to the pagefault entry code later - * and only invoked when really needed. - */ - if (can_schedule) { - kvm_async_pf_task_wait_schedule(token); - } else { - rcu_irq_enter(); - kvm_async_pf_task_wait_halt(token); - rcu_irq_exit(); - } -} - static void apf_task_wake_one(struct kvm_task_sleep_node *n) { hlist_del_init(&n->link); - if (n->use_halt) - smp_send_reschedule(n->cpu); - else if (swq_has_sleeper(&n->wq)) + if (swq_has_sleeper(&n->wq)) swake_up_one(&n->wq); } @@ -337,8 +253,10 @@ bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token) panic("Host injected async #PF in interrupt disabled region\n"); if (reason == KVM_PV_REASON_PAGE_NOT_PRESENT) { - /* page is swapped out by the host. */ - kvm_async_pf_task_wait(token, user_mode(regs)); + if (unlikely(!(user_mode(regs)))) + panic("Host injected async #PF in kernel mode\n"); + /* Page is swapped out by the host. */ + kvm_async_pf_task_wait_schedule(token); } else { rcu_irq_enter(); kvm_async_pf_task_wake(token); @@ -397,10 +315,6 @@ static void kvm_guest_cpu_init(void) WARN_ON_ONCE(!static_branch_likely(&kvm_async_pf_enabled)); pa = slow_virt_to_phys(this_cpu_ptr(&apf_reason)); - -#ifdef CONFIG_PREEMPTION - pa |= KVM_ASYNC_PF_SEND_ALWAYS; -#endif pa |= KVM_ASYNC_PF_ENABLED; if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF_VMEXIT)) From 860f02f13cf9b72fdc48f978723347f54c5364f2 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 19 May 2020 20:28:11 +0800 Subject: [PATCH 0572/1043] MIPS: SGI-IP27: Remove duplicated include in ip27-timer.c After commit 9d0aaf98dc24 ("MIPS: SGI-IP27: Move all shared IP27 declarations to ip27-common.h"), ip27-common.h is included more than once in ip27-timer.c, remove it. Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/sgi-ip27/ip27-timer.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/mips/sgi-ip27/ip27-timer.c b/arch/mips/sgi-ip27/ip27-timer.c index 61f3565f3645..11ffb3e5f3d4 100644 --- a/arch/mips/sgi-ip27/ip27-timer.c +++ b/arch/mips/sgi-ip27/ip27-timer.c @@ -35,8 +35,6 @@ #include #include -#include "ip27-common.h" - static int rt_next_event(unsigned long delta, struct clock_event_device *evt) { unsigned int cpu = smp_processor_id(); From bd6e38983bb76a48604b7a4f0740354158217bd3 Mon Sep 17 00:00:00 2001 From: Zhi Li Date: Mon, 18 May 2020 15:08:08 +0800 Subject: [PATCH 0573/1043] MIPS: Remove useless parameter of bootcmdline_init() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parameter "cmdline_p" is useless in bootcmdline_init()，remove it. Signed-off-by: Zhi Li Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/setup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index 8db533cd816c..7b537fa2035d 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -575,7 +575,7 @@ static int __init bootcmdline_scan_chosen(unsigned long node, const char *uname, #endif /* CONFIG_OF_EARLY_FLATTREE */ -static void __init bootcmdline_init(char **cmdline_p) +static void __init bootcmdline_init(void) { bool dt_bootargs = false; @@ -658,7 +658,7 @@ static void __init arch_mem_init(char **cmdline_p) plat_mem_setup(); memblock_set_bottom_up(true); - bootcmdline_init(cmdline_p); + bootcmdline_init(); strlcpy(command_line, boot_command_line, COMMAND_LINE_SIZE); *cmdline_p = command_line; From 9ee195fd1be87719e5fcda4cbd7ba4454249f04f Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Wed, 6 May 2020 20:42:30 +0300 Subject: [PATCH 0574/1043] mips: MAAR: Add XPA mode support When XPA mode is enabled the normally 32-bits MAAR pair registers are extended to be of 64-bits width as in pure 64-bits MIPS architecture. In this case the MAAR registers can enable the speculative loads/stores for addresses of up to 39-bits width. But in this case the process of the MAAR initialization changes a bit. The upper 32-bits of the registers are supposed to be accessed by mean of the dedicated instructions mfhc0/mthc0 and there is a CP0.MAAR.VH bit which should be set together with CP0.MAAR.VL as indication of the boundary validity. All of these peculiarities were taken into account in this commit so the speculative loads/stores would work when XPA mode is enabled. Co-developed-by: Alexey Malahov Signed-off-by: Alexey Malahov Signed-off-by: Serge Semin Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Arnd Bergmann Cc: Rob Herring Cc: linux-pm@vger.kernel.org Cc: devicetree@vger.kernel.org Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/maar.h | 17 +++++++++++++++-- arch/mips/include/asm/mipsregs.h | 10 ++++++++++ arch/mips/mm/init.c | 8 +++++++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/arch/mips/include/asm/maar.h b/arch/mips/include/asm/maar.h index 6908b93c4ff9..99f1c3e4b11f 100644 --- a/arch/mips/include/asm/maar.h +++ b/arch/mips/include/asm/maar.h @@ -32,7 +32,7 @@ unsigned platform_maar_init(unsigned num_pairs); * @upper: The highest address that the MAAR pair will affect. Must be * aligned to one byte before a 2^16 byte boundary. * @attrs: The accessibility attributes to program, eg. MIPS_MAAR_S. The - * MIPS_MAAR_VL attribute will automatically be set. + * MIPS_MAAR_VL/MIPS_MAAR_VH attributes will automatically be set. * * Program the pair of MAAR registers specified by idx to apply the attributes * specified by attrs to the range of addresses from lower to higher. @@ -48,17 +48,30 @@ static inline void write_maar_pair(unsigned idx, phys_addr_t lower, /* Automatically set MIPS_MAAR_VL */ attrs |= MIPS_MAAR_VL; - /* Write the upper address & attributes (only MIPS_MAAR_VL matters) */ + /* + * Write the upper address & attributes (both MIPS_MAAR_VL and + * MIPS_MAAR_VH matter) + */ write_c0_maari(idx << 1); back_to_back_c0_hazard(); write_c0_maar(((upper >> 4) & MIPS_MAAR_ADDR) | attrs); back_to_back_c0_hazard(); +#ifdef CONFIG_XPA + upper >>= MIPS_MAARX_ADDR_SHIFT; + writex_c0_maar(((upper >> 4) & MIPS_MAARX_ADDR) | MIPS_MAARX_VH); + back_to_back_c0_hazard(); +#endif /* Write the lower address & attributes */ write_c0_maari((idx << 1) | 0x1); back_to_back_c0_hazard(); write_c0_maar((lower >> 4) | attrs); back_to_back_c0_hazard(); +#ifdef CONFIG_XPA + lower >>= MIPS_MAARX_ADDR_SHIFT; + writex_c0_maar(((lower >> 4) & MIPS_MAARX_ADDR) | MIPS_MAARX_VH); + back_to_back_c0_hazard(); +#endif } /** diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h index 90f843c72774..238adce3ccb2 100644 --- a/arch/mips/include/asm/mipsregs.h +++ b/arch/mips/include/asm/mipsregs.h @@ -761,6 +761,14 @@ #define MIPS_MAAR_ADDR_SHIFT 12 #define MIPS_MAAR_S (_ULCAST_(1) << 1) #define MIPS_MAAR_VL (_ULCAST_(1) << 0) +#ifdef CONFIG_XPA +#define MIPS_MAAR_V (MIPS_MAAR_VH | MIPS_MAAR_VL) +#else +#define MIPS_MAAR_V MIPS_MAAR_VL +#endif +#define MIPS_MAARX_VH (_ULCAST_(1) << 31) +#define MIPS_MAARX_ADDR 0xF +#define MIPS_MAARX_ADDR_SHIFT 32 /* MAARI bit definitions */ #define MIPS_MAARI_INDEX (_ULCAST_(0x3f) << 0) @@ -1723,6 +1731,8 @@ do { \ #define write_c0_lladdr(val) __write_ulong_c0_register($17, 0, val) #define read_c0_maar() __read_ulong_c0_register($17, 1) #define write_c0_maar(val) __write_ulong_c0_register($17, 1, val) +#define readx_c0_maar() __readx_32bit_c0_register($17, 1) +#define writex_c0_maar(val) __writex_32bit_c0_register($17, 1, val) #define read_c0_maari() __read_32bit_c0_register($17, 2) #define write_c0_maari(val) __write_32bit_c0_register($17, 2, val) diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c index 79684000de0e..620ebfa45ec1 100644 --- a/arch/mips/mm/init.c +++ b/arch/mips/mm/init.c @@ -358,17 +358,23 @@ void maar_init(void) write_c0_maari(i); back_to_back_c0_hazard(); upper = read_c0_maar(); +#ifdef CONFIG_XPA + upper |= (phys_addr_t)readx_c0_maar() << MIPS_MAARX_ADDR_SHIFT; +#endif write_c0_maari(i + 1); back_to_back_c0_hazard(); lower = read_c0_maar(); +#ifdef CONFIG_XPA + lower |= (phys_addr_t)readx_c0_maar() << MIPS_MAARX_ADDR_SHIFT; +#endif attr = lower & upper; lower = (lower & MIPS_MAAR_ADDR) << 4; upper = ((upper & MIPS_MAAR_ADDR) << 4) | 0xffff; pr_info(" [%d]: ", i / 2); - if (!(attr & MIPS_MAAR_VL)) { + if ((attr & MIPS_MAAR_V) != MIPS_MAAR_V) { pr_cont("disabled\n"); continue; } From f28cef9e4daca11337cb9f144cdebedaab69d78c Mon Sep 17 00:00:00 2001 From: Brian Foster Date: Thu, 14 May 2020 13:50:25 -0700 Subject: [PATCH 0575/1043] xfs: don't fail verifier on empty attr3 leaf block The attr fork can transition from shortform to leaf format while empty if the first xattr doesn't fit in shortform. While this empty leaf block state is intended to be transient, it is technically not due to the transactional implementation of the xattr set operation. We historically have a couple of bandaids to work around this problem. The first is to hold the buffer after the format conversion to prevent premature writeback of the empty leaf buffer and the second is to bypass the xattr count check in the verifier during recovery. The latter assumes that the xattr set is also in the log and will be recovered into the buffer soon after the empty leaf buffer is reconstructed. This is not guaranteed, however. If the filesystem crashes after the format conversion but before the xattr set that induced it, only the format conversion may exist in the log. When recovered, this creates a latent corrupted state on the inode as any subsequent attempts to read the buffer fail due to verifier failure. This includes further attempts to set xattrs on the inode or attempts to destroy the attr fork, which prevents the inode from ever being removed from the unlinked list. To avoid this condition, accept that an empty attr leaf block is a valid state and remove the count check from the verifier. This means that on rare occasions an attr fork might exist in an unexpected state, but is otherwise consistent and functional. Note that we retain the logic to avoid racing with metadata writeback to reduce the window where this can occur. Signed-off-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_attr_leaf.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 863444e2dda7..6d18e86bb9c7 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -308,14 +308,6 @@ xfs_attr3_leaf_verify( if (fa) return fa; - /* - * In recovery there is a transient state where count == 0 is valid - * because we may have transitioned an empty shortform attr to a leaf - * if the attr didn't fit in shortform. - */ - if (!xfs_log_in_recovery(mp) && ichdr.count == 0) - return __this_address; - /* * firstused is the block offset of the first name info structure. * Make sure it doesn't go off the block or crash into the header. @@ -331,6 +323,13 @@ xfs_attr3_leaf_verify( (char *)bp->b_addr + ichdr.firstused) return __this_address; + /* + * NOTE: This verifier historically failed empty leaf buffers because + * we expect the fork to be in another format. Empty attr fork format + * conversions are possible during xattr set, however, and format + * conversion is not atomic with the xattr set that triggers it. We + * cannot assume leaf blocks are non-empty until that is addressed. + */ buf_end = (char *)bp->b_addr + mp->m_attr_geo->blksize; for (i = 0, ent = entries; i < ichdr.count; ent++, i++) { fa = xfs_attr3_leaf_verify_entry(mp, buf_end, leaf, &ichdr, From 6d15120b282e49811a47f2f6d6b749d178be7e99 Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Tue, 19 May 2020 08:51:57 -0700 Subject: [PATCH 0576/1043] clocksource/drivers/timer-ti-dm: Fix warning for set but not used We can get a warning for dmtimer_clocksource_init() with 'pa' set but not used. This was used in the earlier revisions of the code but no longer needed, so let's remove the unused pa and of_translate_address(). Let's also do it for dmtimer_clockevent_init() that has a similar issue. Reported-by: kbuild test robot Signed-off-by: Tony Lindgren Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200519155157.12804-1-tony@atomide.com --- drivers/clocksource/timer-ti-dm-systimer.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/clocksource/timer-ti-dm-systimer.c b/drivers/clocksource/timer-ti-dm-systimer.c index 1495618a5744..7da998d0dd58 100644 --- a/drivers/clocksource/timer-ti-dm-systimer.c +++ b/drivers/clocksource/timer-ti-dm-systimer.c @@ -514,7 +514,6 @@ static int __init dmtimer_clockevent_init(struct device_node *np) struct clock_event_device *dev; struct dmtimer_systimer *t; int error; - u32 pa; clkevt = kzalloc(sizeof(*clkevt), GFP_KERNEL); if (!clkevt) @@ -563,7 +562,6 @@ static int __init dmtimer_clockevent_init(struct device_node *np) writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_ena); writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->wakeup); - pa = of_translate_address(np, of_get_address(np, 0, NULL, NULL)); pr_info("TI gptimer clockevent: %s%lu Hz at %pOF\n", of_find_property(np, "ti,timer-alwon", NULL) ? "always-on " : "", t->rate, np->parent); @@ -637,7 +635,6 @@ static int __init dmtimer_clocksource_init(struct device_node *np) struct dmtimer_systimer *t; struct clocksource *dev; int error; - u32 pa; clksrc = kzalloc(sizeof(*clksrc), GFP_KERNEL); if (!clksrc) @@ -666,7 +663,6 @@ static int __init dmtimer_clocksource_init(struct device_node *np) writel_relaxed(OMAP_TIMER_CTRL_ST | OMAP_TIMER_CTRL_AR, t->base + t->ctrl); - pa = of_translate_address(np, of_get_address(np, 0, NULL, NULL)); pr_info("TI gptimer clocksource: %s%pOF\n", of_find_property(np, "ti,timer-alwon", NULL) ? "always-on " : "", np->parent); From 78bba5c812cc651cee51b64b786be926ab7fe2a9 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Wed, 13 May 2020 15:33:27 -0700 Subject: [PATCH 0577/1043] xfs: use ordered buffers to initialize dquot buffers during quotacheck While QAing the new xfs_repair quotacheck code, I uncovered a quota corruption bug resulting from a bad interaction between dquot buffer initialization and quotacheck. The bug can be reproduced with the following sequence: # mkfs.xfs -f /dev/sdf # mount /dev/sdf /opt -o usrquota # su nobody -s /bin/bash -c 'touch /opt/barf' # sync # xfs_quota -x -c 'report -ahi' /opt User quota on /opt (/dev/sdf) Inodes User ID Used Soft Hard Warn/Grace ---------- --------------------------------- root 3 0 0 00 [------] nobody 1 0 0 00 [------] # xfs_io -x -c 'shutdown' /opt # umount /opt # mount /dev/sdf /opt -o usrquota # touch /opt/man2 # xfs_quota -x -c 'report -ahi' /opt User quota on /opt (/dev/sdf) Inodes User ID Used Soft Hard Warn/Grace ---------- --------------------------------- root 1 0 0 00 [------] nobody 1 0 0 00 [------] # umount /opt Notice how the initial quotacheck set the root dquot icount to 3 (rootino, rbmino, rsumino), but after shutdown -> remount -> recovery, xfs_quota reports that the root dquot has only 1 icount. We haven't deleted anything from the filesystem, which means that quota is now under-counting. This behavior is not limited to icount or the root dquot, but this is the shortest reproducer. I traced the cause of this discrepancy to the way that we handle ondisk dquot updates during quotacheck vs. regular fs activity. Normally, when we allocate a disk block for a dquot, we log the buffer as a regular (dquot) buffer. Subsequent updates to the dquots backed by that block are done via separate dquot log item updates, which means that they depend on the logged buffer update being written to disk before the dquot items. Because individual dquots have their own LSN fields, that initial dquot buffer must always be recovered. However, the story changes for quotacheck, which can cause dquot block allocations but persists the final dquot counter values via a delwri list. Because recovery doesn't gate dquot buffer replay on an LSN, this means that the initial dquot buffer can be replayed over the (newer) contents that were delwritten at the end of quotacheck. In effect, this re-initializes the dquot counters after they've been updated. If the log does not contain any other dquot items to recover, the obsolete dquot contents will not be corrected by log recovery. Because quotacheck uses a transaction to log the setting of the CHKD flags in the superblock, we skip quotacheck during the second mount call, which allows the incorrect icount to remain. Fix this by changing the ondisk dquot initialization function to use ordered buffers to write out fresh dquot blocks if it detects that we're running quotacheck. If the system goes down before quotacheck can complete, the CHKD flags will not be set in the superblock and the next mount will run quotacheck again, which can fix uninitialized dquot buffers. This requires amending the defer code to maintaine ordered buffer state across defer rolls for the sake of the dquot allocation code. For regular operations we preserve the current behavior since the dquot items require properly initialized ondisk dquot records. Signed-off-by: Darrick J. Wong Reviewed-by: Brian Foster Reviewed-by: Christoph Hellwig --- fs/xfs/libxfs/xfs_defer.c | 10 ++++++- fs/xfs/xfs_dquot.c | 56 ++++++++++++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index 1172fbf072d8..d8f586256add 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -240,10 +240,13 @@ xfs_defer_trans_roll( struct xfs_log_item *lip; struct xfs_buf *bplist[XFS_DEFER_OPS_NR_BUFS]; struct xfs_inode *iplist[XFS_DEFER_OPS_NR_INODES]; + unsigned int ordered = 0; /* bitmap */ int bpcount = 0, ipcount = 0; int i; int error; + BUILD_BUG_ON(NBBY * sizeof(ordered) < XFS_DEFER_OPS_NR_BUFS); + list_for_each_entry(lip, &tp->t_items, li_trans) { switch (lip->li_type) { case XFS_LI_BUF: @@ -254,7 +257,10 @@ xfs_defer_trans_roll( ASSERT(0); return -EFSCORRUPTED; } - xfs_trans_dirty_buf(tp, bli->bli_buf); + if (bli->bli_flags & XFS_BLI_ORDERED) + ordered |= (1U << bpcount); + else + xfs_trans_dirty_buf(tp, bli->bli_buf); bplist[bpcount++] = bli->bli_buf; } break; @@ -295,6 +301,8 @@ xfs_defer_trans_roll( /* Rejoin the buffers and dirty them so the log moves forward. */ for (i = 0; i < bpcount; i++) { xfs_trans_bjoin(tp, bplist[i]); + if (ordered & (1U << i)) + xfs_trans_ordered_buf(tp, bplist[i]); xfs_trans_bhold(tp, bplist[i]); } diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 52e0f7245afc..55b95d45303b 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -205,16 +205,18 @@ xfs_qm_adjust_dqtimers( */ STATIC void xfs_qm_init_dquot_blk( - xfs_trans_t *tp, - xfs_mount_t *mp, - xfs_dqid_t id, - uint type, - xfs_buf_t *bp) + struct xfs_trans *tp, + struct xfs_mount *mp, + xfs_dqid_t id, + uint type, + struct xfs_buf *bp) { struct xfs_quotainfo *q = mp->m_quotainfo; - xfs_dqblk_t *d; - xfs_dqid_t curid; - int i; + struct xfs_dqblk *d; + xfs_dqid_t curid; + unsigned int qflag; + unsigned int blftype; + int i; ASSERT(tp); ASSERT(xfs_buf_islocked(bp)); @@ -238,11 +240,39 @@ xfs_qm_init_dquot_blk( } } - xfs_trans_dquot_buf(tp, bp, - (type & XFS_DQ_USER ? XFS_BLF_UDQUOT_BUF : - ((type & XFS_DQ_PROJ) ? XFS_BLF_PDQUOT_BUF : - XFS_BLF_GDQUOT_BUF))); - xfs_trans_log_buf(tp, bp, 0, BBTOB(q->qi_dqchunklen) - 1); + if (type & XFS_DQ_USER) { + qflag = XFS_UQUOTA_CHKD; + blftype = XFS_BLF_UDQUOT_BUF; + } else if (type & XFS_DQ_PROJ) { + qflag = XFS_PQUOTA_CHKD; + blftype = XFS_BLF_PDQUOT_BUF; + } else { + qflag = XFS_GQUOTA_CHKD; + blftype = XFS_BLF_GDQUOT_BUF; + } + + xfs_trans_dquot_buf(tp, bp, blftype); + + /* + * quotacheck uses delayed writes to update all the dquots on disk in an + * efficient manner instead of logging the individual dquot changes as + * they are made. However if we log the buffer allocated here and crash + * after quotacheck while the logged initialisation is still in the + * active region of the log, log recovery can replay the dquot buffer + * initialisation over the top of the checked dquots and corrupt quota + * accounting. + * + * To avoid this problem, quotacheck cannot log the initialised buffer. + * We must still dirty the buffer and write it back before the + * allocation transaction clears the log. Therefore, mark the buffer as + * ordered instead of logging it directly. This is safe for quotacheck + * because it detects and repairs allocated but initialized dquot blocks + * in the quota inodes. + */ + if (!(mp->m_qflags & qflag)) + xfs_trans_ordered_buf(tp, bp); + else + xfs_trans_log_buf(tp, bp, 0, BBTOB(q->qi_dqchunklen) - 1); } /* From 765d3c393c222c3bb281885ea211c8ebc08250b4 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Tue, 12 May 2020 16:42:51 -0700 Subject: [PATCH 0578/1043] xfs: don't allow SWAPEXT if we'd screw up quota accounting Since the old SWAPEXT ioctl doesn't know how to adjust quota ids, bail out of the ids don't match and quotas are enabled. Signed-off-by: Darrick J. Wong Reviewed-by: Eric Sandeen Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_bmap_util.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index cc23a3e23e2d..b8acfd4d3ca6 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -1211,6 +1211,13 @@ xfs_swap_extents_check_format( struct xfs_inode *tip) /* tmp inode */ { + /* User/group/project quota ids must match if quotas are enforced. */ + if (XFS_IS_QUOTA_ON(ip->i_mount) && + (!uid_eq(VFS_I(ip)->i_uid, VFS_I(tip)->i_uid) || + !gid_eq(VFS_I(ip)->i_gid, VFS_I(tip)->i_gid) || + ip->i_d.di_projid != tip->i_d.di_projid)) + return -EINVAL; + /* Should never get a local format */ if (ip->i_d.di_format == XFS_DINODE_FMT_LOCAL || tip->i_d.di_format == XFS_DINODE_FMT_LOCAL) From 14506f7a91d8f4d13fc07126ac8d14c6519f00e3 Mon Sep 17 00:00:00 2001 From: Kaixu Xia Date: Mon, 11 May 2020 08:32:19 -0700 Subject: [PATCH 0579/1043] xfs: fix the warning message in xfs_validate_sb_common() Fix this error message to complain about project and group quota flag bits instead of "PUOTA" and "QUOTA". Signed-off-by: Kaixu Xia Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_sb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_sb.c b/fs/xfs/libxfs/xfs_sb.c index c526c5e5ab76..4df87546bd40 100644 --- a/fs/xfs/libxfs/xfs_sb.c +++ b/fs/xfs/libxfs/xfs_sb.c @@ -243,7 +243,7 @@ xfs_validate_sb_common( } else if (sbp->sb_qflags & (XFS_PQUOTA_ENFD | XFS_GQUOTA_ENFD | XFS_PQUOTA_CHKD | XFS_GQUOTA_CHKD)) { xfs_notice(mp, -"Superblock earlier than Version 5 has XFS_[PQ]UOTA_{ENFD|CHKD} bits."); +"Superblock earlier than Version 5 has XFS_{P|G}QUOTA_{ENFD|CHKD} bits."); return -EFSCORRUPTED; } From b90c2a9c8b4422bb9398b50fe3d6163e46dcddec Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 14 May 2020 13:59:51 -0700 Subject: [PATCH 0580/1043] xfs: xfs_bmapi_read doesn't take a fork id as the last argument The last argument to xfs_bmapi_raad contains XFS_BMAPI_* flags, not the fork. Given that XFS_DATA_FORK evaluates to 0 no real harm is done, but let's fix this anyway. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_rtbitmap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_rtbitmap.c b/fs/xfs/libxfs/xfs_rtbitmap.c index f42c74cb8be5..9498ced947be 100644 --- a/fs/xfs/libxfs/xfs_rtbitmap.c +++ b/fs/xfs/libxfs/xfs_rtbitmap.c @@ -66,7 +66,7 @@ xfs_rtbuf_get( ip = issum ? mp->m_rsumip : mp->m_rbmip; - error = xfs_bmapi_read(ip, block, 1, &map, &nmap, XFS_DATA_FORK); + error = xfs_bmapi_read(ip, block, 1, &map, &nmap, 0); if (error) return error; From cb7d58594412fff106cde550dd9e0a7999cc2a0c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 14 May 2020 14:00:02 -0700 Subject: [PATCH 0581/1043] xfs: call xfs_iformat_fork from xfs_inode_from_disk We always need to fill out the fork structures when reading the inode, so call xfs_iformat_fork from the tail of xfs_inode_from_disk. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_buf.c | 7 ++++--- fs/xfs/libxfs/xfs_inode_buf.h | 2 +- fs/xfs/xfs_inode_item_recover.c | 3 +-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index 81a010422bea..dc00ce6fc4a2 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -180,7 +180,7 @@ xfs_imap_to_bp( return 0; } -void +int xfs_inode_from_disk( struct xfs_inode *ip, struct xfs_dinode *from) @@ -241,6 +241,8 @@ xfs_inode_from_disk( to->di_flags2 = be64_to_cpu(from->di_flags2); to->di_cowextsize = be32_to_cpu(from->di_cowextsize); } + + return xfs_iformat_fork(ip, from); } void @@ -641,8 +643,7 @@ xfs_iread( * Otherwise, just get the truly permanent information. */ if (dip->di_mode) { - xfs_inode_from_disk(ip, dip); - error = xfs_iformat_fork(ip, dip); + error = xfs_inode_from_disk(ip, dip); if (error) { #ifdef DEBUG xfs_alert(mp, "%s: xfs_iformat() returned error %d", diff --git a/fs/xfs/libxfs/xfs_inode_buf.h b/fs/xfs/libxfs/xfs_inode_buf.h index d9b4781ac9fd..0fbb99224ec7 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.h +++ b/fs/xfs/libxfs/xfs_inode_buf.h @@ -54,7 +54,7 @@ int xfs_iread(struct xfs_mount *, struct xfs_trans *, void xfs_dinode_calc_crc(struct xfs_mount *, struct xfs_dinode *); void xfs_inode_to_disk(struct xfs_inode *ip, struct xfs_dinode *to, xfs_lsn_t lsn); -void xfs_inode_from_disk(struct xfs_inode *ip, struct xfs_dinode *from); +int xfs_inode_from_disk(struct xfs_inode *ip, struct xfs_dinode *from); void xfs_log_dinode_to_disk(struct xfs_log_dinode *from, struct xfs_dinode *to); diff --git a/fs/xfs/xfs_inode_item_recover.c b/fs/xfs/xfs_inode_item_recover.c index 2bdba612aa71..82ca5ce312c5 100644 --- a/fs/xfs/xfs_inode_item_recover.c +++ b/fs/xfs/xfs_inode_item_recover.c @@ -89,9 +89,8 @@ xfs_recover_inode_owner_change( /* instantiate the inode */ ASSERT(dip->di_version >= 3); - xfs_inode_from_disk(ip, dip); - error = xfs_iformat_fork(ip, dip); + error = xfs_inode_from_disk(ip, dip); if (error) goto out_free_ip; From 9229d18e801bdbdf79d963d8c944980fc77b5d6b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 14 May 2020 14:01:17 -0700 Subject: [PATCH 0582/1043] xfs: split xfs_iformat_fork xfs_iformat_fork is a weird catchall. Split it into one helper for the data fork and one for the attr fork, and then call both helper as well as the COW fork initialization from xfs_inode_from_disk. Order the COW fork initialization after the attr fork initialization given that it can't fail to simplify the error handling. Note that the newly split helpers are moved down the file in xfs_inode_fork.c to avoid the need for forward declarations. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_buf.c | 20 +++- fs/xfs/libxfs/xfs_inode_fork.c | 186 +++++++++++++++------------------ fs/xfs/libxfs/xfs_inode_fork.h | 3 +- 3 files changed, 103 insertions(+), 106 deletions(-) diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index dc00ce6fc4a2..abdecc80579e 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -187,6 +187,10 @@ xfs_inode_from_disk( { struct xfs_icdinode *to = &ip->i_d; struct inode *inode = VFS_I(ip); + int error; + + ASSERT(ip->i_cowfp == NULL); + ASSERT(ip->i_afp == NULL); /* * Convert v1 inodes immediately to v2 inode format as this is the @@ -242,7 +246,21 @@ xfs_inode_from_disk( to->di_cowextsize = be32_to_cpu(from->di_cowextsize); } - return xfs_iformat_fork(ip, from); + error = xfs_iformat_data_fork(ip, from); + if (error) + return error; + if (XFS_DFORK_Q(from)) { + error = xfs_iformat_attr_fork(ip, from); + if (error) + goto out_destroy_data_fork; + } + if (xfs_is_reflink_inode(ip)) + xfs_ifork_init_cow(ip); + return 0; + +out_destroy_data_fork: + xfs_idestroy_fork(ip, XFS_DATA_FORK); + return error; } void diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index 3e9a42f1e23b..5fadfa9a17eb 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -26,110 +26,6 @@ kmem_zone_t *xfs_ifork_zone; -STATIC int xfs_iformat_local(xfs_inode_t *, xfs_dinode_t *, int, int); -STATIC int xfs_iformat_extents(xfs_inode_t *, xfs_dinode_t *, int); -STATIC int xfs_iformat_btree(xfs_inode_t *, xfs_dinode_t *, int); - -/* - * Copy inode type and data and attr format specific information from the - * on-disk inode to the in-core inode and fork structures. For fifos, devices, - * and sockets this means set i_rdev to the proper value. For files, - * directories, and symlinks this means to bring in the in-line data or extent - * pointers as well as the attribute fork. For a fork in B-tree format, only - * the root is immediately brought in-core. The rest will be read in later when - * first referenced (see xfs_iread_extents()). - */ -int -xfs_iformat_fork( - struct xfs_inode *ip, - struct xfs_dinode *dip) -{ - struct inode *inode = VFS_I(ip); - struct xfs_attr_shortform *atp; - int size; - int error = 0; - xfs_fsize_t di_size; - - switch (inode->i_mode & S_IFMT) { - case S_IFIFO: - case S_IFCHR: - case S_IFBLK: - case S_IFSOCK: - ip->i_d.di_size = 0; - inode->i_rdev = xfs_to_linux_dev_t(xfs_dinode_get_rdev(dip)); - break; - - case S_IFREG: - case S_IFLNK: - case S_IFDIR: - switch (dip->di_format) { - case XFS_DINODE_FMT_LOCAL: - di_size = be64_to_cpu(dip->di_size); - size = (int)di_size; - error = xfs_iformat_local(ip, dip, XFS_DATA_FORK, size); - break; - case XFS_DINODE_FMT_EXTENTS: - error = xfs_iformat_extents(ip, dip, XFS_DATA_FORK); - break; - case XFS_DINODE_FMT_BTREE: - error = xfs_iformat_btree(ip, dip, XFS_DATA_FORK); - break; - default: - xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, - dip, sizeof(*dip), __this_address); - return -EFSCORRUPTED; - } - break; - - default: - xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, dip, - sizeof(*dip), __this_address); - return -EFSCORRUPTED; - } - if (error) - return error; - - if (xfs_is_reflink_inode(ip)) { - ASSERT(ip->i_cowfp == NULL); - xfs_ifork_init_cow(ip); - } - - if (!XFS_DFORK_Q(dip)) - return 0; - - ASSERT(ip->i_afp == NULL); - ip->i_afp = kmem_zone_zalloc(xfs_ifork_zone, KM_NOFS); - - switch (dip->di_aformat) { - case XFS_DINODE_FMT_LOCAL: - atp = (xfs_attr_shortform_t *)XFS_DFORK_APTR(dip); - size = be16_to_cpu(atp->hdr.totsize); - - error = xfs_iformat_local(ip, dip, XFS_ATTR_FORK, size); - break; - case XFS_DINODE_FMT_EXTENTS: - error = xfs_iformat_extents(ip, dip, XFS_ATTR_FORK); - break; - case XFS_DINODE_FMT_BTREE: - error = xfs_iformat_btree(ip, dip, XFS_ATTR_FORK); - break; - default: - xfs_inode_verifier_error(ip, error, __func__, dip, - sizeof(*dip), __this_address); - error = -EFSCORRUPTED; - break; - } - if (error) { - kmem_cache_free(xfs_ifork_zone, ip->i_afp); - ip->i_afp = NULL; - if (ip->i_cowfp) - kmem_cache_free(xfs_ifork_zone, ip->i_cowfp); - ip->i_cowfp = NULL; - xfs_idestroy_fork(ip, XFS_DATA_FORK); - } - return error; -} - void xfs_init_local_fork( struct xfs_inode *ip, @@ -325,6 +221,88 @@ xfs_iformat_btree( return 0; } +int +xfs_iformat_data_fork( + struct xfs_inode *ip, + struct xfs_dinode *dip) +{ + struct inode *inode = VFS_I(ip); + + switch (inode->i_mode & S_IFMT) { + case S_IFIFO: + case S_IFCHR: + case S_IFBLK: + case S_IFSOCK: + ip->i_d.di_size = 0; + inode->i_rdev = xfs_to_linux_dev_t(xfs_dinode_get_rdev(dip)); + return 0; + case S_IFREG: + case S_IFLNK: + case S_IFDIR: + switch (dip->di_format) { + case XFS_DINODE_FMT_LOCAL: + return xfs_iformat_local(ip, dip, XFS_DATA_FORK, + be64_to_cpu(dip->di_size)); + case XFS_DINODE_FMT_EXTENTS: + return xfs_iformat_extents(ip, dip, XFS_DATA_FORK); + case XFS_DINODE_FMT_BTREE: + return xfs_iformat_btree(ip, dip, XFS_DATA_FORK); + default: + xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, + dip, sizeof(*dip), __this_address); + return -EFSCORRUPTED; + } + break; + default: + xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, dip, + sizeof(*dip), __this_address); + return -EFSCORRUPTED; + } +} + +static uint16_t +xfs_dfork_attr_shortform_size( + struct xfs_dinode *dip) +{ + struct xfs_attr_shortform *atp = + (struct xfs_attr_shortform *)XFS_DFORK_APTR(dip); + + return be16_to_cpu(atp->hdr.totsize); +} + +int +xfs_iformat_attr_fork( + struct xfs_inode *ip, + struct xfs_dinode *dip) +{ + int error = 0; + + ip->i_afp = kmem_zone_zalloc(xfs_ifork_zone, KM_NOFS); + switch (dip->di_aformat) { + case XFS_DINODE_FMT_LOCAL: + error = xfs_iformat_local(ip, dip, XFS_ATTR_FORK, + xfs_dfork_attr_shortform_size(dip)); + break; + case XFS_DINODE_FMT_EXTENTS: + error = xfs_iformat_extents(ip, dip, XFS_ATTR_FORK); + break; + case XFS_DINODE_FMT_BTREE: + error = xfs_iformat_btree(ip, dip, XFS_ATTR_FORK); + break; + default: + xfs_inode_verifier_error(ip, error, __func__, dip, + sizeof(*dip), __this_address); + error = -EFSCORRUPTED; + break; + } + + if (error) { + kmem_cache_free(xfs_ifork_zone, ip->i_afp); + ip->i_afp = NULL; + } + return error; +} + /* * Reallocate the space for if_broot based on the number of records * being added or deleted as indicated in rec_diff. Move the records diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index 668ee942be22..8487b0c88a75 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -88,7 +88,8 @@ struct xfs_ifork { struct xfs_ifork *xfs_iext_state_to_fork(struct xfs_inode *ip, int state); -int xfs_iformat_fork(struct xfs_inode *, struct xfs_dinode *); +int xfs_iformat_data_fork(struct xfs_inode *, struct xfs_dinode *); +int xfs_iformat_attr_fork(struct xfs_inode *, struct xfs_dinode *); void xfs_iflush_fork(struct xfs_inode *, struct xfs_dinode *, struct xfs_inode_log_item *, int); void xfs_idestroy_fork(struct xfs_inode *, int); From 0bce8173fdcf203c92a4d57dc7d3bb642ed478a1 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 14 May 2020 14:01:17 -0700 Subject: [PATCH 0583/1043] xfs: handle unallocated inodes in xfs_inode_from_disk Handle inodes with a 0 di_mode in xfs_inode_from_disk, instead of partially duplicating inode reading in xfs_iread. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_buf.c | 50 ++++++++++------------------------- 1 file changed, 14 insertions(+), 36 deletions(-) diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index abdecc80579e..686a026b5f6e 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -192,6 +192,17 @@ xfs_inode_from_disk( ASSERT(ip->i_cowfp == NULL); ASSERT(ip->i_afp == NULL); + /* + * First get the permanent information that is needed to allocate an + * inode. If the inode is unused, mode is zero and we shouldn't mess + * with the unitialized part of it. + */ + to->di_flushiter = be16_to_cpu(from->di_flushiter); + inode->i_generation = be32_to_cpu(from->di_gen); + inode->i_mode = be16_to_cpu(from->di_mode); + if (!inode->i_mode) + return 0; + /* * Convert v1 inodes immediately to v2 inode format as this is the * minimum inode version format we support in the rest of the code. @@ -209,7 +220,6 @@ xfs_inode_from_disk( to->di_format = from->di_format; i_uid_write(inode, be32_to_cpu(from->di_uid)); i_gid_write(inode, be32_to_cpu(from->di_gid)); - to->di_flushiter = be16_to_cpu(from->di_flushiter); /* * Time is signed, so need to convert to signed 32 bit before @@ -223,8 +233,6 @@ xfs_inode_from_disk( inode->i_mtime.tv_nsec = (int)be32_to_cpu(from->di_mtime.t_nsec); inode->i_ctime.tv_sec = (int)be32_to_cpu(from->di_ctime.t_sec); inode->i_ctime.tv_nsec = (int)be32_to_cpu(from->di_ctime.t_nsec); - inode->i_generation = be32_to_cpu(from->di_gen); - inode->i_mode = be16_to_cpu(from->di_mode); to->di_size = be64_to_cpu(from->di_size); to->di_nblocks = be64_to_cpu(from->di_nblocks); @@ -653,39 +661,9 @@ xfs_iread( goto out_brelse; } - /* - * If the on-disk inode is already linked to a directory - * entry, copy all of the inode into the in-core inode. - * xfs_iformat_fork() handles copying in the inode format - * specific information. - * Otherwise, just get the truly permanent information. - */ - if (dip->di_mode) { - error = xfs_inode_from_disk(ip, dip); - if (error) { -#ifdef DEBUG - xfs_alert(mp, "%s: xfs_iformat() returned error %d", - __func__, error); -#endif /* DEBUG */ - goto out_brelse; - } - } else { - /* - * Partial initialisation of the in-core inode. Just the bits - * that xfs_ialloc won't overwrite or relies on being correct. - */ - VFS_I(ip)->i_generation = be32_to_cpu(dip->di_gen); - ip->i_d.di_flushiter = be16_to_cpu(dip->di_flushiter); - - /* - * Make sure to pull in the mode here as well in - * case the inode is released without being used. - * This ensures that xfs_inactive() will see that - * the inode is already free and not try to mess - * with the uninitialized part of it. - */ - VFS_I(ip)->i_mode = 0; - } + error = xfs_inode_from_disk(ip, dip); + if (error) + goto out_brelse; ip->i_delayed_blks = 0; From 2d6051d4965308c3367bf5a2468dff969872a96e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 14 May 2020 14:01:18 -0700 Subject: [PATCH 0584/1043] xfs: call xfs_dinode_verify from xfs_inode_from_disk Keep the code dealing with the dinode together, and also ensure we verify the dinode in the owner change log recovery case as well. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- .../xfs-self-describing-metadata.txt | 10 +++++----- fs/xfs/libxfs/xfs_inode_buf.c | 18 ++++++++---------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Documentation/filesystems/xfs-self-describing-metadata.txt b/Documentation/filesystems/xfs-self-describing-metadata.txt index 8db0121d0980..e912699d7430 100644 --- a/Documentation/filesystems/xfs-self-describing-metadata.txt +++ b/Documentation/filesystems/xfs-self-describing-metadata.txt @@ -337,11 +337,11 @@ buffer. The structure of the verifiers and the identifiers checks is very similar to the buffer code described above. The only difference is where they are called. For -example, inode read verification is done in xfs_iread() when the inode is first -read out of the buffer and the struct xfs_inode is instantiated. The inode is -already extensively verified during writeback in xfs_iflush_int, so the only -addition here is to add the LSN and CRC to the inode as it is copied back into -the buffer. +example, inode read verification is done in xfs_inode_from_disk() when the inode +is first read out of the buffer and the struct xfs_inode is instantiated. The +inode is already extensively verified during writeback in xfs_iflush_int, so the +only addition here is to add the LSN and CRC to the inode as it is copied back +into the buffer. XXX: inode unlinked list modification doesn't recalculate the inode CRC! None of the unlinked list modifications check or update CRCs, neither during unlink nor diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index 686a026b5f6e..3aac22e89298 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -188,10 +188,18 @@ xfs_inode_from_disk( struct xfs_icdinode *to = &ip->i_d; struct inode *inode = VFS_I(ip); int error; + xfs_failaddr_t fa; ASSERT(ip->i_cowfp == NULL); ASSERT(ip->i_afp == NULL); + fa = xfs_dinode_verify(ip->i_mount, ip->i_ino, from); + if (fa) { + xfs_inode_verifier_error(ip, -EFSCORRUPTED, "dinode", from, + sizeof(*from), fa); + return -EFSCORRUPTED; + } + /* * First get the permanent information that is needed to allocate an * inode. If the inode is unused, mode is zero and we shouldn't mess @@ -627,7 +635,6 @@ xfs_iread( { xfs_buf_t *bp; xfs_dinode_t *dip; - xfs_failaddr_t fa; int error; /* @@ -652,15 +659,6 @@ xfs_iread( if (error) return error; - /* even unallocated inodes are verified */ - fa = xfs_dinode_verify(mp, ip->i_ino, dip); - if (fa) { - xfs_inode_verifier_error(ip, -EFSCORRUPTED, "dinode", dip, - sizeof(*dip), fa); - error = -EFSCORRUPTED; - goto out_brelse; - } - error = xfs_inode_from_disk(ip, dip); if (error) goto out_brelse; From 7f0290123506e2b248fe06fa7cdc17c1b5b603b5 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 14 May 2020 14:01:18 -0700 Subject: [PATCH 0585/1043] xfs: don't reset i_delayed_blks in xfs_iread i_delayed_blks is set to 0 in xfs_inode_alloc and can't have anything assigned to it until the inode is visible to the VFS. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_buf.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index 3aac22e89298..329534eebbdc 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -663,8 +663,6 @@ xfs_iread( if (error) goto out_brelse; - ip->i_delayed_blks = 0; - /* * Mark the buffer containing the inode as something to keep * around for a while. This helps to keep recently accessed From bb8a66af4fff1cecb7631c68af761ea8e1a41ac2 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 14 May 2020 14:01:19 -0700 Subject: [PATCH 0586/1043] xfs: remove xfs_iread There is not much point in the xfs_iread function, as it has a single caller and not a whole lot of code. Move it into the only caller, and trim down the overdocumentation to just documenting the important "why" instead of a lot of redundant "what". Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_buf.c | 73 ----------------------------------- fs/xfs/libxfs/xfs_inode_buf.h | 2 - fs/xfs/xfs_icache.c | 33 +++++++++++++++- 3 files changed, 32 insertions(+), 76 deletions(-) diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index 329534eebbdc..05f939adea94 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -614,79 +614,6 @@ xfs_dinode_calc_crc( dip->di_crc = xfs_end_cksum(crc); } -/* - * Read the disk inode attributes into the in-core inode structure. - * - * For version 5 superblocks, if we are initialising a new inode and we are not - * utilising the XFS_MOUNT_IKEEP inode cluster mode, we can simple build the new - * inode core with a random generation number. If we are keeping inodes around, - * we need to read the inode cluster to get the existing generation number off - * disk. Further, if we are using version 4 superblocks (i.e. v1/v2 inode - * format) then log recovery is dependent on the di_flushiter field being - * initialised from the current on-disk value and hence we must also read the - * inode off disk. - */ -int -xfs_iread( - xfs_mount_t *mp, - xfs_trans_t *tp, - xfs_inode_t *ip, - uint iget_flags) -{ - xfs_buf_t *bp; - xfs_dinode_t *dip; - int error; - - /* - * Fill in the location information in the in-core inode. - */ - error = xfs_imap(mp, tp, ip->i_ino, &ip->i_imap, iget_flags); - if (error) - return error; - - /* shortcut IO on inode allocation if possible */ - if ((iget_flags & XFS_IGET_CREATE) && - xfs_sb_version_has_v3inode(&mp->m_sb) && - !(mp->m_flags & XFS_MOUNT_IKEEP)) { - VFS_I(ip)->i_generation = prandom_u32(); - return 0; - } - - /* - * Get pointers to the on-disk inode and the buffer containing it. - */ - error = xfs_imap_to_bp(mp, tp, &ip->i_imap, &dip, &bp, 0); - if (error) - return error; - - error = xfs_inode_from_disk(ip, dip); - if (error) - goto out_brelse; - - /* - * Mark the buffer containing the inode as something to keep - * around for a while. This helps to keep recently accessed - * meta-data in-core longer. - */ - xfs_buf_set_ref(bp, XFS_INO_REF); - - /* - * Use xfs_trans_brelse() to release the buffer containing the on-disk - * inode, because it was acquired with xfs_trans_read_buf() in - * xfs_imap_to_bp() above. If tp is NULL, this is just a normal - * brelse(). If we're within a transaction, then xfs_trans_brelse() - * will only release the buffer if it is not dirty within the - * transaction. It will be OK to release the buffer in this case, - * because inodes on disk are never destroyed and we will be locking the - * new in-core inode before putting it in the cache where other - * processes can find it. Thus we don't have to worry about the inode - * being changed just because we released the buffer. - */ - out_brelse: - xfs_trans_brelse(tp, bp); - return error; -} - /* * Validate di_extsize hint. * diff --git a/fs/xfs/libxfs/xfs_inode_buf.h b/fs/xfs/libxfs/xfs_inode_buf.h index 0fbb99224ec7..e4cbcaf62a32 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.h +++ b/fs/xfs/libxfs/xfs_inode_buf.h @@ -49,8 +49,6 @@ struct xfs_imap { int xfs_imap_to_bp(struct xfs_mount *, struct xfs_trans *, struct xfs_imap *, struct xfs_dinode **, struct xfs_buf **, uint); -int xfs_iread(struct xfs_mount *, struct xfs_trans *, - struct xfs_inode *, uint); void xfs_dinode_calc_crc(struct xfs_mount *, struct xfs_dinode *); void xfs_inode_to_disk(struct xfs_inode *ip, struct xfs_dinode *to, xfs_lsn_t lsn); diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 922a29032e37..af5748f5d927 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -22,6 +22,7 @@ #include "xfs_dquot_item.h" #include "xfs_dquot.h" #include "xfs_reflink.h" +#include "xfs_ialloc.h" #include @@ -508,10 +509,40 @@ xfs_iget_cache_miss( if (!ip) return -ENOMEM; - error = xfs_iread(mp, tp, ip, flags); + error = xfs_imap(mp, tp, ip->i_ino, &ip->i_imap, flags); if (error) goto out_destroy; + /* + * For version 5 superblocks, if we are initialising a new inode and we + * are not utilising the XFS_MOUNT_IKEEP inode cluster mode, we can + * simply build the new inode core with a random generation number. + * + * For version 4 (and older) superblocks, log recovery is dependent on + * the di_flushiter field being initialised from the current on-disk + * value and hence we must also read the inode off disk even when + * initializing new inodes. + */ + if (xfs_sb_version_has_v3inode(&mp->m_sb) && + (flags & XFS_IGET_CREATE) && !(mp->m_flags & XFS_MOUNT_IKEEP)) { + VFS_I(ip)->i_generation = prandom_u32(); + } else { + struct xfs_dinode *dip; + struct xfs_buf *bp; + + error = xfs_imap_to_bp(mp, tp, &ip->i_imap, &dip, &bp, 0); + if (error) + goto out_destroy; + + error = xfs_inode_from_disk(ip, dip); + if (!error) + xfs_buf_set_ref(bp, XFS_INO_REF); + xfs_trans_brelse(tp, bp); + + if (error) + goto out_destroy; + } + if (!xfs_inode_verify_forks(ip)) { error = -EFSCORRUPTED; goto out_destroy; From 1934c8bd81bee4c239478b03a59addf5fe8e2785 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 14 May 2020 14:01:19 -0700 Subject: [PATCH 0587/1043] xfs: remove xfs_ifork_ops xfs_ifork_ops add up to two indirect calls per inode read and flush, despite just having a single instance in the kernel. In xfsprogs phase6 in xfs_repair overrides the verify_dir method to deal with inodes that do not have a valid parent, but that can be fixed pretty easily by ensuring they always have a valid looking parent. Signed-off-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_fork.c | 19 +++++-------------- fs/xfs/libxfs/xfs_inode_fork.h | 15 ++------------- fs/xfs/xfs_inode.c | 4 ++-- 3 files changed, 9 insertions(+), 29 deletions(-) diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index 5fadfa9a17eb..e346e143f105 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -673,18 +673,10 @@ xfs_ifork_init_cow( ip->i_cnextents = 0; } -/* Default fork content verifiers. */ -struct xfs_ifork_ops xfs_default_ifork_ops = { - .verify_attr = xfs_attr_shortform_verify, - .verify_dir = xfs_dir2_sf_verify, - .verify_symlink = xfs_symlink_shortform_verify, -}; - /* Verify the inline contents of the data fork of an inode. */ xfs_failaddr_t xfs_ifork_verify_data( - struct xfs_inode *ip, - struct xfs_ifork_ops *ops) + struct xfs_inode *ip) { /* Non-local data fork, we're done. */ if (ip->i_d.di_format != XFS_DINODE_FMT_LOCAL) @@ -693,9 +685,9 @@ xfs_ifork_verify_data( /* Check the inline data fork if there is one. */ switch (VFS_I(ip)->i_mode & S_IFMT) { case S_IFDIR: - return ops->verify_dir(ip); + return xfs_dir2_sf_verify(ip); case S_IFLNK: - return ops->verify_symlink(ip); + return xfs_symlink_shortform_verify(ip); default: return NULL; } @@ -704,13 +696,12 @@ xfs_ifork_verify_data( /* Verify the inline contents of the attr fork of an inode. */ xfs_failaddr_t xfs_ifork_verify_attr( - struct xfs_inode *ip, - struct xfs_ifork_ops *ops) + struct xfs_inode *ip) { /* There has to be an attr fork allocated if aformat is local. */ if (ip->i_d.di_aformat != XFS_DINODE_FMT_LOCAL) return NULL; if (!XFS_IFORK_PTR(ip, XFS_ATTR_FORK)) return __this_address; - return ops->verify_attr(ip); + return xfs_attr_shortform_verify(ip); } diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index 8487b0c88a75..3f84d33abd3b 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -176,18 +176,7 @@ extern struct kmem_zone *xfs_ifork_zone; extern void xfs_ifork_init_cow(struct xfs_inode *ip); -typedef xfs_failaddr_t (*xfs_ifork_verifier_t)(struct xfs_inode *); - -struct xfs_ifork_ops { - xfs_ifork_verifier_t verify_symlink; - xfs_ifork_verifier_t verify_dir; - xfs_ifork_verifier_t verify_attr; -}; -extern struct xfs_ifork_ops xfs_default_ifork_ops; - -xfs_failaddr_t xfs_ifork_verify_data(struct xfs_inode *ip, - struct xfs_ifork_ops *ops); -xfs_failaddr_t xfs_ifork_verify_attr(struct xfs_inode *ip, - struct xfs_ifork_ops *ops); +xfs_failaddr_t xfs_ifork_verify_data(struct xfs_inode *ip); +xfs_failaddr_t xfs_ifork_verify_attr(struct xfs_inode *ip); #endif /* __XFS_INODE_FORK_H__ */ diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index ab31a5dec7aa..25c00ffe1840 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -3718,7 +3718,7 @@ xfs_inode_verify_forks( struct xfs_ifork *ifp; xfs_failaddr_t fa; - fa = xfs_ifork_verify_data(ip, &xfs_default_ifork_ops); + fa = xfs_ifork_verify_data(ip); if (fa) { ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK); xfs_inode_verifier_error(ip, -EFSCORRUPTED, "data fork", @@ -3726,7 +3726,7 @@ xfs_inode_verify_forks( return false; } - fa = xfs_ifork_verify_attr(ip, &xfs_default_ifork_ops); + fa = xfs_ifork_verify_attr(ip); if (fa) { ifp = XFS_IFORK_PTR(ip, XFS_ATTR_FORK); xfs_inode_verifier_error(ip, -EFSCORRUPTED, "attr fork", From 7c7ba2186305d6bee5eb5b8fb95a61d8de14de4f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 14 May 2020 14:01:19 -0700 Subject: [PATCH 0588/1043] xfs: refactor xfs_inode_verify_forks The split between xfs_inode_verify_forks and the two helpers implementing the actual functionality is a little strange. Reshuffle it so that xfs_inode_verify_forks verifies if the data and attr forks are actually in local format and only call the low-level helpers if that is the case. Handle the actual error reporting in the low-level handlers to streamline the caller. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_fork.c | 51 ++++++++++++++++++++++------------ fs/xfs/libxfs/xfs_inode_fork.h | 4 +-- fs/xfs/xfs_inode.c | 21 +++----------- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index e346e143f105..2d424ab9d734 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -674,34 +674,51 @@ xfs_ifork_init_cow( } /* Verify the inline contents of the data fork of an inode. */ -xfs_failaddr_t -xfs_ifork_verify_data( +int +xfs_ifork_verify_local_data( struct xfs_inode *ip) { - /* Non-local data fork, we're done. */ - if (ip->i_d.di_format != XFS_DINODE_FMT_LOCAL) - return NULL; + xfs_failaddr_t fa = NULL; - /* Check the inline data fork if there is one. */ switch (VFS_I(ip)->i_mode & S_IFMT) { case S_IFDIR: - return xfs_dir2_sf_verify(ip); + fa = xfs_dir2_sf_verify(ip); + break; case S_IFLNK: - return xfs_symlink_shortform_verify(ip); + fa = xfs_symlink_shortform_verify(ip); + break; default: - return NULL; + break; } + + if (fa) { + xfs_inode_verifier_error(ip, -EFSCORRUPTED, "data fork", + ip->i_df.if_u1.if_data, ip->i_df.if_bytes, fa); + return -EFSCORRUPTED; + } + + return 0; } /* Verify the inline contents of the attr fork of an inode. */ -xfs_failaddr_t -xfs_ifork_verify_attr( +int +xfs_ifork_verify_local_attr( struct xfs_inode *ip) { - /* There has to be an attr fork allocated if aformat is local. */ - if (ip->i_d.di_aformat != XFS_DINODE_FMT_LOCAL) - return NULL; - if (!XFS_IFORK_PTR(ip, XFS_ATTR_FORK)) - return __this_address; - return xfs_attr_shortform_verify(ip); + struct xfs_ifork *ifp = ip->i_afp; + xfs_failaddr_t fa; + + if (!ifp) + fa = __this_address; + else + fa = xfs_attr_shortform_verify(ip); + + if (fa) { + xfs_inode_verifier_error(ip, -EFSCORRUPTED, "attr fork", + ifp ? ifp->if_u1.if_data : NULL, + ifp ? ifp->if_bytes : 0, fa); + return -EFSCORRUPTED; + } + + return 0; } diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index 3f84d33abd3b..f46a8c1db596 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -176,7 +176,7 @@ extern struct kmem_zone *xfs_ifork_zone; extern void xfs_ifork_init_cow(struct xfs_inode *ip); -xfs_failaddr_t xfs_ifork_verify_data(struct xfs_inode *ip); -xfs_failaddr_t xfs_ifork_verify_attr(struct xfs_inode *ip); +int xfs_ifork_verify_local_data(struct xfs_inode *ip); +int xfs_ifork_verify_local_attr(struct xfs_inode *ip); #endif /* __XFS_INODE_FORK_H__ */ diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 25c00ffe1840..c8abdefe0037 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -3715,25 +3715,12 @@ bool xfs_inode_verify_forks( struct xfs_inode *ip) { - struct xfs_ifork *ifp; - xfs_failaddr_t fa; - - fa = xfs_ifork_verify_data(ip); - if (fa) { - ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK); - xfs_inode_verifier_error(ip, -EFSCORRUPTED, "data fork", - ifp->if_u1.if_data, ifp->if_bytes, fa); + if (ip->i_d.di_format == XFS_DINODE_FMT_LOCAL && + xfs_ifork_verify_local_data(ip)) return false; - } - - fa = xfs_ifork_verify_attr(ip); - if (fa) { - ifp = XFS_IFORK_PTR(ip, XFS_ATTR_FORK); - xfs_inode_verifier_error(ip, -EFSCORRUPTED, "attr fork", - ifp ? ifp->if_u1.if_data : NULL, - ifp ? ifp->if_bytes : 0, fa); + if (ip->i_d.di_aformat == XFS_DINODE_FMT_LOCAL && + xfs_ifork_verify_local_attr(ip)) return false; - } return true; } From 0f45a1b20cd8f9cfc985a1f91a1e7a86e5e14dd6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 14 May 2020 14:01:31 -0700 Subject: [PATCH 0589/1043] xfs: improve local fork verification Call the data/attr local fork verifiers as soon as we are ready for them. This keeps them close to the code setting up the forks, and avoids a few branches later on. Also open code xfs_inode_verify_forks in the only remaining caller. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_inode_fork.c | 8 +++++++- fs/xfs/xfs_icache.c | 6 ------ fs/xfs/xfs_inode.c | 28 +++++++++------------------- fs/xfs/xfs_inode.h | 2 -- fs/xfs/xfs_inode_item_recover.c | 5 ----- 5 files changed, 16 insertions(+), 33 deletions(-) diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index 2d424ab9d734..6c24c27f5f44 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -227,6 +227,7 @@ xfs_iformat_data_fork( struct xfs_dinode *dip) { struct inode *inode = VFS_I(ip); + int error; switch (inode->i_mode & S_IFMT) { case S_IFIFO: @@ -241,8 +242,11 @@ xfs_iformat_data_fork( case S_IFDIR: switch (dip->di_format) { case XFS_DINODE_FMT_LOCAL: - return xfs_iformat_local(ip, dip, XFS_DATA_FORK, + error = xfs_iformat_local(ip, dip, XFS_DATA_FORK, be64_to_cpu(dip->di_size)); + if (!error) + error = xfs_ifork_verify_local_data(ip); + return error; case XFS_DINODE_FMT_EXTENTS: return xfs_iformat_extents(ip, dip, XFS_DATA_FORK); case XFS_DINODE_FMT_BTREE: @@ -282,6 +286,8 @@ xfs_iformat_attr_fork( case XFS_DINODE_FMT_LOCAL: error = xfs_iformat_local(ip, dip, XFS_ATTR_FORK, xfs_dfork_attr_shortform_size(dip)); + if (!error) + error = xfs_ifork_verify_local_attr(ip); break; case XFS_DINODE_FMT_EXTENTS: error = xfs_iformat_extents(ip, dip, XFS_ATTR_FORK); diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index af5748f5d927..5a3a520b9528 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -543,14 +543,8 @@ xfs_iget_cache_miss( goto out_destroy; } - if (!xfs_inode_verify_forks(ip)) { - error = -EFSCORRUPTED; - goto out_destroy; - } - trace_xfs_iget_miss(ip); - /* * Check the inode free state is valid. This also detects lookup * racing with unlinks. diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index c8abdefe0037..549ff468b7b6 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -3707,23 +3707,6 @@ shutdown: return error; } -/* - * If there are inline format data / attr forks attached to this inode, - * make sure they're not corrupt. - */ -bool -xfs_inode_verify_forks( - struct xfs_inode *ip) -{ - if (ip->i_d.di_format == XFS_DINODE_FMT_LOCAL && - xfs_ifork_verify_local_data(ip)) - return false; - if (ip->i_d.di_aformat == XFS_DINODE_FMT_LOCAL && - xfs_ifork_verify_local_attr(ip)) - return false; - return true; -} - STATIC int xfs_iflush_int( struct xfs_inode *ip, @@ -3808,8 +3791,15 @@ xfs_iflush_int( if (!xfs_sb_version_has_v3inode(&mp->m_sb)) ip->i_d.di_flushiter++; - /* Check the inline fork data before we write out. */ - if (!xfs_inode_verify_forks(ip)) + /* + * If there are inline format data / attr forks attached to this inode, + * make sure they are not corrupt. + */ + if (ip->i_d.di_format == XFS_DINODE_FMT_LOCAL && + xfs_ifork_verify_local_data(ip)) + goto flush_out; + if (ip->i_d.di_aformat == XFS_DINODE_FMT_LOCAL && + xfs_ifork_verify_local_attr(ip)) goto flush_out; /* diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index 83073c883fbf..ff846197941e 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -498,8 +498,6 @@ extern struct kmem_zone *xfs_inode_zone; /* The default CoW extent size hint. */ #define XFS_DEFAULT_COWEXTSZ_HINT 32 -bool xfs_inode_verify_forks(struct xfs_inode *ip); - int xfs_iunlink_init(struct xfs_perag *pag); void xfs_iunlink_destroy(struct xfs_perag *pag); diff --git a/fs/xfs/xfs_inode_item_recover.c b/fs/xfs/xfs_inode_item_recover.c index 82ca5ce312c5..dc3e26ff16c9 100644 --- a/fs/xfs/xfs_inode_item_recover.c +++ b/fs/xfs/xfs_inode_item_recover.c @@ -94,11 +94,6 @@ xfs_recover_inode_owner_change( if (error) goto out_free_ip; - if (!xfs_inode_verify_forks(ip)) { - error = -EFSCORRUPTED; - goto out_free_ip; - } - if (in_f->ilf_fields & XFS_ILOG_DOWNER) { ASSERT(in_f->ilf_fields & XFS_ILOG_DBROOT); error = xfs_bmbt_change_owner(NULL, ip, XFS_DATA_FORK, From 1a1c57b2826f8b408feb733d3321490591a6e4c9 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 14 May 2020 14:06:40 -0700 Subject: [PATCH 0590/1043] xfs: remove the special COW fork handling in xfs_bmapi_read We don't call xfs_bmapi_read for the COW fork anymore, so remove the special casing. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_bmap.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index fda13cd7add0..76be1a18e244 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -3902,8 +3902,7 @@ xfs_bmapi_read( int whichfork = xfs_bmapi_whichfork(flags); ASSERT(*nmap >= 1); - ASSERT(!(flags & ~(XFS_BMAPI_ATTRFORK|XFS_BMAPI_ENTIRE| - XFS_BMAPI_COWFORK))); + ASSERT(!(flags & ~(XFS_BMAPI_ATTRFORK | XFS_BMAPI_ENTIRE))); ASSERT(xfs_isilocked(ip, XFS_ILOCK_SHARED|XFS_ILOCK_EXCL)); if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ip, whichfork)) || @@ -3918,16 +3917,6 @@ xfs_bmapi_read( ifp = XFS_IFORK_PTR(ip, whichfork); if (!ifp) { - /* No CoW fork? Return a hole. */ - if (whichfork == XFS_COW_FORK) { - mval->br_startoff = bno; - mval->br_startblock = HOLESTARTBLOCK; - mval->br_blockcount = len; - mval->br_state = XFS_EXT_NORM; - *nmap = 1; - return 0; - } - /* * A missing attr ifork implies that the inode says we're in * extents or btree format but failed to pass the inode fork From 4b516ff4e772993a99fc9bf36503d23ce5bd5ba9 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 14 May 2020 14:06:41 -0700 Subject: [PATCH 0591/1043] xfs: remove the NULL fork handling in xfs_bmapi_read Now that we fully verify the inode forks before they are added to the inode cache, the crash reported in https://bugzilla.kernel.org/show_bug.cgi?id=204031 can't happen anymore, as we'll never let an inode that has inconsistent nextents counts vs the presence of an in-core attr fork leak into the inactivate code path. So remove the work around to try to handle the case, and just return an error and warn if the fork is not present. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_bmap.c | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 76be1a18e244..34518a6dc737 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -3891,7 +3891,8 @@ xfs_bmapi_read( int flags) { struct xfs_mount *mp = ip->i_mount; - struct xfs_ifork *ifp; + int whichfork = xfs_bmapi_whichfork(flags); + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); struct xfs_bmbt_irec got; xfs_fileoff_t obno; xfs_fileoff_t end; @@ -3899,12 +3900,14 @@ xfs_bmapi_read( int error; bool eof = false; int n = 0; - int whichfork = xfs_bmapi_whichfork(flags); ASSERT(*nmap >= 1); ASSERT(!(flags & ~(XFS_BMAPI_ATTRFORK | XFS_BMAPI_ENTIRE))); ASSERT(xfs_isilocked(ip, XFS_ILOCK_SHARED|XFS_ILOCK_EXCL)); + if (WARN_ON_ONCE(!ifp)) + return -EFSCORRUPTED; + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ip, whichfork)) || XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) { return -EFSCORRUPTED; @@ -3915,21 +3918,6 @@ xfs_bmapi_read( XFS_STATS_INC(mp, xs_blk_mapr); - ifp = XFS_IFORK_PTR(ip, whichfork); - if (!ifp) { - /* - * A missing attr ifork implies that the inode says we're in - * extents or btree format but failed to pass the inode fork - * verifier while trying to load it. Treat that as a file - * corruption too. - */ -#ifdef DEBUG - xfs_alert(mp, "%s: inode %llu missing fork %d", - __func__, ip->i_ino, whichfork); -#endif /* DEBUG */ - return -EFSCORRUPTED; - } - if (!(ifp->if_flags & XFS_IFEXTENTS)) { error = xfs_iread_extents(NULL, ip, whichfork); if (error) From 5fd68bdb5a87c929fff5f7bbd947034368f36f4f Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Sat, 16 May 2020 10:19:29 -0700 Subject: [PATCH 0592/1043] xfs: clean up xchk_bmap_check_rmaps usage of XFS_IFORK_Q XFS_IFORK_Q is supposed to be a predicate, not a function returning a value. Its usage is in xchk_bmap_check_rmaps is incorrect, but that function only cares about whether or not the "size" of the data is zero or not. Convert that logic to use a proper boolean, and teach the caller to skip the call entirely if the end result would be that we'd do nothing anyway. This avoids a crash later in this series. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig [hch: generalized the NULL ifor check] Signed-off-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Reviewed-by: Brian Foster --- fs/xfs/scrub/bmap.c | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/fs/xfs/scrub/bmap.c b/fs/xfs/scrub/bmap.c index add8598eacd5..93d5b8a9d7f7 100644 --- a/fs/xfs/scrub/bmap.c +++ b/fs/xfs/scrub/bmap.c @@ -566,8 +566,8 @@ xchk_bmap_check_rmaps( struct xfs_scrub *sc, int whichfork) { - loff_t size; xfs_agnumber_t agno; + bool zero_size; int error; if (!xfs_sb_version_hasrmapbt(&sc->mp->m_sb) || @@ -579,6 +579,8 @@ xchk_bmap_check_rmaps( if (XFS_IS_REALTIME_INODE(sc->ip) && whichfork == XFS_DATA_FORK) return 0; + ASSERT(XFS_IFORK_PTR(sc->ip, whichfork) != NULL); + /* * Only do this for complex maps that are in btree format, or for * situations where we would seem to have a size but zero extents. @@ -586,19 +588,13 @@ xchk_bmap_check_rmaps( * to flag this bmap as corrupt if there are rmaps that need to be * reattached. */ - switch (whichfork) { - case XFS_DATA_FORK: - size = i_size_read(VFS_I(sc->ip)); - break; - case XFS_ATTR_FORK: - size = XFS_IFORK_Q(sc->ip); - break; - default: - size = 0; - break; - } + if (whichfork == XFS_DATA_FORK) + zero_size = i_size_read(VFS_I(sc->ip)) == 0; + else + zero_size = false; + if (XFS_IFORK_FORMAT(sc->ip, whichfork) != XFS_DINODE_FMT_BTREE && - (size == 0 || XFS_IFORK_NEXTENTS(sc->ip, whichfork) > 0)) + (zero_size || XFS_IFORK_NEXTENTS(sc->ip, whichfork) > 0)) return 0; for (agno = 0; agno < sc->mp->m_sb.sb_agcount; agno++) { @@ -627,12 +623,14 @@ xchk_bmap( struct xchk_bmap_info info = { NULL }; struct xfs_mount *mp = sc->mp; struct xfs_inode *ip = sc->ip; - struct xfs_ifork *ifp; + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); xfs_fileoff_t endoff; struct xfs_iext_cursor icur; int error = 0; - ifp = XFS_IFORK_PTR(ip, whichfork); + /* Non-existent forks can be ignored. */ + if (!ifp) + goto out; info.is_rt = whichfork == XFS_DATA_FORK && XFS_IS_REALTIME_INODE(ip); info.whichfork = whichfork; @@ -641,9 +639,6 @@ xchk_bmap( switch (whichfork) { case XFS_COW_FORK: - /* Non-existent CoW forks are ignorable. */ - if (!ifp) - goto out; /* No CoW forks on non-reflink inodes/filesystems. */ if (!xfs_is_reflink_inode(ip)) { xchk_ino_set_corrupt(sc, sc->ip->i_ino); @@ -651,8 +646,6 @@ xchk_bmap( } break; case XFS_ATTR_FORK: - if (!ifp) - goto out_check_rmap; if (!xfs_sb_version_hasattr(&mp->m_sb) && !xfs_sb_version_hasattr2(&mp->m_sb)) xchk_ino_set_corrupt(sc, sc->ip->i_ino); @@ -717,7 +710,6 @@ xchk_bmap( goto out; } -out_check_rmap: error = xchk_bmap_check_rmaps(sc, whichfork); if (!xchk_fblock_xref_process_error(sc, whichfork, 0, &error)) goto out; From 09c38edd54c16657093a73a3169342f9f9080bb3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 May 2020 10:27:21 -0700 Subject: [PATCH 0593/1043] xfs: remove the XFS_DFORK_Q macro Just checking di_forkoff directly is a little easier to follow. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Chandan Babu R Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_format.h | 5 ++--- fs/xfs/libxfs/xfs_inode_buf.c | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index f2228d9e317a..b42a52bfa1e9 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -964,13 +964,12 @@ enum xfs_dinode_fmt { /* * Inode data & attribute fork sizes, per inode. */ -#define XFS_DFORK_Q(dip) ((dip)->di_forkoff != 0) #define XFS_DFORK_BOFF(dip) ((int)((dip)->di_forkoff << 3)) #define XFS_DFORK_DSIZE(dip,mp) \ - (XFS_DFORK_Q(dip) ? XFS_DFORK_BOFF(dip) : XFS_LITINO(mp)) + ((dip)->di_forkoff ? XFS_DFORK_BOFF(dip) : XFS_LITINO(mp)) #define XFS_DFORK_ASIZE(dip,mp) \ - (XFS_DFORK_Q(dip) ? XFS_LITINO(mp) - XFS_DFORK_BOFF(dip) : 0) + ((dip)->di_forkoff ? XFS_LITINO(mp) - XFS_DFORK_BOFF(dip) : 0) #define XFS_DFORK_SIZE(dip,mp,w) \ ((w) == XFS_DATA_FORK ? \ XFS_DFORK_DSIZE(dip, mp) : \ diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index 05f939adea94..5547bbb3cf94 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -265,7 +265,7 @@ xfs_inode_from_disk( error = xfs_iformat_data_fork(ip, from); if (error) return error; - if (XFS_DFORK_Q(from)) { + if (from->di_forkoff) { error = xfs_iformat_attr_fork(ip, from); if (error) goto out_destroy_data_fork; @@ -435,7 +435,7 @@ xfs_dinode_verify_forkoff( struct xfs_dinode *dip, struct xfs_mount *mp) { - if (!XFS_DFORK_Q(dip)) + if (!dip->di_forkoff) return NULL; switch (dip->di_format) { @@ -538,7 +538,7 @@ xfs_dinode_verify( return __this_address; } - if (XFS_DFORK_Q(dip)) { + if (dip->di_forkoff) { fa = xfs_dinode_verify_fork(dip, mp, XFS_ATTR_FORK); if (fa) return fa; From b2c20045b67bf37aa63be9bd9463708dfb38cbcc Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 May 2020 10:27:21 -0700 Subject: [PATCH 0594/1043] xfs: remove xfs_ifree_local_data xfs_ifree only need to free inline data in the data fork, as we've already taken care of the attr fork before (and in fact freed the fork structure). Just open code the freeing of the inline data. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Chandan Babu R Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_inode.c | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 549ff468b7b6..7d3144dc99b7 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -2711,24 +2711,6 @@ xfs_ifree_cluster( return 0; } -/* - * Free any local-format buffers sitting around before we reset to - * extents format. - */ -static inline void -xfs_ifree_local_data( - struct xfs_inode *ip, - int whichfork) -{ - struct xfs_ifork *ifp; - - if (XFS_IFORK_FORMAT(ip, whichfork) != XFS_DINODE_FMT_LOCAL) - return; - - ifp = XFS_IFORK_PTR(ip, whichfork); - xfs_idata_realloc(ip, -ifp->if_bytes, whichfork); -} - /* * This is called to return an inode to the inode free list. * The inode should already be truncated to 0 length and have @@ -2765,8 +2747,16 @@ xfs_ifree( if (error) return error; - xfs_ifree_local_data(ip, XFS_DATA_FORK); - xfs_ifree_local_data(ip, XFS_ATTR_FORK); + /* + * Free any local-format data sitting around before we reset the + * data fork to extents format. Note that the attr fork data has + * already been freed by xfs_attr_inactive. + */ + if (ip->i_d.di_format == XFS_DINODE_FMT_LOCAL) { + kmem_free(ip->i_df.if_u1.if_data); + ip->i_df.if_u1.if_data = NULL; + ip->i_df.if_bytes = 0; + } VFS_I(ip)->i_mode = 0; /* mark incore inode as free */ ip->i_d.di_flags = 0; From daf83964a3681cf1f1f255ad6095c0b60cba7dca Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 May 2020 10:27:22 -0700 Subject: [PATCH 0595/1043] xfs: move the per-fork nextents fields into struct xfs_ifork There are there are three extents counters per inode, one for each of the forks. Two are in the legacy icdinode and one is directly in struct xfs_inode. Switch to a single counter in the xfs_ifork structure where it uses up padding at the end of the structure. This simplifies various bits of code that just wants the number of extents counter and can now directly dereference it. Signed-off-by: Christoph Hellwig Reviewed-by: Chandan Babu R Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_attr.c | 4 +- fs/xfs/libxfs/xfs_attr_leaf.c | 3 +- fs/xfs/libxfs/xfs_bmap.c | 126 ++++++++++++++------------------- fs/xfs/libxfs/xfs_dir2_block.c | 2 +- fs/xfs/libxfs/xfs_inode_buf.c | 6 +- fs/xfs/libxfs/xfs_inode_buf.h | 2 - fs/xfs/libxfs/xfs_inode_fork.c | 20 ++++-- fs/xfs/libxfs/xfs_inode_fork.h | 20 +++--- fs/xfs/scrub/bmap.c | 3 +- fs/xfs/scrub/parent.c | 2 +- fs/xfs/xfs_bmap_util.c | 28 ++++---- fs/xfs/xfs_file.c | 2 +- fs/xfs/xfs_icache.c | 1 - fs/xfs/xfs_inode.c | 19 +++-- fs/xfs/xfs_inode.h | 1 - fs/xfs/xfs_inode_item.c | 14 ++-- fs/xfs/xfs_ioctl.c | 25 +++---- fs/xfs/xfs_iomap.c | 2 +- fs/xfs/xfs_iops.c | 2 +- fs/xfs/xfs_itable.c | 4 +- fs/xfs/xfs_qm_syscalls.c | 2 +- fs/xfs/xfs_quotaops.c | 2 +- fs/xfs/xfs_symlink.c | 2 +- fs/xfs/xfs_trace.h | 2 +- 24 files changed, 132 insertions(+), 162 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index e4fe3dca9883..1b01675e9c80 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -62,7 +62,7 @@ xfs_inode_hasattr( { if (!XFS_IFORK_Q(ip) || (ip->i_d.di_aformat == XFS_DINODE_FMT_EXTENTS && - ip->i_d.di_anextents == 0)) + ip->i_afp->if_nextents == 0)) return 0; return 1; } @@ -214,7 +214,7 @@ xfs_attr_set_args( */ if (dp->i_d.di_aformat == XFS_DINODE_FMT_LOCAL || (dp->i_d.di_aformat == XFS_DINODE_FMT_EXTENTS && - dp->i_d.di_anextents == 0)) { + dp->i_afp->if_nextents == 0)) { /* * Build initial attribute list (if required). diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 6d18e86bb9c7..11ff6dd08512 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -718,11 +718,12 @@ xfs_attr_fork_remove( struct xfs_inode *ip, struct xfs_trans *tp) { + ASSERT(ip->i_afp->if_nextents == 0); + xfs_idestroy_fork(ip, XFS_ATTR_FORK); ip->i_d.di_forkoff = 0; ip->i_d.di_aformat = XFS_DINODE_FMT_EXTENTS; - ASSERT(ip->i_d.di_anextents == 0); ASSERT(ip->i_afp == NULL); xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index 34518a6dc737..c1136be49abe 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -61,10 +61,10 @@ xfs_bmap_compute_maxlevels( int sz; /* root block size */ /* - * The maximum number of extents in a file, hence the maximum - * number of leaf entries, is controlled by the type of di_nextents - * (a signed 32-bit number, xfs_extnum_t), or by di_anextents - * (a signed 16-bit number, xfs_aextnum_t). + * The maximum number of extents in a file, hence the maximum number of + * leaf entries, is controlled by the size of the on-disk extent count, + * either a signed 32-bit number for the data fork, or a signed 16-bit + * number for the attr fork. * * Note that we can no longer assume that if we are in ATTR1 that * the fork offset of all the inodes will be @@ -120,10 +120,11 @@ xfs_bmbt_lookup_first( */ static inline bool xfs_bmap_needs_btree(struct xfs_inode *ip, int whichfork) { + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); + return whichfork != XFS_COW_FORK && XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_EXTENTS && - XFS_IFORK_NEXTENTS(ip, whichfork) > - XFS_IFORK_MAXEXT(ip, whichfork); + ifp->if_nextents > XFS_IFORK_MAXEXT(ip, whichfork); } /* @@ -131,10 +132,11 @@ static inline bool xfs_bmap_needs_btree(struct xfs_inode *ip, int whichfork) */ static inline bool xfs_bmap_wants_extents(struct xfs_inode *ip, int whichfork) { + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); + return whichfork != XFS_COW_FORK && XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_BTREE && - XFS_IFORK_NEXTENTS(ip, whichfork) <= - XFS_IFORK_MAXEXT(ip, whichfork); + ifp->if_nextents <= XFS_IFORK_MAXEXT(ip, whichfork); } /* @@ -334,7 +336,7 @@ xfs_bmap_check_leaf_extents( } /* skip large extent count inodes */ - if (ip->i_d.di_nextents > 10000) + if (ip->i_df.if_nextents > 10000) return; bno = NULLFSBLOCK; @@ -750,7 +752,7 @@ xfs_bmap_extents_to_btree( xfs_bmbt_disk_set_all(arp, &rec); cnt++; } - ASSERT(cnt == XFS_IFORK_NEXTENTS(ip, whichfork)); + ASSERT(cnt == ifp->if_nextents); xfs_btree_set_numrecs(ablock, cnt); /* @@ -802,7 +804,7 @@ xfs_bmap_local_to_extents_empty( ASSERT(whichfork != XFS_COW_FORK); ASSERT(XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_LOCAL); ASSERT(ifp->if_bytes == 0); - ASSERT(XFS_IFORK_NEXTENTS(ip, whichfork) == 0); + ASSERT(ifp->if_nextents == 0); xfs_bmap_forkoff_reset(ip, whichfork); ifp->if_flags &= ~XFS_IFINLINE; @@ -907,7 +909,7 @@ xfs_bmap_local_to_extents( xfs_iext_first(ifp, &icur); xfs_iext_insert(ip, &icur, &rec, 0); - XFS_IFORK_NEXT_SET(ip, whichfork, 1); + ifp->if_nextents = 1; ip->i_d.di_nblocks = 1; xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, 1L); @@ -972,7 +974,8 @@ xfs_bmap_add_attrfork_extents( xfs_btree_cur_t *cur; /* bmap btree cursor */ int error; /* error return value */ - if (ip->i_d.di_nextents * sizeof(xfs_bmbt_rec_t) <= XFS_IFORK_DSIZE(ip)) + if (ip->i_df.if_nextents * sizeof(struct xfs_bmbt_rec) <= + XFS_IFORK_DSIZE(ip)) return 0; cur = NULL; error = xfs_bmap_extents_to_btree(tp, ip, &cur, 0, flags, @@ -1091,10 +1094,6 @@ xfs_bmap_add_attrfork( goto trans_cancel; if (XFS_IFORK_Q(ip)) goto trans_cancel; - if (XFS_IS_CORRUPT(mp, ip->i_d.di_anextents != 0)) { - error = -EFSCORRUPTED; - goto trans_cancel; - } if (ip->i_d.di_aformat != XFS_DINODE_FMT_EXTENTS) { /* * For inodes coming from pre-6.2 filesystems. @@ -1183,13 +1182,13 @@ xfs_iread_bmbt_block( xfs_extnum_t num_recs; xfs_extnum_t j; int whichfork = cur->bc_ino.whichfork; + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); block = xfs_btree_get_block(cur, level, &bp); /* Abort if we find more records than nextents. */ num_recs = xfs_btree_get_numrecs(block); - if (unlikely(ir->loaded + num_recs > - XFS_IFORK_NEXTENTS(ip, whichfork))) { + if (unlikely(ir->loaded + num_recs > ifp->if_nextents)) { xfs_warn(ip->i_mount, "corrupt dinode %llu, (btree extents).", (unsigned long long)ip->i_ino); xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, block, @@ -1215,7 +1214,7 @@ xfs_iread_bmbt_block( xfs_bmap_fork_to_state(whichfork)); trace_xfs_read_extent(ip, &ir->icur, xfs_bmap_fork_to_state(whichfork), _THIS_IP_); - xfs_iext_next(XFS_IFORK_PTR(ip, whichfork), &ir->icur); + xfs_iext_next(ifp, &ir->icur); } return 0; @@ -1254,8 +1253,7 @@ xfs_iread_extents( if (error) goto out; - if (XFS_IS_CORRUPT(mp, - ir.loaded != XFS_IFORK_NEXTENTS(ip, whichfork))) { + if (XFS_IS_CORRUPT(mp, ir.loaded != ifp->if_nextents)) { error = -EFSCORRUPTED; goto out; } @@ -1463,23 +1461,22 @@ xfs_bmap_last_offset( */ int /* 1=>1 block, 0=>otherwise */ xfs_bmap_one_block( - xfs_inode_t *ip, /* incore inode */ - int whichfork) /* data or attr fork */ + struct xfs_inode *ip, /* incore inode */ + int whichfork) /* data or attr fork */ { - struct xfs_ifork *ifp; /* inode fork pointer */ - int rval; /* return value */ - xfs_bmbt_irec_t s; /* internal version of extent */ + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); + int rval; /* return value */ + struct xfs_bmbt_irec s; /* internal version of extent */ struct xfs_iext_cursor icur; #ifndef DEBUG if (whichfork == XFS_DATA_FORK) return XFS_ISIZE(ip) == ip->i_mount->m_sb.sb_blocksize; #endif /* !DEBUG */ - if (XFS_IFORK_NEXTENTS(ip, whichfork) != 1) + if (ifp->if_nextents != 1) return 0; if (XFS_IFORK_FORMAT(ip, whichfork) != XFS_DINODE_FMT_EXTENTS) return 0; - ifp = XFS_IFORK_PTR(ip, whichfork); ASSERT(ifp->if_flags & XFS_IFEXTENTS); xfs_iext_first(ifp, &icur); xfs_iext_get_extent(ifp, &icur, &s); @@ -1501,10 +1498,11 @@ xfs_bmap_add_extent_delay_real( struct xfs_bmalloca *bma, int whichfork) { + struct xfs_mount *mp = bma->ip->i_mount; + struct xfs_ifork *ifp = XFS_IFORK_PTR(bma->ip, whichfork); struct xfs_bmbt_irec *new = &bma->got; int error; /* error return value */ int i; /* temp state */ - struct xfs_ifork *ifp; /* inode fork pointer */ xfs_fileoff_t new_endoff; /* end offset of new entry */ xfs_bmbt_irec_t r[3]; /* neighbor extent entries */ /* left is 0, right is 1, prev is 2 */ @@ -1514,16 +1512,9 @@ xfs_bmap_add_extent_delay_real( xfs_filblks_t da_old; /* old count del alloc blocks used */ xfs_filblks_t temp=0; /* value for da_new calculations */ int tmp_rval; /* partial logging flags */ - struct xfs_mount *mp; - xfs_extnum_t *nextents; struct xfs_bmbt_irec old; - mp = bma->ip->i_mount; - ifp = XFS_IFORK_PTR(bma->ip, whichfork); ASSERT(whichfork != XFS_ATTR_FORK); - nextents = (whichfork == XFS_COW_FORK ? &bma->ip->i_cnextents : - &bma->ip->i_d.di_nextents); - ASSERT(!isnullstartblock(new->br_startblock)); ASSERT(!bma->cur || (bma->cur->bc_ino.flags & XFS_BTCUR_BMBT_WASDEL)); @@ -1614,7 +1605,7 @@ xfs_bmap_add_extent_delay_real( xfs_iext_remove(bma->ip, &bma->icur, state); xfs_iext_prev(ifp, &bma->icur); xfs_iext_update_extent(bma->ip, state, &bma->icur, &LEFT); - (*nextents)--; + ifp->if_nextents--; if (bma->cur == NULL) rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; @@ -1718,8 +1709,8 @@ xfs_bmap_add_extent_delay_real( PREV.br_startblock = new->br_startblock; PREV.br_state = new->br_state; xfs_iext_update_extent(bma->ip, state, &bma->icur, &PREV); + ifp->if_nextents++; - (*nextents)++; if (bma->cur == NULL) rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; else { @@ -1784,7 +1775,8 @@ xfs_bmap_add_extent_delay_real( * The left neighbor is not contiguous. */ xfs_iext_update_extent(bma->ip, state, &bma->icur, new); - (*nextents)++; + ifp->if_nextents++; + if (bma->cur == NULL) rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; else { @@ -1870,7 +1862,8 @@ xfs_bmap_add_extent_delay_real( * The right neighbor is not contiguous. */ xfs_iext_update_extent(bma->ip, state, &bma->icur, new); - (*nextents)++; + ifp->if_nextents++; + if (bma->cur == NULL) rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; else { @@ -1955,7 +1948,7 @@ xfs_bmap_add_extent_delay_real( xfs_iext_next(ifp, &bma->icur); xfs_iext_insert(bma->ip, &bma->icur, &RIGHT, state); xfs_iext_insert(bma->ip, &bma->icur, &LEFT, state); - (*nextents)++; + ifp->if_nextents++; if (bma->cur == NULL) rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; @@ -2159,8 +2152,7 @@ xfs_bmap_add_extent_unwritten_real( xfs_iext_remove(ip, icur, state); xfs_iext_prev(ifp, icur); xfs_iext_update_extent(ip, state, icur, &LEFT); - XFS_IFORK_NEXT_SET(ip, whichfork, - XFS_IFORK_NEXTENTS(ip, whichfork) - 2); + ifp->if_nextents -= 2; if (cur == NULL) rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; else { @@ -2212,8 +2204,7 @@ xfs_bmap_add_extent_unwritten_real( xfs_iext_remove(ip, icur, state); xfs_iext_prev(ifp, icur); xfs_iext_update_extent(ip, state, icur, &LEFT); - XFS_IFORK_NEXT_SET(ip, whichfork, - XFS_IFORK_NEXTENTS(ip, whichfork) - 1); + ifp->if_nextents--; if (cur == NULL) rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; else { @@ -2255,9 +2246,8 @@ xfs_bmap_add_extent_unwritten_real( xfs_iext_remove(ip, icur, state); xfs_iext_prev(ifp, icur); xfs_iext_update_extent(ip, state, icur, &PREV); + ifp->if_nextents--; - XFS_IFORK_NEXT_SET(ip, whichfork, - XFS_IFORK_NEXTENTS(ip, whichfork) - 1); if (cur == NULL) rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; else { @@ -2364,8 +2354,8 @@ xfs_bmap_add_extent_unwritten_real( xfs_iext_update_extent(ip, state, icur, &PREV); xfs_iext_insert(ip, icur, new, state); - XFS_IFORK_NEXT_SET(ip, whichfork, - XFS_IFORK_NEXTENTS(ip, whichfork) + 1); + ifp->if_nextents++; + if (cur == NULL) rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; else { @@ -2440,9 +2430,8 @@ xfs_bmap_add_extent_unwritten_real( xfs_iext_update_extent(ip, state, icur, &PREV); xfs_iext_next(ifp, icur); xfs_iext_insert(ip, icur, new, state); + ifp->if_nextents++; - XFS_IFORK_NEXT_SET(ip, whichfork, - XFS_IFORK_NEXTENTS(ip, whichfork) + 1); if (cur == NULL) rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; else { @@ -2493,9 +2482,8 @@ xfs_bmap_add_extent_unwritten_real( xfs_iext_next(ifp, icur); xfs_iext_insert(ip, icur, &r[1], state); xfs_iext_insert(ip, icur, &r[0], state); + ifp->if_nextents += 2; - XFS_IFORK_NEXT_SET(ip, whichfork, - XFS_IFORK_NEXTENTS(ip, whichfork) + 2); if (cur == NULL) rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; else { @@ -2810,9 +2798,8 @@ xfs_bmap_add_extent_hole_real( xfs_iext_remove(ip, icur, state); xfs_iext_prev(ifp, icur); xfs_iext_update_extent(ip, state, icur, &left); + ifp->if_nextents--; - XFS_IFORK_NEXT_SET(ip, whichfork, - XFS_IFORK_NEXTENTS(ip, whichfork) - 1); if (cur == NULL) { rval = XFS_ILOG_CORE | xfs_ilog_fext(whichfork); } else { @@ -2910,8 +2897,8 @@ xfs_bmap_add_extent_hole_real( * Insert a new entry. */ xfs_iext_insert(ip, icur, new, state); - XFS_IFORK_NEXT_SET(ip, whichfork, - XFS_IFORK_NEXTENTS(ip, whichfork) + 1); + ifp->if_nextents++; + if (cur == NULL) { rval = XFS_ILOG_CORE | xfs_ilog_fext(whichfork); } else { @@ -4512,8 +4499,7 @@ xfs_bmapi_write( goto error0; ASSERT(XFS_IFORK_FORMAT(ip, whichfork) != XFS_DINODE_FMT_BTREE || - XFS_IFORK_NEXTENTS(ip, whichfork) > - XFS_IFORK_MAXEXT(ip, whichfork)); + ifp->if_nextents > XFS_IFORK_MAXEXT(ip, whichfork)); xfs_bmapi_finish(&bma, whichfork, 0); xfs_bmap_validate_ret(orig_bno, orig_len, orig_flags, orig_mval, orig_nmap, *nmap); @@ -5056,8 +5042,7 @@ xfs_bmap_del_extent_real( */ if (tp->t_blk_res == 0 && XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_EXTENTS && - XFS_IFORK_NEXTENTS(ip, whichfork) >= - XFS_IFORK_MAXEXT(ip, whichfork) && + ifp->if_nextents >= XFS_IFORK_MAXEXT(ip, whichfork) && del->br_startoff > got.br_startoff && del_endoff < got_endoff) return -ENOSPC; @@ -5109,8 +5094,8 @@ xfs_bmap_del_extent_real( */ xfs_iext_remove(ip, icur, state); xfs_iext_prev(ifp, icur); - XFS_IFORK_NEXT_SET(ip, whichfork, - XFS_IFORK_NEXTENTS(ip, whichfork) - 1); + ifp->if_nextents--; + flags |= XFS_ILOG_CORE; if (!cur) { flags |= xfs_ilog_fext(whichfork); @@ -5218,8 +5203,8 @@ xfs_bmap_del_extent_real( } } else flags |= xfs_ilog_fext(whichfork); - XFS_IFORK_NEXT_SET(ip, whichfork, - XFS_IFORK_NEXTENTS(ip, whichfork) + 1); + + ifp->if_nextents++; xfs_iext_next(ifp, icur); xfs_iext_insert(ip, icur, &new, state); break; @@ -5667,6 +5652,7 @@ xfs_bmse_merge( struct xfs_btree_cur *cur, int *logflags) /* output */ { + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); struct xfs_bmbt_irec new; xfs_filblks_t blockcount; int error, i; @@ -5685,8 +5671,7 @@ xfs_bmse_merge( * Update the on-disk extent count, the btree if necessary and log the * inode. */ - XFS_IFORK_NEXT_SET(ip, whichfork, - XFS_IFORK_NEXTENTS(ip, whichfork) - 1); + ifp->if_nextents--; *logflags |= XFS_ILOG_CORE; if (!cur) { *logflags |= XFS_ILOG_DEXT; @@ -5724,7 +5709,7 @@ xfs_bmse_merge( done: xfs_iext_remove(ip, icur, 0); - xfs_iext_prev(XFS_IFORK_PTR(ip, whichfork), icur); + xfs_iext_prev(ifp, icur); xfs_iext_update_extent(ip, xfs_bmap_fork_to_state(whichfork), icur, &new); @@ -6074,8 +6059,7 @@ xfs_bmap_split_extent( /* Add new extent */ xfs_iext_next(ifp, &icur); xfs_iext_insert(ip, &icur, &new, 0); - XFS_IFORK_NEXT_SET(ip, whichfork, - XFS_IFORK_NEXTENTS(ip, whichfork) + 1); + ifp->if_nextents++; if (cur) { error = xfs_bmbt_lookup_eq(cur, &new, &i); diff --git a/fs/xfs/libxfs/xfs_dir2_block.c b/fs/xfs/libxfs/xfs_dir2_block.c index 1dbf2f980a26..5b59d3f7746b 100644 --- a/fs/xfs/libxfs/xfs_dir2_block.c +++ b/fs/xfs/libxfs/xfs_dir2_block.c @@ -1104,7 +1104,7 @@ xfs_dir2_sf_to_block( ASSERT(ifp->if_bytes == dp->i_d.di_size); ASSERT(ifp->if_u1.if_data != NULL); ASSERT(dp->i_d.di_size >= xfs_dir2_sf_hdr_size(oldsfp->i8count)); - ASSERT(dp->i_d.di_nextents == 0); + ASSERT(dp->i_df.if_nextents == 0); /* * Copy the directory into a temporary buffer. diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index 5547bbb3cf94..a374e2a81e76 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -245,8 +245,6 @@ xfs_inode_from_disk( to->di_size = be64_to_cpu(from->di_size); to->di_nblocks = be64_to_cpu(from->di_nblocks); to->di_extsize = be32_to_cpu(from->di_extsize); - to->di_nextents = be32_to_cpu(from->di_nextents); - to->di_anextents = be16_to_cpu(from->di_anextents); to->di_forkoff = from->di_forkoff; to->di_aformat = from->di_aformat; to->di_dmevmask = be32_to_cpu(from->di_dmevmask); @@ -311,8 +309,8 @@ xfs_inode_to_disk( to->di_size = cpu_to_be64(from->di_size); to->di_nblocks = cpu_to_be64(from->di_nblocks); to->di_extsize = cpu_to_be32(from->di_extsize); - to->di_nextents = cpu_to_be32(from->di_nextents); - to->di_anextents = cpu_to_be16(from->di_anextents); + to->di_nextents = cpu_to_be32(xfs_ifork_nextents(&ip->i_df)); + to->di_anextents = cpu_to_be16(xfs_ifork_nextents(ip->i_afp)); to->di_forkoff = from->di_forkoff; to->di_aformat = from->di_aformat; to->di_dmevmask = cpu_to_be32(from->di_dmevmask); diff --git a/fs/xfs/libxfs/xfs_inode_buf.h b/fs/xfs/libxfs/xfs_inode_buf.h index e4cbcaf62a32..fecccfb26463 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.h +++ b/fs/xfs/libxfs/xfs_inode_buf.h @@ -22,8 +22,6 @@ struct xfs_icdinode { xfs_fsize_t di_size; /* number of bytes in file */ xfs_rfsblock_t di_nblocks; /* # of direct & btree blocks used */ xfs_extlen_t di_extsize; /* basic/minimum extent size for file */ - xfs_extnum_t di_nextents; /* number of extents in data fork */ - xfs_aextnum_t di_anextents; /* number of extents in attribute fork*/ uint8_t di_forkoff; /* attr fork offs, <<3 for 64b align */ int8_t di_aformat; /* format of attr fork's data */ uint32_t di_dmevmask; /* DMIG event mask */ diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index 6c24c27f5f44..2702ad5ba995 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -188,12 +188,11 @@ xfs_iformat_btree( * or the number of extents is greater than the number of * blocks. */ - if (unlikely(XFS_IFORK_NEXTENTS(ip, whichfork) <= - XFS_IFORK_MAXEXT(ip, whichfork) || + if (unlikely(ifp->if_nextents <= XFS_IFORK_MAXEXT(ip, whichfork) || nrecs == 0 || XFS_BMDR_SPACE_CALC(nrecs) > XFS_DFORK_SIZE(dip, mp, whichfork) || - XFS_IFORK_NEXTENTS(ip, whichfork) > ip->i_d.di_nblocks) || + ifp->if_nextents > ip->i_d.di_nblocks) || level == 0 || level > XFS_BTREE_MAXLEVELS) { xfs_warn(mp, "corrupt inode %Lu (btree).", (unsigned long long) ip->i_ino); @@ -229,6 +228,12 @@ xfs_iformat_data_fork( struct inode *inode = VFS_I(ip); int error; + /* + * Initialize the extent count early, as the per-format routines may + * depend on it. + */ + ip->i_df.if_nextents = be32_to_cpu(dip->di_nextents); + switch (inode->i_mode & S_IFMT) { case S_IFIFO: case S_IFCHR: @@ -281,7 +286,13 @@ xfs_iformat_attr_fork( { int error = 0; + /* + * Initialize the extent count early, as the per-format routines may + * depend on it. + */ ip->i_afp = kmem_zone_zalloc(xfs_ifork_zone, KM_NOFS); + ip->i_afp->if_nextents = be16_to_cpu(dip->di_anextents); + switch (dip->di_aformat) { case XFS_DINODE_FMT_LOCAL: error = xfs_iformat_local(ip, dip, XFS_ATTR_FORK, @@ -617,7 +628,7 @@ xfs_iflush_fork( !(iip->ili_fields & extflag[whichfork])); if ((iip->ili_fields & extflag[whichfork]) && (ifp->if_bytes > 0)) { - ASSERT(XFS_IFORK_NEXTENTS(ip, whichfork) > 0); + ASSERT(ifp->if_nextents > 0); (void)xfs_iextents_copy(ip, (xfs_bmbt_rec_t *)cp, whichfork); } @@ -676,7 +687,6 @@ xfs_ifork_init_cow( KM_NOFS); ip->i_cowfp->if_flags = XFS_IFEXTENTS; ip->i_cformat = XFS_DINODE_FMT_EXTENTS; - ip->i_cnextents = 0; } /* Verify the inline contents of the data fork of an inode. */ diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index f46a8c1db596..a69d425fe68d 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -23,6 +23,7 @@ struct xfs_ifork { } if_u1; short if_broot_bytes; /* bytes allocated for root */ unsigned char if_flags; /* per-fork flags */ + xfs_extnum_t if_nextents; /* # of extents in this fork */ }; /* @@ -67,18 +68,6 @@ struct xfs_ifork { ((w) == XFS_ATTR_FORK ? \ ((ip)->i_d.di_aformat = (n)) : \ ((ip)->i_cformat = (n)))) -#define XFS_IFORK_NEXTENTS(ip,w) \ - ((w) == XFS_DATA_FORK ? \ - (ip)->i_d.di_nextents : \ - ((w) == XFS_ATTR_FORK ? \ - (ip)->i_d.di_anextents : \ - (ip)->i_cnextents)) -#define XFS_IFORK_NEXT_SET(ip,w,n) \ - ((w) == XFS_DATA_FORK ? \ - ((ip)->i_d.di_nextents = (n)) : \ - ((w) == XFS_ATTR_FORK ? \ - ((ip)->i_d.di_anextents = (n)) : \ - ((ip)->i_cnextents = (n)))) #define XFS_IFORK_MAXEXT(ip, w) \ (XFS_IFORK_SIZE(ip, w) / sizeof(xfs_bmbt_rec_t)) @@ -86,6 +75,13 @@ struct xfs_ifork { (XFS_IFORK_FORMAT((ip), (w)) == XFS_DINODE_FMT_EXTENTS || \ XFS_IFORK_FORMAT((ip), (w)) == XFS_DINODE_FMT_BTREE) +static inline xfs_extnum_t xfs_ifork_nextents(struct xfs_ifork *ifp) +{ + if (!ifp) + return 0; + return ifp->if_nextents; +} + struct xfs_ifork *xfs_iext_state_to_fork(struct xfs_inode *ip, int state); int xfs_iformat_data_fork(struct xfs_inode *, struct xfs_dinode *); diff --git a/fs/xfs/scrub/bmap.c b/fs/xfs/scrub/bmap.c index 93d5b8a9d7f7..162912c5080a 100644 --- a/fs/xfs/scrub/bmap.c +++ b/fs/xfs/scrub/bmap.c @@ -566,6 +566,7 @@ xchk_bmap_check_rmaps( struct xfs_scrub *sc, int whichfork) { + struct xfs_ifork *ifp = XFS_IFORK_PTR(sc->ip, whichfork); xfs_agnumber_t agno; bool zero_size; int error; @@ -594,7 +595,7 @@ xchk_bmap_check_rmaps( zero_size = false; if (XFS_IFORK_FORMAT(sc->ip, whichfork) != XFS_DINODE_FMT_BTREE && - (zero_size || XFS_IFORK_NEXTENTS(sc->ip, whichfork) > 0)) + (zero_size || ifp->if_nextents > 0)) return 0; for (agno = 0; agno < sc->mp->m_sb.sb_agcount; agno++) { diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index 5705adc43a75..855aa8bcab64 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -90,7 +90,7 @@ xchk_parent_count_parent_dentries( * if there is one. */ lock_mode = xfs_ilock_data_map_shared(parent); - if (parent->i_d.di_nextents > 0) + if (parent->i_df.if_nextents > 0) error = xfs_dir3_data_readahead(parent, 0, 0); xfs_iunlock(parent, lock_mode); if (error) diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index b8acfd4d3ca6..ee2ea571c853 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -1227,7 +1227,7 @@ xfs_swap_extents_check_format( * if the target inode has less extents that then temporary inode then * why did userspace call us? */ - if (ip->i_d.di_nextents < tip->i_d.di_nextents) + if (ip->i_df.if_nextents < tip->i_df.if_nextents) return -EINVAL; /* @@ -1248,14 +1248,12 @@ xfs_swap_extents_check_format( /* Check temp in extent form to max in target */ if (tip->i_d.di_format == XFS_DINODE_FMT_EXTENTS && - XFS_IFORK_NEXTENTS(tip, XFS_DATA_FORK) > - XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)) + tip->i_df.if_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)) return -EINVAL; /* Check target in extent form to max in temp */ if (ip->i_d.di_format == XFS_DINODE_FMT_EXTENTS && - XFS_IFORK_NEXTENTS(ip, XFS_DATA_FORK) > - XFS_IFORK_MAXEXT(tip, XFS_DATA_FORK)) + ip->i_df.if_nextents > XFS_IFORK_MAXEXT(tip, XFS_DATA_FORK)) return -EINVAL; /* @@ -1271,7 +1269,7 @@ xfs_swap_extents_check_format( if (XFS_IFORK_Q(ip) && XFS_BMAP_BMDR_SPACE(tip->i_df.if_broot) > XFS_IFORK_BOFF(ip)) return -EINVAL; - if (XFS_IFORK_NEXTENTS(tip, XFS_DATA_FORK) <= + if (tip->i_df.if_nextents <= XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)) return -EINVAL; } @@ -1281,7 +1279,7 @@ xfs_swap_extents_check_format( if (XFS_IFORK_Q(tip) && XFS_BMAP_BMDR_SPACE(ip->i_df.if_broot) > XFS_IFORK_BOFF(tip)) return -EINVAL; - if (XFS_IFORK_NEXTENTS(ip, XFS_DATA_FORK) <= + if (ip->i_df.if_nextents <= XFS_IFORK_MAXEXT(tip, XFS_DATA_FORK)) return -EINVAL; } @@ -1434,15 +1432,15 @@ xfs_swap_extent_forks( /* * Count the number of extended attribute blocks */ - if ( ((XFS_IFORK_Q(ip) != 0) && (ip->i_d.di_anextents > 0)) && - (ip->i_d.di_aformat != XFS_DINODE_FMT_LOCAL)) { + if (XFS_IFORK_Q(ip) && ip->i_afp->if_nextents > 0 && + ip->i_d.di_aformat != XFS_DINODE_FMT_LOCAL) { error = xfs_bmap_count_blocks(tp, ip, XFS_ATTR_FORK, &junk, &aforkblks); if (error) return error; } - if ( ((XFS_IFORK_Q(tip) != 0) && (tip->i_d.di_anextents > 0)) && - (tip->i_d.di_aformat != XFS_DINODE_FMT_LOCAL)) { + if (XFS_IFORK_Q(tip) && tip->i_afp->if_nextents > 0 && + tip->i_d.di_aformat != XFS_DINODE_FMT_LOCAL) { error = xfs_bmap_count_blocks(tp, tip, XFS_ATTR_FORK, &junk, &taforkblks); if (error) @@ -1475,7 +1473,6 @@ xfs_swap_extent_forks( ip->i_d.di_nblocks = tip->i_d.di_nblocks - taforkblks + aforkblks; tip->i_d.di_nblocks = tmp + taforkblks - aforkblks; - swap(ip->i_d.di_nextents, tip->i_d.di_nextents); swap(ip->i_d.di_format, tip->i_d.di_format); /* @@ -1622,9 +1619,9 @@ xfs_swap_extents( * performed with log redo items! */ if (xfs_sb_version_hasrmapbt(&mp->m_sb)) { - int w = XFS_DATA_FORK; - uint32_t ipnext = XFS_IFORK_NEXTENTS(ip, w); - uint32_t tipnext = XFS_IFORK_NEXTENTS(tip, w); + int w = XFS_DATA_FORK; + uint32_t ipnext = ip->i_df.if_nextents; + uint32_t tipnext = tip->i_df.if_nextents; /* * Conceptually this shouldn't affect the shape of either bmbt, @@ -1727,7 +1724,6 @@ xfs_swap_extents( ASSERT(ip->i_cformat == XFS_DINODE_FMT_EXTENTS); ASSERT(tip->i_cformat == XFS_DINODE_FMT_EXTENTS); - swap(ip->i_cnextents, tip->i_cnextents); swap(ip->i_cowfp, tip->i_cowfp); if (ip->i_cowfp && ip->i_cowfp->if_bytes) diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 4b8bdecc3863..403c90309a8f 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -1102,7 +1102,7 @@ xfs_dir_open( * certain to have the next operation be a read there. */ mode = xfs_ilock_data_map_shared(ip); - if (ip->i_d.di_nextents > 0) + if (ip->i_df.if_nextents > 0) error = xfs_dir3_data_readahead(ip, 0, 0); xfs_iunlock(ip, mode); return error; diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 5a3a520b9528..791d5d5e318c 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -63,7 +63,6 @@ xfs_inode_alloc( memset(&ip->i_imap, 0, sizeof(struct xfs_imap)); ip->i_afp = NULL; ip->i_cowfp = NULL; - ip->i_cnextents = 0; ip->i_cformat = XFS_DINODE_FMT_EXTENTS; memset(&ip->i_df, 0, sizeof(ip->i_df)); ip->i_flags = 0; diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 7d3144dc99b7..1677c4e7207e 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -825,7 +825,7 @@ xfs_ialloc( inode->i_mode &= ~S_ISGID; ip->i_d.di_size = 0; - ip->i_d.di_nextents = 0; + ip->i_df.if_nextents = 0; ASSERT(ip->i_d.di_nblocks == 0); tv = current_time(inode); @@ -919,7 +919,6 @@ xfs_ialloc( * Attribute fork settings for new inode. */ ip->i_d.di_aformat = XFS_DINODE_FMT_EXTENTS; - ip->i_d.di_anextents = 0; /* * Log the new values stuffed into the inode. @@ -1686,7 +1685,7 @@ xfs_inactive_truncate( if (error) goto error_trans_cancel; - ASSERT(ip->i_d.di_nextents == 0); + ASSERT(ip->i_df.if_nextents == 0); error = xfs_trans_commit(tp); if (error) @@ -1836,7 +1835,7 @@ xfs_inactive( if (S_ISREG(VFS_I(ip)->i_mode) && (ip->i_d.di_size != 0 || XFS_ISIZE(ip) != 0 || - ip->i_d.di_nextents > 0 || ip->i_delayed_blks > 0)) + ip->i_df.if_nextents > 0 || ip->i_delayed_blks > 0)) truncate = 1; error = xfs_qm_dqattach(ip); @@ -1862,7 +1861,6 @@ xfs_inactive( } ASSERT(!ip->i_afp); - ASSERT(ip->i_d.di_anextents == 0); ASSERT(ip->i_d.di_forkoff == 0); /* @@ -2731,8 +2729,7 @@ xfs_ifree( ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL)); ASSERT(VFS_I(ip)->i_nlink == 0); - ASSERT(ip->i_d.di_nextents == 0); - ASSERT(ip->i_d.di_anextents == 0); + ASSERT(ip->i_df.if_nextents == 0); ASSERT(ip->i_d.di_size == 0 || !S_ISREG(VFS_I(ip)->i_mode)); ASSERT(ip->i_d.di_nblocks == 0); @@ -3628,7 +3625,7 @@ xfs_iflush( ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL|XFS_ILOCK_SHARED)); ASSERT(xfs_isiflocked(ip)); ASSERT(ip->i_d.di_format != XFS_DINODE_FMT_BTREE || - ip->i_d.di_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)); + ip->i_df.if_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)); *bpp = NULL; @@ -3710,7 +3707,7 @@ xfs_iflush_int( ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL|XFS_ILOCK_SHARED)); ASSERT(xfs_isiflocked(ip)); ASSERT(ip->i_d.di_format != XFS_DINODE_FMT_BTREE || - ip->i_d.di_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)); + ip->i_df.if_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)); ASSERT(iip != NULL && iip->ili_fields != 0); dip = xfs_buf_offset(bp, ip->i_imap.im_boffset); @@ -3751,13 +3748,13 @@ xfs_iflush_int( goto flush_out; } } - if (XFS_TEST_ERROR(ip->i_d.di_nextents + ip->i_d.di_anextents > + if (XFS_TEST_ERROR(ip->i_df.if_nextents + xfs_ifork_nextents(ip->i_afp) > ip->i_d.di_nblocks, mp, XFS_ERRTAG_IFLUSH_5)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: detected corrupt incore inode %Lu, " "total extents = %d, nblocks = %Ld, ptr "PTR_FMT, __func__, ip->i_ino, - ip->i_d.di_nextents + ip->i_d.di_anextents, + ip->i_df.if_nextents + xfs_ifork_nextents(ip->i_afp), ip->i_d.di_nblocks, ip); goto flush_out; } diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index ff846197941e..24dae63ba16c 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -57,7 +57,6 @@ typedef struct xfs_inode { struct xfs_icdinode i_d; /* most of ondisk inode */ - xfs_extnum_t i_cnextents; /* # of extents in cow fork */ unsigned int i_cformat; /* format of cow fork */ /* VFS inode */ diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index cefa2484f0db..401ba26aeed7 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -39,7 +39,7 @@ xfs_inode_item_data_fork_size( switch (ip->i_d.di_format) { case XFS_DINODE_FMT_EXTENTS: if ((iip->ili_fields & XFS_ILOG_DEXT) && - ip->i_d.di_nextents > 0 && + ip->i_df.if_nextents > 0 && ip->i_df.if_bytes > 0) { /* worst case, doesn't subtract delalloc extents */ *nbytes += XFS_IFORK_DSIZE(ip); @@ -80,7 +80,7 @@ xfs_inode_item_attr_fork_size( switch (ip->i_d.di_aformat) { case XFS_DINODE_FMT_EXTENTS: if ((iip->ili_fields & XFS_ILOG_AEXT) && - ip->i_d.di_anextents > 0 && + ip->i_afp->if_nextents > 0 && ip->i_afp->if_bytes > 0) { /* worst case, doesn't subtract unused space */ *nbytes += XFS_IFORK_ASIZE(ip); @@ -148,7 +148,7 @@ xfs_inode_item_format_data_fork( ~(XFS_ILOG_DDATA | XFS_ILOG_DBROOT | XFS_ILOG_DEV); if ((iip->ili_fields & XFS_ILOG_DEXT) && - ip->i_d.di_nextents > 0 && + ip->i_df.if_nextents > 0 && ip->i_df.if_bytes > 0) { struct xfs_bmbt_rec *p; @@ -233,12 +233,12 @@ xfs_inode_item_format_attr_fork( ~(XFS_ILOG_ADATA | XFS_ILOG_ABROOT); if ((iip->ili_fields & XFS_ILOG_AEXT) && - ip->i_d.di_anextents > 0 && + ip->i_afp->if_nextents > 0 && ip->i_afp->if_bytes > 0) { struct xfs_bmbt_rec *p; ASSERT(xfs_iext_count(ip->i_afp) == - ip->i_d.di_anextents); + ip->i_afp->if_nextents); p = xlog_prepare_iovec(lv, vecp, XLOG_REG_TYPE_IATTR_EXT); data_bytes = xfs_iextents_copy(ip, p, XFS_ATTR_FORK); @@ -326,8 +326,8 @@ xfs_inode_to_log_dinode( to->di_size = from->di_size; to->di_nblocks = from->di_nblocks; to->di_extsize = from->di_extsize; - to->di_nextents = from->di_nextents; - to->di_anextents = from->di_anextents; + to->di_nextents = xfs_ifork_nextents(&ip->i_df); + to->di_anextents = xfs_ifork_nextents(ip->i_afp); to->di_forkoff = from->di_forkoff; to->di_aformat = from->di_aformat; to->di_dmevmask = from->di_dmevmask; diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 4ee0d13232f3..7a71c03e9022 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -1104,26 +1104,17 @@ xfs_fill_fsxattr( bool attr, struct fsxattr *fa) { + struct xfs_ifork *ifp = attr ? ip->i_afp : &ip->i_df; + simple_fill_fsxattr(fa, xfs_ip2xflags(ip)); fa->fsx_extsize = ip->i_d.di_extsize << ip->i_mount->m_sb.sb_blocklog; fa->fsx_cowextsize = ip->i_d.di_cowextsize << ip->i_mount->m_sb.sb_blocklog; fa->fsx_projid = ip->i_d.di_projid; - - if (attr) { - if (ip->i_afp) { - if (ip->i_afp->if_flags & XFS_IFEXTENTS) - fa->fsx_nextents = xfs_iext_count(ip->i_afp); - else - fa->fsx_nextents = ip->i_d.di_anextents; - } else - fa->fsx_nextents = 0; - } else { - if (ip->i_df.if_flags & XFS_IFEXTENTS) - fa->fsx_nextents = xfs_iext_count(&ip->i_df); - else - fa->fsx_nextents = ip->i_d.di_nextents; - } + if (ifp && (ifp->if_flags & XFS_IFEXTENTS)) + fa->fsx_nextents = xfs_iext_count(ifp); + else + fa->fsx_nextents = xfs_ifork_nextents(ifp); } STATIC int @@ -1211,7 +1202,7 @@ xfs_ioctl_setattr_xflags( uint64_t di_flags2; /* Can't change realtime flag if any extents are allocated. */ - if ((ip->i_d.di_nextents || ip->i_delayed_blks) && + if ((ip->i_df.if_nextents || ip->i_delayed_blks) && XFS_IS_REALTIME_INODE(ip) != (fa->fsx_xflags & FS_XFLAG_REALTIME)) return -EINVAL; @@ -1389,7 +1380,7 @@ xfs_ioctl_setattr_check_extsize( xfs_extlen_t size; xfs_fsblock_t extsize_fsb; - if (S_ISREG(VFS_I(ip)->i_mode) && ip->i_d.di_nextents && + if (S_ISREG(VFS_I(ip)->i_mode) && ip->i_df.if_nextents && ((ip->i_d.di_extsize << mp->m_sb.sb_blocklog) != fa->fsx_extsize)) return -EINVAL; diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index bb590a267a7f..b4fd918749e5 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -1258,7 +1258,7 @@ xfs_xattr_iomap_begin( lockmode = xfs_ilock_attr_map_shared(ip); /* if there are no attribute fork or extents, return ENOENT */ - if (!XFS_IFORK_Q(ip) || !ip->i_d.di_anextents) { + if (!XFS_IFORK_Q(ip) || !ip->i_afp->if_nextents) { error = -ENOENT; goto out_unlock; } diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c index 26a71237d70f..d66528fa3657 100644 --- a/fs/xfs/xfs_iops.c +++ b/fs/xfs/xfs_iops.c @@ -872,7 +872,7 @@ xfs_setattr_size( /* * Short circuit the truncate case for zero length files. */ - if (newsize == 0 && oldsize == 0 && ip->i_d.di_nextents == 0) { + if (newsize == 0 && oldsize == 0 && ip->i_df.if_nextents == 0) { if (!(iattr->ia_valid & (ATTR_CTIME|ATTR_MTIME))) return 0; diff --git a/fs/xfs/xfs_itable.c b/fs/xfs/xfs_itable.c index ff2da28fed90..80da86c5703f 100644 --- a/fs/xfs/xfs_itable.c +++ b/fs/xfs/xfs_itable.c @@ -104,9 +104,9 @@ xfs_bulkstat_one_int( buf->bs_xflags = xfs_ip2xflags(ip); buf->bs_extsize_blks = dic->di_extsize; - buf->bs_extents = dic->di_nextents; + buf->bs_extents = xfs_ifork_nextents(&ip->i_df); xfs_bulkstat_health(ip, buf); - buf->bs_aextents = dic->di_anextents; + buf->bs_aextents = xfs_ifork_nextents(ip->i_afp); buf->bs_forkoff = XFS_IFORK_BOFF(ip); buf->bs_version = XFS_BULKSTAT_VERSION_V5; diff --git a/fs/xfs/xfs_qm_syscalls.c b/fs/xfs/xfs_qm_syscalls.c index 944486f2b287..9edf761eec73 100644 --- a/fs/xfs/xfs_qm_syscalls.c +++ b/fs/xfs/xfs_qm_syscalls.c @@ -302,7 +302,7 @@ xfs_qm_scall_trunc_qfile( goto out_unlock; } - ASSERT(ip->i_d.di_nextents == 0); + ASSERT(ip->i_df.if_nextents == 0); xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); error = xfs_trans_commit(tp); diff --git a/fs/xfs/xfs_quotaops.c b/fs/xfs/xfs_quotaops.c index 38669e827206..b5d10ecb5474 100644 --- a/fs/xfs/xfs_quotaops.c +++ b/fs/xfs/xfs_quotaops.c @@ -36,7 +36,7 @@ xfs_qm_fill_state( } tstate->flags |= QCI_SYSFILE; tstate->blocks = ip->i_d.di_nblocks; - tstate->nextents = ip->i_d.di_nextents; + tstate->nextents = ip->i_df.if_nextents; tstate->spc_timelimit = (u32)q->qi_btimelimit; tstate->ino_timelimit = (u32)q->qi_itimelimit; tstate->rt_spc_timelimit = (u32)q->qi_rtbtimelimit; diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index 973441992b08..8cf2fcb509c1 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -384,7 +384,7 @@ xfs_inactive_symlink_rmt( * either 1 or 2 extents and that we can * free them all in one bunmapi call. */ - ASSERT(ip->i_d.di_nextents > 0 && ip->i_d.di_nextents <= 2); + ASSERT(ip->i_df.if_nextents > 0 && ip->i_df.if_nextents <= 2); error = xfs_trans_alloc(mp, &M_RES(mp)->tr_itruncate, 0, 0, 0, &tp); if (error) diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index a4323a63438d..ba2ab69e1fc7 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -1898,7 +1898,7 @@ DECLARE_EVENT_CLASS(xfs_swap_extent_class, __entry->which = which; __entry->ino = ip->i_ino; __entry->format = ip->i_d.di_format; - __entry->nex = ip->i_d.di_nextents; + __entry->nex = ip->i_df.if_nextents; __entry->broot_size = ip->i_df.if_broot_bytes; __entry->fork_off = XFS_IFORK_BOFF(ip); ), From f7e67b20ecbbcb9180c888a5c4fde267935e075f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 May 2020 10:28:05 -0700 Subject: [PATCH 0596/1043] xfs: move the fork format fields into struct xfs_ifork Both the data and attr fork have a format that is stored in the legacy idinode. Move it into the xfs_ifork structure instead, where it uses up padding. Signed-off-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Chandan Babu R Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_attr.c | 12 +-- fs/xfs/libxfs/xfs_attr_leaf.c | 32 ++++---- fs/xfs/libxfs/xfs_bmap.c | 120 +++++++++++++---------------- fs/xfs/libxfs/xfs_bmap_btree.c | 5 +- fs/xfs/libxfs/xfs_dir2.c | 8 +- fs/xfs/libxfs/xfs_dir2_sf.c | 13 ++-- fs/xfs/libxfs/xfs_inode_buf.c | 6 +- fs/xfs/libxfs/xfs_inode_buf.h | 2 - fs/xfs/libxfs/xfs_inode_fork.c | 14 ++-- fs/xfs/libxfs/xfs_inode_fork.h | 28 ++++--- fs/xfs/libxfs/xfs_symlink_remote.c | 14 ++-- fs/xfs/scrub/bmap.c | 5 +- fs/xfs/scrub/dabtree.c | 2 +- fs/xfs/scrub/dir.c | 7 +- fs/xfs/xfs_aops.c | 2 +- fs/xfs/xfs_attr_inactive.c | 2 +- fs/xfs/xfs_attr_list.c | 4 +- fs/xfs/xfs_bmap_util.c | 56 +++++++------- fs/xfs/xfs_dir2_readdir.c | 2 +- fs/xfs/xfs_icache.c | 1 - fs/xfs/xfs_inode.c | 36 ++++----- fs/xfs/xfs_inode.h | 2 - fs/xfs/xfs_inode_item.c | 12 +-- fs/xfs/xfs_iomap.c | 4 +- fs/xfs/xfs_itable.c | 2 +- fs/xfs/xfs_symlink.c | 2 +- fs/xfs/xfs_trace.h | 2 +- 27 files changed, 181 insertions(+), 214 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c index 1b01675e9c80..3b1bd6e112f8 100644 --- a/fs/xfs/libxfs/xfs_attr.c +++ b/fs/xfs/libxfs/xfs_attr.c @@ -61,7 +61,7 @@ xfs_inode_hasattr( struct xfs_inode *ip) { if (!XFS_IFORK_Q(ip) || - (ip->i_d.di_aformat == XFS_DINODE_FMT_EXTENTS && + (ip->i_afp->if_format == XFS_DINODE_FMT_EXTENTS && ip->i_afp->if_nextents == 0)) return 0; return 1; @@ -84,7 +84,7 @@ xfs_attr_get_ilocked( if (!xfs_inode_hasattr(args->dp)) return -ENOATTR; - if (args->dp->i_d.di_aformat == XFS_DINODE_FMT_LOCAL) + if (args->dp->i_afp->if_format == XFS_DINODE_FMT_LOCAL) return xfs_attr_shortform_getvalue(args); if (xfs_bmap_one_block(args->dp, XFS_ATTR_FORK)) return xfs_attr_leaf_get(args); @@ -212,14 +212,14 @@ xfs_attr_set_args( * If the attribute list is non-existent or a shortform list, * upgrade it to a single-leaf-block attribute list. */ - if (dp->i_d.di_aformat == XFS_DINODE_FMT_LOCAL || - (dp->i_d.di_aformat == XFS_DINODE_FMT_EXTENTS && + if (dp->i_afp->if_format == XFS_DINODE_FMT_LOCAL || + (dp->i_afp->if_format == XFS_DINODE_FMT_EXTENTS && dp->i_afp->if_nextents == 0)) { /* * Build initial attribute list (if required). */ - if (dp->i_d.di_aformat == XFS_DINODE_FMT_EXTENTS) + if (dp->i_afp->if_format == XFS_DINODE_FMT_EXTENTS) xfs_attr_shortform_create(args); /* @@ -272,7 +272,7 @@ xfs_attr_remove_args( if (!xfs_inode_hasattr(dp)) { error = -ENOATTR; - } else if (dp->i_d.di_aformat == XFS_DINODE_FMT_LOCAL) { + } else if (dp->i_afp->if_format == XFS_DINODE_FMT_LOCAL) { ASSERT(dp->i_afp->if_flags & XFS_IFINLINE); error = xfs_attr_shortform_remove(args); } else if (xfs_bmap_one_block(dp, XFS_ATTR_FORK)) { diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 11ff6dd08512..9ca33d064f83 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -538,7 +538,7 @@ xfs_attr_shortform_bytesfit( /* rounded down */ offset = (XFS_LITINO(mp) - bytes) >> 3; - if (dp->i_d.di_format == XFS_DINODE_FMT_DEV) { + if (dp->i_df.if_format == XFS_DINODE_FMT_DEV) { minforkoff = roundup(sizeof(xfs_dev_t), 8) >> 3; return (offset >= minforkoff) ? minforkoff : 0; } @@ -566,7 +566,7 @@ xfs_attr_shortform_bytesfit( dsize = dp->i_df.if_bytes; - switch (dp->i_d.di_format) { + switch (dp->i_df.if_format) { case XFS_DINODE_FMT_EXTENTS: /* * If there is no attr fork and the data fork is extents, @@ -635,22 +635,19 @@ xfs_sbversion_add_attr2(xfs_mount_t *mp, xfs_trans_t *tp) * Create the initial contents of a shortform attribute list. */ void -xfs_attr_shortform_create(xfs_da_args_t *args) +xfs_attr_shortform_create( + struct xfs_da_args *args) { - xfs_attr_sf_hdr_t *hdr; - xfs_inode_t *dp; - struct xfs_ifork *ifp; + struct xfs_inode *dp = args->dp; + struct xfs_ifork *ifp = dp->i_afp; + struct xfs_attr_sf_hdr *hdr; trace_xfs_attr_sf_create(args); - dp = args->dp; - ASSERT(dp != NULL); - ifp = dp->i_afp; - ASSERT(ifp != NULL); ASSERT(ifp->if_bytes == 0); - if (dp->i_d.di_aformat == XFS_DINODE_FMT_EXTENTS) { + if (ifp->if_format == XFS_DINODE_FMT_EXTENTS) { ifp->if_flags &= ~XFS_IFEXTENTS; /* just in case */ - dp->i_d.di_aformat = XFS_DINODE_FMT_LOCAL; + ifp->if_format = XFS_DINODE_FMT_LOCAL; ifp->if_flags |= XFS_IFINLINE; } else { ASSERT(ifp->if_flags & XFS_IFINLINE); @@ -722,7 +719,6 @@ xfs_attr_fork_remove( xfs_idestroy_fork(ip, XFS_ATTR_FORK); ip->i_d.di_forkoff = 0; - ip->i_d.di_aformat = XFS_DINODE_FMT_EXTENTS; ASSERT(ip->i_afp == NULL); @@ -775,7 +771,7 @@ xfs_attr_shortform_remove(xfs_da_args_t *args) totsize -= size; if (totsize == sizeof(xfs_attr_sf_hdr_t) && (mp->m_flags & XFS_MOUNT_ATTR2) && - (dp->i_d.di_format != XFS_DINODE_FMT_BTREE) && + (dp->i_df.if_format != XFS_DINODE_FMT_BTREE) && !(args->op_flags & XFS_DA_OP_ADDNAME)) { xfs_attr_fork_remove(dp, args->trans); } else { @@ -785,7 +781,7 @@ xfs_attr_shortform_remove(xfs_da_args_t *args) ASSERT(totsize > sizeof(xfs_attr_sf_hdr_t) || (args->op_flags & XFS_DA_OP_ADDNAME) || !(mp->m_flags & XFS_MOUNT_ATTR2) || - dp->i_d.di_format == XFS_DINODE_FMT_BTREE); + dp->i_df.if_format == XFS_DINODE_FMT_BTREE); xfs_trans_log_inode(args->trans, dp, XFS_ILOG_CORE | XFS_ILOG_ADATA); } @@ -962,7 +958,7 @@ xfs_attr_shortform_allfit( + be16_to_cpu(name_loc->valuelen); } if ((dp->i_mount->m_flags & XFS_MOUNT_ATTR2) && - (dp->i_d.di_format != XFS_DINODE_FMT_BTREE) && + (dp->i_df.if_format != XFS_DINODE_FMT_BTREE) && (bytes == sizeof(struct xfs_attr_sf_hdr))) return -1; return xfs_attr_shortform_bytesfit(dp, bytes); @@ -981,7 +977,7 @@ xfs_attr_shortform_verify( int i; int64_t size; - ASSERT(ip->i_d.di_aformat == XFS_DINODE_FMT_LOCAL); + ASSERT(ip->i_afp->if_format == XFS_DINODE_FMT_LOCAL); ifp = XFS_IFORK_PTR(ip, XFS_ATTR_FORK); sfp = (struct xfs_attr_shortform *)ifp->if_u1.if_data; size = ifp->if_bytes; @@ -1085,7 +1081,7 @@ xfs_attr3_leaf_to_shortform( if (forkoff == -1) { ASSERT(dp->i_mount->m_flags & XFS_MOUNT_ATTR2); - ASSERT(dp->i_d.di_format != XFS_DINODE_FMT_BTREE); + ASSERT(dp->i_df.if_format != XFS_DINODE_FMT_BTREE); xfs_attr_fork_remove(dp, args->trans); goto out; } diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index c1136be49abe..edc63dba007f 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -123,7 +123,7 @@ static inline bool xfs_bmap_needs_btree(struct xfs_inode *ip, int whichfork) struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); return whichfork != XFS_COW_FORK && - XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_EXTENTS && + ifp->if_format == XFS_DINODE_FMT_EXTENTS && ifp->if_nextents > XFS_IFORK_MAXEXT(ip, whichfork); } @@ -135,7 +135,7 @@ static inline bool xfs_bmap_wants_extents(struct xfs_inode *ip, int whichfork) struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); return whichfork != XFS_COW_FORK && - XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_BTREE && + ifp->if_format == XFS_DINODE_FMT_BTREE && ifp->if_nextents <= XFS_IFORK_MAXEXT(ip, whichfork); } @@ -215,8 +215,8 @@ xfs_bmap_forkoff_reset( int whichfork) { if (whichfork == XFS_ATTR_FORK && - ip->i_d.di_format != XFS_DINODE_FMT_DEV && - ip->i_d.di_format != XFS_DINODE_FMT_BTREE) { + ip->i_df.if_format != XFS_DINODE_FMT_DEV && + ip->i_df.if_format != XFS_DINODE_FMT_BTREE) { uint dfl_forkoff = xfs_default_attroffset(ip) >> 3; if (dfl_forkoff > ip->i_d.di_forkoff) @@ -317,31 +317,28 @@ xfs_bmap_check_leaf_extents( xfs_inode_t *ip, /* incore inode pointer */ int whichfork) /* data or attr fork */ { + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); struct xfs_btree_block *block; /* current btree block */ xfs_fsblock_t bno; /* block # of "block" */ xfs_buf_t *bp; /* buffer for "block" */ int error; /* error return value */ xfs_extnum_t i=0, j; /* index into the extents list */ - struct xfs_ifork *ifp; /* fork structure */ int level; /* btree level, for checking */ - xfs_mount_t *mp; /* file system mount structure */ __be64 *pp; /* pointer to block address */ xfs_bmbt_rec_t *ep; /* pointer to current extent */ xfs_bmbt_rec_t last = {0, 0}; /* last extent in prev block */ xfs_bmbt_rec_t *nextp; /* pointer to next extent */ int bp_release = 0; - if (XFS_IFORK_FORMAT(ip, whichfork) != XFS_DINODE_FMT_BTREE) { + if (ifp->if_format != XFS_DINODE_FMT_BTREE) return; - } /* skip large extent count inodes */ if (ip->i_df.if_nextents > 10000) return; bno = NULLFSBLOCK; - mp = ip->i_mount; - ifp = XFS_IFORK_PTR(ip, whichfork); block = ifp->if_broot; /* * Root level must use BMAP_BROOT_PTR_ADDR macro to get ptr out. @@ -606,7 +603,7 @@ xfs_bmap_btree_to_extents( ASSERT(cur); ASSERT(whichfork != XFS_COW_FORK); ASSERT(ifp->if_flags & XFS_IFEXTENTS); - ASSERT(XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_BTREE); + ASSERT(ifp->if_format == XFS_DINODE_FMT_BTREE); ASSERT(be16_to_cpu(rblock->bb_level) == 1); ASSERT(be16_to_cpu(rblock->bb_numrecs) == 1); ASSERT(xfs_bmbt_maxrecs(mp, ifp->if_broot_bytes, 0) == 1); @@ -634,7 +631,7 @@ xfs_bmap_btree_to_extents( xfs_iroot_realloc(ip, -1, whichfork); ASSERT(ifp->if_broot == NULL); ASSERT((ifp->if_flags & XFS_IFBROOT) == 0); - XFS_IFORK_FMT_SET(ip, whichfork, XFS_DINODE_FMT_EXTENTS); + ifp->if_format = XFS_DINODE_FMT_EXTENTS; *logflagsp |= XFS_ILOG_CORE | xfs_ilog_fext(whichfork); return 0; } @@ -670,7 +667,7 @@ xfs_bmap_extents_to_btree( mp = ip->i_mount; ASSERT(whichfork != XFS_COW_FORK); ifp = XFS_IFORK_PTR(ip, whichfork); - ASSERT(XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_EXTENTS); + ASSERT(ifp->if_format == XFS_DINODE_FMT_EXTENTS); /* * Make space in the inode incore. This needs to be undone if we fail @@ -694,7 +691,7 @@ xfs_bmap_extents_to_btree( /* * Convert to a btree with two levels, one record in root. */ - XFS_IFORK_FMT_SET(ip, whichfork, XFS_DINODE_FMT_BTREE); + ifp->if_format = XFS_DINODE_FMT_BTREE; memset(&args, 0, sizeof(args)); args.tp = tp; args.mp = mp; @@ -780,7 +777,7 @@ out_unreserve_dquot: xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, -1L); out_root_realloc: xfs_iroot_realloc(ip, -1, whichfork); - XFS_IFORK_FMT_SET(ip, whichfork, XFS_DINODE_FMT_EXTENTS); + ifp->if_format = XFS_DINODE_FMT_EXTENTS; ASSERT(ifp->if_broot == NULL); xfs_btree_del_cursor(cur, XFS_BTREE_ERROR); @@ -802,7 +799,7 @@ xfs_bmap_local_to_extents_empty( struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); ASSERT(whichfork != XFS_COW_FORK); - ASSERT(XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_LOCAL); + ASSERT(ifp->if_format == XFS_DINODE_FMT_LOCAL); ASSERT(ifp->if_bytes == 0); ASSERT(ifp->if_nextents == 0); @@ -811,7 +808,7 @@ xfs_bmap_local_to_extents_empty( ifp->if_flags |= XFS_IFEXTENTS; ifp->if_u1.if_root = NULL; ifp->if_height = 0; - XFS_IFORK_FMT_SET(ip, whichfork, XFS_DINODE_FMT_EXTENTS); + ifp->if_format = XFS_DINODE_FMT_EXTENTS; xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); } @@ -842,7 +839,7 @@ xfs_bmap_local_to_extents( */ ASSERT(!(S_ISREG(VFS_I(ip)->i_mode) && whichfork == XFS_DATA_FORK)); ifp = XFS_IFORK_PTR(ip, whichfork); - ASSERT(XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_LOCAL); + ASSERT(ifp->if_format == XFS_DINODE_FMT_LOCAL); if (!ifp->if_bytes) { xfs_bmap_local_to_extents_empty(tp, ip, whichfork); @@ -1036,7 +1033,7 @@ xfs_bmap_set_attrforkoff( int size, int *version) { - switch (ip->i_d.di_format) { + switch (ip->i_df.if_format) { case XFS_DINODE_FMT_DEV: ip->i_d.di_forkoff = roundup(sizeof(xfs_dev_t), 8) >> 3; break; @@ -1094,13 +1091,6 @@ xfs_bmap_add_attrfork( goto trans_cancel; if (XFS_IFORK_Q(ip)) goto trans_cancel; - if (ip->i_d.di_aformat != XFS_DINODE_FMT_EXTENTS) { - /* - * For inodes coming from pre-6.2 filesystems. - */ - ASSERT(ip->i_d.di_aformat == 0); - ip->i_d.di_aformat = XFS_DINODE_FMT_EXTENTS; - } xfs_trans_ijoin(tp, ip, 0); xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); @@ -1109,9 +1099,10 @@ xfs_bmap_add_attrfork( goto trans_cancel; ASSERT(ip->i_afp == NULL); ip->i_afp = kmem_zone_zalloc(xfs_ifork_zone, 0); + ip->i_afp->if_format = XFS_DINODE_FMT_EXTENTS; ip->i_afp->if_flags = XFS_IFEXTENTS; logflags = 0; - switch (ip->i_d.di_format) { + switch (ip->i_df.if_format) { case XFS_DINODE_FMT_LOCAL: error = xfs_bmap_add_attrfork_local(tp, ip, &logflags); break; @@ -1237,9 +1228,7 @@ xfs_iread_extents( ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL)); - if (XFS_IS_CORRUPT(mp, - XFS_IFORK_FORMAT(ip, whichfork) != - XFS_DINODE_FMT_BTREE)) { + if (XFS_IS_CORRUPT(mp, ifp->if_format != XFS_DINODE_FMT_BTREE)) { error = -EFSCORRUPTED; goto out; } @@ -1287,14 +1276,13 @@ xfs_bmap_first_unused( xfs_fileoff_t lowest, max; int error; - ASSERT(xfs_ifork_has_extents(ip, whichfork) || - XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_LOCAL); - - if (XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_LOCAL) { + if (ifp->if_format == XFS_DINODE_FMT_LOCAL) { *first_unused = 0; return 0; } + ASSERT(xfs_ifork_has_extents(ifp)); + if (!(ifp->if_flags & XFS_IFEXTENTS)) { error = xfs_iread_extents(tp, ip, whichfork); if (error) @@ -1335,7 +1323,7 @@ xfs_bmap_last_before( struct xfs_iext_cursor icur; int error; - switch (XFS_IFORK_FORMAT(ip, whichfork)) { + switch (ifp->if_format) { case XFS_DINODE_FMT_LOCAL: *last_block = 0; return 0; @@ -1434,16 +1422,17 @@ xfs_bmap_last_offset( xfs_fileoff_t *last_block, int whichfork) { + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); struct xfs_bmbt_irec rec; int is_empty; int error; *last_block = 0; - if (XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_LOCAL) + if (ifp->if_format == XFS_DINODE_FMT_LOCAL) return 0; - if (XFS_IS_CORRUPT(ip->i_mount, !xfs_ifork_has_extents(ip, whichfork))) + if (XFS_IS_CORRUPT(ip->i_mount, !xfs_ifork_has_extents(ifp))) return -EFSCORRUPTED; error = xfs_bmap_last_extent(NULL, ip, whichfork, &rec, &is_empty); @@ -1475,7 +1464,7 @@ xfs_bmap_one_block( #endif /* !DEBUG */ if (ifp->if_nextents != 1) return 0; - if (XFS_IFORK_FORMAT(ip, whichfork) != XFS_DINODE_FMT_EXTENTS) + if (ifp->if_format != XFS_DINODE_FMT_EXTENTS) return 0; ASSERT(ifp->if_flags & XFS_IFEXTENTS); xfs_iext_first(ifp, &icur); @@ -3895,10 +3884,9 @@ xfs_bmapi_read( if (WARN_ON_ONCE(!ifp)) return -EFSCORRUPTED; - if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ip, whichfork)) || - XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) { + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) || + XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) return -EFSCORRUPTED; - } if (XFS_FORCED_SHUTDOWN(mp)) return -EIO; @@ -4281,11 +4269,13 @@ xfs_bmapi_minleft( struct xfs_inode *ip, int fork) { + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, fork); + if (tp && tp->t_firstblock != NULLFSBLOCK) return 0; - if (XFS_IFORK_FORMAT(ip, fork) != XFS_DINODE_FMT_BTREE) + if (ifp->if_format != XFS_DINODE_FMT_BTREE) return 1; - return be16_to_cpu(XFS_IFORK_PTR(ip, fork)->if_broot->bb_level) + 1; + return be16_to_cpu(ifp->if_broot->bb_level) + 1; } /* @@ -4300,11 +4290,13 @@ xfs_bmapi_finish( int whichfork, int error) { + struct xfs_ifork *ifp = XFS_IFORK_PTR(bma->ip, whichfork); + if ((bma->logflags & xfs_ilog_fext(whichfork)) && - XFS_IFORK_FORMAT(bma->ip, whichfork) != XFS_DINODE_FMT_EXTENTS) + ifp->if_format != XFS_DINODE_FMT_EXTENTS) bma->logflags &= ~xfs_ilog_fext(whichfork); else if ((bma->logflags & xfs_ilog_fbroot(whichfork)) && - XFS_IFORK_FORMAT(bma->ip, whichfork) != XFS_DINODE_FMT_BTREE) + ifp->if_format != XFS_DINODE_FMT_BTREE) bma->logflags &= ~xfs_ilog_fbroot(whichfork); if (bma->logflags) @@ -4336,13 +4328,13 @@ xfs_bmapi_write( .total = total, }; struct xfs_mount *mp = ip->i_mount; - struct xfs_ifork *ifp; + int whichfork = xfs_bmapi_whichfork(flags); + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); xfs_fileoff_t end; /* end of mapped file region */ bool eof = false; /* after the end of extents */ int error; /* error return */ int n; /* current extent index */ xfs_fileoff_t obno; /* old block number (offset) */ - int whichfork; /* data or attr fork */ #ifdef DEBUG xfs_fileoff_t orig_bno; /* original block number value */ @@ -4357,13 +4349,12 @@ xfs_bmapi_write( orig_mval = mval; orig_nmap = *nmap; #endif - whichfork = xfs_bmapi_whichfork(flags); ASSERT(*nmap >= 1); ASSERT(*nmap <= XFS_BMAP_MAX_NMAP); ASSERT(tp != NULL); ASSERT(len > 0); - ASSERT(XFS_IFORK_FORMAT(ip, whichfork) != XFS_DINODE_FMT_LOCAL); + ASSERT(ifp->if_format != XFS_DINODE_FMT_LOCAL); ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL)); ASSERT(!(flags & XFS_BMAPI_REMAP)); @@ -4379,7 +4370,7 @@ xfs_bmapi_write( ASSERT((flags & (XFS_BMAPI_PREALLOC | XFS_BMAPI_ZERO)) != (XFS_BMAPI_PREALLOC | XFS_BMAPI_ZERO)); - if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ip, whichfork)) || + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) || XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) { return -EFSCORRUPTED; } @@ -4387,8 +4378,6 @@ xfs_bmapi_write( if (XFS_FORCED_SHUTDOWN(mp)) return -EIO; - ifp = XFS_IFORK_PTR(ip, whichfork); - XFS_STATS_INC(mp, xs_blk_mapw); if (!(ifp->if_flags & XFS_IFEXTENTS)) { @@ -4498,7 +4487,7 @@ xfs_bmapi_write( if (error) goto error0; - ASSERT(XFS_IFORK_FORMAT(ip, whichfork) != XFS_DINODE_FMT_BTREE || + ASSERT(ifp->if_format != XFS_DINODE_FMT_BTREE || ifp->if_nextents > XFS_IFORK_MAXEXT(ip, whichfork)); xfs_bmapi_finish(&bma, whichfork, 0); xfs_bmap_validate_ret(orig_bno, orig_len, orig_flags, orig_mval, @@ -4645,7 +4634,7 @@ xfs_bmapi_remap( ASSERT((flags & (XFS_BMAPI_ATTRFORK | XFS_BMAPI_PREALLOC)) != (XFS_BMAPI_ATTRFORK | XFS_BMAPI_PREALLOC)); - if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ip, whichfork)) || + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) || XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) { return -EFSCORRUPTED; } @@ -4689,9 +4678,9 @@ xfs_bmapi_remap( error = xfs_bmap_btree_to_extents(tp, ip, cur, &logflags, whichfork); error0: - if (ip->i_d.di_format != XFS_DINODE_FMT_EXTENTS) + if (ip->i_df.if_format != XFS_DINODE_FMT_EXTENTS) logflags &= ~XFS_ILOG_DEXT; - else if (ip->i_d.di_format != XFS_DINODE_FMT_BTREE) + else if (ip->i_df.if_format != XFS_DINODE_FMT_BTREE) logflags &= ~XFS_ILOG_DBROOT; if (logflags) @@ -5041,7 +5030,7 @@ xfs_bmap_del_extent_real( * conversion to btree format, since the transaction will be dirty then. */ if (tp->t_blk_res == 0 && - XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_EXTENTS && + ifp->if_format == XFS_DINODE_FMT_EXTENTS && ifp->if_nextents >= XFS_IFORK_MAXEXT(ip, whichfork) && del->br_startoff > got.br_startoff && del_endoff < got_endoff) return -ENOSPC; @@ -5284,7 +5273,7 @@ __xfs_bunmapi( whichfork = xfs_bmapi_whichfork(flags); ASSERT(whichfork != XFS_COW_FORK); ifp = XFS_IFORK_PTR(ip, whichfork); - if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ip, whichfork))) + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp))) return -EFSCORRUPTED; if (XFS_FORCED_SHUTDOWN(mp)) return -EIO; @@ -5322,7 +5311,7 @@ __xfs_bunmapi( logflags = 0; if (ifp->if_flags & XFS_IFBROOT) { - ASSERT(XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_BTREE); + ASSERT(ifp->if_format == XFS_DINODE_FMT_BTREE); cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); cur->bc_ino.flags = 0; } else @@ -5567,10 +5556,10 @@ error0: * logging the extent records if we've converted to btree format. */ if ((logflags & xfs_ilog_fext(whichfork)) && - XFS_IFORK_FORMAT(ip, whichfork) != XFS_DINODE_FMT_EXTENTS) + ifp->if_format != XFS_DINODE_FMT_EXTENTS) logflags &= ~xfs_ilog_fext(whichfork); else if ((logflags & xfs_ilog_fbroot(whichfork)) && - XFS_IFORK_FORMAT(ip, whichfork) != XFS_DINODE_FMT_BTREE) + ifp->if_format != XFS_DINODE_FMT_BTREE) logflags &= ~xfs_ilog_fbroot(whichfork); /* * Log inode even in the error case, if the transaction @@ -5781,7 +5770,7 @@ xfs_bmap_collapse_extents( int error = 0; int logflags = 0; - if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ip, whichfork)) || + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) || XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) { return -EFSCORRUPTED; } @@ -5898,7 +5887,7 @@ xfs_bmap_insert_extents( int error = 0; int logflags = 0; - if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ip, whichfork)) || + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) || XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) { return -EFSCORRUPTED; } @@ -5992,18 +5981,18 @@ xfs_bmap_split_extent( xfs_fileoff_t split_fsb) { int whichfork = XFS_DATA_FORK; + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); struct xfs_btree_cur *cur = NULL; struct xfs_bmbt_irec got; struct xfs_bmbt_irec new; /* split extent */ struct xfs_mount *mp = ip->i_mount; - struct xfs_ifork *ifp; xfs_fsblock_t gotblkcnt; /* new block count for got */ struct xfs_iext_cursor icur; int error = 0; int logflags = 0; int i = 0; - if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ip, whichfork)) || + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) || XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) { return -EFSCORRUPTED; } @@ -6011,7 +6000,6 @@ xfs_bmap_split_extent( if (XFS_FORCED_SHUTDOWN(mp)) return -EIO; - ifp = XFS_IFORK_PTR(ip, whichfork); if (!(ifp->if_flags & XFS_IFEXTENTS)) { /* Read in all the extents */ error = xfs_iread_extents(tp, ip, whichfork); diff --git a/fs/xfs/libxfs/xfs_bmap_btree.c b/fs/xfs/libxfs/xfs_bmap_btree.c index 295a59cf8840..d9c63f17d2de 100644 --- a/fs/xfs/libxfs/xfs_bmap_btree.c +++ b/fs/xfs/libxfs/xfs_bmap_btree.c @@ -636,10 +636,7 @@ xfs_bmbt_change_owner( ASSERT(tp || buffer_list); ASSERT(!(tp && buffer_list)); - if (whichfork == XFS_DATA_FORK) - ASSERT(ip->i_d.di_format == XFS_DINODE_FMT_BTREE); - else - ASSERT(ip->i_d.di_aformat == XFS_DINODE_FMT_BTREE); + ASSERT(XFS_IFORK_PTR(ip, whichfork)->if_format == XFS_DINODE_FMT_BTREE); cur = xfs_bmbt_init_cursor(ip->i_mount, tp, ip, whichfork); if (!cur) diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c index dd6fcaaea318..612a9c5e41b1 100644 --- a/fs/xfs/libxfs/xfs_dir2.c +++ b/fs/xfs/libxfs/xfs_dir2.c @@ -278,7 +278,7 @@ xfs_dir_createname( if (!inum) args->op_flags |= XFS_DA_OP_JUSTCHECK; - if (dp->i_d.di_format == XFS_DINODE_FMT_LOCAL) { + if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { rval = xfs_dir2_sf_addname(args); goto out_free; } @@ -373,7 +373,7 @@ xfs_dir_lookup( args->op_flags |= XFS_DA_OP_CILOOKUP; lock_mode = xfs_ilock_data_map_shared(dp); - if (dp->i_d.di_format == XFS_DINODE_FMT_LOCAL) { + if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { rval = xfs_dir2_sf_lookup(args); goto out_check_rval; } @@ -443,7 +443,7 @@ xfs_dir_removename( args->whichfork = XFS_DATA_FORK; args->trans = tp; - if (dp->i_d.di_format == XFS_DINODE_FMT_LOCAL) { + if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { rval = xfs_dir2_sf_removename(args); goto out_free; } @@ -504,7 +504,7 @@ xfs_dir_replace( args->whichfork = XFS_DATA_FORK; args->trans = tp; - if (dp->i_d.di_format == XFS_DINODE_FMT_LOCAL) { + if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { rval = xfs_dir2_sf_replace(args); goto out_free; } diff --git a/fs/xfs/libxfs/xfs_dir2_sf.c b/fs/xfs/libxfs/xfs_dir2_sf.c index 7b7f6fb2ea3b..2463b5d73447 100644 --- a/fs/xfs/libxfs/xfs_dir2_sf.c +++ b/fs/xfs/libxfs/xfs_dir2_sf.c @@ -343,7 +343,7 @@ xfs_dir2_block_to_sf( */ ASSERT(dp->i_df.if_bytes == 0); xfs_init_local_fork(dp, XFS_DATA_FORK, sfp, size); - dp->i_d.di_format = XFS_DINODE_FMT_LOCAL; + dp->i_df.if_format = XFS_DINODE_FMT_LOCAL; dp->i_d.di_size = size; logflags |= XFS_ILOG_DDATA; @@ -710,11 +710,11 @@ xfs_dir2_sf_verify( struct xfs_inode *ip) { struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK); struct xfs_dir2_sf_hdr *sfp; struct xfs_dir2_sf_entry *sfep; struct xfs_dir2_sf_entry *next_sfep; char *endp; - struct xfs_ifork *ifp; xfs_ino_t ino; int i; int i8count; @@ -723,9 +723,8 @@ xfs_dir2_sf_verify( int error; uint8_t filetype; - ASSERT(ip->i_d.di_format == XFS_DINODE_FMT_LOCAL); + ASSERT(ifp->if_format == XFS_DINODE_FMT_LOCAL); - ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK); sfp = (struct xfs_dir2_sf_hdr *)ifp->if_u1.if_data; size = ifp->if_bytes; @@ -827,9 +826,9 @@ xfs_dir2_sf_create( * If it's currently a zero-length extent file, * convert it to local format. */ - if (dp->i_d.di_format == XFS_DINODE_FMT_EXTENTS) { + if (dp->i_df.if_format == XFS_DINODE_FMT_EXTENTS) { dp->i_df.if_flags &= ~XFS_IFEXTENTS; /* just in case */ - dp->i_d.di_format = XFS_DINODE_FMT_LOCAL; + dp->i_df.if_format = XFS_DINODE_FMT_LOCAL; xfs_trans_log_inode(args->trans, dp, XFS_ILOG_CORE); dp->i_df.if_flags |= XFS_IFINLINE; } @@ -1027,7 +1026,7 @@ xfs_dir2_sf_replace_needblock( int newsize; struct xfs_dir2_sf_hdr *sfp; - if (dp->i_d.di_format != XFS_DINODE_FMT_LOCAL) + if (dp->i_df.if_format != XFS_DINODE_FMT_LOCAL) return false; sfp = (struct xfs_dir2_sf_hdr *)dp->i_df.if_u1.if_data; diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index a374e2a81e76..ab555671e154 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -225,7 +225,6 @@ xfs_inode_from_disk( be16_to_cpu(from->di_projid_lo); } - to->di_format = from->di_format; i_uid_write(inode, be32_to_cpu(from->di_uid)); i_gid_write(inode, be32_to_cpu(from->di_gid)); @@ -246,7 +245,6 @@ xfs_inode_from_disk( to->di_nblocks = be64_to_cpu(from->di_nblocks); to->di_extsize = be32_to_cpu(from->di_extsize); to->di_forkoff = from->di_forkoff; - to->di_aformat = from->di_aformat; to->di_dmevmask = be32_to_cpu(from->di_dmevmask); to->di_dmstate = be16_to_cpu(from->di_dmstate); to->di_flags = be16_to_cpu(from->di_flags); @@ -289,7 +287,7 @@ xfs_inode_to_disk( to->di_magic = cpu_to_be16(XFS_DINODE_MAGIC); to->di_onlink = 0; - to->di_format = from->di_format; + to->di_format = xfs_ifork_format(&ip->i_df); to->di_uid = cpu_to_be32(i_uid_read(inode)); to->di_gid = cpu_to_be32(i_gid_read(inode)); to->di_projid_lo = cpu_to_be16(from->di_projid & 0xffff); @@ -312,7 +310,7 @@ xfs_inode_to_disk( to->di_nextents = cpu_to_be32(xfs_ifork_nextents(&ip->i_df)); to->di_anextents = cpu_to_be16(xfs_ifork_nextents(ip->i_afp)); to->di_forkoff = from->di_forkoff; - to->di_aformat = from->di_aformat; + to->di_aformat = xfs_ifork_format(ip->i_afp); to->di_dmevmask = cpu_to_be32(from->di_dmevmask); to->di_dmstate = cpu_to_be16(from->di_dmstate); to->di_flags = cpu_to_be16(from->di_flags); diff --git a/fs/xfs/libxfs/xfs_inode_buf.h b/fs/xfs/libxfs/xfs_inode_buf.h index fecccfb26463..865ac493c72a 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.h +++ b/fs/xfs/libxfs/xfs_inode_buf.h @@ -16,14 +16,12 @@ struct xfs_dinode; * format specific structures at the appropriate time. */ struct xfs_icdinode { - int8_t di_format; /* format of di_c data */ uint16_t di_flushiter; /* incremented on flush */ uint32_t di_projid; /* owner's project id */ xfs_fsize_t di_size; /* number of bytes in file */ xfs_rfsblock_t di_nblocks; /* # of direct & btree blocks used */ xfs_extlen_t di_extsize; /* basic/minimum extent size for file */ uint8_t di_forkoff; /* attr fork offs, <<3 for 64b align */ - int8_t di_aformat; /* format of attr fork's data */ uint32_t di_dmevmask; /* DMIG event mask */ uint16_t di_dmstate; /* DMIG state info */ uint16_t di_flags; /* random flags, XFS_DIFLAG_... */ diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index 2702ad5ba995..ef43b4893766 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -232,6 +232,7 @@ xfs_iformat_data_fork( * Initialize the extent count early, as the per-format routines may * depend on it. */ + ip->i_df.if_format = dip->di_format; ip->i_df.if_nextents = be32_to_cpu(dip->di_nextents); switch (inode->i_mode & S_IFMT) { @@ -245,7 +246,7 @@ xfs_iformat_data_fork( case S_IFREG: case S_IFLNK: case S_IFDIR: - switch (dip->di_format) { + switch (ip->i_df.if_format) { case XFS_DINODE_FMT_LOCAL: error = xfs_iformat_local(ip, dip, XFS_DATA_FORK, be64_to_cpu(dip->di_size)); @@ -291,9 +292,12 @@ xfs_iformat_attr_fork( * depend on it. */ ip->i_afp = kmem_zone_zalloc(xfs_ifork_zone, KM_NOFS); + ip->i_afp->if_format = dip->di_aformat; + if (unlikely(ip->i_afp->if_format == 0)) /* pre IRIX 6.2 file system */ + ip->i_afp->if_format = XFS_DINODE_FMT_EXTENTS; ip->i_afp->if_nextents = be16_to_cpu(dip->di_anextents); - switch (dip->di_aformat) { + switch (ip->i_afp->if_format) { case XFS_DINODE_FMT_LOCAL: error = xfs_iformat_local(ip, dip, XFS_ATTR_FORK, xfs_dfork_attr_shortform_size(dip)); @@ -516,7 +520,7 @@ xfs_idestroy_fork( * not local then we may or may not have an extents list, * so check and free it up if we do. */ - if (XFS_IFORK_FORMAT(ip, whichfork) == XFS_DINODE_FMT_LOCAL) { + if (ifp->if_format == XFS_DINODE_FMT_LOCAL) { if (ifp->if_u1.if_data != NULL) { kmem_free(ifp->if_u1.if_data); ifp->if_u1.if_data = NULL; @@ -613,7 +617,7 @@ xfs_iflush_fork( } cp = XFS_DFORK_PTR(dip, whichfork); mp = ip->i_mount; - switch (XFS_IFORK_FORMAT(ip, whichfork)) { + switch (ifp->if_format) { case XFS_DINODE_FMT_LOCAL: if ((iip->ili_fields & dataflag[whichfork]) && (ifp->if_bytes > 0)) { @@ -686,7 +690,7 @@ xfs_ifork_init_cow( ip->i_cowfp = kmem_zone_zalloc(xfs_ifork_zone, KM_NOFS); ip->i_cowfp->if_flags = XFS_IFEXTENTS; - ip->i_cformat = XFS_DINODE_FMT_EXTENTS; + ip->i_cowfp->if_format = XFS_DINODE_FMT_EXTENTS; } /* Verify the inline contents of the data fork of an inode. */ diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index a69d425fe68d..d849cca103ed 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -23,6 +23,7 @@ struct xfs_ifork { } if_u1; short if_broot_bytes; /* bytes allocated for root */ unsigned char if_flags; /* per-fork flags */ + int8_t if_format; /* format of this fork */ xfs_extnum_t if_nextents; /* # of extents in this fork */ }; @@ -56,24 +57,14 @@ struct xfs_ifork { ((w) == XFS_ATTR_FORK ? \ XFS_IFORK_ASIZE(ip) : \ 0)) -#define XFS_IFORK_FORMAT(ip,w) \ - ((w) == XFS_DATA_FORK ? \ - (ip)->i_d.di_format : \ - ((w) == XFS_ATTR_FORK ? \ - (ip)->i_d.di_aformat : \ - (ip)->i_cformat)) -#define XFS_IFORK_FMT_SET(ip,w,n) \ - ((w) == XFS_DATA_FORK ? \ - ((ip)->i_d.di_format = (n)) : \ - ((w) == XFS_ATTR_FORK ? \ - ((ip)->i_d.di_aformat = (n)) : \ - ((ip)->i_cformat = (n)))) #define XFS_IFORK_MAXEXT(ip, w) \ (XFS_IFORK_SIZE(ip, w) / sizeof(xfs_bmbt_rec_t)) -#define xfs_ifork_has_extents(ip, w) \ - (XFS_IFORK_FORMAT((ip), (w)) == XFS_DINODE_FMT_EXTENTS || \ - XFS_IFORK_FORMAT((ip), (w)) == XFS_DINODE_FMT_BTREE) +static inline bool xfs_ifork_has_extents(struct xfs_ifork *ifp) +{ + return ifp->if_format == XFS_DINODE_FMT_EXTENTS || + ifp->if_format == XFS_DINODE_FMT_BTREE; +} static inline xfs_extnum_t xfs_ifork_nextents(struct xfs_ifork *ifp) { @@ -82,6 +73,13 @@ static inline xfs_extnum_t xfs_ifork_nextents(struct xfs_ifork *ifp) return ifp->if_nextents; } +static inline int8_t xfs_ifork_format(struct xfs_ifork *ifp) +{ + if (!ifp) + return XFS_DINODE_FMT_EXTENTS; + return ifp->if_format; +} + struct xfs_ifork *xfs_iext_state_to_fork(struct xfs_inode *ip, int state); int xfs_iformat_data_fork(struct xfs_inode *, struct xfs_dinode *); diff --git a/fs/xfs/libxfs/xfs_symlink_remote.c b/fs/xfs/libxfs/xfs_symlink_remote.c index 3b8260ca7d1b..594bc447a7dd 100644 --- a/fs/xfs/libxfs/xfs_symlink_remote.c +++ b/fs/xfs/libxfs/xfs_symlink_remote.c @@ -204,16 +204,12 @@ xfs_failaddr_t xfs_symlink_shortform_verify( struct xfs_inode *ip) { - char *sfp; - char *endp; - struct xfs_ifork *ifp; - int size; + struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK); + char *sfp = (char *)ifp->if_u1.if_data; + int size = ifp->if_bytes; + char *endp = sfp + size; - ASSERT(ip->i_d.di_format == XFS_DINODE_FMT_LOCAL); - ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK); - sfp = (char *)ifp->if_u1.if_data; - size = ifp->if_bytes; - endp = sfp + size; + ASSERT(ifp->if_format == XFS_DINODE_FMT_LOCAL); /* * Zero length symlinks should never occur in memory as they are diff --git a/fs/xfs/scrub/bmap.c b/fs/xfs/scrub/bmap.c index 162912c5080a..7badd6dfe544 100644 --- a/fs/xfs/scrub/bmap.c +++ b/fs/xfs/scrub/bmap.c @@ -589,12 +589,13 @@ xchk_bmap_check_rmaps( * to flag this bmap as corrupt if there are rmaps that need to be * reattached. */ + if (whichfork == XFS_DATA_FORK) zero_size = i_size_read(VFS_I(sc->ip)) == 0; else zero_size = false; - if (XFS_IFORK_FORMAT(sc->ip, whichfork) != XFS_DINODE_FMT_BTREE && + if (ifp->if_format != XFS_DINODE_FMT_BTREE && (zero_size || ifp->if_nextents > 0)) return 0; @@ -657,7 +658,7 @@ xchk_bmap( } /* Check the fork values */ - switch (XFS_IFORK_FORMAT(ip, whichfork)) { + switch (ifp->if_format) { case XFS_DINODE_FMT_UUID: case XFS_DINODE_FMT_DEV: case XFS_DINODE_FMT_LOCAL: diff --git a/fs/xfs/scrub/dabtree.c b/fs/xfs/scrub/dabtree.c index 9a2e27ac1300..44b15015021f 100644 --- a/fs/xfs/scrub/dabtree.c +++ b/fs/xfs/scrub/dabtree.c @@ -468,7 +468,7 @@ xchk_da_btree( int error; /* Skip short format data structures; no btree to scan. */ - if (!xfs_ifork_has_extents(sc->ip, whichfork)) + if (!xfs_ifork_has_extents(XFS_IFORK_PTR(sc->ip, whichfork))) return 0; /* Set up initial da state. */ diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index fe2a6e030c8a..7c432997edad 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -635,7 +635,7 @@ xchk_directory_blocks( { struct xfs_bmbt_irec got; struct xfs_da_args args; - struct xfs_ifork *ifp; + struct xfs_ifork *ifp = XFS_IFORK_PTR(sc->ip, XFS_DATA_FORK); struct xfs_mount *mp = sc->mp; xfs_fileoff_t leaf_lblk; xfs_fileoff_t free_lblk; @@ -647,11 +647,10 @@ xchk_directory_blocks( int error; /* Ignore local format directories. */ - if (sc->ip->i_d.di_format != XFS_DINODE_FMT_EXTENTS && - sc->ip->i_d.di_format != XFS_DINODE_FMT_BTREE) + if (ifp->if_format != XFS_DINODE_FMT_EXTENTS && + ifp->if_format != XFS_DINODE_FMT_BTREE) return 0; - ifp = XFS_IFORK_PTR(sc->ip, XFS_DATA_FORK); lblk = XFS_B_TO_FSB(mp, XFS_DIR2_DATA_OFFSET); leaf_lblk = XFS_B_TO_FSB(mp, XFS_DIR2_LEAF_OFFSET); free_lblk = XFS_B_TO_FSB(mp, XFS_DIR2_FREE_OFFSET); diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index 9d9cebf18726..2834cbf1212e 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -382,7 +382,7 @@ xfs_map_blocks( */ retry: xfs_ilock(ip, XFS_ILOCK_SHARED); - ASSERT(ip->i_d.di_format != XFS_DINODE_FMT_BTREE || + ASSERT(ip->i_df.if_format != XFS_DINODE_FMT_BTREE || (ip->i_df.if_flags & XFS_IFEXTENTS)); /* diff --git a/fs/xfs/xfs_attr_inactive.c b/fs/xfs/xfs_attr_inactive.c index c42f90e16b4f..00ffc46c0bf7 100644 --- a/fs/xfs/xfs_attr_inactive.c +++ b/fs/xfs/xfs_attr_inactive.c @@ -367,7 +367,7 @@ xfs_attr_inactive( * removal below. */ if (xfs_inode_hasattr(dp) && - dp->i_d.di_aformat != XFS_DINODE_FMT_LOCAL) { + dp->i_afp->if_format != XFS_DINODE_FMT_LOCAL) { error = xfs_attr3_root_inactive(&trans, dp); if (error) goto out_cancel; diff --git a/fs/xfs/xfs_attr_list.c b/fs/xfs/xfs_attr_list.c index 5ff1d929d3b5..e380bd1a9bfc 100644 --- a/fs/xfs/xfs_attr_list.c +++ b/fs/xfs/xfs_attr_list.c @@ -512,9 +512,9 @@ xfs_attr_list_ilocked( */ if (!xfs_inode_hasattr(dp)) return 0; - else if (dp->i_d.di_aformat == XFS_DINODE_FMT_LOCAL) + if (dp->i_afp->if_format == XFS_DINODE_FMT_LOCAL) return xfs_attr_shortform_list(context); - else if (xfs_bmap_one_block(dp, XFS_ATTR_FORK)) + if (xfs_bmap_one_block(dp, XFS_ATTR_FORK)) return xfs_attr_leaf_list(context); return xfs_attr_node_list(context); } diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index ee2ea571c853..f37f5cc4b19f 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -223,7 +223,7 @@ xfs_bmap_count_blocks( if (!ifp) return 0; - switch (XFS_IFORK_FORMAT(ip, whichfork)) { + switch (ifp->if_format) { case XFS_DINODE_FMT_BTREE: if (!(ifp->if_flags & XFS_IFEXTENTS)) { error = xfs_iread_extents(tp, ip, whichfork); @@ -449,7 +449,7 @@ xfs_getbmap( break; } - switch (XFS_IFORK_FORMAT(ip, whichfork)) { + switch (ifp->if_format) { case XFS_DINODE_FMT_EXTENTS: case XFS_DINODE_FMT_BTREE: break; @@ -1210,6 +1210,8 @@ xfs_swap_extents_check_format( struct xfs_inode *ip, /* target inode */ struct xfs_inode *tip) /* tmp inode */ { + struct xfs_ifork *ifp = &ip->i_df; + struct xfs_ifork *tifp = &tip->i_df; /* User/group/project quota ids must match if quotas are enforced. */ if (XFS_IS_QUOTA_ON(ip->i_mount) && @@ -1219,15 +1221,15 @@ xfs_swap_extents_check_format( return -EINVAL; /* Should never get a local format */ - if (ip->i_d.di_format == XFS_DINODE_FMT_LOCAL || - tip->i_d.di_format == XFS_DINODE_FMT_LOCAL) + if (ifp->if_format == XFS_DINODE_FMT_LOCAL || + tifp->if_format == XFS_DINODE_FMT_LOCAL) return -EINVAL; /* * if the target inode has less extents that then temporary inode then * why did userspace call us? */ - if (ip->i_df.if_nextents < tip->i_df.if_nextents) + if (ifp->if_nextents < tifp->if_nextents) return -EINVAL; /* @@ -1242,18 +1244,18 @@ xfs_swap_extents_check_format( * form then we will end up with the target inode in the wrong format * as we already know there are less extents in the temp inode. */ - if (ip->i_d.di_format == XFS_DINODE_FMT_EXTENTS && - tip->i_d.di_format == XFS_DINODE_FMT_BTREE) + if (ifp->if_format == XFS_DINODE_FMT_EXTENTS && + tifp->if_format == XFS_DINODE_FMT_BTREE) return -EINVAL; /* Check temp in extent form to max in target */ - if (tip->i_d.di_format == XFS_DINODE_FMT_EXTENTS && - tip->i_df.if_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)) + if (tifp->if_format == XFS_DINODE_FMT_EXTENTS && + tifp->if_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)) return -EINVAL; /* Check target in extent form to max in temp */ - if (ip->i_d.di_format == XFS_DINODE_FMT_EXTENTS && - ip->i_df.if_nextents > XFS_IFORK_MAXEXT(tip, XFS_DATA_FORK)) + if (ifp->if_format == XFS_DINODE_FMT_EXTENTS && + ifp->if_nextents > XFS_IFORK_MAXEXT(tip, XFS_DATA_FORK)) return -EINVAL; /* @@ -1265,22 +1267,20 @@ xfs_swap_extents_check_format( * (a common defrag case) which will occur when the temp inode is in * extent format... */ - if (tip->i_d.di_format == XFS_DINODE_FMT_BTREE) { + if (tifp->if_format == XFS_DINODE_FMT_BTREE) { if (XFS_IFORK_Q(ip) && - XFS_BMAP_BMDR_SPACE(tip->i_df.if_broot) > XFS_IFORK_BOFF(ip)) + XFS_BMAP_BMDR_SPACE(tifp->if_broot) > XFS_IFORK_BOFF(ip)) return -EINVAL; - if (tip->i_df.if_nextents <= - XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)) + if (tifp->if_nextents <= XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)) return -EINVAL; } /* Reciprocal target->temp btree format checks */ - if (ip->i_d.di_format == XFS_DINODE_FMT_BTREE) { + if (ifp->if_format == XFS_DINODE_FMT_BTREE) { if (XFS_IFORK_Q(tip) && XFS_BMAP_BMDR_SPACE(ip->i_df.if_broot) > XFS_IFORK_BOFF(tip)) return -EINVAL; - if (ip->i_df.if_nextents <= - XFS_IFORK_MAXEXT(tip, XFS_DATA_FORK)) + if (ifp->if_nextents <= XFS_IFORK_MAXEXT(tip, XFS_DATA_FORK)) return -EINVAL; } @@ -1433,14 +1433,14 @@ xfs_swap_extent_forks( * Count the number of extended attribute blocks */ if (XFS_IFORK_Q(ip) && ip->i_afp->if_nextents > 0 && - ip->i_d.di_aformat != XFS_DINODE_FMT_LOCAL) { + ip->i_afp->if_format != XFS_DINODE_FMT_LOCAL) { error = xfs_bmap_count_blocks(tp, ip, XFS_ATTR_FORK, &junk, &aforkblks); if (error) return error; } if (XFS_IFORK_Q(tip) && tip->i_afp->if_nextents > 0 && - tip->i_d.di_aformat != XFS_DINODE_FMT_LOCAL) { + tip->i_afp->if_format != XFS_DINODE_FMT_LOCAL) { error = xfs_bmap_count_blocks(tp, tip, XFS_ATTR_FORK, &junk, &taforkblks); if (error) @@ -1455,9 +1455,9 @@ xfs_swap_extent_forks( * bmbt scan as the last step. */ if (xfs_sb_version_has_v3inode(&ip->i_mount->m_sb)) { - if (ip->i_d.di_format == XFS_DINODE_FMT_BTREE) + if (ip->i_df.if_format == XFS_DINODE_FMT_BTREE) (*target_log_flags) |= XFS_ILOG_DOWNER; - if (tip->i_d.di_format == XFS_DINODE_FMT_BTREE) + if (tip->i_df.if_format == XFS_DINODE_FMT_BTREE) (*src_log_flags) |= XFS_ILOG_DOWNER; } @@ -1473,8 +1473,6 @@ xfs_swap_extent_forks( ip->i_d.di_nblocks = tip->i_d.di_nblocks - taforkblks + aforkblks; tip->i_d.di_nblocks = tmp + taforkblks - aforkblks; - swap(ip->i_d.di_format, tip->i_d.di_format); - /* * The extents in the source inode could still contain speculative * preallocation beyond EOF (e.g. the file is open but not modified @@ -1488,7 +1486,7 @@ xfs_swap_extent_forks( tip->i_delayed_blks = ip->i_delayed_blks; ip->i_delayed_blks = 0; - switch (ip->i_d.di_format) { + switch (ip->i_df.if_format) { case XFS_DINODE_FMT_EXTENTS: (*src_log_flags) |= XFS_ILOG_DEXT; break; @@ -1499,7 +1497,7 @@ xfs_swap_extent_forks( break; } - switch (tip->i_d.di_format) { + switch (tip->i_df.if_format) { case XFS_DINODE_FMT_EXTENTS: (*target_log_flags) |= XFS_ILOG_DEXT; break; @@ -1721,8 +1719,10 @@ xfs_swap_extents( /* Swap the cow forks. */ if (xfs_sb_version_hasreflink(&mp->m_sb)) { - ASSERT(ip->i_cformat == XFS_DINODE_FMT_EXTENTS); - ASSERT(tip->i_cformat == XFS_DINODE_FMT_EXTENTS); + ASSERT(!ip->i_cowfp || + ip->i_cowfp->if_format == XFS_DINODE_FMT_EXTENTS); + ASSERT(!tip->i_cowfp || + tip->i_cowfp->if_format == XFS_DINODE_FMT_EXTENTS); swap(ip->i_cowfp, tip->i_cowfp); diff --git a/fs/xfs/xfs_dir2_readdir.c b/fs/xfs/xfs_dir2_readdir.c index 871ec22c9aee..66deddd5e296 100644 --- a/fs/xfs/xfs_dir2_readdir.c +++ b/fs/xfs/xfs_dir2_readdir.c @@ -524,7 +524,7 @@ xfs_readdir( args.geo = dp->i_mount->m_dir_geo; args.trans = tp; - if (dp->i_d.di_format == XFS_DINODE_FMT_LOCAL) + if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) rval = xfs_dir2_sf_getdents(&args, ctx); else if ((rval = xfs_dir2_isblock(&args, &v))) ; diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 791d5d5e318c..c09b3e9eab1d 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -63,7 +63,6 @@ xfs_inode_alloc( memset(&ip->i_imap, 0, sizeof(struct xfs_imap)); ip->i_afp = NULL; ip->i_cowfp = NULL; - ip->i_cformat = XFS_DINODE_FMT_EXTENTS; memset(&ip->i_df, 0, sizeof(ip->i_df)); ip->i_flags = 0; ip->i_delayed_blks = 0; diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 1677c4e7207e..64f5f9a440ae 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -112,7 +112,7 @@ xfs_ilock_data_map_shared( { uint lock_mode = XFS_ILOCK_SHARED; - if (ip->i_d.di_format == XFS_DINODE_FMT_BTREE && + if (ip->i_df.if_format == XFS_DINODE_FMT_BTREE && (ip->i_df.if_flags & XFS_IFEXTENTS) == 0) lock_mode = XFS_ILOCK_EXCL; xfs_ilock(ip, lock_mode); @@ -125,7 +125,8 @@ xfs_ilock_attr_map_shared( { uint lock_mode = XFS_ILOCK_SHARED; - if (ip->i_d.di_aformat == XFS_DINODE_FMT_BTREE && + if (ip->i_afp && + ip->i_afp->if_format == XFS_DINODE_FMT_BTREE && (ip->i_afp->if_flags & XFS_IFEXTENTS) == 0) lock_mode = XFS_ILOCK_EXCL; xfs_ilock(ip, lock_mode); @@ -851,7 +852,7 @@ xfs_ialloc( case S_IFCHR: case S_IFBLK: case S_IFSOCK: - ip->i_d.di_format = XFS_DINODE_FMT_DEV; + ip->i_df.if_format = XFS_DINODE_FMT_DEV; ip->i_df.if_flags = 0; flags |= XFS_ILOG_DEV; break; @@ -907,7 +908,7 @@ xfs_ialloc( } /* FALLTHROUGH */ case S_IFLNK: - ip->i_d.di_format = XFS_DINODE_FMT_EXTENTS; + ip->i_df.if_format = XFS_DINODE_FMT_EXTENTS; ip->i_df.if_flags = XFS_IFEXTENTS; ip->i_df.if_bytes = 0; ip->i_df.if_u1.if_root = NULL; @@ -915,10 +916,6 @@ xfs_ialloc( default: ASSERT(0); } - /* - * Attribute fork settings for new inode. - */ - ip->i_d.di_aformat = XFS_DINODE_FMT_EXTENTS; /* * Log the new values stuffed into the inode. @@ -2749,7 +2746,7 @@ xfs_ifree( * data fork to extents format. Note that the attr fork data has * already been freed by xfs_attr_inactive. */ - if (ip->i_d.di_format == XFS_DINODE_FMT_LOCAL) { + if (ip->i_df.if_format == XFS_DINODE_FMT_LOCAL) { kmem_free(ip->i_df.if_u1.if_data); ip->i_df.if_u1.if_data = NULL; ip->i_df.if_bytes = 0; @@ -2760,8 +2757,7 @@ xfs_ifree( ip->i_d.di_flags2 = 0; ip->i_d.di_dmevmask = 0; ip->i_d.di_forkoff = 0; /* mark the attr fork not in use */ - ip->i_d.di_format = XFS_DINODE_FMT_EXTENTS; - ip->i_d.di_aformat = XFS_DINODE_FMT_EXTENTS; + ip->i_df.if_format = XFS_DINODE_FMT_EXTENTS; /* Don't attempt to replay owner changes for a deleted inode */ ip->i_itemp->ili_fields &= ~(XFS_ILOG_AOWNER|XFS_ILOG_DOWNER); @@ -3624,7 +3620,7 @@ xfs_iflush( ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL|XFS_ILOCK_SHARED)); ASSERT(xfs_isiflocked(ip)); - ASSERT(ip->i_d.di_format != XFS_DINODE_FMT_BTREE || + ASSERT(ip->i_df.if_format != XFS_DINODE_FMT_BTREE || ip->i_df.if_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)); *bpp = NULL; @@ -3706,7 +3702,7 @@ xfs_iflush_int( ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL|XFS_ILOCK_SHARED)); ASSERT(xfs_isiflocked(ip)); - ASSERT(ip->i_d.di_format != XFS_DINODE_FMT_BTREE || + ASSERT(ip->i_df.if_format != XFS_DINODE_FMT_BTREE || ip->i_df.if_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)); ASSERT(iip != NULL && iip->ili_fields != 0); @@ -3728,8 +3724,8 @@ xfs_iflush_int( } if (S_ISREG(VFS_I(ip)->i_mode)) { if (XFS_TEST_ERROR( - (ip->i_d.di_format != XFS_DINODE_FMT_EXTENTS) && - (ip->i_d.di_format != XFS_DINODE_FMT_BTREE), + ip->i_df.if_format != XFS_DINODE_FMT_EXTENTS && + ip->i_df.if_format != XFS_DINODE_FMT_BTREE, mp, XFS_ERRTAG_IFLUSH_3)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: Bad regular inode %Lu, ptr "PTR_FMT, @@ -3738,9 +3734,9 @@ xfs_iflush_int( } } else if (S_ISDIR(VFS_I(ip)->i_mode)) { if (XFS_TEST_ERROR( - (ip->i_d.di_format != XFS_DINODE_FMT_EXTENTS) && - (ip->i_d.di_format != XFS_DINODE_FMT_BTREE) && - (ip->i_d.di_format != XFS_DINODE_FMT_LOCAL), + ip->i_df.if_format != XFS_DINODE_FMT_EXTENTS && + ip->i_df.if_format != XFS_DINODE_FMT_BTREE && + ip->i_df.if_format != XFS_DINODE_FMT_LOCAL, mp, XFS_ERRTAG_IFLUSH_4)) { xfs_alert_tag(mp, XFS_PTAG_IFLUSH, "%s: Bad directory inode %Lu, ptr "PTR_FMT, @@ -3782,10 +3778,10 @@ xfs_iflush_int( * If there are inline format data / attr forks attached to this inode, * make sure they are not corrupt. */ - if (ip->i_d.di_format == XFS_DINODE_FMT_LOCAL && + if (ip->i_df.if_format == XFS_DINODE_FMT_LOCAL && xfs_ifork_verify_local_data(ip)) goto flush_out; - if (ip->i_d.di_aformat == XFS_DINODE_FMT_LOCAL && + if (ip->i_afp && ip->i_afp->if_format == XFS_DINODE_FMT_LOCAL && xfs_ifork_verify_local_attr(ip)) goto flush_out; diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h index 24dae63ba16c..dadcf1945896 100644 --- a/fs/xfs/xfs_inode.h +++ b/fs/xfs/xfs_inode.h @@ -57,8 +57,6 @@ typedef struct xfs_inode { struct xfs_icdinode i_d; /* most of ondisk inode */ - unsigned int i_cformat; /* format of cow fork */ - /* VFS inode */ struct inode i_vnode; /* embedded VFS inode */ diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c index 401ba26aeed7..ba47bf65b772 100644 --- a/fs/xfs/xfs_inode_item.c +++ b/fs/xfs/xfs_inode_item.c @@ -36,7 +36,7 @@ xfs_inode_item_data_fork_size( { struct xfs_inode *ip = iip->ili_inode; - switch (ip->i_d.di_format) { + switch (ip->i_df.if_format) { case XFS_DINODE_FMT_EXTENTS: if ((iip->ili_fields & XFS_ILOG_DEXT) && ip->i_df.if_nextents > 0 && @@ -77,7 +77,7 @@ xfs_inode_item_attr_fork_size( { struct xfs_inode *ip = iip->ili_inode; - switch (ip->i_d.di_aformat) { + switch (ip->i_afp->if_format) { case XFS_DINODE_FMT_EXTENTS: if ((iip->ili_fields & XFS_ILOG_AEXT) && ip->i_afp->if_nextents > 0 && @@ -142,7 +142,7 @@ xfs_inode_item_format_data_fork( struct xfs_inode *ip = iip->ili_inode; size_t data_bytes; - switch (ip->i_d.di_format) { + switch (ip->i_df.if_format) { case XFS_DINODE_FMT_EXTENTS: iip->ili_fields &= ~(XFS_ILOG_DDATA | XFS_ILOG_DBROOT | XFS_ILOG_DEV); @@ -227,7 +227,7 @@ xfs_inode_item_format_attr_fork( struct xfs_inode *ip = iip->ili_inode; size_t data_bytes; - switch (ip->i_d.di_aformat) { + switch (ip->i_afp->if_format) { case XFS_DINODE_FMT_EXTENTS: iip->ili_fields &= ~(XFS_ILOG_ADATA | XFS_ILOG_ABROOT); @@ -305,7 +305,7 @@ xfs_inode_to_log_dinode( struct inode *inode = VFS_I(ip); to->di_magic = XFS_DINODE_MAGIC; - to->di_format = from->di_format; + to->di_format = xfs_ifork_format(&ip->i_df); to->di_uid = i_uid_read(inode); to->di_gid = i_gid_read(inode); to->di_projid_lo = from->di_projid & 0xffff; @@ -329,7 +329,7 @@ xfs_inode_to_log_dinode( to->di_nextents = xfs_ifork_nextents(&ip->i_df); to->di_anextents = xfs_ifork_nextents(ip->i_afp); to->di_forkoff = from->di_forkoff; - to->di_aformat = from->di_aformat; + to->di_aformat = xfs_ifork_format(ip->i_afp); to->di_dmevmask = from->di_dmevmask; to->di_dmstate = from->di_dmstate; to->di_flags = from->di_flags; diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index b4fd918749e5..6ae3a2457777 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -856,7 +856,7 @@ xfs_buffered_write_iomap_begin( xfs_ilock(ip, XFS_ILOCK_EXCL); - if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ip, XFS_DATA_FORK)) || + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(&ip->i_df)) || XFS_TEST_ERROR(false, mp, XFS_ERRTAG_BMAPIFORMAT)) { error = -EFSCORRUPTED; goto out_unlock; @@ -1263,7 +1263,7 @@ xfs_xattr_iomap_begin( goto out_unlock; } - ASSERT(ip->i_d.di_aformat != XFS_DINODE_FMT_LOCAL); + ASSERT(ip->i_afp->if_format != XFS_DINODE_FMT_LOCAL); error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap, &nimaps, XFS_BMAPI_ATTRFORK); out_unlock: diff --git a/fs/xfs/xfs_itable.c b/fs/xfs/xfs_itable.c index 80da86c5703f..16ca97a7ff00 100644 --- a/fs/xfs/xfs_itable.c +++ b/fs/xfs/xfs_itable.c @@ -115,7 +115,7 @@ xfs_bulkstat_one_int( buf->bs_cowextsize_blks = dic->di_cowextsize; } - switch (dic->di_format) { + switch (ip->i_df.if_format) { case XFS_DINODE_FMT_DEV: buf->bs_rdev = sysv_encode_dev(inode->i_rdev); buf->bs_blksize = BLKDEV_IOSIZE; diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c index 8cf2fcb509c1..8e88a7ca387e 100644 --- a/fs/xfs/xfs_symlink.c +++ b/fs/xfs/xfs_symlink.c @@ -251,7 +251,7 @@ xfs_symlink( xfs_init_local_fork(ip, XFS_DATA_FORK, target_path, pathlen); ip->i_d.di_size = pathlen; - ip->i_d.di_format = XFS_DINODE_FMT_LOCAL; + ip->i_df.if_format = XFS_DINODE_FMT_LOCAL; xfs_trans_log_inode(tp, ip, XFS_ILOG_DDATA | XFS_ILOG_CORE); } else { int offset; diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index ba2ab69e1fc7..460136628a79 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -1897,7 +1897,7 @@ DECLARE_EVENT_CLASS(xfs_swap_extent_class, __entry->dev = VFS_I(ip)->i_sb->s_dev; __entry->which = which; __entry->ino = ip->i_ino; - __entry->format = ip->i_d.di_format; + __entry->format = ip->i_df.if_format; __entry->nex = ip->i_df.if_nextents; __entry->broot_size = ip->i_df.if_broot_bytes; __entry->fork_off = XFS_IFORK_BOFF(ip); From ef8385128d4b31a382d496b1c433697993bd0bfb Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 18 May 2020 10:29:27 -0700 Subject: [PATCH 0597/1043] xfs: cleanup xfs_idestroy_fork Move freeing the dynamically allocated attr and COW fork, as well as zeroing the pointers where actually needed into the callers, and just pass the xfs_ifork structure to xfs_idestroy_fork. Also simplify the kmem_free calls by not checking for NULL first. Signed-off-by: Christoph Hellwig Reviewed-by: Chandan Babu R Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_attr_leaf.c | 7 +++---- fs/xfs/libxfs/xfs_inode_buf.c | 2 +- fs/xfs/libxfs/xfs_inode_fork.c | 32 +++++++++----------------------- fs/xfs/libxfs/xfs_inode_fork.h | 2 +- fs/xfs/xfs_attr_inactive.c | 7 +++++-- fs/xfs/xfs_icache.c | 15 +++++++++------ 6 files changed, 28 insertions(+), 37 deletions(-) diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index 9ca33d064f83..f3d18a1f5b20 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -717,11 +717,10 @@ xfs_attr_fork_remove( { ASSERT(ip->i_afp->if_nextents == 0); - xfs_idestroy_fork(ip, XFS_ATTR_FORK); + xfs_idestroy_fork(ip->i_afp); + kmem_cache_free(xfs_ifork_zone, ip->i_afp); + ip->i_afp = NULL; ip->i_d.di_forkoff = 0; - - ASSERT(ip->i_afp == NULL); - xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); } diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c index ab555671e154..6f84ea85fdd8 100644 --- a/fs/xfs/libxfs/xfs_inode_buf.c +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -271,7 +271,7 @@ xfs_inode_from_disk( return 0; out_destroy_data_fork: - xfs_idestroy_fork(ip, XFS_DATA_FORK); + xfs_idestroy_fork(&ip->i_df); return error; } diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c index ef43b4893766..28b366275ae0 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.c +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -503,38 +503,24 @@ xfs_idata_realloc( void xfs_idestroy_fork( - xfs_inode_t *ip, - int whichfork) + struct xfs_ifork *ifp) { - struct xfs_ifork *ifp; - - ifp = XFS_IFORK_PTR(ip, whichfork); if (ifp->if_broot != NULL) { kmem_free(ifp->if_broot); ifp->if_broot = NULL; } /* - * If the format is local, then we can't have an extents - * array so just look for an inline data array. If we're - * not local then we may or may not have an extents list, - * so check and free it up if we do. + * If the format is local, then we can't have an extents array so just + * look for an inline data array. If we're not local then we may or may + * not have an extents list, so check and free it up if we do. */ if (ifp->if_format == XFS_DINODE_FMT_LOCAL) { - if (ifp->if_u1.if_data != NULL) { - kmem_free(ifp->if_u1.if_data); - ifp->if_u1.if_data = NULL; - } - } else if ((ifp->if_flags & XFS_IFEXTENTS) && ifp->if_height) { - xfs_iext_destroy(ifp); - } - - if (whichfork == XFS_ATTR_FORK) { - kmem_cache_free(xfs_ifork_zone, ip->i_afp); - ip->i_afp = NULL; - } else if (whichfork == XFS_COW_FORK) { - kmem_cache_free(xfs_ifork_zone, ip->i_cowfp); - ip->i_cowfp = NULL; + kmem_free(ifp->if_u1.if_data); + ifp->if_u1.if_data = NULL; + } else if (ifp->if_flags & XFS_IFEXTENTS) { + if (ifp->if_height) + xfs_iext_destroy(ifp); } } diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h index d849cca103ed..a4953e95c4f3 100644 --- a/fs/xfs/libxfs/xfs_inode_fork.h +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -86,7 +86,7 @@ int xfs_iformat_data_fork(struct xfs_inode *, struct xfs_dinode *); int xfs_iformat_attr_fork(struct xfs_inode *, struct xfs_dinode *); void xfs_iflush_fork(struct xfs_inode *, struct xfs_dinode *, struct xfs_inode_log_item *, int); -void xfs_idestroy_fork(struct xfs_inode *, int); +void xfs_idestroy_fork(struct xfs_ifork *ifp); void xfs_idata_realloc(struct xfs_inode *ip, int64_t byte_diff, int whichfork); void xfs_iroot_realloc(struct xfs_inode *, int, int); diff --git a/fs/xfs/xfs_attr_inactive.c b/fs/xfs/xfs_attr_inactive.c index 00ffc46c0bf7..bfad669e6b2f 100644 --- a/fs/xfs/xfs_attr_inactive.c +++ b/fs/xfs/xfs_attr_inactive.c @@ -388,8 +388,11 @@ out_cancel: xfs_trans_cancel(trans); out_destroy_fork: /* kill the in-core attr fork before we drop the inode lock */ - if (dp->i_afp) - xfs_idestroy_fork(dp, XFS_ATTR_FORK); + if (dp->i_afp) { + xfs_idestroy_fork(dp->i_afp); + kmem_cache_free(xfs_ifork_zone, dp->i_afp); + dp->i_afp = NULL; + } if (lock_mode) xfs_iunlock(dp, lock_mode); return error; diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index c09b3e9eab1d..d806d3bfa893 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -87,15 +87,18 @@ xfs_inode_free_callback( case S_IFREG: case S_IFDIR: case S_IFLNK: - xfs_idestroy_fork(ip, XFS_DATA_FORK); + xfs_idestroy_fork(&ip->i_df); break; } - if (ip->i_afp) - xfs_idestroy_fork(ip, XFS_ATTR_FORK); - if (ip->i_cowfp) - xfs_idestroy_fork(ip, XFS_COW_FORK); - + if (ip->i_afp) { + xfs_idestroy_fork(ip->i_afp); + kmem_cache_free(xfs_ifork_zone, ip->i_afp); + } + if (ip->i_cowfp) { + xfs_idestroy_fork(ip->i_cowfp); + kmem_cache_free(xfs_ifork_zone, ip->i_cowfp); + } if (ip->i_itemp) { ASSERT(!test_bit(XFS_LI_IN_AIL, &ip->i_itemp->ili_item.li_flags)); From 7d148be69e3a0eaa9d029a3c51b545e322116a99 Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Wed, 13 May 2020 15:55:02 +0200 Subject: [PATCH 0598/1043] sched/fair: Optimize enqueue_task_fair() enqueue_task_fair jumps to enqueue_throttle label when cfs_rq_of(se) is throttled which means that se can't be NULL in such case and we can move the label after the if (!se) statement. Futhermore, the latter can be removed because se is always NULL when reaching this point. Signed-off-by: Vincent Guittot Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Phil Auld Link: https://lkml.kernel.org/r/20200513135502.4672-1-vincent.guittot@linaro.org --- kernel/sched/fair.c | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 9a58874ef104..4e586863827b 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5512,28 +5512,27 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) list_add_leaf_cfs_rq(cfs_rq); } + /* At this point se is NULL and we are at root level*/ + add_nr_running(rq, 1); + + /* + * Since new tasks are assigned an initial util_avg equal to + * half of the spare capacity of their CPU, tiny tasks have the + * ability to cross the overutilized threshold, which will + * result in the load balancer ruining all the task placement + * done by EAS. As a way to mitigate that effect, do not account + * for the first enqueue operation of new tasks during the + * overutilized flag detection. + * + * A better way of solving this problem would be to wait for + * the PELT signals of tasks to converge before taking them + * into account, but that is not straightforward to implement, + * and the following generally works well enough in practice. + */ + if (flags & ENQUEUE_WAKEUP) + update_overutilized_status(rq); + enqueue_throttle: - if (!se) { - add_nr_running(rq, 1); - /* - * Since new tasks are assigned an initial util_avg equal to - * half of the spare capacity of their CPU, tiny tasks have the - * ability to cross the overutilized threshold, which will - * result in the load balancer ruining all the task placement - * done by EAS. As a way to mitigate that effect, do not account - * for the first enqueue operation of new tasks during the - * overutilized flag detection. - * - * A better way of solving this problem would be to wait for - * the PELT signals of tasks to converge before taking them - * into account, but that is not straightforward to implement, - * and the following generally works well enough in practice. - */ - if (flags & ENQUEUE_WAKEUP) - update_overutilized_status(rq); - - } - if (cfs_bandwidth_used()) { /* * When bandwidth control is enabled; the cfs_rq_throttled() From 12aa2587388de6697fd2e585ae6a90f70249540b Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Thu, 7 May 2020 11:10:39 +0800 Subject: [PATCH 0599/1043] sched/cpuacct: Use __this_cpu_add() instead of this_cpu_ptr() The cpuacct_charge() and cpuacct_account_field() are called with rq->lock held, and this means preemption(and IRQs) are indeed disabled, so it is safe to use __this_cpu_*() to allow for better code-generation. Signed-off-by: Muchun Song Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200507031039.32615-1-songmuchun@bytedance.com --- kernel/sched/cpuacct.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/cpuacct.c b/kernel/sched/cpuacct.c index 9fbb10383434..6448b0438ffb 100644 --- a/kernel/sched/cpuacct.c +++ b/kernel/sched/cpuacct.c @@ -347,7 +347,7 @@ void cpuacct_charge(struct task_struct *tsk, u64 cputime) rcu_read_lock(); for (ca = task_ca(tsk); ca; ca = parent_ca(ca)) - this_cpu_ptr(ca->cpuusage)->usages[index] += cputime; + __this_cpu_add(ca->cpuusage->usages[index], cputime); rcu_read_unlock(); } @@ -363,7 +363,7 @@ void cpuacct_account_field(struct task_struct *tsk, int index, u64 val) rcu_read_lock(); for (ca = task_ca(tsk); ca != &root_cpuacct; ca = parent_ca(ca)) - this_cpu_ptr(ca->cpustat)->cpustat[index] += val; + __this_cpu_add(ca->cpustat->cpustat[index], val); rcu_read_unlock(); } From 95d685935a2edf209fc68f52494ede4a382a6c2b Mon Sep 17 00:00:00 2001 From: Vincent Guittot Date: Wed, 6 May 2020 17:53:01 +0200 Subject: [PATCH 0600/1043] sched/pelt: Sync util/runnable_sum with PELT window when propagating update_tg_cfs_*() propagate the impact of the attach/detach of an entity down into the cfs_rq hierarchy and must keep the sync with the current pelt window. Even if we can't sync child cfs_rq and its group se, we can sync the group se and its parent cfs_rq with current position in the PELT window. In fact, we must keep them sync in order to stay also synced with others entities and group entities that are already attached to the cfs_rq. Signed-off-by: Vincent Guittot Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200506155301.14288-1-vincent.guittot@linaro.org --- kernel/sched/fair.c | 49 +++++++++++++++++++++++++-------------------- kernel/sched/pelt.c | 24 ++++++++++++++++++++++ 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 4e586863827b..44b0c8edc260 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -3441,52 +3441,46 @@ static inline void update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) { long delta = gcfs_rq->avg.util_avg - se->avg.util_avg; + /* + * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. + * See ___update_load_avg() for details. + */ + u32 divider = LOAD_AVG_MAX - 1024 + cfs_rq->avg.period_contrib; /* Nothing to update */ if (!delta) return; - /* - * The relation between sum and avg is: - * - * LOAD_AVG_MAX - 1024 + sa->period_contrib - * - * however, the PELT windows are not aligned between grq and gse. - */ - /* Set new sched_entity's utilization */ se->avg.util_avg = gcfs_rq->avg.util_avg; - se->avg.util_sum = se->avg.util_avg * LOAD_AVG_MAX; + se->avg.util_sum = se->avg.util_avg * divider; /* Update parent cfs_rq utilization */ add_positive(&cfs_rq->avg.util_avg, delta); - cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * LOAD_AVG_MAX; + cfs_rq->avg.util_sum = cfs_rq->avg.util_avg * divider; } static inline void update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) { long delta = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; + /* + * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. + * See ___update_load_avg() for details. + */ + u32 divider = LOAD_AVG_MAX - 1024 + cfs_rq->avg.period_contrib; /* Nothing to update */ if (!delta) return; - /* - * The relation between sum and avg is: - * - * LOAD_AVG_MAX - 1024 + sa->period_contrib - * - * however, the PELT windows are not aligned between grq and gse. - */ - /* Set new sched_entity's runnable */ se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; - se->avg.runnable_sum = se->avg.runnable_avg * LOAD_AVG_MAX; + se->avg.runnable_sum = se->avg.runnable_avg * divider; /* Update parent cfs_rq runnable */ add_positive(&cfs_rq->avg.runnable_avg, delta); - cfs_rq->avg.runnable_sum = cfs_rq->avg.runnable_avg * LOAD_AVG_MAX; + cfs_rq->avg.runnable_sum = cfs_rq->avg.runnable_avg * divider; } static inline void @@ -3496,19 +3490,26 @@ update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq unsigned long load_avg; u64 load_sum = 0; s64 delta_sum; + u32 divider; if (!runnable_sum) return; gcfs_rq->prop_runnable_sum = 0; + /* + * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. + * See ___update_load_avg() for details. + */ + divider = LOAD_AVG_MAX - 1024 + cfs_rq->avg.period_contrib; + if (runnable_sum >= 0) { /* * Add runnable; clip at LOAD_AVG_MAX. Reflects that until * the CPU is saturated running == runnable. */ runnable_sum += se->avg.load_sum; - runnable_sum = min(runnable_sum, (long)LOAD_AVG_MAX); + runnable_sum = min_t(long, runnable_sum, divider); } else { /* * Estimate the new unweighted runnable_sum of the gcfs_rq by @@ -3533,7 +3534,7 @@ update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq runnable_sum = max(runnable_sum, running_sum); load_sum = (s64)se_weight(se) * runnable_sum; - load_avg = div_s64(load_sum, LOAD_AVG_MAX); + load_avg = div_s64(load_sum, divider); delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum; delta_avg = load_avg - se->avg.load_avg; @@ -3697,6 +3698,10 @@ update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) */ static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { + /* + * cfs_rq->avg.period_contrib can be used for both cfs_rq and se. + * See ___update_load_avg() for details. + */ u32 divider = LOAD_AVG_MAX - 1024 + cfs_rq->avg.period_contrib; /* diff --git a/kernel/sched/pelt.c b/kernel/sched/pelt.c index b647d04d9c8b..b4b1ff96642f 100644 --- a/kernel/sched/pelt.c +++ b/kernel/sched/pelt.c @@ -237,6 +237,30 @@ ___update_load_sum(u64 now, struct sched_avg *sa, return 1; } +/* + * When syncing *_avg with *_sum, we must take into account the current + * position in the PELT segment otherwise the remaining part of the segment + * will be considered as idle time whereas it's not yet elapsed and this will + * generate unwanted oscillation in the range [1002..1024[. + * + * The max value of *_sum varies with the position in the time segment and is + * equals to : + * + * LOAD_AVG_MAX*y + sa->period_contrib + * + * which can be simplified into: + * + * LOAD_AVG_MAX - 1024 + sa->period_contrib + * + * because LOAD_AVG_MAX*y == LOAD_AVG_MAX-1024 + * + * The same care must be taken when a sched entity is added, updated or + * removed from a cfs_rq and we need to update sched_avg. Scheduler entities + * and the cfs rq, to which they are attached, have the same position in the + * time segment because they use the same clock. This means that we can use + * the period_contrib of cfs_rq when updating the sched_avg of a sched_entity + * if it's more convenient. + */ static __always_inline void ___update_load_avg(struct sched_avg *sa, unsigned long load) { From 04f5c362ec6d3ff0e14f1c05230b550da7f528a4 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 7 May 2020 14:21:41 -0500 Subject: [PATCH 0601/1043] sched/fair: Replace zero-length array with flexible-array The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] sizeof(flexible-array-member) triggers a warning because flexible array members have incomplete type[1]. There are some instances of code in which the sizeof operator is being incorrectly/erroneously applied to zero-length arrays and the result is zero. Such instances may be hiding some bugs. So, this work (flexible-array member conversions) will also help to get completely rid of those sorts of issues. This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200507192141.GA16183@embeddedor --- kernel/sched/fair.c | 2 +- kernel/sched/sched.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 44b0c8edc260..01f94cf52783 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1094,7 +1094,7 @@ struct numa_group { * more by CPU use than by memory faults. */ unsigned long *faults_cpu; - unsigned long faults[0]; + unsigned long faults[]; }; /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 21416b30c520..2bd2a222318a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1462,7 +1462,7 @@ struct sched_group { * by attaching extra space to the end of the structure, * depending on how many CPUs the kernel has booted up with) */ - unsigned long cpumask[0]; + unsigned long cpumask[]; }; static inline struct cpumask *sched_group_span(struct sched_group *sg) From dbe9337109c2705f08e6a00392f991eb2d2570a5 Mon Sep 17 00:00:00 2001 From: Muchun Song Date: Mon, 20 Apr 2020 15:04:53 +0800 Subject: [PATCH 0602/1043] sched/cpuacct: Fix charge cpuacct.usage_sys The user_mode(task_pt_regs(tsk)) always return true for user thread, and false for kernel thread. So it means that the cpuacct.usage_sys is the time that kernel thread uses not the time that thread uses in the kernel mode. We can try get_irq_regs() first, if it is NULL, then we can fall back to task_pt_regs(). Signed-off-by: Muchun Song Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20200420070453.76815-1-songmuchun@bytedance.com --- kernel/sched/cpuacct.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/sched/cpuacct.c b/kernel/sched/cpuacct.c index 6448b0438ffb..941c28cf9738 100644 --- a/kernel/sched/cpuacct.c +++ b/kernel/sched/cpuacct.c @@ -5,6 +5,7 @@ * Based on the work by Paul Menage (menage@google.com) and Balbir Singh * (balbir@in.ibm.com). */ +#include #include "sched.h" /* Time spent by the tasks of the CPU accounting group executing in ... */ @@ -339,7 +340,7 @@ void cpuacct_charge(struct task_struct *tsk, u64 cputime) { struct cpuacct *ca; int index = CPUACCT_STAT_SYSTEM; - struct pt_regs *regs = task_pt_regs(tsk); + struct pt_regs *regs = get_irq_regs() ? : task_pt_regs(tsk); if (regs && user_mode(regs)) index = CPUACCT_STAT_USER; From d505b8af58912ae1e1a211fabc9995b19bd40828 Mon Sep 17 00:00:00 2001 From: Huaixin Chang Date: Sat, 25 Apr 2020 18:52:48 +0800 Subject: [PATCH 0603/1043] sched: Defend cfs and rt bandwidth quota against overflow When users write some huge number into cpu.cfs_quota_us or cpu.rt_runtime_us, overflow might happen during to_ratio() shifts of schedulable checks. to_ratio() could be altered to avoid unnecessary internal overflow, but min_cfs_quota_period is less than 1 << BW_SHIFT, so a cutoff would still be needed. Set a cap MAX_BW for cfs_quota_us and rt_runtime_us to prevent overflow. Signed-off-by: Huaixin Chang Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Ben Segall Link: https://lkml.kernel.org/r/20200425105248.60093-1-changhuaixin@linux.alibaba.com --- kernel/sched/core.c | 8 ++++++++ kernel/sched/rt.c | 12 +++++++++++- kernel/sched/sched.h | 2 ++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 74fb89b5ce3e..fa905b6daafa 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7379,6 +7379,8 @@ static DEFINE_MUTEX(cfs_constraints_mutex); const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */ static const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */ +/* More than 203 days if BW_SHIFT equals 20. */ +static const u64 max_cfs_runtime = MAX_BW * NSEC_PER_USEC; static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); @@ -7406,6 +7408,12 @@ static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) if (period > max_cfs_quota_period) return -EINVAL; + /* + * Bound quota to defend quota against overflow during bandwidth shift. + */ + if (quota != RUNTIME_INF && quota > max_cfs_runtime) + return -EINVAL; + /* * Prevent race between setting of cfs_rq->runtime_enabled and * unthrottle_offline_cfs_rqs(). diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index df11d88c9895..6d60ba21ed29 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -9,6 +9,8 @@ int sched_rr_timeslice = RR_TIMESLICE; int sysctl_sched_rr_timeslice = (MSEC_PER_SEC / HZ) * RR_TIMESLICE; +/* More than 4 hours if BW_SHIFT equals 20. */ +static const u64 max_rt_runtime = MAX_BW; static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun); @@ -2585,6 +2587,12 @@ static int tg_set_rt_bandwidth(struct task_group *tg, if (rt_period == 0) return -EINVAL; + /* + * Bound quota to defend quota against overflow during bandwidth shift. + */ + if (rt_runtime != RUNTIME_INF && rt_runtime > max_rt_runtime) + return -EINVAL; + mutex_lock(&rt_constraints_mutex); err = __rt_schedulable(tg, rt_period, rt_runtime); if (err) @@ -2702,7 +2710,9 @@ static int sched_rt_global_validate(void) return -EINVAL; if ((sysctl_sched_rt_runtime != RUNTIME_INF) && - (sysctl_sched_rt_runtime > sysctl_sched_rt_period)) + ((sysctl_sched_rt_runtime > sysctl_sched_rt_period) || + ((u64)sysctl_sched_rt_runtime * + NSEC_PER_USEC > max_rt_runtime))) return -EINVAL; return 0; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 2bd2a222318a..f7ab6334e992 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1915,6 +1915,8 @@ extern void init_dl_inactive_task_timer(struct sched_dl_entity *dl_se); #define BW_SHIFT 20 #define BW_UNIT (1 << BW_SHIFT) #define RATIO_SHIFT 8 +#define MAX_BW_BITS (64 - BW_SHIFT) +#define MAX_BW ((1ULL << MAX_BW_BITS) - 1) unsigned long to_ratio(u64 period, u64 runtime); extern void init_entity_runnable_average(struct sched_entity *se); From 866c70f28e49e6d757a2af67f300dcd343c28e7e Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Wed, 20 May 2020 13:12:37 +0800 Subject: [PATCH 0604/1043] MIPS: SGI-IP27: Remove not used includes and comment in ip27-timer.c After commit 0ce5ebd24d25 ("mfd: ioc3: Add driver for SGI IOC3 chip"), the related includes and comment about ioc3 are not used any more in ip27-timer.c, remove them. Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/sgi-ip27/ip27-timer.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/arch/mips/sgi-ip27/ip27-timer.c b/arch/mips/sgi-ip27/ip27-timer.c index 11ffb3e5f3d4..115b1d9dc76d 100644 --- a/arch/mips/sgi-ip27/ip27-timer.c +++ b/arch/mips/sgi-ip27/ip27-timer.c @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -31,10 +30,6 @@ #define TICK_SIZE (tick_nsec / 1000) -/* Includes for ioc3_init(). */ -#include -#include - static int rt_next_event(unsigned long delta, struct clock_event_device *evt) { unsigned int cpu = smp_processor_id(); From c9c2e9c596e90fcf9ee0bf16672f7b938f39b913 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Tue, 19 May 2020 23:22:30 +0200 Subject: [PATCH 0605/1043] MIPS: ingenic: Add missing include Add missing include which adds the prototype to plat_time_init(). Fixes: f932449c11da ("MIPS: ingenic: Drop obsolete code, merge the rest in setup.c") Signed-off-by: Paul Cercueil Reported-by: kbuild test robot Signed-off-by: Thomas Bogendoerfer --- arch/mips/jz4740/setup.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/mips/jz4740/setup.c b/arch/mips/jz4740/setup.c index 142cf127bf9e..61468a87775c 100644 --- a/arch/mips/jz4740/setup.c +++ b/arch/mips/jz4740/setup.c @@ -20,6 +20,7 @@ #include #include #include +#include #define JZ4740_EMC_BASE_ADDR 0x13010000 From febd668d375caf13a7fcd93b3498366854de854a Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 18 May 2020 06:30:09 -0400 Subject: [PATCH 0606/1043] rcuwait: avoid lockdep splats from rcuwait_active() rcuwait_active only returns whether w->task is not NULL. This is exactly one of the usecases that are mentioned in the documentation for rcu_access_pointer() where it is correct to bypass lockdep checks. This avoids a splat from kvm_vcpu_on_spin(). Reported-by: Wanpeng Li Tested-by: Wanpeng Li Acked-by: Davidlohr Bueso Cc: Peter Zijlstra Signed-off-by: Paolo Bonzini --- include/linux/rcuwait.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/rcuwait.h b/include/linux/rcuwait.h index c1414ce44abc..61c56cca95c4 100644 --- a/include/linux/rcuwait.h +++ b/include/linux/rcuwait.h @@ -31,7 +31,7 @@ static inline void rcuwait_init(struct rcuwait *w) */ static inline int rcuwait_active(struct rcuwait *w) { - return !!rcu_dereference(w->task); + return !!rcu_access_pointer(w->task); } extern int rcuwait_wake_up(struct rcuwait *w); From 7769e18c201aa88eade5556faf9da7f2bc15bb8a Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Mon, 6 Apr 2020 02:15:14 +0200 Subject: [PATCH 0607/1043] scsi: storvsc: Re-init stor_chns when a channel interrupt is re-assigned For each storvsc_device, storvsc keeps track of the channel target CPUs associated to the device (alloced_cpus) and it uses this information to fill a "cache" (stor_chns) mapping CPU->channel according to a certain heuristic. Update the alloced_cpus mask and the stor_chns array when a channel of the storvsc device is re-assigned to a different CPU. Signed-off-by: Andrea Parri (Microsoft) Cc: "James E.J. Bottomley" Cc: "Martin K. Petersen" Cc: Link: https://lore.kernel.org/r/20200406001514.19876-12-parri.andrea@gmail.com Reviewed-by; Long Li Reviewed-by: Michael Kelley [ wei: fix a small issue reported by kbuild test robot ] Signed-off-by: Wei Liu --- drivers/hv/vmbus_drv.c | 4 ++ drivers/scsi/storvsc_drv.c | 96 ++++++++++++++++++++++++++++++++++---- include/linux/hyperv.h | 3 ++ 3 files changed, 95 insertions(+), 8 deletions(-) diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 5d24b25fb5aa..28c009c7a9cd 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1777,6 +1777,10 @@ static ssize_t target_cpu_store(struct vmbus_channel *channel, * in on a CPU that is different from the channel target_cpu value. */ + if (channel->change_target_cpu_callback) + (*channel->change_target_cpu_callback)(channel, + channel->target_cpu, target_cpu); + channel->target_cpu = target_cpu; channel->target_vp = hv_cpu_number_to_vp_number(target_cpu); channel->numa_node = cpu_to_node(target_cpu); diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c index fb41636519ee..072ed8728657 100644 --- a/drivers/scsi/storvsc_drv.c +++ b/drivers/scsi/storvsc_drv.c @@ -621,6 +621,64 @@ get_in_err: } +static void storvsc_change_target_cpu(struct vmbus_channel *channel, u32 old, + u32 new) +{ + struct storvsc_device *stor_device; + struct vmbus_channel *cur_chn; + bool old_is_alloced = false; + struct hv_device *device; + unsigned long flags; + int cpu; + + device = channel->primary_channel ? + channel->primary_channel->device_obj + : channel->device_obj; + stor_device = get_out_stor_device(device); + if (!stor_device) + return; + + /* See storvsc_do_io() -> get_og_chn(). */ + spin_lock_irqsave(&device->channel->lock, flags); + + /* + * Determines if the storvsc device has other channels assigned to + * the "old" CPU to update the alloced_cpus mask and the stor_chns + * array. + */ + if (device->channel != channel && device->channel->target_cpu == old) { + cur_chn = device->channel; + old_is_alloced = true; + goto old_is_alloced; + } + list_for_each_entry(cur_chn, &device->channel->sc_list, sc_list) { + if (cur_chn == channel) + continue; + if (cur_chn->target_cpu == old) { + old_is_alloced = true; + goto old_is_alloced; + } + } + +old_is_alloced: + if (old_is_alloced) + WRITE_ONCE(stor_device->stor_chns[old], cur_chn); + else + cpumask_clear_cpu(old, &stor_device->alloced_cpus); + + /* "Flush" the stor_chns array. */ + for_each_possible_cpu(cpu) { + if (stor_device->stor_chns[cpu] && !cpumask_test_cpu( + cpu, &stor_device->alloced_cpus)) + WRITE_ONCE(stor_device->stor_chns[cpu], NULL); + } + + WRITE_ONCE(stor_device->stor_chns[new], channel); + cpumask_set_cpu(new, &stor_device->alloced_cpus); + + spin_unlock_irqrestore(&device->channel->lock, flags); +} + static void handle_sc_creation(struct vmbus_channel *new_sc) { struct hv_device *device = new_sc->primary_channel->device_obj; @@ -648,6 +706,8 @@ static void handle_sc_creation(struct vmbus_channel *new_sc) return; } + new_sc->change_target_cpu_callback = storvsc_change_target_cpu; + /* Add the sub-channel to the array of available channels. */ stor_device->stor_chns[new_sc->target_cpu] = new_sc; cpumask_set_cpu(new_sc->target_cpu, &stor_device->alloced_cpus); @@ -876,6 +936,8 @@ static int storvsc_channel_init(struct hv_device *device, bool is_fc) if (stor_device->stor_chns == NULL) return -ENOMEM; + device->channel->change_target_cpu_callback = storvsc_change_target_cpu; + stor_device->stor_chns[device->channel->target_cpu] = device->channel; cpumask_set_cpu(device->channel->target_cpu, &stor_device->alloced_cpus); @@ -1248,8 +1310,10 @@ static struct vmbus_channel *get_og_chn(struct storvsc_device *stor_device, const struct cpumask *node_mask; int num_channels, tgt_cpu; - if (stor_device->num_sc == 0) + if (stor_device->num_sc == 0) { + stor_device->stor_chns[q_num] = stor_device->device->channel; return stor_device->device->channel; + } /* * Our channel array is sparsley populated and we @@ -1258,7 +1322,6 @@ static struct vmbus_channel *get_og_chn(struct storvsc_device *stor_device, * The strategy is simple: * I. Ensure NUMA locality * II. Distribute evenly (best effort) - * III. Mapping is persistent. */ node_mask = cpumask_of_node(cpu_to_node(q_num)); @@ -1268,8 +1331,10 @@ static struct vmbus_channel *get_og_chn(struct storvsc_device *stor_device, if (cpumask_test_cpu(tgt_cpu, node_mask)) num_channels++; } - if (num_channels == 0) + if (num_channels == 0) { + stor_device->stor_chns[q_num] = stor_device->device->channel; return stor_device->device->channel; + } hash_qnum = q_num; while (hash_qnum >= num_channels) @@ -1295,6 +1360,7 @@ static int storvsc_do_io(struct hv_device *device, struct storvsc_device *stor_device; struct vstor_packet *vstor_packet; struct vmbus_channel *outgoing_channel, *channel; + unsigned long flags; int ret = 0; const struct cpumask *node_mask; int tgt_cpu; @@ -1308,10 +1374,11 @@ static int storvsc_do_io(struct hv_device *device, request->device = device; /* - * Select an an appropriate channel to send the request out. + * Select an appropriate channel to send the request out. */ - if (stor_device->stor_chns[q_num] != NULL) { - outgoing_channel = stor_device->stor_chns[q_num]; + /* See storvsc_change_target_cpu(). */ + outgoing_channel = READ_ONCE(stor_device->stor_chns[q_num]); + if (outgoing_channel != NULL) { if (outgoing_channel->target_cpu == q_num) { /* * Ideally, we want to pick a different channel if @@ -1324,7 +1391,10 @@ static int storvsc_do_io(struct hv_device *device, continue; if (tgt_cpu == q_num) continue; - channel = stor_device->stor_chns[tgt_cpu]; + channel = READ_ONCE( + stor_device->stor_chns[tgt_cpu]); + if (channel == NULL) + continue; if (hv_get_avail_to_write_percent( &channel->outbound) > ring_avail_percent_lowater) { @@ -1350,7 +1420,10 @@ static int storvsc_do_io(struct hv_device *device, for_each_cpu(tgt_cpu, &stor_device->alloced_cpus) { if (cpumask_test_cpu(tgt_cpu, node_mask)) continue; - channel = stor_device->stor_chns[tgt_cpu]; + channel = READ_ONCE( + stor_device->stor_chns[tgt_cpu]); + if (channel == NULL) + continue; if (hv_get_avail_to_write_percent( &channel->outbound) > ring_avail_percent_lowater) { @@ -1360,7 +1433,14 @@ static int storvsc_do_io(struct hv_device *device, } } } else { + spin_lock_irqsave(&device->channel->lock, flags); + outgoing_channel = stor_device->stor_chns[q_num]; + if (outgoing_channel != NULL) { + spin_unlock_irqrestore(&device->channel->lock, flags); + goto found_channel; + } outgoing_channel = get_og_chn(stor_device, q_num); + spin_unlock_irqrestore(&device->channel->lock, flags); } found_channel: diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index b85d7580f2c1..cd64ab7bb3b7 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -773,6 +773,9 @@ struct vmbus_channel { void (*onchannel_callback)(void *context); void *channel_callback_context; + void (*change_target_cpu_callback)(struct vmbus_channel *channel, + u32 old, u32 new); + /* * Synchronize channel scheduling and channel removal; see the inline * comments in vmbus_chan_sched() and vmbus_reset_channel_cb(). From 677b0ce5d66c5445440f3cedd3c20ff1665c0f46 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 14 Apr 2020 16:23:43 +0100 Subject: [PATCH 0608/1043] drivers: hv: remove redundant assignment to pointer primary_channel The pointer primary_channel is being assigned with a value that is never used. The assignment is redundant and can be removed. Move the definition of primary_channel to a narrower scope. Addresses-Coverity: ("Unused value") Signed-off-by: Colin Ian King Link: https://lore.kernel.org/r/20200414152343.243166-1-colin.king@canonical.com [ wei: move primary_channel and update commit message ] Signed-off-by: Wei Liu --- drivers/hv/channel_mgmt.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index ffd7fffa5f83..fde806d6525b 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -400,7 +400,6 @@ static void vmbus_release_relid(u32 relid) void hv_process_channel_removal(struct vmbus_channel *channel) { - struct vmbus_channel *primary_channel; unsigned long flags; lockdep_assert_held(&vmbus_connection.channel_mutex); @@ -425,10 +424,8 @@ void hv_process_channel_removal(struct vmbus_channel *channel) if (channel->primary_channel == NULL) { list_del(&channel->listentry); - - primary_channel = channel; } else { - primary_channel = channel->primary_channel; + struct vmbus_channel *primary_channel = channel->primary_channel; spin_lock_irqsave(&primary_channel->lock, flags); list_del(&channel->sc_list); spin_unlock_irqrestore(&primary_channel->lock, flags); From 7357b1df744c2a3bcbe00cea0eef1509d004f488 Mon Sep 17 00:00:00 2001 From: Michael Kelley Date: Wed, 22 Apr 2020 12:57:34 -0700 Subject: [PATCH 0609/1043] KVM: x86: hyperv: Remove duplicate definitions of Reference TSC Page The Hyper-V Reference TSC Page structure is defined twice. struct ms_hyperv_tsc_page has padding out to a full 4 Kbyte page size. But the padding is not needed because the declaration includes a union with HV_HYP_PAGE_SIZE. KVM uses the second definition, which is struct _HV_REFERENCE_TSC_PAGE, because it does not have the padding. Fix the duplication by removing the padding from ms_hyperv_tsc_page. Fix up the KVM code to use it. Remove the no longer used struct _HV_REFERENCE_TSC_PAGE. There is no functional change. Signed-off-by: Michael Kelley Acked-by: Paolo Bonzini Reviewed-by: Vitaly Kuznetsov Link: https://lore.kernel.org/r/20200422195737.10223-2-mikelley@microsoft.com Signed-off-by: Wei Liu --- arch/x86/include/asm/hyperv-tlfs.h | 8 -------- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/hyperv.c | 4 ++-- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h index 29336574d0bc..0e4d76920957 100644 --- a/arch/x86/include/asm/hyperv-tlfs.h +++ b/arch/x86/include/asm/hyperv-tlfs.h @@ -303,7 +303,6 @@ struct ms_hyperv_tsc_page { u32 reserved1; volatile u64 tsc_scale; volatile s64 tsc_offset; - u64 reserved2[509]; } __packed; /* @@ -433,13 +432,6 @@ enum HV_GENERIC_SET_FORMAT { */ #define HV_CLOCK_HZ (NSEC_PER_SEC/100) -typedef struct _HV_REFERENCE_TSC_PAGE { - __u32 tsc_sequence; - __u32 res1; - __u64 tsc_scale; - __s64 tsc_offset; -} __packed HV_REFERENCE_TSC_PAGE, *PHV_REFERENCE_TSC_PAGE; - /* Define the number of synthetic interrupt sources. */ #define HV_SYNIC_SINT_COUNT (16) /* Define the expected SynIC version. */ diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 42a2d0d3984a..4698343b9a05 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -865,7 +865,7 @@ struct kvm_hv { u64 hv_crash_param[HV_X64_MSR_CRASH_PARAMS]; u64 hv_crash_ctl; - HV_REFERENCE_TSC_PAGE tsc_ref; + struct ms_hyperv_tsc_page tsc_ref; struct idr conn_to_evt; diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index bcefa9d4e57e..1f3c6fd3cdaa 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -900,7 +900,7 @@ static int kvm_hv_msr_set_crash_data(struct kvm_vcpu *vcpu, * These two equivalencies are implemented in this function. */ static bool compute_tsc_page_parameters(struct pvclock_vcpu_time_info *hv_clock, - HV_REFERENCE_TSC_PAGE *tsc_ref) + struct ms_hyperv_tsc_page *tsc_ref) { u64 max_mul; @@ -941,7 +941,7 @@ void kvm_hv_setup_tsc_page(struct kvm *kvm, u64 gfn; BUILD_BUG_ON(sizeof(tsc_seq) != sizeof(hv->tsc_ref.tsc_sequence)); - BUILD_BUG_ON(offsetof(HV_REFERENCE_TSC_PAGE, tsc_sequence) != 0); + BUILD_BUG_ON(offsetof(struct ms_hyperv_tsc_page, tsc_sequence) != 0); if (!(hv->hv_tsc_page & HV_X64_MSR_TSC_REFERENCE_ENABLE)) return; From a8a42d0284f18a64d540a604aecec9f961964e3b Mon Sep 17 00:00:00 2001 From: Michael Kelley Date: Wed, 22 Apr 2020 12:57:35 -0700 Subject: [PATCH 0610/1043] x86/hyperv: Remove HV_PROCESSOR_POWER_STATE #defines The HV_PROCESSOR_POWER_STATE_C #defines date back to year 2010, but they are not in the TLFS v6.0 document and are not used anywhere in Linux. Remove them. Signed-off-by: Michael Kelley Link: https://lore.kernel.org/r/20200422195737.10223-3-mikelley@microsoft.com Signed-off-by: Wei Liu --- arch/x86/include/asm/hyperv-tlfs.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h index 0e4d76920957..2dd1ceb2bcf8 100644 --- a/arch/x86/include/asm/hyperv-tlfs.h +++ b/arch/x86/include/asm/hyperv-tlfs.h @@ -390,11 +390,6 @@ struct hv_tsc_emulation_status { #define HV_X64_MSR_TSC_REFERENCE_ENABLE 0x00000001 #define HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT 12 -#define HV_PROCESSOR_POWER_STATE_C0 0 -#define HV_PROCESSOR_POWER_STATE_C1 1 -#define HV_PROCESSOR_POWER_STATE_C2 2 -#define HV_PROCESSOR_POWER_STATE_C3 3 - #define HV_FLUSH_ALL_PROCESSORS BIT(0) #define HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES BIT(1) #define HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY BIT(2) From c55a844f46f958b4df804eccd734b86795e40359 Mon Sep 17 00:00:00 2001 From: Michael Kelley Date: Wed, 22 Apr 2020 12:57:36 -0700 Subject: [PATCH 0611/1043] x86/hyperv: Split hyperv-tlfs.h into arch dependent and independent files In preparation for adding ARM64 support, split hyperv-tlfs.h into architecture dependent and architecture independent files, similar to what has been done with mshyperv.h. Move architecture independent definitions into include/asm-generic/hyperv-tlfs.h. The split will avoid duplicating significant lines of code in the ARM64 version of hyperv-tlfs.h. The split has no functional impact. Some of the common definitions have "X64" in the symbol name. Change these to remove the "X64" in the architecture independent version of hyperv-tlfs.h, but add aliases with the "X64" in the x86 version so that x86 code will continue to compile. A later patch set will change all the references and allow removal of the aliases. Signed-off-by: Michael Kelley Link: https://lore.kernel.org/r/20200422195737.10223-4-mikelley@microsoft.com Signed-off-by: Wei Liu --- MAINTAINERS | 1 + arch/x86/include/asm/hyperv-tlfs.h | 459 +++-------------------------- include/asm-generic/hyperv-tlfs.h | 442 +++++++++++++++++++++++++++ 3 files changed, 479 insertions(+), 423 deletions(-) create mode 100644 include/asm-generic/hyperv-tlfs.h diff --git a/MAINTAINERS b/MAINTAINERS index b816a453b10e..bfb9f384b3f6 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -7883,6 +7883,7 @@ F: drivers/pci/controller/pci-hyperv.c F: drivers/scsi/storvsc_drv.c F: drivers/uio/uio_hv_generic.c F: drivers/video/fbdev/hyperv_fb.c +F: include/asm-generic/hyperv-tlfs.h F: include/asm-generic/mshyperv.h F: include/clocksource/hyperv_timer.h F: include/linux/hyperv.h diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h index 2dd1ceb2bcf8..4e91f6118d5d 100644 --- a/arch/x86/include/asm/hyperv-tlfs.h +++ b/arch/x86/include/asm/hyperv-tlfs.h @@ -11,17 +11,6 @@ #include #include - -/* - * While not explicitly listed in the TLFS, Hyper-V always runs with a page size - * of 4096. These definitions are used when communicating with Hyper-V using - * guest physical pages and guest physical page addresses, since the guest page - * size may not be 4096 on all architectures. - */ -#define HV_HYP_PAGE_SHIFT 12 -#define HV_HYP_PAGE_SIZE BIT(HV_HYP_PAGE_SHIFT) -#define HV_HYP_PAGE_MASK (~(HV_HYP_PAGE_SIZE - 1)) - /* * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent * is set by CPUID(HvCpuIdFunctionVersionAndFeatures). @@ -39,78 +28,41 @@ #define HYPERV_CPUID_MAX 0x4000ffff /* - * Feature identification. EAX indicates which features are available - * to the partition based upon the current partition privileges. - * These are HYPERV_CPUID_FEATURES.EAX bits. + * Aliases for Group A features that have X64 in the name. + * On x86/x64 these are HYPERV_CPUID_FEATURES.EAX bits. */ -/* VP Runtime (HV_X64_MSR_VP_RUNTIME) available */ -#define HV_X64_MSR_VP_RUNTIME_AVAILABLE BIT(0) -/* Partition Reference Counter (HV_X64_MSR_TIME_REF_COUNT) available*/ -#define HV_MSR_TIME_REF_COUNT_AVAILABLE BIT(1) -/* - * Basic SynIC MSRs (HV_X64_MSR_SCONTROL through HV_X64_MSR_EOM - * and HV_X64_MSR_SINT0 through HV_X64_MSR_SINT15) available - */ -#define HV_X64_MSR_SYNIC_AVAILABLE BIT(2) -/* - * Synthetic Timer MSRs (HV_X64_MSR_STIMER0_CONFIG through - * HV_X64_MSR_STIMER3_COUNT) available - */ -#define HV_MSR_SYNTIMER_AVAILABLE BIT(3) -/* - * APIC access MSRs (HV_X64_MSR_EOI, HV_X64_MSR_ICR and HV_X64_MSR_TPR) - * are available - */ -#define HV_X64_MSR_APIC_ACCESS_AVAILABLE BIT(4) -/* Hypercall MSRs (HV_X64_MSR_GUEST_OS_ID and HV_X64_MSR_HYPERCALL) available*/ -#define HV_X64_MSR_HYPERCALL_AVAILABLE BIT(5) -/* Access virtual processor index MSR (HV_X64_MSR_VP_INDEX) available*/ -#define HV_X64_MSR_VP_INDEX_AVAILABLE BIT(6) -/* Virtual system reset MSR (HV_X64_MSR_RESET) is available*/ -#define HV_X64_MSR_RESET_AVAILABLE BIT(7) -/* - * Access statistics pages MSRs (HV_X64_MSR_STATS_PARTITION_RETAIL_PAGE, - * HV_X64_MSR_STATS_PARTITION_INTERNAL_PAGE, HV_X64_MSR_STATS_VP_RETAIL_PAGE, - * HV_X64_MSR_STATS_VP_INTERNAL_PAGE) available - */ -#define HV_X64_MSR_STAT_PAGES_AVAILABLE BIT(8) -/* Partition reference TSC MSR is available */ -#define HV_MSR_REFERENCE_TSC_AVAILABLE BIT(9) -/* Partition Guest IDLE MSR is available */ -#define HV_X64_MSR_GUEST_IDLE_AVAILABLE BIT(10) -/* - * There is a single feature flag that signifies if the partition has access - * to MSRs with local APIC and TSC frequencies. - */ -#define HV_X64_ACCESS_FREQUENCY_MSRS BIT(11) -/* AccessReenlightenmentControls privilege */ -#define HV_X64_ACCESS_REENLIGHTENMENT BIT(13) -/* AccessTscInvariantControls privilege */ -#define HV_X64_ACCESS_TSC_INVARIANT BIT(15) +#define HV_X64_MSR_VP_RUNTIME_AVAILABLE \ + HV_MSR_VP_RUNTIME_AVAILABLE +#define HV_X64_MSR_SYNIC_AVAILABLE \ + HV_MSR_SYNIC_AVAILABLE +#define HV_X64_MSR_APIC_ACCESS_AVAILABLE \ + HV_MSR_APIC_ACCESS_AVAILABLE +#define HV_X64_MSR_HYPERCALL_AVAILABLE \ + HV_MSR_HYPERCALL_AVAILABLE +#define HV_X64_MSR_VP_INDEX_AVAILABLE \ + HV_MSR_VP_INDEX_AVAILABLE +#define HV_X64_MSR_RESET_AVAILABLE \ + HV_MSR_RESET_AVAILABLE +#define HV_X64_MSR_GUEST_IDLE_AVAILABLE \ + HV_MSR_GUEST_IDLE_AVAILABLE +#define HV_X64_ACCESS_FREQUENCY_MSRS \ + HV_ACCESS_FREQUENCY_MSRS +#define HV_X64_ACCESS_REENLIGHTENMENT \ + HV_ACCESS_REENLIGHTENMENT +#define HV_X64_ACCESS_TSC_INVARIANT \ + HV_ACCESS_TSC_INVARIANT /* - * Feature identification: indicates which flags were specified at partition - * creation. The format is the same as the partition creation flag structure - * defined in section Partition Creation Flags. - * These are HYPERV_CPUID_FEATURES.EBX bits. + * Aliases for Group B features that have X64 in the name. + * On x86/x64 these are HYPERV_CPUID_FEATURES.EBX bits. */ -#define HV_X64_CREATE_PARTITIONS BIT(0) -#define HV_X64_ACCESS_PARTITION_ID BIT(1) -#define HV_X64_ACCESS_MEMORY_POOL BIT(2) -#define HV_X64_ADJUST_MESSAGE_BUFFERS BIT(3) -#define HV_X64_POST_MESSAGES BIT(4) -#define HV_X64_SIGNAL_EVENTS BIT(5) -#define HV_X64_CREATE_PORT BIT(6) -#define HV_X64_CONNECT_PORT BIT(7) -#define HV_X64_ACCESS_STATS BIT(8) -#define HV_X64_DEBUGGING BIT(11) -#define HV_X64_CPU_POWER_MANAGEMENT BIT(12) +#define HV_X64_POST_MESSAGES HV_POST_MESSAGES +#define HV_X64_SIGNAL_EVENTS HV_SIGNAL_EVENTS /* - * Feature identification. EDX indicates which miscellaneous features - * are available to the partition. - * These are HYPERV_CPUID_FEATURES.EDX bits. + * Group D Features. The bit assignments are custom to each architecture. + * On x86/x64 these are HYPERV_CPUID_FEATURES.EDX bits. */ /* The MWAIT instruction is available (per section MONITOR / MWAIT) */ #define HV_X64_MWAIT_AVAILABLE BIT(0) @@ -187,7 +139,7 @@ * processor, except for virtual processors that are reported as sibling SMT * threads. */ -#define HV_X64_NO_NONARCH_CORESHARING BIT(18) +#define HV_X64_NO_NONARCH_CORESHARING BIT(18) /* Nested features. These are HYPERV_CPUID_NESTED_FEATURES.EAX bits. */ #define HV_X64_NESTED_DIRECT_FLUSH BIT(17) @@ -295,42 +247,6 @@ union hv_x64_msr_hypercall_contents { } __packed; }; -/* - * TSC page layout. - */ -struct ms_hyperv_tsc_page { - volatile u32 tsc_sequence; - u32 reserved1; - volatile u64 tsc_scale; - volatile s64 tsc_offset; -} __packed; - -/* - * The guest OS needs to register the guest ID with the hypervisor. - * The guest ID is a 64 bit entity and the structure of this ID is - * specified in the Hyper-V specification: - * - * msdn.microsoft.com/en-us/library/windows/hardware/ff542653%28v=vs.85%29.aspx - * - * While the current guideline does not specify how Linux guest ID(s) - * need to be generated, our plan is to publish the guidelines for - * Linux and other guest operating systems that currently are hosted - * on Hyper-V. The implementation here conforms to this yet - * unpublished guidelines. - * - * - * Bit(s) - * 63 - Indicates if the OS is Open Source or not; 1 is Open Source - * 62:56 - Os Type; Linux is 0x100 - * 55:48 - Distro specific identification - * 47:16 - Linux kernel version number - * 15:0 - Distro specific identification - * - * - */ - -#define HV_LINUX_VENDOR_ID 0x8100 - struct hv_reenlightenment_control { __u64 vector:8; __u64 reserved1:8; @@ -354,31 +270,12 @@ struct hv_tsc_emulation_status { #define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK \ (~((1ull << HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT) - 1)) -/* - * Crash notification (HV_X64_MSR_CRASH_CTL) flags. - */ -#define HV_CRASH_CTL_CRASH_NOTIFY_MSG BIT_ULL(62) -#define HV_CRASH_CTL_CRASH_NOTIFY BIT_ULL(63) #define HV_X64_MSR_CRASH_PARAMS \ (1 + (HV_X64_MSR_CRASH_P4 - HV_X64_MSR_CRASH_P0)) #define HV_IPI_LOW_VECTOR 0x10 #define HV_IPI_HIGH_VECTOR 0xff -/* Declare the various hypercall operations. */ -#define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE 0x0002 -#define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST 0x0003 -#define HVCALL_NOTIFY_LONG_SPIN_WAIT 0x0008 -#define HVCALL_SEND_IPI 0x000b -#define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX 0x0013 -#define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX 0x0014 -#define HVCALL_SEND_IPI_EX 0x0015 -#define HVCALL_POST_MESSAGE 0x005c -#define HVCALL_SIGNAL_EVENT 0x005d -#define HVCALL_RETARGET_INTERRUPT 0x007e -#define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE 0x00af -#define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_LIST 0x00b0 - #define HV_X64_MSR_VP_ASSIST_PAGE_ENABLE 0x00000001 #define HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT 12 #define HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_MASK \ @@ -390,63 +287,6 @@ struct hv_tsc_emulation_status { #define HV_X64_MSR_TSC_REFERENCE_ENABLE 0x00000001 #define HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT 12 -#define HV_FLUSH_ALL_PROCESSORS BIT(0) -#define HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES BIT(1) -#define HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY BIT(2) -#define HV_FLUSH_USE_EXTENDED_RANGE_FORMAT BIT(3) - -enum HV_GENERIC_SET_FORMAT { - HV_GENERIC_SET_SPARSE_4K, - HV_GENERIC_SET_ALL, -}; - -#define HV_PARTITION_ID_SELF ((u64)-1) - -#define HV_HYPERCALL_RESULT_MASK GENMASK_ULL(15, 0) -#define HV_HYPERCALL_FAST_BIT BIT(16) -#define HV_HYPERCALL_VARHEAD_OFFSET 17 -#define HV_HYPERCALL_REP_COMP_OFFSET 32 -#define HV_HYPERCALL_REP_COMP_MASK GENMASK_ULL(43, 32) -#define HV_HYPERCALL_REP_START_OFFSET 48 -#define HV_HYPERCALL_REP_START_MASK GENMASK_ULL(59, 48) - -/* hypercall status code */ -#define HV_STATUS_SUCCESS 0 -#define HV_STATUS_INVALID_HYPERCALL_CODE 2 -#define HV_STATUS_INVALID_HYPERCALL_INPUT 3 -#define HV_STATUS_INVALID_ALIGNMENT 4 -#define HV_STATUS_INVALID_PARAMETER 5 -#define HV_STATUS_INSUFFICIENT_MEMORY 11 -#define HV_STATUS_INVALID_PORT_ID 17 -#define HV_STATUS_INVALID_CONNECTION_ID 18 -#define HV_STATUS_INSUFFICIENT_BUFFERS 19 - -/* - * The Hyper-V TimeRefCount register and the TSC - * page provide a guest VM clock with 100ns tick rate - */ -#define HV_CLOCK_HZ (NSEC_PER_SEC/100) - -/* Define the number of synthetic interrupt sources. */ -#define HV_SYNIC_SINT_COUNT (16) -/* Define the expected SynIC version. */ -#define HV_SYNIC_VERSION_1 (0x1) -/* Valid SynIC vectors are 16-255. */ -#define HV_SYNIC_FIRST_VALID_VECTOR (16) - -#define HV_SYNIC_CONTROL_ENABLE (1ULL << 0) -#define HV_SYNIC_SIMP_ENABLE (1ULL << 0) -#define HV_SYNIC_SIEFP_ENABLE (1ULL << 0) -#define HV_SYNIC_SINT_MASKED (1ULL << 16) -#define HV_SYNIC_SINT_AUTO_EOI (1ULL << 17) -#define HV_SYNIC_SINT_VECTOR_MASK (0xFF) - -#define HV_SYNIC_STIMER_COUNT (4) - -/* Define synthetic interrupt controller message constants. */ -#define HV_MESSAGE_SIZE (256) -#define HV_MESSAGE_PAYLOAD_BYTE_COUNT (240) -#define HV_MESSAGE_PAYLOAD_QWORD_COUNT (30) /* Define hypervisor message types. */ enum hv_message_type { @@ -457,76 +297,25 @@ enum hv_message_type { HVMSG_GPA_INTERCEPT = 0x80000001, /* Timer notification messages. */ - HVMSG_TIMER_EXPIRED = 0x80000010, + HVMSG_TIMER_EXPIRED = 0x80000010, /* Error messages. */ HVMSG_INVALID_VP_REGISTER_VALUE = 0x80000020, HVMSG_UNRECOVERABLE_EXCEPTION = 0x80000021, - HVMSG_UNSUPPORTED_FEATURE = 0x80000022, + HVMSG_UNSUPPORTED_FEATURE = 0x80000022, /* Trace buffer complete messages. */ HVMSG_EVENTLOG_BUFFERCOMPLETE = 0x80000040, /* Platform-specific processor intercept messages. */ - HVMSG_X64_IOPORT_INTERCEPT = 0x80010000, + HVMSG_X64_IOPORT_INTERCEPT = 0x80010000, HVMSG_X64_MSR_INTERCEPT = 0x80010001, - HVMSG_X64_CPUID_INTERCEPT = 0x80010002, + HVMSG_X64_CPUID_INTERCEPT = 0x80010002, HVMSG_X64_EXCEPTION_INTERCEPT = 0x80010003, - HVMSG_X64_APIC_EOI = 0x80010004, - HVMSG_X64_LEGACY_FP_ERROR = 0x80010005 + HVMSG_X64_APIC_EOI = 0x80010004, + HVMSG_X64_LEGACY_FP_ERROR = 0x80010005 }; -/* Define synthetic interrupt controller message flags. */ -union hv_message_flags { - __u8 asu8; - struct { - __u8 msg_pending:1; - __u8 reserved:7; - } __packed; -}; - -/* Define port identifier type. */ -union hv_port_id { - __u32 asu32; - struct { - __u32 id:24; - __u32 reserved:8; - } __packed u; -}; - -/* Define synthetic interrupt controller message header. */ -struct hv_message_header { - __u32 message_type; - __u8 payload_size; - union hv_message_flags message_flags; - __u8 reserved[2]; - union { - __u64 sender; - union hv_port_id port; - }; -} __packed; - -/* Define synthetic interrupt controller message format. */ -struct hv_message { - struct hv_message_header header; - union { - __u64 payload[HV_MESSAGE_PAYLOAD_QWORD_COUNT]; - } u; -} __packed; - -/* Define the synthetic interrupt message page layout. */ -struct hv_message_page { - struct hv_message sint_message[HV_SYNIC_SINT_COUNT]; -} __packed; - -/* Define timer message payload structure. */ -struct hv_timer_message_payload { - __u32 timer_index; - __u32 reserved; - __u64 expiration_time; /* When the timer expired */ - __u64 delivery_time; /* When the message was delivered */ -} __packed; - struct hv_nested_enlightenments_control { struct { __u32 directhypercall:1; @@ -754,187 +543,11 @@ struct hv_enlightened_vmcs { #define HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL 0xFFFF -/* Define synthetic interrupt controller flag constants. */ -#define HV_EVENT_FLAGS_COUNT (256 * 8) -#define HV_EVENT_FLAGS_LONG_COUNT (256 / sizeof(unsigned long)) - -/* - * Synthetic timer configuration. - */ -union hv_stimer_config { - u64 as_uint64; - struct { - u64 enable:1; - u64 periodic:1; - u64 lazy:1; - u64 auto_enable:1; - u64 apic_vector:8; - u64 direct_mode:1; - u64 reserved_z0:3; - u64 sintx:4; - u64 reserved_z1:44; - } __packed; -}; - - -/* Define the synthetic interrupt controller event flags format. */ -union hv_synic_event_flags { - unsigned long flags[HV_EVENT_FLAGS_LONG_COUNT]; -}; - -/* Define SynIC control register. */ -union hv_synic_scontrol { - u64 as_uint64; - struct { - u64 enable:1; - u64 reserved:63; - } __packed; -}; - -/* Define synthetic interrupt source. */ -union hv_synic_sint { - u64 as_uint64; - struct { - u64 vector:8; - u64 reserved1:8; - u64 masked:1; - u64 auto_eoi:1; - u64 polling:1; - u64 reserved2:45; - } __packed; -}; - -/* Define the format of the SIMP register */ -union hv_synic_simp { - u64 as_uint64; - struct { - u64 simp_enabled:1; - u64 preserved:11; - u64 base_simp_gpa:52; - } __packed; -}; - -/* Define the format of the SIEFP register */ -union hv_synic_siefp { - u64 as_uint64; - struct { - u64 siefp_enabled:1; - u64 preserved:11; - u64 base_siefp_gpa:52; - } __packed; -}; - -struct hv_vpset { - u64 format; - u64 valid_bank_mask; - u64 bank_contents[]; -} __packed; - -/* HvCallSendSyntheticClusterIpi hypercall */ -struct hv_send_ipi { - u32 vector; - u32 reserved; - u64 cpu_mask; -} __packed; - -/* HvCallSendSyntheticClusterIpiEx hypercall */ -struct hv_send_ipi_ex { - u32 vector; - u32 reserved; - struct hv_vpset vp_set; -} __packed; - -/* HvFlushGuestPhysicalAddressSpace hypercalls */ -struct hv_guest_mapping_flush { - u64 address_space; - u64 flags; -} __packed; - -/* - * HV_MAX_FLUSH_PAGES = "additional_pages" + 1. It's limited - * by the bitwidth of "additional_pages" in union hv_gpa_page_range. - */ -#define HV_MAX_FLUSH_PAGES (2048) - -/* HvFlushGuestPhysicalAddressList hypercall */ -union hv_gpa_page_range { - u64 address_space; - struct { - u64 additional_pages:11; - u64 largepage:1; - u64 basepfn:52; - } page; -}; - -/* - * All input flush parameters should be in single page. The max flush - * count is equal with how many entries of union hv_gpa_page_range can - * be populated into the input parameter page. - */ -#define HV_MAX_FLUSH_REP_COUNT ((HV_HYP_PAGE_SIZE - 2 * sizeof(u64)) / \ - sizeof(union hv_gpa_page_range)) - -struct hv_guest_mapping_flush_list { - u64 address_space; - u64 flags; - union hv_gpa_page_range gpa_list[HV_MAX_FLUSH_REP_COUNT]; -}; - -/* HvFlushVirtualAddressSpace, HvFlushVirtualAddressList hypercalls */ -struct hv_tlb_flush { - u64 address_space; - u64 flags; - u64 processor_mask; - u64 gva_list[]; -} __packed; - -/* HvFlushVirtualAddressSpaceEx, HvFlushVirtualAddressListEx hypercalls */ -struct hv_tlb_flush_ex { - u64 address_space; - u64 flags; - struct hv_vpset hv_vp_set; - u64 gva_list[]; -} __packed; - struct hv_partition_assist_pg { u32 tlb_lock_count; }; -union hv_msi_entry { - u64 as_uint64; - struct { - u32 address; - u32 data; - } __packed; -}; -struct hv_interrupt_entry { - u32 source; /* 1 for MSI(-X) */ - u32 reserved1; - union hv_msi_entry msi_entry; -} __packed; +#include -/* - * flags for hv_device_interrupt_target.flags - */ -#define HV_DEVICE_INTERRUPT_TARGET_MULTICAST 1 -#define HV_DEVICE_INTERRUPT_TARGET_PROCESSOR_SET 2 - -struct hv_device_interrupt_target { - u32 vector; - u32 flags; - union { - u64 vp_mask; - struct hv_vpset vp_set; - }; -} __packed; - -/* HvRetargetDeviceInterrupt hypercall */ -struct hv_retarget_device_interrupt { - u64 partition_id; /* use "self" */ - u64 device_id; - struct hv_interrupt_entry int_entry; - u64 reserved2; - struct hv_device_interrupt_target int_target; -} __packed __aligned(8); #endif diff --git a/include/asm-generic/hyperv-tlfs.h b/include/asm-generic/hyperv-tlfs.h new file mode 100644 index 000000000000..1f92ef92eb56 --- /dev/null +++ b/include/asm-generic/hyperv-tlfs.h @@ -0,0 +1,442 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * This file contains definitions from Hyper-V Hypervisor Top-Level Functional + * Specification (TLFS): + * https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/reference/tlfs + */ + +#ifndef _ASM_GENERIC_HYPERV_TLFS_H +#define _ASM_GENERIC_HYPERV_TLFS_H + +#include +#include +#include + +/* + * While not explicitly listed in the TLFS, Hyper-V always runs with a page size + * of 4096. These definitions are used when communicating with Hyper-V using + * guest physical pages and guest physical page addresses, since the guest page + * size may not be 4096 on all architectures. + */ +#define HV_HYP_PAGE_SHIFT 12 +#define HV_HYP_PAGE_SIZE BIT(HV_HYP_PAGE_SHIFT) +#define HV_HYP_PAGE_MASK (~(HV_HYP_PAGE_SIZE - 1)) + +/* + * Hyper-V provides two categories of flags relevant to guest VMs. The + * "Features" category indicates specific functionality that is available + * to guests on this particular instance of Hyper-V. The "Features" + * are presented in four groups, each of which is 32 bits. The group A + * and B definitions are common across architectures and are listed here. + * However, not all flags are relevant on all architectures. + * + * Groups C and D vary across architectures and are listed in the + * architecture specific portion of hyperv-tlfs.h. Some of these flags exist + * on multiple architectures, but the bit positions are different so they + * cannot appear in the generic portion of hyperv-tlfs.h. + * + * The "Enlightenments" category provides recommendations on whether to use + * specific enlightenments that are available. The Enlighenments are a single + * group of 32 bits, but they vary across architectures and are listed in + * the architecture specific portion of hyperv-tlfs.h. + */ + +/* + * Group A Features. + */ + +/* VP Runtime register available */ +#define HV_MSR_VP_RUNTIME_AVAILABLE BIT(0) +/* Partition Reference Counter available*/ +#define HV_MSR_TIME_REF_COUNT_AVAILABLE BIT(1) +/* Basic SynIC register available */ +#define HV_MSR_SYNIC_AVAILABLE BIT(2) +/* Synthetic Timer registers available */ +#define HV_MSR_SYNTIMER_AVAILABLE BIT(3) +/* Virtual APIC assist and VP assist page registers available */ +#define HV_MSR_APIC_ACCESS_AVAILABLE BIT(4) +/* Hypercall and Guest OS ID registers available*/ +#define HV_MSR_HYPERCALL_AVAILABLE BIT(5) +/* Access virtual processor index register available*/ +#define HV_MSR_VP_INDEX_AVAILABLE BIT(6) +/* Virtual system reset register available*/ +#define HV_MSR_RESET_AVAILABLE BIT(7) +/* Access statistics page registers available */ +#define HV_MSR_STAT_PAGES_AVAILABLE BIT(8) +/* Partition reference TSC register is available */ +#define HV_MSR_REFERENCE_TSC_AVAILABLE BIT(9) +/* Partition Guest IDLE register is available */ +#define HV_MSR_GUEST_IDLE_AVAILABLE BIT(10) +/* Partition local APIC and TSC frequency registers available */ +#define HV_ACCESS_FREQUENCY_MSRS BIT(11) +/* AccessReenlightenmentControls privilege */ +#define HV_ACCESS_REENLIGHTENMENT BIT(13) +/* AccessTscInvariantControls privilege */ +#define HV_ACCESS_TSC_INVARIANT BIT(15) + +/* + * Group B features. + */ +#define HV_CREATE_PARTITIONS BIT(0) +#define HV_ACCESS_PARTITION_ID BIT(1) +#define HV_ACCESS_MEMORY_POOL BIT(2) +#define HV_ADJUST_MESSAGE_BUFFERS BIT(3) +#define HV_POST_MESSAGES BIT(4) +#define HV_SIGNAL_EVENTS BIT(5) +#define HV_CREATE_PORT BIT(6) +#define HV_CONNECT_PORT BIT(7) +#define HV_ACCESS_STATS BIT(8) +#define HV_DEBUGGING BIT(11) +#define HV_CPU_POWER_MANAGEMENT BIT(12) + + +/* + * TSC page layout. + */ +struct ms_hyperv_tsc_page { + volatile u32 tsc_sequence; + u32 reserved1; + volatile u64 tsc_scale; + volatile s64 tsc_offset; +} __packed; + +/* + * The guest OS needs to register the guest ID with the hypervisor. + * The guest ID is a 64 bit entity and the structure of this ID is + * specified in the Hyper-V specification: + * + * msdn.microsoft.com/en-us/library/windows/hardware/ff542653%28v=vs.85%29.aspx + * + * While the current guideline does not specify how Linux guest ID(s) + * need to be generated, our plan is to publish the guidelines for + * Linux and other guest operating systems that currently are hosted + * on Hyper-V. The implementation here conforms to this yet + * unpublished guidelines. + * + * + * Bit(s) + * 63 - Indicates if the OS is Open Source or not; 1 is Open Source + * 62:56 - Os Type; Linux is 0x100 + * 55:48 - Distro specific identification + * 47:16 - Linux kernel version number + * 15:0 - Distro specific identification + * + * + */ + +#define HV_LINUX_VENDOR_ID 0x8100 + +/* + * Crash notification flags. + */ +#define HV_CRASH_CTL_CRASH_NOTIFY_MSG BIT_ULL(62) +#define HV_CRASH_CTL_CRASH_NOTIFY BIT_ULL(63) + +/* Declare the various hypercall operations. */ +#define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE 0x0002 +#define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST 0x0003 +#define HVCALL_NOTIFY_LONG_SPIN_WAIT 0x0008 +#define HVCALL_SEND_IPI 0x000b +#define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX 0x0013 +#define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX 0x0014 +#define HVCALL_SEND_IPI_EX 0x0015 +#define HVCALL_POST_MESSAGE 0x005c +#define HVCALL_SIGNAL_EVENT 0x005d +#define HVCALL_RETARGET_INTERRUPT 0x007e +#define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE 0x00af +#define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_LIST 0x00b0 + +#define HV_FLUSH_ALL_PROCESSORS BIT(0) +#define HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES BIT(1) +#define HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY BIT(2) +#define HV_FLUSH_USE_EXTENDED_RANGE_FORMAT BIT(3) + +enum HV_GENERIC_SET_FORMAT { + HV_GENERIC_SET_SPARSE_4K, + HV_GENERIC_SET_ALL, +}; + +#define HV_PARTITION_ID_SELF ((u64)-1) +#define HV_VP_INDEX_SELF ((u32)-2) + +#define HV_HYPERCALL_RESULT_MASK GENMASK_ULL(15, 0) +#define HV_HYPERCALL_FAST_BIT BIT(16) +#define HV_HYPERCALL_VARHEAD_OFFSET 17 +#define HV_HYPERCALL_REP_COMP_OFFSET 32 +#define HV_HYPERCALL_REP_COMP_1 BIT_ULL(32) +#define HV_HYPERCALL_REP_COMP_MASK GENMASK_ULL(43, 32) +#define HV_HYPERCALL_REP_START_OFFSET 48 +#define HV_HYPERCALL_REP_START_MASK GENMASK_ULL(59, 48) + +/* hypercall status code */ +#define HV_STATUS_SUCCESS 0 +#define HV_STATUS_INVALID_HYPERCALL_CODE 2 +#define HV_STATUS_INVALID_HYPERCALL_INPUT 3 +#define HV_STATUS_INVALID_ALIGNMENT 4 +#define HV_STATUS_INVALID_PARAMETER 5 +#define HV_STATUS_INSUFFICIENT_MEMORY 11 +#define HV_STATUS_INVALID_PORT_ID 17 +#define HV_STATUS_INVALID_CONNECTION_ID 18 +#define HV_STATUS_INSUFFICIENT_BUFFERS 19 + +/* + * The Hyper-V TimeRefCount register and the TSC + * page provide a guest VM clock with 100ns tick rate + */ +#define HV_CLOCK_HZ (NSEC_PER_SEC/100) + +/* Define the number of synthetic interrupt sources. */ +#define HV_SYNIC_SINT_COUNT (16) +/* Define the expected SynIC version. */ +#define HV_SYNIC_VERSION_1 (0x1) +/* Valid SynIC vectors are 16-255. */ +#define HV_SYNIC_FIRST_VALID_VECTOR (16) + +#define HV_SYNIC_CONTROL_ENABLE (1ULL << 0) +#define HV_SYNIC_SIMP_ENABLE (1ULL << 0) +#define HV_SYNIC_SIEFP_ENABLE (1ULL << 0) +#define HV_SYNIC_SINT_MASKED (1ULL << 16) +#define HV_SYNIC_SINT_AUTO_EOI (1ULL << 17) +#define HV_SYNIC_SINT_VECTOR_MASK (0xFF) + +#define HV_SYNIC_STIMER_COUNT (4) + +/* Define synthetic interrupt controller message constants. */ +#define HV_MESSAGE_SIZE (256) +#define HV_MESSAGE_PAYLOAD_BYTE_COUNT (240) +#define HV_MESSAGE_PAYLOAD_QWORD_COUNT (30) + +/* Define synthetic interrupt controller message flags. */ +union hv_message_flags { + __u8 asu8; + struct { + __u8 msg_pending:1; + __u8 reserved:7; + } __packed; +}; + +/* Define port identifier type. */ +union hv_port_id { + __u32 asu32; + struct { + __u32 id:24; + __u32 reserved:8; + } __packed u; +}; + +/* Define synthetic interrupt controller message header. */ +struct hv_message_header { + __u32 message_type; + __u8 payload_size; + union hv_message_flags message_flags; + __u8 reserved[2]; + union { + __u64 sender; + union hv_port_id port; + }; +} __packed; + +/* Define synthetic interrupt controller message format. */ +struct hv_message { + struct hv_message_header header; + union { + __u64 payload[HV_MESSAGE_PAYLOAD_QWORD_COUNT]; + } u; +} __packed; + +/* Define the synthetic interrupt message page layout. */ +struct hv_message_page { + struct hv_message sint_message[HV_SYNIC_SINT_COUNT]; +} __packed; + +/* Define timer message payload structure. */ +struct hv_timer_message_payload { + __u32 timer_index; + __u32 reserved; + __u64 expiration_time; /* When the timer expired */ + __u64 delivery_time; /* When the message was delivered */ +} __packed; + + +/* Define synthetic interrupt controller flag constants. */ +#define HV_EVENT_FLAGS_COUNT (256 * 8) +#define HV_EVENT_FLAGS_LONG_COUNT (256 / sizeof(unsigned long)) + +/* + * Synthetic timer configuration. + */ +union hv_stimer_config { + u64 as_uint64; + struct { + u64 enable:1; + u64 periodic:1; + u64 lazy:1; + u64 auto_enable:1; + u64 apic_vector:8; + u64 direct_mode:1; + u64 reserved_z0:3; + u64 sintx:4; + u64 reserved_z1:44; + } __packed; +}; + + +/* Define the synthetic interrupt controller event flags format. */ +union hv_synic_event_flags { + unsigned long flags[HV_EVENT_FLAGS_LONG_COUNT]; +}; + +/* Define SynIC control register. */ +union hv_synic_scontrol { + u64 as_uint64; + struct { + u64 enable:1; + u64 reserved:63; + } __packed; +}; + +/* Define synthetic interrupt source. */ +union hv_synic_sint { + u64 as_uint64; + struct { + u64 vector:8; + u64 reserved1:8; + u64 masked:1; + u64 auto_eoi:1; + u64 polling:1; + u64 reserved2:45; + } __packed; +}; + +/* Define the format of the SIMP register */ +union hv_synic_simp { + u64 as_uint64; + struct { + u64 simp_enabled:1; + u64 preserved:11; + u64 base_simp_gpa:52; + } __packed; +}; + +/* Define the format of the SIEFP register */ +union hv_synic_siefp { + u64 as_uint64; + struct { + u64 siefp_enabled:1; + u64 preserved:11; + u64 base_siefp_gpa:52; + } __packed; +}; + +struct hv_vpset { + u64 format; + u64 valid_bank_mask; + u64 bank_contents[]; +} __packed; + +/* HvCallSendSyntheticClusterIpi hypercall */ +struct hv_send_ipi { + u32 vector; + u32 reserved; + u64 cpu_mask; +} __packed; + +/* HvCallSendSyntheticClusterIpiEx hypercall */ +struct hv_send_ipi_ex { + u32 vector; + u32 reserved; + struct hv_vpset vp_set; +} __packed; + +/* HvFlushGuestPhysicalAddressSpace hypercalls */ +struct hv_guest_mapping_flush { + u64 address_space; + u64 flags; +} __packed; + +/* + * HV_MAX_FLUSH_PAGES = "additional_pages" + 1. It's limited + * by the bitwidth of "additional_pages" in union hv_gpa_page_range. + */ +#define HV_MAX_FLUSH_PAGES (2048) + +/* HvFlushGuestPhysicalAddressList hypercall */ +union hv_gpa_page_range { + u64 address_space; + struct { + u64 additional_pages:11; + u64 largepage:1; + u64 basepfn:52; + } page; +}; + +/* + * All input flush parameters should be in single page. The max flush + * count is equal with how many entries of union hv_gpa_page_range can + * be populated into the input parameter page. + */ +#define HV_MAX_FLUSH_REP_COUNT ((HV_HYP_PAGE_SIZE - 2 * sizeof(u64)) / \ + sizeof(union hv_gpa_page_range)) + +struct hv_guest_mapping_flush_list { + u64 address_space; + u64 flags; + union hv_gpa_page_range gpa_list[HV_MAX_FLUSH_REP_COUNT]; +}; + +/* HvFlushVirtualAddressSpace, HvFlushVirtualAddressList hypercalls */ +struct hv_tlb_flush { + u64 address_space; + u64 flags; + u64 processor_mask; + u64 gva_list[]; +} __packed; + +/* HvFlushVirtualAddressSpaceEx, HvFlushVirtualAddressListEx hypercalls */ +struct hv_tlb_flush_ex { + u64 address_space; + u64 flags; + struct hv_vpset hv_vp_set; + u64 gva_list[]; +} __packed; + +/* HvRetargetDeviceInterrupt hypercall */ +union hv_msi_entry { + u64 as_uint64; + struct { + u32 address; + u32 data; + } __packed; +}; + +struct hv_interrupt_entry { + u32 source; /* 1 for MSI(-X) */ + u32 reserved1; + union hv_msi_entry msi_entry; +} __packed; + +/* + * flags for hv_device_interrupt_target.flags + */ +#define HV_DEVICE_INTERRUPT_TARGET_MULTICAST 1 +#define HV_DEVICE_INTERRUPT_TARGET_PROCESSOR_SET 2 + +struct hv_device_interrupt_target { + u32 vector; + u32 flags; + union { + u64 vp_mask; + struct hv_vpset vp_set; + }; +} __packed; + +struct hv_retarget_device_interrupt { + u64 partition_id; /* use "self" */ + u64 device_id; + struct hv_interrupt_entry int_entry; + u64 reserved2; + struct hv_device_interrupt_target int_target; +} __packed __aligned(8); + +#endif From 88b42da6e3dc9adbdc3d7e1a9293ddf3d84f1021 Mon Sep 17 00:00:00 2001 From: Michael Kelley Date: Wed, 22 Apr 2020 12:57:37 -0700 Subject: [PATCH 0612/1043] asm-generic/hyperv: Add definitions for Get/SetVpRegister hypercalls Add definitions for GetVpRegister and SetVpRegister hypercalls, which are implemented for both x86 and ARM64. Signed-off-by: Michael Kelley Link: https://lore.kernel.org/r/20200422195737.10223-5-mikelley@microsoft.com Signed-off-by: Wei Liu --- include/asm-generic/hyperv-tlfs.h | 51 +++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/include/asm-generic/hyperv-tlfs.h b/include/asm-generic/hyperv-tlfs.h index 1f92ef92eb56..262fae9526b1 100644 --- a/include/asm-generic/hyperv-tlfs.h +++ b/include/asm-generic/hyperv-tlfs.h @@ -141,6 +141,8 @@ struct ms_hyperv_tsc_page { #define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX 0x0013 #define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX 0x0014 #define HVCALL_SEND_IPI_EX 0x0015 +#define HVCALL_GET_VP_REGISTERS 0x0050 +#define HVCALL_SET_VP_REGISTERS 0x0051 #define HVCALL_POST_MESSAGE 0x005c #define HVCALL_SIGNAL_EVENT 0x005d #define HVCALL_RETARGET_INTERRUPT 0x007e @@ -439,4 +441,53 @@ struct hv_retarget_device_interrupt { struct hv_device_interrupt_target int_target; } __packed __aligned(8); + +/* HvGetVpRegisters hypercall input with variable size reg name list*/ +struct hv_get_vp_registers_input { + struct { + u64 partitionid; + u32 vpindex; + u8 inputvtl; + u8 padding[3]; + } header; + struct input { + u32 name0; + u32 name1; + } element[]; +} __packed; + + +/* HvGetVpRegisters returns an array of these output elements */ +struct hv_get_vp_registers_output { + union { + struct { + u32 a; + u32 b; + u32 c; + u32 d; + } as32 __packed; + struct { + u64 low; + u64 high; + } as64 __packed; + }; +}; + +/* HvSetVpRegisters hypercall with variable size reg name/value list*/ +struct hv_set_vp_registers_input { + struct { + u64 partitionid; + u32 vpindex; + u8 inputvtl; + u8 padding[3]; + } header; + struct { + u32 name; + u32 padding1; + u64 padding2; + u64 valuelow; + u64 valuehigh; + } element[]; +} __packed; + #endif From 69f57058badded5c523871bbaaaf60b637bcf623 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 23 Apr 2020 16:45:02 +0300 Subject: [PATCH 0613/1043] hyper-v: Use UUID API for exporting the GUID (part 2) This is a follow up to the commit 1d3c9c075462 ("hyper-v: Use UUID API for exporting the GUID") which starts the conversion. There is export_guid() function which exports guid_t to the u8 array. Use it instead of open coding variant. This allows to hide the uuid_t internals. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20200423134505.78221-1-andriy.shevchenko@linux.intel.com Signed-off-by: Wei Liu --- drivers/hv/hv_trace.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/hv/hv_trace.h b/drivers/hv/hv_trace.h index a43bc76c2d5d..a888784552c8 100644 --- a/drivers/hv/hv_trace.h +++ b/drivers/hv/hv_trace.h @@ -44,10 +44,8 @@ TRACE_EVENT(vmbus_onoffer, __entry->monitorid = offer->monitorid; __entry->is_ddc_int = offer->is_dedicated_interrupt; __entry->connection_id = offer->connection_id; - memcpy(__entry->if_type, - &offer->offer.if_type.b, 16); - memcpy(__entry->if_instance, - &offer->offer.if_instance.b, 16); + export_guid(__entry->if_type, &offer->offer.if_type); + export_guid(__entry->if_instance, &offer->offer.if_instance); __entry->chn_flags = offer->offer.chn_flags; __entry->mmio_mb = offer->offer.mmio_megabytes; __entry->sub_idx = offer->offer.sub_channel_index; From 458c4475be9ae42e248963b2db732266d40408b7 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 23 Apr 2020 16:45:03 +0300 Subject: [PATCH 0614/1043] hyper-v: Supply GUID pointer to printf() like functions Drop dereference when printing the GUID with printf() like functions. This allows to hide the uuid_t internals. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20200423134505.78221-2-andriy.shevchenko@linux.intel.com Signed-off-by: Wei Liu --- drivers/hv/vmbus_drv.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 28c009c7a9cd..c1b7f4c344df 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -201,7 +201,7 @@ static ssize_t class_id_show(struct device *dev, if (!hv_dev->channel) return -ENODEV; return sprintf(buf, "{%pUl}\n", - hv_dev->channel->offermsg.offer.if_type.b); + &hv_dev->channel->offermsg.offer.if_type); } static DEVICE_ATTR_RO(class_id); @@ -213,7 +213,7 @@ static ssize_t device_id_show(struct device *dev, if (!hv_dev->channel) return -ENODEV; return sprintf(buf, "{%pUl}\n", - hv_dev->channel->offermsg.offer.if_instance.b); + &hv_dev->channel->offermsg.offer.if_instance); } static DEVICE_ATTR_RO(device_id); @@ -1991,7 +1991,7 @@ int vmbus_device_register(struct hv_device *child_device_obj) int ret; dev_set_name(&child_device_obj->device, "%pUl", - child_device_obj->channel->offermsg.offer.if_instance.b); + &child_device_obj->channel->offermsg.offer.if_instance); child_device_obj->device.bus = &hv_bus; child_device_obj->device.parent = &hv_acpi_dev->dev; From 0027e3fd6d907c81097ab656de2521854f37b9a6 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 23 Apr 2020 16:45:04 +0300 Subject: [PATCH 0615/1043] hyper-v: Replace open-coded variant of %*phN specifier printf() like functions in the kernel have extensions, such as %*phN to dump small pieces of memory as hex values. Replace print_alias_name() with the direct use of %*phN. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20200423134505.78221-3-andriy.shevchenko@linux.intel.com Signed-off-by: Wei Liu --- drivers/hv/vmbus_drv.c | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index c1b7f4c344df..48e62028213b 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -117,14 +117,6 @@ static int vmbus_exists(void) return 0; } -#define VMBUS_ALIAS_LEN ((sizeof((struct hv_vmbus_device_id *)0)->guid) * 2) -static void print_alias_name(struct hv_device *hv_dev, char *alias_name) -{ - int i; - for (i = 0; i < VMBUS_ALIAS_LEN; i += 2) - sprintf(&alias_name[i], "%02x", hv_dev->dev_type.b[i/2]); -} - static u8 channel_monitor_group(const struct vmbus_channel *channel) { return (u8)channel->offermsg.monitorid / 32; @@ -221,10 +213,8 @@ static ssize_t modalias_show(struct device *dev, struct device_attribute *dev_attr, char *buf) { struct hv_device *hv_dev = device_to_hv_device(dev); - char alias_name[VMBUS_ALIAS_LEN + 1]; - print_alias_name(hv_dev, alias_name); - return sprintf(buf, "vmbus:%s\n", alias_name); + return sprintf(buf, "vmbus:%*phN\n", UUID_SIZE, &hv_dev->dev_type); } static DEVICE_ATTR_RO(modalias); @@ -693,12 +683,9 @@ __ATTRIBUTE_GROUPS(vmbus_dev); static int vmbus_uevent(struct device *device, struct kobj_uevent_env *env) { struct hv_device *dev = device_to_hv_device(device); - int ret; - char alias_name[VMBUS_ALIAS_LEN + 1]; + const char *format = "MODALIAS=vmbus:%*phN"; - print_alias_name(dev, alias_name); - ret = add_uevent_var(env, "MODALIAS=vmbus:%s", alias_name); - return ret; + return add_uevent_var(env, format, UUID_SIZE, &dev->dev_type); } static const struct hv_vmbus_device_id * From b7d18c57c94a38a81c1dbcfee0d63153ffaf1856 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 23 Apr 2020 16:45:05 +0300 Subject: [PATCH 0616/1043] hyper-v: Switch to use UUID types directly uuid_le is an alias for guid_t and is going to be removed in the future. Replace it with original type. Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20200423134505.78221-4-andriy.shevchenko@linux.intel.com Signed-off-by: Wei Liu --- include/linux/mod_devicetable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h index 4c2ddd0941a7..f36b1cf5ffe3 100644 --- a/include/linux/mod_devicetable.h +++ b/include/linux/mod_devicetable.h @@ -434,7 +434,7 @@ struct virtio_device_id { * For Hyper-V devices we use the device guid as the id. */ struct hv_vmbus_device_id { - uuid_le guid; + guid_t guid; kernel_ulong_t driver_data; /* Data private to the driver */ }; From 723c425f2947dd0ecdde270cb8463b22b3e59603 Mon Sep 17 00:00:00 2001 From: Wei Liu Date: Wed, 6 May 2020 16:08:05 +0000 Subject: [PATCH 0617/1043] Driver: hv: vmbus: drop a no long applicable comment None of the things mentioned in the comment is initialized in hv_init. They've been moved elsewhere. Signed-off-by: Wei Liu Link: https://lore.kernel.org/r/20200506160806.118965-1-wei.liu@kernel.org Reviewed-by: Michael Kelley --- drivers/hv/vmbus_drv.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 48e62028213b..c2a4a7c0b99a 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1404,7 +1404,6 @@ static int vmbus_bus_init(void) { int ret; - /* Hypervisor initialization...setup hypercall page..etc */ ret = hv_init(); if (ret != 0) { pr_err("Unable to initialize the hypervisor - 0x%x\n", ret); From db5871e85533f06ffe1795c1cb9b9153860d1e4f Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 7 May 2020 13:53:23 -0500 Subject: [PATCH 0618/1043] vmbus: Replace zero-length array with flexible-array The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] sizeof(flexible-array-member) triggers a warning because flexible array members have incomplete type[1]. There are some instances of code in which the sizeof operator is being incorrectly/erroneously applied to zero-length arrays and the result is zero. Such instances may be hiding some bugs. So, this work (flexible-array member conversions) will also help to get completely rid of those sorts of issues. This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Link: https://lore.kernel.org/r/20200507185323.GA14416@embeddedor Signed-off-by: Wei Liu --- include/linux/hyperv.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index cd64ab7bb3b7..d783847d8cb4 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -117,7 +117,7 @@ struct hv_ring_buffer { * Ring data starts here + RingDataStartOffset * !!! DO NOT place any fields below this !!! */ - u8 buffer[0]; + u8 buffer[]; } __packed; struct hv_ring_buffer_info { @@ -313,7 +313,7 @@ struct vmadd_remove_transfer_page_set { struct gpa_range { u32 byte_count; u32 byte_offset; - u64 pfn_array[0]; + u64 pfn_array[]; }; /* @@ -563,7 +563,7 @@ struct vmbus_channel_gpadl_header { u32 gpadl; u16 range_buflen; u16 rangecount; - struct gpa_range range[0]; + struct gpa_range range[]; } __packed; /* This is the followup packet that contains more PFNs. */ @@ -571,7 +571,7 @@ struct vmbus_channel_gpadl_body { struct vmbus_channel_message_header header; u32 msgnumber; u32 gpadl; - u64 pfn[0]; + u64 pfn[]; } __packed; struct vmbus_channel_gpadl_created { @@ -679,7 +679,7 @@ struct vmbus_channel_msginfo { * The channel message that goes out on the "wire". * It will contain at minimum the VMBUS_CHANNEL_MESSAGE_HEADER header */ - unsigned char msg[0]; + unsigned char msg[]; }; struct vmbus_close_msg { From c5d6082d35e0bcc20a26a067ffcfddcb5257e580 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 15 May 2020 17:57:52 +0100 Subject: [PATCH 0619/1043] irqchip/gic-v3-its: Balance initial LPI affinity across CPUs When mapping a LPI, the ITS driver picks the first possible affinity, which is in most cases CPU0, assuming that if that's not suitable, someone will come and set the affinity to something more interesting. It apparently isn't the case, and people complain of poor performance when many interrupts are glued to the same CPU. So let's place the interrupts by finding the "least loaded" CPU (that is, the one that has the fewer LPIs mapped to it). So called 'managed' interrupts are an interesting case where the affinity is actually dictated by the kernel itself, and we should honor this. Reported-by: John Garry Signed-off-by: Marc Zyngier Tested-by: John Garry Link: https://lore.kernel.org/r/1575642904-58295-1-git-send-email-john.garry@huawei.com Link: https://lore.kernel.org/r/20200515165752.121296-3-maz@kernel.org --- drivers/irqchip/irq-gic-v3-its.c | 127 ++++++++++++++++++++++++------- 1 file changed, 100 insertions(+), 27 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index 4eb8441d0c2b..cd685f521c77 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -1541,15 +1541,104 @@ static void its_dec_lpi_count(struct irq_data *d, int cpu) atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); } +static unsigned int cpumask_pick_least_loaded(struct irq_data *d, + const struct cpumask *cpu_mask) +{ + unsigned int cpu = nr_cpu_ids, tmp; + int count = S32_MAX; + + for_each_cpu(tmp, cpu_mask) { + int this_count = its_read_lpi_count(d, tmp); + if (this_count < count) { + cpu = tmp; + count = this_count; + } + } + + return cpu; +} + +/* + * As suggested by Thomas Gleixner in: + * https://lore.kernel.org/r/87h80q2aoc.fsf@nanos.tec.linutronix.de + */ +static int its_select_cpu(struct irq_data *d, + const struct cpumask *aff_mask) +{ + struct its_device *its_dev = irq_data_get_irq_chip_data(d); + cpumask_var_t tmpmask; + int cpu, node; + + if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC)) + return -ENOMEM; + + node = its_dev->its->numa_node; + + if (!irqd_affinity_is_managed(d)) { + /* First try the NUMA node */ + if (node != NUMA_NO_NODE) { + /* + * Try the intersection of the affinity mask and the + * node mask (and the online mask, just to be safe). + */ + cpumask_and(tmpmask, cpumask_of_node(node), aff_mask); + cpumask_and(tmpmask, tmpmask, cpu_online_mask); + + /* + * Ideally, we would check if the mask is empty, and + * try again on the full node here. + * + * But it turns out that the way ACPI describes the + * affinity for ITSs only deals about memory, and + * not target CPUs, so it cannot describe a single + * ITS placed next to two NUMA nodes. + * + * Instead, just fallback on the online mask. This + * diverges from Thomas' suggestion above. + */ + cpu = cpumask_pick_least_loaded(d, tmpmask); + if (cpu < nr_cpu_ids) + goto out; + + /* If we can't cross sockets, give up */ + if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144)) + goto out; + + /* If the above failed, expand the search */ + } + + /* Try the intersection of the affinity and online masks */ + cpumask_and(tmpmask, aff_mask, cpu_online_mask); + + /* If that doesn't fly, the online mask is the last resort */ + if (cpumask_empty(tmpmask)) + cpumask_copy(tmpmask, cpu_online_mask); + + cpu = cpumask_pick_least_loaded(d, tmpmask); + } else { + cpumask_and(tmpmask, irq_data_get_affinity_mask(d), cpu_online_mask); + + /* If we cannot cross sockets, limit the search to that node */ + if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) && + node != NUMA_NO_NODE) + cpumask_and(tmpmask, tmpmask, cpumask_of_node(node)); + + cpu = cpumask_pick_least_loaded(d, tmpmask); + } +out: + free_cpumask_var(tmpmask); + + pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu); + return cpu; +} + static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, bool force) { - unsigned int cpu; - const struct cpumask *cpu_mask = cpu_online_mask; struct its_device *its_dev = irq_data_get_irq_chip_data(d); struct its_collection *target_col; u32 id = its_get_event_id(d); - int prev_cpu; + int cpu, prev_cpu; /* A forwarded interrupt should use irq_set_vcpu_affinity */ if (irqd_is_forwarded_to_vcpu(d)) @@ -1558,18 +1647,12 @@ static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, prev_cpu = its_dev->event_map.col_map[id]; its_dec_lpi_count(d, prev_cpu); - /* lpi cannot be routed to a redistributor that is on a foreign node */ - if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) { - if (its_dev->its->numa_node >= 0) { - cpu_mask = cpumask_of_node(its_dev->its->numa_node); - if (!cpumask_intersects(mask_val, cpu_mask)) - goto err; - } - } + if (!force) + cpu = its_select_cpu(d, mask_val); + else + cpu = cpumask_pick_least_loaded(d, mask_val); - cpu = cpumask_any_and(mask_val, cpu_mask); - - if (cpu >= nr_cpu_ids) + if (cpu < 0 || cpu >= nr_cpu_ids) goto err; /* don't set the affinity when the target cpu is same as current one */ @@ -3473,21 +3556,11 @@ static int its_irq_domain_activate(struct irq_domain *domain, { struct its_device *its_dev = irq_data_get_irq_chip_data(d); u32 event = its_get_event_id(d); - const struct cpumask *cpu_mask = cpu_online_mask; int cpu; - /* get the cpu_mask of local node */ - if (its_dev->its->numa_node >= 0) - cpu_mask = cpumask_of_node(its_dev->its->numa_node); - - /* Bind the LPI to the first possible CPU */ - cpu = cpumask_first_and(cpu_mask, cpu_online_mask); - if (cpu >= nr_cpu_ids) { - if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) - return -EINVAL; - - cpu = cpumask_first(cpu_online_mask); - } + cpu = its_select_cpu(d, cpu_online_mask); + if (cpu < 0 || cpu >= nr_cpu_ids) + return -EINVAL; its_inc_lpi_count(d, cpu); its_dev->event_map.col_map[event] = cpu; From 128516e49de67d10d52fba62ef8d482b220ac4b0 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 20 May 2020 15:03:06 +0300 Subject: [PATCH 0620/1043] iio: dummy_evgen: Fix use after free on error in iio_dummy_evgen_create() We need to preserve the "iio_evgen->irq_sim_domain" error code before we free "iio_evgen" otherwise it leads to a use after free. Fixes: 337cbeb2c13e ("genirq/irq_sim: Simplify the API") Signed-off-by: Dan Carpenter Signed-off-by: Marc Zyngier --- drivers/iio/dummy/iio_dummy_evgen.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/iio/dummy/iio_dummy_evgen.c b/drivers/iio/dummy/iio_dummy_evgen.c index 409fe0f7df1c..ee85d596e528 100644 --- a/drivers/iio/dummy/iio_dummy_evgen.c +++ b/drivers/iio/dummy/iio_dummy_evgen.c @@ -45,6 +45,8 @@ static struct iio_dummy_eventgen *iio_evgen; static int iio_dummy_evgen_create(void) { + int ret; + iio_evgen = kzalloc(sizeof(*iio_evgen), GFP_KERNEL); if (!iio_evgen) return -ENOMEM; @@ -52,8 +54,9 @@ static int iio_dummy_evgen_create(void) iio_evgen->irq_sim_domain = irq_domain_create_sim(NULL, IIO_EVENTGEN_NO); if (IS_ERR(iio_evgen->irq_sim_domain)) { + ret = PTR_ERR(iio_evgen->irq_sim_domain); kfree(iio_evgen); - return PTR_ERR(iio_evgen->irq_sim_domain); + return ret; } mutex_init(&iio_evgen->lock); From 6b668c9b7fc6fc0c313cdaee8b75d17f4d954ab5 Mon Sep 17 00:00:00 2001 From: Xiaoguang Wang Date: Wed, 20 May 2020 15:35:03 +0800 Subject: [PATCH 0621/1043] io_uring: don't submit sqes when ctx->refs is dying When IORING_SETUP_SQPOLL is enabled, io_ring_ctx_wait_and_kill() will wait for sq thread to idle by busy loop: while (ctx->sqo_thread && !wq_has_sleeper(&ctx->sqo_wait)) cond_resched(); Above loop isn't very CPU friendly, it may introduce a short cpu burst on the current cpu. If ctx->refs is dying, we forbid sq_thread from submitting any further SQEs. Instead they just get discarded when we exit. Signed-off-by: Xiaoguang Wang Signed-off-by: Jens Axboe --- fs/io_uring.c | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 50f079417911..0b51f21e5432 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -6175,7 +6175,8 @@ static int io_sq_thread(void *data) } mutex_lock(&ctx->uring_lock); - ret = io_submit_sqes(ctx, to_submit, NULL, -1); + if (likely(!percpu_ref_is_dying(&ctx->refs))) + ret = io_submit_sqes(ctx, to_submit, NULL, -1); mutex_unlock(&ctx->uring_lock); timeout = jiffies + ctx->sq_thread_idle; } @@ -7465,16 +7466,6 @@ static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx) percpu_ref_kill(&ctx->refs); mutex_unlock(&ctx->uring_lock); - /* - * Wait for sq thread to idle, if we have one. It won't spin on new - * work after we've killed the ctx ref above. This is important to do - * before we cancel existing commands, as the thread could otherwise - * be queueing new work post that. If that's work we need to cancel, - * it could cause shutdown to hang. - */ - while (ctx->sqo_thread && !wq_has_sleeper(&ctx->sqo_wait)) - cond_resched(); - io_kill_timeouts(ctx); io_poll_remove_all(ctx); From 9d44a121c5a79bc8a9d67c058456bd52a83c79e7 Mon Sep 17 00:00:00 2001 From: Richard Guy Briggs Date: Wed, 20 May 2020 14:47:13 -0400 Subject: [PATCH 0622/1043] audit: add subj creds to NETFILTER_CFG record to Some table unregister actions seem to be initiated by the kernel to garbage collect unused tables that are not initiated by any userspace actions. It was found to be necessary to add the subject credentials to cover this case to reveal the source of these actions. A sample record: The uid, auid, tty, ses and exe fields have not been included since they are in the SYSCALL record and contain nothing useful in the non-user context. Here are two sample orphaned records: type=NETFILTER_CFG msg=audit(2020-05-20 12:14:36.505:5) : table=filter family=ipv4 entries=0 op=register pid=1 subj=kernel comm=swapper/0 type=NETFILTER_CFG msg=audit(2020-05-20 12:15:27.701:301) : table=nat family=bridge entries=0 op=unregister pid=30 subj=system_u:system_r:kernel_t:s0 comm=kworker/u4:1 Signed-off-by: Richard Guy Briggs Signed-off-by: Paul Moore --- kernel/auditsc.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kernel/auditsc.c b/kernel/auditsc.c index cfe3486e5f31..468a23390457 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -2557,12 +2557,18 @@ void __audit_log_nfcfg(const char *name, u8 af, unsigned int nentries, enum audit_nfcfgop op) { struct audit_buffer *ab; + char comm[sizeof(current->comm)]; ab = audit_log_start(audit_context(), GFP_KERNEL, AUDIT_NETFILTER_CFG); if (!ab) return; audit_log_format(ab, "table=%s family=%u entries=%u op=%s", name, af, nentries, audit_nfcfgs[op].s); + + audit_log_format(ab, " pid=%u", task_pid_nr(current)); + audit_log_task_context(ab); /* subj= */ + audit_log_format(ab, " comm="); + audit_log_untrustedstring(ab, get_task_comm(comm, current)); audit_log_end(ab); } EXPORT_SYMBOL_GPL(__audit_log_nfcfg); From 37e2bc4433a9d68f6d4c8e3fe5ef3a77609bbe8d Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Wed, 20 May 2020 15:08:02 +0800 Subject: [PATCH 0623/1043] MIPS: SGI-IP27: Remove not used definition TICK_SIZE in ip27-timer.c After commit f5ff0a280201 ("[MIPS] Use generic NTP code for all MIPS platforms"), TICK_SIZE is not used in ip27-timer.c for many years, remove it. Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/sgi-ip27/ip27-timer.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/mips/sgi-ip27/ip27-timer.c b/arch/mips/sgi-ip27/ip27-timer.c index 115b1d9dc76d..c0e33632bc37 100644 --- a/arch/mips/sgi-ip27/ip27-timer.c +++ b/arch/mips/sgi-ip27/ip27-timer.c @@ -28,8 +28,6 @@ #include "ip27-common.h" -#define TICK_SIZE (tick_nsec / 1000) - static int rt_next_event(unsigned long delta, struct clock_event_device *evt) { unsigned int cpu = smp_processor_id(); From bbb5946eb545fab8ad8f46bce8a803e1c0c39d47 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 03:34:37 +0300 Subject: [PATCH 0624/1043] mips: MAAR: Use more precise address mask Indeed according to the MIPS32 Privileged Resource Architecgture the MAAR pair register address field either takes [12:31] bits for non-XPA systems and [12:55] otherwise. In any case the current address mask is just wrong for 64-bit and 32-bits XPA chips. So lets extend it to 59-bits of physical address value. This shall cover the 64-bits architecture and systems with XPA enabled, and won't cause any problem for non-XPA 32-bit systems, since address values exceeding the architecture specific MAAR mask will be just truncated with setting zeros in the unsupported upper bits. Co-developed-by: Alexey Malahov Signed-off-by: Alexey Malahov Signed-off-by: Serge Semin Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Arnd Bergmann Cc: Rob Herring Cc: devicetree@vger.kernel.org Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mipsregs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h index 238adce3ccb2..246b8b2ebdf8 100644 --- a/arch/mips/include/asm/mipsregs.h +++ b/arch/mips/include/asm/mipsregs.h @@ -757,7 +757,7 @@ /* MAAR bit definitions */ #define MIPS_MAAR_VH (_U64CAST_(1) << 63) -#define MIPS_MAAR_ADDR ((BIT_ULL(BITS_PER_LONG - 12) - 1) << 12) +#define MIPS_MAAR_ADDR GENMASK_ULL(55, 12) #define MIPS_MAAR_ADDR_SHIFT 12 #define MIPS_MAAR_S (_ULCAST_(1) << 1) #define MIPS_MAAR_VL (_ULCAST_(1) << 0) From 181e9d4efaf6aa8d1e7d510aeb7114c0f276fad7 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 20 May 2020 19:49:25 +0300 Subject: [PATCH 0625/1043] irqdomain: Make __irq_domain_add() less OF-dependent __irq_domain_add() relies in some places on the fact that the fwnode can be only of type OF. This prevents refactoring of the code to support other types of fwnode. Make it less OF-dependent by switching it to use the fwnode directly where it makes sense. Signed-off-by: Andy Shevchenko Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200520164927.39090-1-andriy.shevchenko@linux.intel.com --- kernel/irq/irqdomain.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index e2aa128ea3ee..7649f3848ab5 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -132,14 +132,13 @@ struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, int size, const struct irq_domain_ops *ops, void *host_data) { - struct device_node *of_node = to_of_node(fwnode); struct irqchip_fwid *fwid; struct irq_domain *domain; static atomic_t unknown_domains; domain = kzalloc_node(sizeof(*domain) + (sizeof(unsigned int) * size), - GFP_KERNEL, of_node_to_nid(of_node)); + GFP_KERNEL, of_node_to_nid(to_of_node(fwnode))); if (!domain) return NULL; @@ -177,15 +176,15 @@ struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, int size, domain->fwnode = fwnode; #endif - } else if (of_node) { + } else if (is_of_node(fwnode)) { char *name; /* - * DT paths contain '/', which debugfs is legitimately + * fwnode paths contain '/', which debugfs is legitimately * unhappy about. Replace them with ':', which does * the trick and is not as offensive as '\'... */ - name = kasprintf(GFP_KERNEL, "%pOF", of_node); + name = kasprintf(GFP_KERNEL, "%pfw", fwnode); if (!name) { kfree(domain); return NULL; @@ -210,7 +209,7 @@ struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, int size, domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED; } - of_node_get(of_node); + fwnode_handle_get(fwnode); /* Fill structure */ INIT_RADIX_TREE(&domain->revmap_tree, GFP_KERNEL); @@ -259,7 +258,7 @@ void irq_domain_remove(struct irq_domain *domain) pr_debug("Removed domain %s\n", domain->name); - of_node_put(irq_domain_get_of_node(domain)); + fwnode_handle_put(domain->fwnode); if (domain->flags & IRQ_DOMAIN_NAME_ALLOCATED) kfree(domain->name); kfree(domain); From 87526603c89256e18ad2c23821fdaf376b072fc8 Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 20 May 2020 19:49:26 +0300 Subject: [PATCH 0626/1043] irqdomain: Get rid of special treatment for ACPI in __irq_domain_add() Now that __irq_domain_add() is able to better deals with generic fwnodes, there is no need to special-case ACPI anymore. Get rid of the special treatment for ACPI. Signed-off-by: Andy Shevchenko Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200520164927.39090-2-andriy.shevchenko@linux.intel.com --- kernel/irq/irqdomain.c | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 7649f3848ab5..5d14d910ab64 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -161,22 +161,7 @@ struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, int size, domain->name = fwid->name; break; } -#ifdef CONFIG_ACPI - } else if (is_acpi_device_node(fwnode)) { - struct acpi_buffer buf = { - .length = ACPI_ALLOCATE_BUFFER, - }; - acpi_handle handle; - - handle = acpi_device_handle(to_acpi_device_node(fwnode)); - if (acpi_get_name(handle, ACPI_FULL_PATHNAME, &buf) == AE_OK) { - domain->name = buf.pointer; - domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED; - } - - domain->fwnode = fwnode; -#endif - } else if (is_of_node(fwnode)) { + } else if (is_of_node(fwnode) || is_acpi_device_node(fwnode)) { char *name; /* From 9ed78b05f998050784ae863bd5ba4aea2e2141ed Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Wed, 20 May 2020 19:49:27 +0300 Subject: [PATCH 0627/1043] irqdomain: Allow software nodes for IRQ domain creation In some cases we need to have an IRQ domain created out of software node. One of such cases is DesignWare GPIO driver when it's instantiated from half-baked ACPI table (alas, we can't fix it for devices which are few years on market) and thus using software nodes to quirk this. But the driver is using IRQ domains based on per GPIO port firmware nodes, which are in the above case software ones. This brings a warning message to be printed [ 73.957183] irq: Invalid fwnode type for irqdomain and creates an anonymous IRQ domain without a debugfs entry. Allowing software nodes to be valid for IRQ domains rids us of the warning and debugs gets correctly populated. % ls -1 /sys/kernel/debug/irq/domains/ ... intel-quark-dw-apb-gpio:portA Signed-off-by: Andy Shevchenko [maz: refactored commit message] Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200520164927.39090-3-andriy.shevchenko@linux.intel.com --- kernel/irq/irqdomain.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 5d14d910ab64..a4c2c915511d 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -161,7 +161,8 @@ struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, int size, domain->name = fwid->name; break; } - } else if (is_of_node(fwnode) || is_acpi_device_node(fwnode)) { + } else if (is_of_node(fwnode) || is_acpi_device_node(fwnode) || + is_software_node(fwnode)) { char *name; /* From 7c5014b0987a30e4989c90633c198aced454c0ec Mon Sep 17 00:00:00 2001 From: Martijn Coenen Date: Wed, 13 May 2020 15:38:35 +0200 Subject: [PATCH 0628/1043] loop: Call loop_config_discard() only after new config is applied loop_set_status() calls loop_config_discard() to configure discard for the loop device; however, the discard configuration depends on whether the loop device uses encryption, and when we call it the encryption configuration has not been updated yet. Move the call down so we apply the correct discard configuration based on the new configuration. Signed-off-by: Martijn Coenen Reviewed-by: Christoph Hellwig Reviewed-by: Bob Liu Reviewed-by: Bart Van Assche Signed-off-by: Jens Axboe --- drivers/block/loop.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index da693e6a834e..f1754262fc94 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -1334,8 +1334,6 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info) } } - loop_config_discard(lo); - memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE); memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE); lo->lo_file_name[LO_NAME_SIZE-1] = 0; @@ -1359,6 +1357,8 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info) lo->lo_key_owner = uid; } + loop_config_discard(lo); + /* update dio if lo_offset or transfer is changed */ __loop_update_dio(lo, lo->use_dio); From 083a6a50783ef54256eec3499e6575237e0e3d53 Mon Sep 17 00:00:00 2001 From: Martijn Coenen Date: Wed, 13 May 2020 15:38:36 +0200 Subject: [PATCH 0629/1043] loop: Remove sector_t truncation checks sector_t is now always u64, so we don't need to check for truncation. Signed-off-by: Martijn Coenen Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/block/loop.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index f1754262fc94..00de7fec0ed5 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -228,24 +228,20 @@ static void __loop_update_dio(struct loop_device *lo, bool dio) blk_mq_unfreeze_queue(lo->lo_queue); } -static int +static void figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit) { loff_t size = get_size(offset, sizelimit, lo->lo_backing_file); - sector_t x = (sector_t)size; struct block_device *bdev = lo->lo_device; - if (unlikely((loff_t)x != size)) - return -EFBIG; if (lo->lo_offset != offset) lo->lo_offset = offset; if (lo->lo_sizelimit != sizelimit) lo->lo_sizelimit = sizelimit; - set_capacity(lo->lo_disk, x); + set_capacity(lo->lo_disk, size); bd_set_size(bdev, (loff_t)get_capacity(bdev->bd_disk) << 9); /* let user-space know about the new size */ kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE); - return 0; } static inline int @@ -1003,10 +999,8 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode, !file->f_op->write_iter) lo_flags |= LO_FLAGS_READ_ONLY; - error = -EFBIG; size = get_loop_size(lo, file); - if ((loff_t)(sector_t)size != size) - goto out_unlock; + error = loop_prepare_queue(lo); if (error) goto out_unlock; @@ -1328,10 +1322,7 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info) lo->lo_device->bd_inode->i_mapping->nrpages); goto out_unfreeze; } - if (figure_loop_size(lo, info->lo_offset, info->lo_sizelimit)) { - err = -EFBIG; - goto out_unfreeze; - } + figure_loop_size(lo, info->lo_offset, info->lo_sizelimit); } memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE); @@ -1534,7 +1525,9 @@ static int loop_set_capacity(struct loop_device *lo) if (unlikely(lo->lo_state != Lo_bound)) return -ENXIO; - return figure_loop_size(lo, lo->lo_offset, lo->lo_sizelimit); + figure_loop_size(lo, lo->lo_offset, lo->lo_sizelimit); + + return 0; } static int loop_set_dio(struct loop_device *lo, unsigned long arg) From 5795b6f5607f7e4db62ddea144727780cb351a9b Mon Sep 17 00:00:00 2001 From: Martijn Coenen Date: Wed, 13 May 2020 15:38:37 +0200 Subject: [PATCH 0630/1043] loop: Factor out setting loop device size This code is used repeatedly. Signed-off-by: Martijn Coenen Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/block/loop.c | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 00de7fec0ed5..e69ff3c19eff 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -228,20 +228,35 @@ static void __loop_update_dio(struct loop_device *lo, bool dio) blk_mq_unfreeze_queue(lo->lo_queue); } +/** + * loop_set_size() - sets device size and notifies userspace + * @lo: struct loop_device to set the size for + * @size: new size of the loop device + * + * Callers must validate that the size passed into this function fits into + * a sector_t, eg using loop_validate_size() + */ +static void loop_set_size(struct loop_device *lo, loff_t size) +{ + struct block_device *bdev = lo->lo_device; + + set_capacity(lo->lo_disk, size); + bd_set_size(bdev, size << SECTOR_SHIFT); + /* let user-space know about the new size */ + kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE); +} + static void figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit) { loff_t size = get_size(offset, sizelimit, lo->lo_backing_file); - struct block_device *bdev = lo->lo_device; if (lo->lo_offset != offset) lo->lo_offset = offset; if (lo->lo_sizelimit != sizelimit) lo->lo_sizelimit = sizelimit; - set_capacity(lo->lo_disk, size); - bd_set_size(bdev, (loff_t)get_capacity(bdev->bd_disk) << 9); - /* let user-space know about the new size */ - kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE); + + loop_set_size(lo, size); } static inline int @@ -1034,11 +1049,8 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode, loop_update_rotational(lo); loop_update_dio(lo); - set_capacity(lo->lo_disk, size); - bd_set_size(bdev, size << 9); loop_sysfs_init(lo); - /* let user-space know about the new size */ - kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE); + loop_set_size(lo, size); set_blocksize(bdev, S_ISBLK(inode->i_mode) ? block_size(inode->i_bdev) : PAGE_SIZE); From 716ad0986cbd1d3b2ab3f6d23144a94638dac20b Mon Sep 17 00:00:00 2001 From: Martijn Coenen Date: Wed, 13 May 2020 15:38:38 +0200 Subject: [PATCH 0631/1043] loop: Switch to set_capacity_revalidate_and_notify() This was recently added to block/genhd.c, and takes care of both updating the capacity and notifying userspace of the new size. Signed-off-by: Martijn Coenen Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/block/loop.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index e69ff3c19eff..d9a756f8abd5 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -240,10 +240,9 @@ static void loop_set_size(struct loop_device *lo, loff_t size) { struct block_device *bdev = lo->lo_device; - set_capacity(lo->lo_disk, size); bd_set_size(bdev, size << SECTOR_SHIFT); - /* let user-space know about the new size */ - kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE); + + set_capacity_revalidate_and_notify(lo->lo_disk, size, false); } static void From b0bd158dd630bd47640e0e418c062cda1e0da5ad Mon Sep 17 00:00:00 2001 From: Martijn Coenen Date: Wed, 13 May 2020 15:38:39 +0200 Subject: [PATCH 0632/1043] loop: Refactor loop_set_status() size calculation figure_loop_size() calculates the loop size based on the passed in parameters, but at the same time it updates the offset and sizelimit parameters in the loop device configuration. That is a somewhat unexpected side effect of a function with this name, and it is only only needed by one of the two callers of this function - loop_set_status(). Move the lo_offset and lo_sizelimit assignment back into loop_set_status(), and use the newly factored out functions to validate and apply the newly calculated size. This allows us to get rid of figure_loop_size() in a follow-up commit. Signed-off-by: Martijn Coenen Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/block/loop.c | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index d9a756f8abd5..c134b3439483 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -250,11 +250,6 @@ figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit) { loff_t size = get_size(offset, sizelimit, lo->lo_backing_file); - if (lo->lo_offset != offset) - lo->lo_offset = offset; - if (lo->lo_sizelimit != sizelimit) - lo->lo_sizelimit = sizelimit; - loop_set_size(lo, size); } @@ -1272,6 +1267,7 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info) kuid_t uid = current_uid(); struct block_device *bdev; bool partscan = false; + bool size_changed = false; err = mutex_lock_killable(&loop_ctl_mutex); if (err) @@ -1293,6 +1289,7 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info) if (lo->lo_offset != info->lo_offset || lo->lo_sizelimit != info->lo_sizelimit) { + size_changed = true; sync_blockdev(lo->lo_device); kill_bdev(lo->lo_device); } @@ -1300,6 +1297,15 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info) /* I/O need to be drained during transfer transition */ blk_mq_freeze_queue(lo->lo_queue); + if (size_changed && lo->lo_device->bd_inode->i_mapping->nrpages) { + /* If any pages were dirtied after kill_bdev(), try again */ + err = -EAGAIN; + pr_warn("%s: loop%d (%s) has still dirty pages (nrpages=%lu)\n", + __func__, lo->lo_number, lo->lo_file_name, + lo->lo_device->bd_inode->i_mapping->nrpages); + goto out_unfreeze; + } + err = loop_release_xfer(lo); if (err) goto out_unfreeze; @@ -1323,19 +1329,8 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info) if (err) goto out_unfreeze; - if (lo->lo_offset != info->lo_offset || - lo->lo_sizelimit != info->lo_sizelimit) { - /* kill_bdev should have truncated all the pages */ - if (lo->lo_device->bd_inode->i_mapping->nrpages) { - err = -EAGAIN; - pr_warn("%s: loop%d (%s) has still dirty pages (nrpages=%lu)\n", - __func__, lo->lo_number, lo->lo_file_name, - lo->lo_device->bd_inode->i_mapping->nrpages); - goto out_unfreeze; - } - figure_loop_size(lo, info->lo_offset, info->lo_sizelimit); - } - + lo->lo_offset = info->lo_offset; + lo->lo_sizelimit = info->lo_sizelimit; memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE); memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE); lo->lo_file_name[LO_NAME_SIZE-1] = 0; @@ -1359,6 +1354,12 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info) lo->lo_key_owner = uid; } + if (size_changed) { + loff_t new_size = get_size(lo->lo_offset, lo->lo_sizelimit, + lo->lo_backing_file); + loop_set_size(lo, new_size); + } + loop_config_discard(lo); /* update dio if lo_offset or transfer is changed */ From 0a6ed1b5ff6757f11ad2d57906ceb40488a5ee52 Mon Sep 17 00:00:00 2001 From: Martijn Coenen Date: Wed, 13 May 2020 15:38:40 +0200 Subject: [PATCH 0633/1043] loop: Remove figure_loop_size() This function was now only used by loop_set_capacity(). Just open code the remaining code in the caller instead. Signed-off-by: Martijn Coenen Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/block/loop.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index c134b3439483..e281a9f03d96 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -245,14 +245,6 @@ static void loop_set_size(struct loop_device *lo, loff_t size) set_capacity_revalidate_and_notify(lo->lo_disk, size, false); } -static void -figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit) -{ - loff_t size = get_size(offset, sizelimit, lo->lo_backing_file); - - loop_set_size(lo, size); -} - static inline int lo_do_transfer(struct loop_device *lo, int cmd, struct page *rpage, unsigned roffs, @@ -1534,10 +1526,13 @@ loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) { static int loop_set_capacity(struct loop_device *lo) { + loff_t size; + if (unlikely(lo->lo_state != Lo_bound)) return -ENXIO; - figure_loop_size(lo, lo->lo_offset, lo->lo_sizelimit); + size = get_loop_size(lo, lo->lo_backing_file); + loop_set_size(lo, size); return 0; } From 0c3796c244598122a5d59d56f30d19390096817f Mon Sep 17 00:00:00 2001 From: Martijn Coenen Date: Wed, 13 May 2020 15:38:41 +0200 Subject: [PATCH 0634/1043] loop: Factor out configuring loop from status Factor out this code into a separate function, so it can be reused by other code more easily. Signed-off-by: Martijn Coenen Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/block/loop.c | 117 +++++++++++++++++++++++++------------------ 1 file changed, 67 insertions(+), 50 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index e281a9f03d96..6a4c0ba225ca 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -1251,75 +1251,43 @@ static int loop_clr_fd(struct loop_device *lo) return __loop_clr_fd(lo, false); } +/** + * loop_set_status_from_info - configure device from loop_info + * @lo: struct loop_device to configure + * @info: struct loop_info64 to configure the device with + * + * Configures the loop device parameters according to the passed + * in loop_info64 configuration. + */ static int -loop_set_status(struct loop_device *lo, const struct loop_info64 *info) +loop_set_status_from_info(struct loop_device *lo, + const struct loop_info64 *info) { int err; struct loop_func_table *xfer; kuid_t uid = current_uid(); - struct block_device *bdev; - bool partscan = false; - bool size_changed = false; - err = mutex_lock_killable(&loop_ctl_mutex); - if (err) - return err; - if (lo->lo_encrypt_key_size && - !uid_eq(lo->lo_key_owner, uid) && - !capable(CAP_SYS_ADMIN)) { - err = -EPERM; - goto out_unlock; - } - if (lo->lo_state != Lo_bound) { - err = -ENXIO; - goto out_unlock; - } - if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE) { - err = -EINVAL; - goto out_unlock; - } - - if (lo->lo_offset != info->lo_offset || - lo->lo_sizelimit != info->lo_sizelimit) { - size_changed = true; - sync_blockdev(lo->lo_device); - kill_bdev(lo->lo_device); - } - - /* I/O need to be drained during transfer transition */ - blk_mq_freeze_queue(lo->lo_queue); - - if (size_changed && lo->lo_device->bd_inode->i_mapping->nrpages) { - /* If any pages were dirtied after kill_bdev(), try again */ - err = -EAGAIN; - pr_warn("%s: loop%d (%s) has still dirty pages (nrpages=%lu)\n", - __func__, lo->lo_number, lo->lo_file_name, - lo->lo_device->bd_inode->i_mapping->nrpages); - goto out_unfreeze; - } + if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE) + return -EINVAL; err = loop_release_xfer(lo); if (err) - goto out_unfreeze; + return err; if (info->lo_encrypt_type) { unsigned int type = info->lo_encrypt_type; - if (type >= MAX_LO_CRYPT) { - err = -EINVAL; - goto out_unfreeze; - } + if (type >= MAX_LO_CRYPT) + return -EINVAL; xfer = xfer_funcs[type]; - if (xfer == NULL) { - err = -EINVAL; - goto out_unfreeze; - } + if (xfer == NULL) + return -EINVAL; } else xfer = NULL; err = loop_init_xfer(lo, xfer, info); if (err) - goto out_unfreeze; + return err; lo->lo_offset = info->lo_offset; lo->lo_sizelimit = info->lo_sizelimit; @@ -1346,6 +1314,55 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info) lo->lo_key_owner = uid; } + return 0; +} + +static int +loop_set_status(struct loop_device *lo, const struct loop_info64 *info) +{ + int err; + struct block_device *bdev; + kuid_t uid = current_uid(); + bool partscan = false; + bool size_changed = false; + + err = mutex_lock_killable(&loop_ctl_mutex); + if (err) + return err; + if (lo->lo_encrypt_key_size && + !uid_eq(lo->lo_key_owner, uid) && + !capable(CAP_SYS_ADMIN)) { + err = -EPERM; + goto out_unlock; + } + if (lo->lo_state != Lo_bound) { + err = -ENXIO; + goto out_unlock; + } + + if (lo->lo_offset != info->lo_offset || + lo->lo_sizelimit != info->lo_sizelimit) { + size_changed = true; + sync_blockdev(lo->lo_device); + kill_bdev(lo->lo_device); + } + + /* I/O need to be drained during transfer transition */ + blk_mq_freeze_queue(lo->lo_queue); + + if (size_changed && lo->lo_device->bd_inode->i_mapping->nrpages) { + /* If any pages were dirtied after kill_bdev(), try again */ + err = -EAGAIN; + pr_warn("%s: loop%d (%s) has still dirty pages (nrpages=%lu)\n", + __func__, lo->lo_number, lo->lo_file_name, + lo->lo_device->bd_inode->i_mapping->nrpages); + goto out_unfreeze; + } + + err = loop_set_status_from_info(lo, info); + if (err) + goto out_unfreeze; + if (size_changed) { loff_t new_size = get_size(lo->lo_offset, lo->lo_sizelimit, lo->lo_backing_file); From 62ab466ca881fe200c21aa74b65f8bd83ec482dc Mon Sep 17 00:00:00 2001 From: Martijn Coenen Date: Wed, 13 May 2020 15:38:42 +0200 Subject: [PATCH 0635/1043] loop: Move loop_set_status_from_info() and friends up So we can use it without forward declaration. This is a separate commit to make it easier to verify that this is just a move, without functional modifications. Signed-off-by: Martijn Coenen Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/block/loop.c | 206 +++++++++++++++++++++---------------------- 1 file changed, 103 insertions(+), 103 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 6a4c0ba225ca..4dc11d954169 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -949,6 +949,109 @@ static void loop_update_rotational(struct loop_device *lo) blk_queue_flag_clear(QUEUE_FLAG_NONROT, q); } +static int +loop_release_xfer(struct loop_device *lo) +{ + int err = 0; + struct loop_func_table *xfer = lo->lo_encryption; + + if (xfer) { + if (xfer->release) + err = xfer->release(lo); + lo->transfer = NULL; + lo->lo_encryption = NULL; + module_put(xfer->owner); + } + return err; +} + +static int +loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer, + const struct loop_info64 *i) +{ + int err = 0; + + if (xfer) { + struct module *owner = xfer->owner; + + if (!try_module_get(owner)) + return -EINVAL; + if (xfer->init) + err = xfer->init(lo, i); + if (err) + module_put(owner); + else + lo->lo_encryption = xfer; + } + return err; +} + +/** + * loop_set_status_from_info - configure device from loop_info + * @lo: struct loop_device to configure + * @info: struct loop_info64 to configure the device with + * + * Configures the loop device parameters according to the passed + * in loop_info64 configuration. + */ +static int +loop_set_status_from_info(struct loop_device *lo, + const struct loop_info64 *info) +{ + int err; + struct loop_func_table *xfer; + kuid_t uid = current_uid(); + + if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE) + return -EINVAL; + + err = loop_release_xfer(lo); + if (err) + return err; + + if (info->lo_encrypt_type) { + unsigned int type = info->lo_encrypt_type; + + if (type >= MAX_LO_CRYPT) + return -EINVAL; + xfer = xfer_funcs[type]; + if (xfer == NULL) + return -EINVAL; + } else + xfer = NULL; + + err = loop_init_xfer(lo, xfer, info); + if (err) + return err; + + lo->lo_offset = info->lo_offset; + lo->lo_sizelimit = info->lo_sizelimit; + memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE); + memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE); + lo->lo_file_name[LO_NAME_SIZE-1] = 0; + lo->lo_crypt_name[LO_NAME_SIZE-1] = 0; + + if (!xfer) + xfer = &none_funcs; + lo->transfer = xfer->transfer; + lo->ioctl = xfer->ioctl; + + if ((lo->lo_flags & LO_FLAGS_AUTOCLEAR) != + (info->lo_flags & LO_FLAGS_AUTOCLEAR)) + lo->lo_flags ^= LO_FLAGS_AUTOCLEAR; + + lo->lo_encrypt_key_size = info->lo_encrypt_key_size; + lo->lo_init[0] = info->lo_init[0]; + lo->lo_init[1] = info->lo_init[1]; + if (info->lo_encrypt_key_size) { + memcpy(lo->lo_encrypt_key, info->lo_encrypt_key, + info->lo_encrypt_key_size); + lo->lo_key_owner = uid; + } + + return 0; +} + static int loop_set_fd(struct loop_device *lo, fmode_t mode, struct block_device *bdev, unsigned int arg) { @@ -1070,43 +1173,6 @@ out: return error; } -static int -loop_release_xfer(struct loop_device *lo) -{ - int err = 0; - struct loop_func_table *xfer = lo->lo_encryption; - - if (xfer) { - if (xfer->release) - err = xfer->release(lo); - lo->transfer = NULL; - lo->lo_encryption = NULL; - module_put(xfer->owner); - } - return err; -} - -static int -loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer, - const struct loop_info64 *i) -{ - int err = 0; - - if (xfer) { - struct module *owner = xfer->owner; - - if (!try_module_get(owner)) - return -EINVAL; - if (xfer->init) - err = xfer->init(lo, i); - if (err) - module_put(owner); - else - lo->lo_encryption = xfer; - } - return err; -} - static int __loop_clr_fd(struct loop_device *lo, bool release) { struct file *filp = NULL; @@ -1251,72 +1317,6 @@ static int loop_clr_fd(struct loop_device *lo) return __loop_clr_fd(lo, false); } -/** - * loop_set_status_from_info - configure device from loop_info - * @lo: struct loop_device to configure - * @info: struct loop_info64 to configure the device with - * - * Configures the loop device parameters according to the passed - * in loop_info64 configuration. - */ -static int -loop_set_status_from_info(struct loop_device *lo, - const struct loop_info64 *info) -{ - int err; - struct loop_func_table *xfer; - kuid_t uid = current_uid(); - - if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE) - return -EINVAL; - - err = loop_release_xfer(lo); - if (err) - return err; - - if (info->lo_encrypt_type) { - unsigned int type = info->lo_encrypt_type; - - if (type >= MAX_LO_CRYPT) - return -EINVAL; - xfer = xfer_funcs[type]; - if (xfer == NULL) - return -EINVAL; - } else - xfer = NULL; - - err = loop_init_xfer(lo, xfer, info); - if (err) - return err; - - lo->lo_offset = info->lo_offset; - lo->lo_sizelimit = info->lo_sizelimit; - memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE); - memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE); - lo->lo_file_name[LO_NAME_SIZE-1] = 0; - lo->lo_crypt_name[LO_NAME_SIZE-1] = 0; - - if (!xfer) - xfer = &none_funcs; - lo->transfer = xfer->transfer; - lo->ioctl = xfer->ioctl; - - if ((lo->lo_flags & LO_FLAGS_AUTOCLEAR) != - (info->lo_flags & LO_FLAGS_AUTOCLEAR)) - lo->lo_flags ^= LO_FLAGS_AUTOCLEAR; - - lo->lo_encrypt_key_size = info->lo_encrypt_key_size; - lo->lo_init[0] = info->lo_init[0]; - lo->lo_init[1] = info->lo_init[1]; - if (info->lo_encrypt_key_size) { - memcpy(lo->lo_encrypt_key, info->lo_encrypt_key, - info->lo_encrypt_key_size); - lo->lo_key_owner = uid; - } - - return 0; -} - static int loop_set_status(struct loop_device *lo, const struct loop_info64 *info) { From 571fae6e290d64a3e8132c455e7786c99c467ed1 Mon Sep 17 00:00:00 2001 From: Martijn Coenen Date: Wed, 13 May 2020 15:38:43 +0200 Subject: [PATCH 0636/1043] loop: Rework lo_ioctl() __user argument casting In preparation for a new ioctl that needs to copy_from_user(); makes the code easier to read as well. Signed-off-by: Martijn Coenen Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/block/loop.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 4dc11d954169..31f10da4945e 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -1634,6 +1634,7 @@ static int lo_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg) { struct loop_device *lo = bdev->bd_disk->private_data; + void __user *argp = (void __user *) arg; int err; switch (cmd) { @@ -1646,21 +1647,19 @@ static int lo_ioctl(struct block_device *bdev, fmode_t mode, case LOOP_SET_STATUS: err = -EPERM; if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) { - err = loop_set_status_old(lo, - (struct loop_info __user *)arg); + err = loop_set_status_old(lo, argp); } break; case LOOP_GET_STATUS: - return loop_get_status_old(lo, (struct loop_info __user *) arg); + return loop_get_status_old(lo, argp); case LOOP_SET_STATUS64: err = -EPERM; if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) { - err = loop_set_status64(lo, - (struct loop_info64 __user *) arg); + err = loop_set_status64(lo, argp); } break; case LOOP_GET_STATUS64: - return loop_get_status64(lo, (struct loop_info64 __user *) arg); + return loop_get_status64(lo, argp); case LOOP_SET_CAPACITY: case LOOP_SET_DIRECT_IO: case LOOP_SET_BLOCK_SIZE: From faf1d25440d6ad06d509dada4b6fe62fea844370 Mon Sep 17 00:00:00 2001 From: Martijn Coenen Date: Wed, 13 May 2020 15:38:44 +0200 Subject: [PATCH 0637/1043] loop: Clean up LOOP_SET_STATUS lo_flags handling LOOP_SET_STATUS(64) will actually allow some lo_flags to be modified; in particular, LO_FLAGS_AUTOCLEAR can be set and cleared, whereas LO_FLAGS_PARTSCAN can be set to request a partition scan. Make this explicit by updating the UAPI to include the flags that can be set/cleared using this ioctl. The implementation can then blindly take over the passed in flags, and use the previous flags for those flags that can't be set / cleared using LOOP_SET_STATUS. Signed-off-by: Martijn Coenen Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/block/loop.c | 19 +++++++++++++------ include/uapi/linux/loop.h | 10 ++++++++-- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 31f10da4945e..13518ba191f5 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -1036,9 +1036,7 @@ loop_set_status_from_info(struct loop_device *lo, lo->transfer = xfer->transfer; lo->ioctl = xfer->ioctl; - if ((lo->lo_flags & LO_FLAGS_AUTOCLEAR) != - (info->lo_flags & LO_FLAGS_AUTOCLEAR)) - lo->lo_flags ^= LO_FLAGS_AUTOCLEAR; + lo->lo_flags = info->lo_flags; lo->lo_encrypt_key_size = info->lo_encrypt_key_size; lo->lo_init[0] = info->lo_init[0]; @@ -1323,6 +1321,7 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info) int err; struct block_device *bdev; kuid_t uid = current_uid(); + int prev_lo_flags; bool partscan = false; bool size_changed = false; @@ -1359,10 +1358,19 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info) goto out_unfreeze; } + prev_lo_flags = lo->lo_flags; + err = loop_set_status_from_info(lo, info); if (err) goto out_unfreeze; + /* Mask out flags that can't be set using LOOP_SET_STATUS. */ + lo->lo_flags &= ~LOOP_SET_STATUS_SETTABLE_FLAGS; + /* For those flags, use the previous values instead */ + lo->lo_flags |= prev_lo_flags & ~LOOP_SET_STATUS_SETTABLE_FLAGS; + /* For flags that can't be cleared, use previous values too */ + lo->lo_flags |= prev_lo_flags & ~LOOP_SET_STATUS_CLEARABLE_FLAGS; + if (size_changed) { loff_t new_size = get_size(lo->lo_offset, lo->lo_sizelimit, lo->lo_backing_file); @@ -1377,9 +1385,8 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info) out_unfreeze: blk_mq_unfreeze_queue(lo->lo_queue); - if (!err && (info->lo_flags & LO_FLAGS_PARTSCAN) && - !(lo->lo_flags & LO_FLAGS_PARTSCAN)) { - lo->lo_flags |= LO_FLAGS_PARTSCAN; + if (!err && (lo->lo_flags & LO_FLAGS_PARTSCAN) && + !(prev_lo_flags & LO_FLAGS_PARTSCAN)) { lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN; bdev = lo->lo_device; partscan = true; diff --git a/include/uapi/linux/loop.h b/include/uapi/linux/loop.h index 080a8df134ef..6b32fee80ce0 100644 --- a/include/uapi/linux/loop.h +++ b/include/uapi/linux/loop.h @@ -25,6 +25,12 @@ enum { LO_FLAGS_DIRECT_IO = 16, }; +/* LO_FLAGS that can be set using LOOP_SET_STATUS(64) */ +#define LOOP_SET_STATUS_SETTABLE_FLAGS (LO_FLAGS_AUTOCLEAR | LO_FLAGS_PARTSCAN) + +/* LO_FLAGS that can be cleared using LOOP_SET_STATUS(64) */ +#define LOOP_SET_STATUS_CLEARABLE_FLAGS (LO_FLAGS_AUTOCLEAR) + #include /* for __kernel_old_dev_t */ #include /* for __u64 */ @@ -37,7 +43,7 @@ struct loop_info { int lo_offset; int lo_encrypt_type; int lo_encrypt_key_size; /* ioctl w/o */ - int lo_flags; /* ioctl r/o */ + int lo_flags; char lo_name[LO_NAME_SIZE]; unsigned char lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */ unsigned long lo_init[2]; @@ -53,7 +59,7 @@ struct loop_info64 { __u32 lo_number; /* ioctl r/o */ __u32 lo_encrypt_type; __u32 lo_encrypt_key_size; /* ioctl w/o */ - __u32 lo_flags; /* ioctl r/o */ + __u32 lo_flags; __u8 lo_file_name[LO_NAME_SIZE]; __u8 lo_crypt_name[LO_NAME_SIZE]; __u8 lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */ From 3448914e8cc550ba792d4ccc74471d1ca4293aae Mon Sep 17 00:00:00 2001 From: Martijn Coenen Date: Wed, 13 May 2020 15:38:45 +0200 Subject: [PATCH 0638/1043] loop: Add LOOP_CONFIGURE ioctl This allows userspace to completely setup a loop device with a single ioctl, removing the in-between state where the device can be partially configured - eg the loop device has a backing file associated with it, but is reading from the wrong offset. Besides removing the intermediate state, another big benefit of this ioctl is that LOOP_SET_STATUS can be slow; the main reason for this slowness is that LOOP_SET_STATUS(64) calls blk_mq_freeze_queue() to freeze the associated queue; this requires waiting for RCU synchronization, which I've measured can take about 15-20ms on this device on average. In addition to doing what LOOP_SET_STATUS can do, LOOP_CONFIGURE can also be used to: - Set the correct block size immediately by setting loop_config.block_size (avoids LOOP_SET_BLOCK_SIZE) - Explicitly request direct I/O mode by setting LO_FLAGS_DIRECT_IO in loop_config.info.lo_flags (avoids LOOP_SET_DIRECT_IO) - Explicitly request read-only mode by setting LO_FLAGS_READ_ONLY in loop_config.info.lo_flags Here's setting up ~70 regular loop devices with an offset on an x86 Android device, using LOOP_SET_FD and LOOP_SET_STATUS: vsoc_x86:/system/apex # time for i in `seq 30 100`; do losetup -r -o 4096 /dev/block/loop$i com.android.adbd.apex; done 0m03.40s real 0m00.02s user 0m00.03s system Here's configuring ~70 devices in the same way, but using a modified losetup that uses the new LOOP_CONFIGURE ioctl: vsoc_x86:/system/apex # time for i in `seq 30 100`; do losetup -r -o 4096 /dev/block/loop$i com.android.adbd.apex; done 0m01.94s real 0m00.01s user 0m00.01s system Signed-off-by: Martijn Coenen Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/block/loop.c | 104 ++++++++++++++++++++++++++++---------- include/uapi/linux/loop.h | 21 ++++++++ 2 files changed, 97 insertions(+), 28 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 13518ba191f5..a565c5aafa52 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -228,6 +228,19 @@ static void __loop_update_dio(struct loop_device *lo, bool dio) blk_mq_unfreeze_queue(lo->lo_queue); } +/** + * loop_validate_block_size() - validates the passed in block size + * @bsize: size to validate + */ +static int +loop_validate_block_size(unsigned short bsize) +{ + if (bsize < 512 || bsize > PAGE_SIZE || !is_power_of_2(bsize)) + return -EINVAL; + + return 0; +} + /** * loop_set_size() - sets device size and notifies userspace * @lo: struct loop_device to set the size for @@ -1050,23 +1063,24 @@ loop_set_status_from_info(struct loop_device *lo, return 0; } -static int loop_set_fd(struct loop_device *lo, fmode_t mode, - struct block_device *bdev, unsigned int arg) +static int loop_configure(struct loop_device *lo, fmode_t mode, + struct block_device *bdev, + const struct loop_config *config) { struct file *file; struct inode *inode; struct address_space *mapping; struct block_device *claimed_bdev = NULL; - int lo_flags = 0; int error; loff_t size; bool partscan; + unsigned short bsize; /* This is safe, since we have a reference from open(). */ __module_get(THIS_MODULE); error = -EBADF; - file = fget(arg); + file = fget(config->fd); if (!file) goto out; @@ -1075,7 +1089,7 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode, * here to avoid changing device under exclusive owner. */ if (!(mode & FMODE_EXCL)) { - claimed_bdev = bd_start_claiming(bdev, loop_set_fd); + claimed_bdev = bd_start_claiming(bdev, loop_configure); if (IS_ERR(claimed_bdev)) { error = PTR_ERR(claimed_bdev); goto out_putf; @@ -1097,11 +1111,26 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode, mapping = file->f_mapping; inode = mapping->host; + size = get_loop_size(lo, file); + + if ((config->info.lo_flags & ~LOOP_CONFIGURE_SETTABLE_FLAGS) != 0) { + error = -EINVAL; + goto out_unlock; + } + + if (config->block_size) { + error = loop_validate_block_size(config->block_size); + if (error) + goto out_unlock; + } + + error = loop_set_status_from_info(lo, &config->info); + if (error) + goto out_unlock; + if (!(file->f_mode & FMODE_WRITE) || !(mode & FMODE_WRITE) || !file->f_op->write_iter) - lo_flags |= LO_FLAGS_READ_ONLY; - - size = get_loop_size(lo, file); + lo->lo_flags |= LO_FLAGS_READ_ONLY; error = loop_prepare_queue(lo); if (error) @@ -1109,30 +1138,28 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode, error = 0; - set_device_ro(bdev, (lo_flags & LO_FLAGS_READ_ONLY) != 0); + set_device_ro(bdev, (lo->lo_flags & LO_FLAGS_READ_ONLY) != 0); - lo->use_dio = false; + lo->use_dio = lo->lo_flags & LO_FLAGS_DIRECT_IO; lo->lo_device = bdev; - lo->lo_flags = lo_flags; lo->lo_backing_file = file; - lo->transfer = NULL; - lo->ioctl = NULL; - lo->lo_sizelimit = 0; lo->old_gfp_mask = mapping_gfp_mask(mapping); mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS)); - if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync) + if (!(lo->lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync) blk_queue_write_cache(lo->lo_queue, true, false); - if (io_is_direct(lo->lo_backing_file) && inode->i_sb->s_bdev) { + if (config->block_size) + bsize = config->block_size; + else if (io_is_direct(lo->lo_backing_file) && inode->i_sb->s_bdev) /* In case of direct I/O, match underlying block size */ - unsigned short bsize = bdev_logical_block_size( - inode->i_sb->s_bdev); + bsize = bdev_logical_block_size(inode->i_sb->s_bdev); + else + bsize = 512; - blk_queue_logical_block_size(lo->lo_queue, bsize); - blk_queue_physical_block_size(lo->lo_queue, bsize); - blk_queue_io_min(lo->lo_queue, bsize); - } + blk_queue_logical_block_size(lo->lo_queue, bsize); + blk_queue_physical_block_size(lo->lo_queue, bsize); + blk_queue_io_min(lo->lo_queue, bsize); loop_update_rotational(lo); loop_update_dio(lo); @@ -1155,14 +1182,14 @@ static int loop_set_fd(struct loop_device *lo, fmode_t mode, if (partscan) loop_reread_partitions(lo, bdev); if (claimed_bdev) - bd_abort_claiming(bdev, claimed_bdev, loop_set_fd); + bd_abort_claiming(bdev, claimed_bdev, loop_configure); return 0; out_unlock: mutex_unlock(&loop_ctl_mutex); out_bdev: if (claimed_bdev) - bd_abort_claiming(bdev, claimed_bdev, loop_set_fd); + bd_abort_claiming(bdev, claimed_bdev, loop_configure); out_putf: fput(file); out: @@ -1582,8 +1609,9 @@ static int loop_set_block_size(struct loop_device *lo, unsigned long arg) if (lo->lo_state != Lo_bound) return -ENXIO; - if (arg < 512 || arg > PAGE_SIZE || !is_power_of_2(arg)) - return -EINVAL; + err = loop_validate_block_size(arg); + if (err) + return err; if (lo->lo_queue->limits.logical_block_size == arg) return 0; @@ -1645,8 +1673,27 @@ static int lo_ioctl(struct block_device *bdev, fmode_t mode, int err; switch (cmd) { - case LOOP_SET_FD: - return loop_set_fd(lo, mode, bdev, arg); + case LOOP_SET_FD: { + /* + * Legacy case - pass in a zeroed out struct loop_config with + * only the file descriptor set , which corresponds with the + * default parameters we'd have used otherwise. + */ + struct loop_config config; + + memset(&config, 0, sizeof(config)); + config.fd = arg; + + return loop_configure(lo, mode, bdev, &config); + } + case LOOP_CONFIGURE: { + struct loop_config config; + + if (copy_from_user(&config, argp, sizeof(config))) + return -EFAULT; + + return loop_configure(lo, mode, bdev, &config); + } case LOOP_CHANGE_FD: return loop_change_fd(lo, bdev, arg); case LOOP_CLR_FD: @@ -1818,6 +1865,7 @@ static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode, case LOOP_CLR_FD: case LOOP_GET_STATUS64: case LOOP_SET_STATUS64: + case LOOP_CONFIGURE: arg = (unsigned long) compat_ptr(arg); /* fall through */ case LOOP_SET_FD: diff --git a/include/uapi/linux/loop.h b/include/uapi/linux/loop.h index 6b32fee80ce0..24a1c45bd1ae 100644 --- a/include/uapi/linux/loop.h +++ b/include/uapi/linux/loop.h @@ -31,6 +31,10 @@ enum { /* LO_FLAGS that can be cleared using LOOP_SET_STATUS(64) */ #define LOOP_SET_STATUS_CLEARABLE_FLAGS (LO_FLAGS_AUTOCLEAR) +/* LO_FLAGS that can be set using LOOP_CONFIGURE */ +#define LOOP_CONFIGURE_SETTABLE_FLAGS (LO_FLAGS_READ_ONLY | LO_FLAGS_AUTOCLEAR \ + | LO_FLAGS_PARTSCAN | LO_FLAGS_DIRECT_IO) + #include /* for __kernel_old_dev_t */ #include /* for __u64 */ @@ -66,6 +70,22 @@ struct loop_info64 { __u64 lo_init[2]; }; +/** + * struct loop_config - Complete configuration for a loop device. + * @fd: fd of the file to be used as a backing file for the loop device. + * @block_size: block size to use; ignored if 0. + * @info: struct loop_info64 to configure the loop device with. + * + * This structure is used with the LOOP_CONFIGURE ioctl, and can be used to + * atomically setup and configure all loop device parameters at once. + */ +struct loop_config { + __u32 fd; + __u32 block_size; + struct loop_info64 info; + __u64 __reserved[8]; +}; + /* * Loop filter types */ @@ -96,6 +116,7 @@ struct loop_info64 { #define LOOP_SET_CAPACITY 0x4C07 #define LOOP_SET_DIRECT_IO 0x4C08 #define LOOP_SET_BLOCK_SIZE 0x4C09 +#define LOOP_CONFIGURE 0x4C0A /* /dev/loop-control interface */ #define LOOP_CTL_ADD 0x4C80 From 9353848c6589ffe6373d03f3a58feaeda1009641 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 19 May 2020 16:22:58 +0200 Subject: [PATCH 0639/1043] dasd: refactor dasd_ioctl_information Prepare for in-kernel callers of this functionality. Signed-off-by: Christoph Hellwig [sth@de.ibm.com: remove leftover kfree] Signed-off-by: Stefan Haberland Reviewed-by: Peter Oberparleiter Reviewed-by: Jan Hoeppner Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_ioctl.c | 42 ++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/drivers/s390/block/dasd_ioctl.c b/drivers/s390/block/dasd_ioctl.c index 9a5f3add325f..9b7782395c37 100644 --- a/drivers/s390/block/dasd_ioctl.c +++ b/drivers/s390/block/dasd_ioctl.c @@ -457,10 +457,9 @@ static int dasd_ioctl_read_profile(struct dasd_block *block, void __user *argp) /* * Return dasd information. Used for BIODASDINFO and BIODASDINFO2. */ -static int dasd_ioctl_information(struct dasd_block *block, - unsigned int cmd, void __user *argp) +static int __dasd_ioctl_information(struct dasd_block *block, + struct dasd_information2_t *dasd_info) { - struct dasd_information2_t *dasd_info; struct subchannel_id sch_id; struct ccw_dev_id dev_id; struct dasd_device *base; @@ -473,15 +472,9 @@ static int dasd_ioctl_information(struct dasd_block *block, if (!base->discipline || !base->discipline->fill_info) return -EINVAL; - dasd_info = kzalloc(sizeof(struct dasd_information2_t), GFP_KERNEL); - if (dasd_info == NULL) - return -ENOMEM; - rc = base->discipline->fill_info(base, dasd_info); - if (rc) { - kfree(dasd_info); + if (rc) return rc; - } cdev = base->cdev; ccw_device_get_id(cdev, &dev_id); @@ -520,15 +513,24 @@ static int dasd_ioctl_information(struct dasd_block *block, list_for_each(l, &base->ccw_queue) dasd_info->chanq_len++; spin_unlock_irqrestore(&block->queue_lock, flags); + return 0; +} - rc = 0; - if (copy_to_user(argp, dasd_info, - ((cmd == (unsigned int) BIODASDINFO2) ? - sizeof(struct dasd_information2_t) : - sizeof(struct dasd_information_t)))) - rc = -EFAULT; +static int dasd_ioctl_information(struct dasd_block *block, void __user *argp, + size_t copy_size) +{ + struct dasd_information2_t *dasd_info; + int error; + + dasd_info = kzalloc(sizeof(*dasd_info), GFP_KERNEL); + if (!dasd_info) + return -ENOMEM; + + error = __dasd_ioctl_information(block, dasd_info); + if (!error && copy_to_user(argp, dasd_info, copy_size)) + error = -EFAULT; kfree(dasd_info); - return rc; + return error; } /* @@ -622,10 +624,12 @@ int dasd_ioctl(struct block_device *bdev, fmode_t mode, rc = dasd_ioctl_check_format(bdev, argp); break; case BIODASDINFO: - rc = dasd_ioctl_information(block, cmd, argp); + rc = dasd_ioctl_information(block, argp, + sizeof(struct dasd_information_t)); break; case BIODASDINFO2: - rc = dasd_ioctl_information(block, cmd, argp); + rc = dasd_ioctl_information(block, argp, + sizeof(struct dasd_information2_t)); break; case BIODASDPRRD: rc = dasd_ioctl_read_profile(block, argp); From 26d7e28e38206b1b3207af1409eee2269ab36f82 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Tue, 19 May 2020 16:22:59 +0200 Subject: [PATCH 0640/1043] s390/dasd: remove ioctl_by_bdev calls The IBM partition parser requires device type specific information only available to the DASD driver to correctly register partitions. The current approach of using ioctl_by_bdev with a fake user space pointer is discouraged. Fix this by replacing IOCTL calls with direct in-kernel function calls. Suggested-by: Christoph Hellwig Signed-off-by: Stefan Haberland Reviewed-by: Jan Hoeppner Reviewed-by: Peter Oberparleiter Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- MAINTAINERS | 1 + block/partitions/ibm.c | 24 +++++++++++++++++------ drivers/s390/block/dasd_ioctl.c | 34 +++++++++++++++++++++++++++++++++ include/linux/dasd_mod.h | 9 +++++++++ 4 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 include/linux/dasd_mod.h diff --git a/MAINTAINERS b/MAINTAINERS index b816a453b10e..927fd9845d2f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14628,6 +14628,7 @@ S: Supported W: http://www.ibm.com/developerworks/linux/linux390/ F: block/partitions/ibm.c F: drivers/s390/block/dasd* +F: include/linux/dasd_mod.h S390 IOMMU (PCI) M: Gerald Schaefer diff --git a/block/partitions/ibm.c b/block/partitions/ibm.c index 073faa6a69b8..d6e18df9c53c 100644 --- a/block/partitions/ibm.c +++ b/block/partitions/ibm.c @@ -13,10 +13,11 @@ #include #include #include +#include +#include #include "check.h" - union label_t { struct vtoc_volume_label_cdl vol; struct vtoc_volume_label_ldl lnx; @@ -288,7 +289,9 @@ static int find_cms1_partitions(struct parsed_partitions *state, */ int ibm_partition(struct parsed_partitions *state) { + int (*fn)(struct gendisk *disk, dasd_information2_t *info); struct block_device *bdev = state->bdev; + struct gendisk *disk = bdev->bd_disk; int blocksize, res; loff_t i_size, offset, size; dasd_information2_t *info; @@ -299,24 +302,31 @@ int ibm_partition(struct parsed_partitions *state) union label_t *label; res = 0; + if (!disk->fops->getgeo) + goto out_exit; + fn = symbol_get(dasd_biodasdinfo); + if (!fn) + goto out_exit; blocksize = bdev_logical_block_size(bdev); if (blocksize <= 0) - goto out_exit; + goto out_symbol; i_size = i_size_read(bdev->bd_inode); if (i_size == 0) - goto out_exit; + goto out_symbol; info = kmalloc(sizeof(dasd_information2_t), GFP_KERNEL); if (info == NULL) - goto out_exit; + goto out_symbol; geo = kmalloc(sizeof(struct hd_geometry), GFP_KERNEL); if (geo == NULL) goto out_nogeo; label = kmalloc(sizeof(union label_t), GFP_KERNEL); if (label == NULL) goto out_nolab; - if (ioctl_by_bdev(bdev, HDIO_GETGEO, (unsigned long)geo) != 0) + /* set start if not filled by getgeo function e.g. virtblk */ + geo->start = get_start_sect(bdev); + if (disk->fops->getgeo(bdev, geo)) goto out_freeall; - if (ioctl_by_bdev(bdev, BIODASDINFO2, (unsigned long)info) != 0) { + if (fn(disk, info)) { kfree(info); info = NULL; } @@ -359,6 +369,8 @@ out_nolab: kfree(geo); out_nogeo: kfree(info); +out_symbol: + symbol_put(dasd_biodasdinfo); out_exit: return res; } diff --git a/drivers/s390/block/dasd_ioctl.c b/drivers/s390/block/dasd_ioctl.c index 9b7782395c37..777734d1b4e5 100644 --- a/drivers/s390/block/dasd_ioctl.c +++ b/drivers/s390/block/dasd_ioctl.c @@ -22,6 +22,7 @@ #include #include #include +#include /* This is ugly... */ #define PRINTK_HEADER "dasd_ioctl:" @@ -664,3 +665,36 @@ int dasd_ioctl(struct block_device *bdev, fmode_t mode, dasd_put_device(base); return rc; } + + +/** + * dasd_biodasdinfo() - fill out the dasd information structure + * @disk [in]: pointer to gendisk structure that references a DASD + * @info [out]: pointer to the dasd_information2_t structure + * + * Provide access to DASD specific information. + * The gendisk structure is checked if it belongs to the DASD driver by + * comparing the gendisk->fops pointer. + * If it does not belong to the DASD driver -EINVAL is returned. + * Otherwise the provided dasd_information2_t structure is filled out. + * + * Returns: + * %0 on success and a negative error value on failure. + */ +int dasd_biodasdinfo(struct gendisk *disk, struct dasd_information2_t *info) +{ + struct dasd_device *base; + int error; + + if (disk->fops != &dasd_device_operations) + return -EINVAL; + + base = dasd_device_from_gendisk(disk); + if (!base) + return -ENODEV; + error = __dasd_ioctl_information(base->block, info); + dasd_put_device(base); + return error; +} +/* export that symbol_get in partition detection is possible */ +EXPORT_SYMBOL_GPL(dasd_biodasdinfo); diff --git a/include/linux/dasd_mod.h b/include/linux/dasd_mod.h new file mode 100644 index 000000000000..d39abad2ff6e --- /dev/null +++ b/include/linux/dasd_mod.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef DASD_MOD_H +#define DASD_MOD_H + +#include + +extern int dasd_biodasdinfo(struct gendisk *disk, dasd_information2_t *info); + +#endif From 3783daeb1d24696ff00125050353cfce4f5b6239 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 19 May 2020 16:33:21 +0200 Subject: [PATCH 0641/1043] block: remove ioctl_by_bdev No callers left. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- fs/block_dev.c | 12 ------------ include/linux/fs.h | 1 - 2 files changed, 13 deletions(-) diff --git a/fs/block_dev.c b/fs/block_dev.c index 7cbb7b79935e..3003831c771d 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -2166,18 +2166,6 @@ const struct file_operations def_blk_fops = { .fallocate = blkdev_fallocate, }; -int ioctl_by_bdev(struct block_device *bdev, unsigned cmd, unsigned long arg) -{ - int res; - mm_segment_t old_fs = get_fs(); - set_fs(KERNEL_DS); - res = blkdev_ioctl(bdev, 0, cmd, arg); - set_fs(old_fs); - return res; -} - -EXPORT_SYMBOL(ioctl_by_bdev); - /** * lookup_bdev - lookup a struct block_device by name * @pathname: special file representing the block device diff --git a/include/linux/fs.h b/include/linux/fs.h index 1a95e5158811..861ca61d728b 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -2636,7 +2636,6 @@ extern int sync_filesystem(struct super_block *); extern const struct file_operations def_blk_fops; extern const struct file_operations def_chr_fops; #ifdef CONFIG_BLOCK -extern int ioctl_by_bdev(struct block_device *, unsigned, unsigned long); extern int blkdev_ioctl(struct block_device *, fmode_t, unsigned, unsigned long); extern long compat_blkdev_ioctl(struct file *, unsigned, unsigned long); extern int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder); From bd35c77e32e4359580207891c0f7a438ad4b42df Mon Sep 17 00:00:00 2001 From: Krzysztof Piecuch Date: Thu, 23 Jan 2020 16:09:26 +0000 Subject: [PATCH 0642/1043] x86/tsc: Add tsc_early_khz command line parameter Changing base clock frequency directly impacts TSC Hz but not CPUID.16h value. An overclocked CPU supporting CPUID.16h and with partial CPUID.15h support will set TSC KHZ according to "best guess" given by CPUID.16h relying on tsc_refine_calibration_work to give better numbers later. tsc_refine_calibration_work will refuse to do its work when the outcome is off the early TSC KHZ value by more than 1% which is certain to happen on an overclocked system. Fix this by adding a tsc_early_khz command line parameter that makes the kernel skip early TSC calibration and use the given value instead. This allows the user to provide the expected TSC frequency that is closer to reality than the one reported by the hardware, enabling tsc_refine_calibration_work to do meaningful error checking. [ tglx: Made the variable __initdata as it's only used on init and removed the error checking in the argument parser because kstrto*() only stores to the variable if the string is valid ] Signed-off-by: Krzysztof Piecuch Signed-off-by: Thomas Gleixner Link: https://lkml.kernel.org/r/O2CpIOrqLZHgNRkfjRpz_LGqnc1ix_seNIiOCvHY4RHoulOVRo6kMXKuLOfBVTi0SMMevg6Go1uZ_cL9fLYtYdTRNH78ChaFaZyG3VAyYz8=@protonmail.com --- Documentation/admin-guide/kernel-parameters.txt | 6 ++++++ arch/x86/kernel/tsc.c | 12 +++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 7bc83f3d9bdf..409ee4ae467c 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5067,6 +5067,12 @@ interruptions from clocksource watchdog are not acceptable). + tsc_early_khz= [X86] Skip early TSC calibration and use the given + value instead. Useful when the early TSC frequency discovery + procedure is not reliable, such as on overclocked systems + with CPUID.16h support and partial CPUID.15h support. + Format: + tsx= [X86] Control Transactional Synchronization Extensions (TSX) feature in Intel processors that support TSX control. diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index fdd4c1078632..49d925043171 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -41,6 +41,7 @@ EXPORT_SYMBOL(tsc_khz); * TSC can be unstable due to cpufreq or due to unsynced TSCs */ static int __read_mostly tsc_unstable; +static unsigned int __initdata tsc_early_khz; static DEFINE_STATIC_KEY_FALSE(__use_tsc); @@ -59,6 +60,12 @@ struct cyc2ns { static DEFINE_PER_CPU_ALIGNED(struct cyc2ns, cyc2ns); +static int __init tsc_early_khz_setup(char *buf) +{ + return kstrtouint(buf, 0, &tsc_early_khz); +} +early_param("tsc_early_khz", tsc_early_khz_setup); + __always_inline void cyc2ns_read_begin(struct cyc2ns_data *data) { int seq, idx; @@ -1412,7 +1419,10 @@ static bool __init determine_cpu_tsc_frequencies(bool early) if (early) { cpu_khz = x86_platform.calibrate_cpu(); - tsc_khz = x86_platform.calibrate_tsc(); + if (tsc_early_khz) + tsc_khz = tsc_early_khz; + else + tsc_khz = x86_platform.calibrate_tsc(); } else { /* We should not be here with non-native cpu calibration */ WARN_ON(x86_platform.calibrate_cpu != native_calibrate_cpu); From ab7c01fdc3cfe02256e777a36366b70e2a539c27 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 17:07:14 +0300 Subject: [PATCH 0643/1043] mips: Add MIPS Release 5 support There are five MIPS32/64 architecture releases currently available: from 1 to 6 except fourth one, which was intentionally skipped. Three of them can be called as major: 1st, 2nd and 6th, that not only have some system level alterations, but also introduced significant core/ISA level updates. The rest of the MIPS architecture releases are minor. Even though they don't have as much ISA/system/core level changes as the major ones with respect to the previous releases, they still provide a set of updates (I'd say they were intended to be the intermediate releases before a major one) that might be useful for the kernel and user-level code, when activated by the kernel or compiler. In particular the following features were introduced or ended up being available at/after MIPS32/64 Release 5 architecture: + the last release of the misaligned memory access instructions, + virtualisation - VZ ASE - is optional component of the arch, + SIMD - MSA ASE - is optional component of the arch, + DSP ASE is optional component of the arch, + CP0.Status.FR=1 for CP1.FIR.F64=1 (pure 64-bit FPU general registers) must be available if FPU is implemented, + CP1.FIR.Has2008 support is required so CP1.FCSR.{ABS2008,NAN2008} bits are available. + UFR/UNFR aliases to access CP0.Status.FR from user-space by means of ctc1/cfc1 instructions (enabled by CP0.Config5.UFR), + CP0.COnfig5.LLB=1 and eretnc instruction are implemented to without accidentally clearing LL-bit when returning from an interrupt, exception, or error trap, + XPA feature together with extended versions of CPx registers is introduced, which needs to have mfhc0/mthc0 instructions available. So due to these changes GNU GCC provides an extended instructions set support for MIPS32/64 Release 5 by default like eretnc/mfhc0/mthc0. Even though the architecture alteration isn't that big, it still worth to be taken into account by the kernel software. Finally we can't deny that some optimization/limitations might be found in future and implemented on some level in kernel or compiler. In this case having even intermediate MIPS architecture releases support would be more than useful. So the most of the changes provided by this commit can be split into either compile- or runtime configs related. The compile-time related changes are caused by adding the new CONFIG_CPU_MIPS32_R5/CONFIG_CPU_MIPSR5 configs and concern the code activating MIPSR2 or MIPSR6 already implemented features (like eretnc/LLbit, mthc0/mfhc0). In addition CPU_HAS_MSA can be now freely enabled for MIPS32/64 release 5 based platforms as this is done for CPU_MIPS32_R6 CPUs. The runtime changes concerns the features which are handled with respect to the MIPS ISA revision detected at run-time by means of CP0.Config.{AT,AR} bits. Alas these fields can be used to detect either r1 or r2 or r6 releases. But since we know which CPUs in fact support the R5 arch, we can manually set MIPS_CPU_ISA_M32R5/MIPS_CPU_ISA_M64R5 bit of c->isa_level and then use cpu_has_mips32r5/cpu_has_mips64r5 where it's appropriate. Since XPA/EVA provide too complex alterationss and to have them used with MIPS32 Release 2 charged kernels (for compatibility with current platform configs) they are left to be setup as a separate kernel configs. Co-developed-by: Alexey Malahov Signed-off-by: Alexey Malahov Signed-off-by: Serge Semin Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Arnd Bergmann Cc: Rob Herring Cc: devicetree@vger.kernel.org Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kconfig | 56 +++++++++++++++++++++++++--- arch/mips/Makefile | 2 + arch/mips/include/asm/asmmacro.h | 18 +++++---- arch/mips/include/asm/compiler.h | 5 +++ arch/mips/include/asm/cpu-features.h | 27 ++++++++++---- arch/mips/include/asm/cpu-info.h | 2 +- arch/mips/include/asm/cpu-type.h | 7 +++- arch/mips/include/asm/cpu.h | 10 +++-- arch/mips/include/asm/fpu.h | 4 +- arch/mips/include/asm/hazards.h | 8 ++-- arch/mips/include/asm/module.h | 4 ++ arch/mips/include/asm/stackframe.h | 2 +- arch/mips/include/asm/switch_to.h | 8 ++-- arch/mips/kernel/cpu-probe.c | 17 +++++++++ arch/mips/kernel/entry.S | 6 +-- arch/mips/kernel/proc.c | 4 ++ arch/mips/kernel/r4k_fpu.S | 14 +++---- arch/mips/kvm/vz.c | 6 +-- arch/mips/lib/csum_partial.S | 6 ++- arch/mips/mm/c-r4k.c | 7 ++-- arch/mips/mm/sc-mips.c | 7 ++-- 21 files changed, 163 insertions(+), 57 deletions(-) diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index b2ff77f8366f..9dc173ff7293 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1516,6 +1516,21 @@ config CPU_MIPS32_R2 specific type of processor in your system, choose those that one otherwise CPU_MIPS32_R1 is a safe bet for any MIPS32 system. +config CPU_MIPS32_R5 + bool "MIPS32 Release 5" + depends on SYS_HAS_CPU_MIPS32_R5 + select CPU_HAS_PREFETCH + select CPU_SUPPORTS_32BIT_KERNEL + select CPU_SUPPORTS_HIGHMEM + select CPU_SUPPORTS_MSA + select HAVE_KVM + select MIPS_O32_FP64_SUPPORT + help + Choose this option to build a kernel for release 5 or later of the + MIPS32 architecture. New MIPS processors, starting with the Warrior + family, are based on a MIPS32r5 processor. If you own an older + processor, you probably need to select MIPS32r1 or MIPS32r2 instead. + config CPU_MIPS32_R6 bool "MIPS32 Release 6" depends on SYS_HAS_CPU_MIPS32_R6 @@ -1568,6 +1583,23 @@ config CPU_MIPS64_R2 specific type of processor in your system, choose those that one otherwise CPU_MIPS64_R1 is a safe bet for any MIPS64 system. +config CPU_MIPS64_R5 + bool "MIPS64 Release 5" + depends on SYS_HAS_CPU_MIPS64_R5 + select CPU_HAS_PREFETCH + select CPU_SUPPORTS_32BIT_KERNEL + select CPU_SUPPORTS_64BIT_KERNEL + select CPU_SUPPORTS_HIGHMEM + select CPU_SUPPORTS_HUGEPAGES + select CPU_SUPPORTS_MSA + select MIPS_O32_FP64_SUPPORT if 32BIT || MIPS32_O32 + select HAVE_KVM + help + Choose this option to build a kernel for release 5 or later of the + MIPS64 architecture. This is a intermediate MIPS architecture + release partly implementing release 6 features. Though there is no + any hardware known to be based on this release. + config CPU_MIPS64_R6 bool "MIPS64 Release 6" depends on SYS_HAS_CPU_MIPS64_R6 @@ -1762,7 +1794,7 @@ endchoice config CPU_MIPS32_3_5_FEATURES bool "MIPS32 Release 3.5 Features" depends on SYS_HAS_CPU_MIPS32_R3_5 - depends on CPU_MIPS32_R2 || CPU_MIPS32_R6 + depends on CPU_MIPS32_R2 || CPU_MIPS32_R5 || CPU_MIPS32_R6 help Choose this option to build a kernel for release 2 or later of the MIPS32 architecture including features from the 3.5 release such as @@ -1782,7 +1814,7 @@ config CPU_MIPS32_3_5_EVA config CPU_MIPS32_R5_FEATURES bool "MIPS32 Release 5 Features" depends on SYS_HAS_CPU_MIPS32_R5 - depends on CPU_MIPS32_R2 + depends on CPU_MIPS32_R2 || CPU_MIPS32_R5 help Choose this option to build a kernel for release 2 or later of the MIPS32 architecture including features from release 5 such as @@ -2020,11 +2052,13 @@ endmenu # config CPU_MIPS32 bool - default y if CPU_MIPS32_R1 || CPU_MIPS32_R2 || CPU_MIPS32_R6 + default y if CPU_MIPS32_R1 || CPU_MIPS32_R2 || CPU_MIPS32_R5 || \ + CPU_MIPS32_R6 config CPU_MIPS64 bool - default y if CPU_MIPS64_R1 || CPU_MIPS64_R2 || CPU_MIPS64_R6 + default y if CPU_MIPS64_R1 || CPU_MIPS64_R2 || CPU_MIPS64_R5 || \ + CPU_MIPS64_R6 # # These indicate the revision of the architecture @@ -2040,6 +2074,13 @@ config CPU_MIPSR2 select CPU_HAS_DIEI if !CPU_DIEI_BROKEN select MIPS_SPRAM +config CPU_MIPSR5 + bool + default y if CPU_MIPS32_R5 || CPU_MIPS64_R5 + select CPU_HAS_RIXI + select CPU_HAS_DIEI if !CPU_DIEI_BROKEN + select MIPS_SPRAM + config CPU_MIPSR6 bool default y if CPU_MIPS32_R6 || CPU_MIPS64_R6 @@ -2054,6 +2095,7 @@ config TARGET_ISA_REV int default 1 if CPU_MIPSR1 default 2 if CPU_MIPSR2 + default 5 if CPU_MIPSR5 default 6 if CPU_MIPSR6 default 0 help @@ -2643,7 +2685,11 @@ config NEED_PER_CPU_EMBED_FIRST_CHUNK config RELOCATABLE bool "Relocatable kernel" - depends on SYS_SUPPORTS_RELOCATABLE && (CPU_MIPS32_R2 || CPU_MIPS64_R2 || CPU_MIPS32_R6 || CPU_MIPS64_R6 || CAVIUM_OCTEON_SOC) + depends on SYS_SUPPORTS_RELOCATABLE + depends on CPU_MIPS32_R2 || CPU_MIPS64_R2 || \ + CPU_MIPS32_R5 || CPU_MIPS64_R5 || \ + CPU_MIPS32_R6 || CPU_MIPS64_R6 || \ + CAVIUM_OCTEON_SOC help This builds a kernel image that retains relocation information so it can be loaded someplace besides the default 1MB. diff --git a/arch/mips/Makefile b/arch/mips/Makefile index b50377ec3ab5..5d7a33ae86a4 100644 --- a/arch/mips/Makefile +++ b/arch/mips/Makefile @@ -146,9 +146,11 @@ cflags-$(CONFIG_CPU_R4X00) += -march=r4600 -Wa,--trap cflags-$(CONFIG_CPU_TX49XX) += -march=r4600 -Wa,--trap cflags-$(CONFIG_CPU_MIPS32_R1) += -march=mips32 -Wa,--trap cflags-$(CONFIG_CPU_MIPS32_R2) += -march=mips32r2 -Wa,--trap +cflags-$(CONFIG_CPU_MIPS32_R5) += -march=mips32r5 -Wa,--trap -modd-spreg cflags-$(CONFIG_CPU_MIPS32_R6) += -march=mips32r6 -Wa,--trap -modd-spreg cflags-$(CONFIG_CPU_MIPS64_R1) += -march=mips64 -Wa,--trap cflags-$(CONFIG_CPU_MIPS64_R2) += -march=mips64r2 -Wa,--trap +cflags-$(CONFIG_CPU_MIPS64_R5) += -march=mips64r5 -Wa,--trap cflags-$(CONFIG_CPU_MIPS64_R6) += -march=mips64r6 -Wa,--trap cflags-$(CONFIG_CPU_R5000) += -march=r5000 -Wa,--trap cflags-$(CONFIG_CPU_R5500) += $(call cc-option,-march=r5500,-march=r5000) \ diff --git a/arch/mips/include/asm/asmmacro.h b/arch/mips/include/asm/asmmacro.h index 655f40ddb6d1..86f2323ebe6b 100644 --- a/arch/mips/include/asm/asmmacro.h +++ b/arch/mips/include/asm/asmmacro.h @@ -44,7 +44,8 @@ .endm #endif -#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6) +#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \ + defined(CONFIG_CPU_MIPSR6) .macro local_irq_enable reg=t0 ei irq_enable_hazard @@ -54,7 +55,7 @@ di irq_disable_hazard .endm -#else +#else /* !CONFIG_CPU_MIPSR2 && !CONFIG_CPU_MIPSR5 && !CONFIG_CPU_MIPSR6 */ .macro local_irq_enable reg=t0 mfc0 \reg, CP0_STATUS ori \reg, \reg, 1 @@ -79,7 +80,7 @@ sw \reg, TI_PRE_COUNT($28) #endif .endm -#endif /* CONFIG_CPU_MIPSR2 */ +#endif /* !CONFIG_CPU_MIPSR2 && !CONFIG_CPU_MIPSR5 && !CONFIG_CPU_MIPSR6 */ .macro fpu_save_16even thread tmp=t0 .set push @@ -131,7 +132,7 @@ .macro fpu_save_double thread status tmp #if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPSR2) || \ - defined(CONFIG_CPU_MIPSR6) + defined(CONFIG_CPU_MIPSR5) || defined(CONFIG_CPU_MIPSR6) sll \tmp, \status, 5 bgez \tmp, 10f fpu_save_16odd \thread @@ -190,7 +191,7 @@ .macro fpu_restore_double thread status tmp #if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPSR2) || \ - defined(CONFIG_CPU_MIPSR6) + defined(CONFIG_CPU_MIPSR5) || defined(CONFIG_CPU_MIPSR6) sll \tmp, \status, 5 bgez \tmp, 10f # 16 register mode? @@ -200,16 +201,17 @@ fpu_restore_16even \thread \tmp .endm -#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6) +#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \ + defined(CONFIG_CPU_MIPSR6) .macro _EXT rd, rs, p, s ext \rd, \rs, \p, \s .endm -#else /* !CONFIG_CPU_MIPSR2 || !CONFIG_CPU_MIPSR6 */ +#else /* !CONFIG_CPU_MIPSR2 && !CONFIG_CPU_MIPSR5 && !CONFIG_CPU_MIPSR6 */ .macro _EXT rd, rs, p, s srl \rd, \rs, \p andi \rd, \rd, (1 << \s) - 1 .endm -#endif /* !CONFIG_CPU_MIPSR2 || !CONFIG_CPU_MIPSR6 */ +#endif /* !CONFIG_CPU_MIPSR2 && !CONFIG_CPU_MIPSR5 && !CONFIG_CPU_MIPSR6 */ /* * Temporary until all gas have MT ASE support diff --git a/arch/mips/include/asm/compiler.h b/arch/mips/include/asm/compiler.h index f77e99f1722e..a2cb2d2b1c07 100644 --- a/arch/mips/include/asm/compiler.h +++ b/arch/mips/include/asm/compiler.h @@ -57,6 +57,11 @@ #define MIPS_ISA_ARCH_LEVEL MIPS_ISA_LEVEL #define MIPS_ISA_LEVEL_RAW mips64r6 #define MIPS_ISA_ARCH_LEVEL_RAW MIPS_ISA_LEVEL_RAW +#elif defined(CONFIG_CPU_MIPSR5) +#define MIPS_ISA_LEVEL "mips64r5" +#define MIPS_ISA_ARCH_LEVEL MIPS_ISA_LEVEL +#define MIPS_ISA_LEVEL_RAW mips64r5 +#define MIPS_ISA_ARCH_LEVEL_RAW MIPS_ISA_LEVEL_RAW #else /* MIPS64 is a superset of MIPS32 */ #define MIPS_ISA_LEVEL "mips64r2" diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h index 400b123cb6da..227d7416591c 100644 --- a/arch/mips/include/asm/cpu-features.h +++ b/arch/mips/include/asm/cpu-features.h @@ -284,6 +284,9 @@ #ifndef cpu_has_mips32r2 # define cpu_has_mips32r2 __isa_range_or_flag(2, 6, MIPS_CPU_ISA_M32R2) #endif +#ifndef cpu_has_mips32r5 +# define cpu_has_mips32r5 __isa_range_or_flag(5, 6, MIPS_CPU_ISA_M32R5) +#endif #ifndef cpu_has_mips32r6 # define cpu_has_mips32r6 __isa_ge_or_flag(6, MIPS_CPU_ISA_M32R6) #endif @@ -293,6 +296,10 @@ #ifndef cpu_has_mips64r2 # define cpu_has_mips64r2 __isa_range_or_flag(2, 6, MIPS_CPU_ISA_M64R2) #endif +#ifndef cpu_has_mips64r5 +# define cpu_has_mips64r5 (cpu_has_64bits && \ + __isa_range_or_flag(5, 6, MIPS_CPU_ISA_M64R5)) +#endif #ifndef cpu_has_mips64r6 # define cpu_has_mips64r6 __isa_ge_and_flag(6, MIPS_CPU_ISA_M64R6) #endif @@ -313,19 +320,25 @@ (cpu_has_mips_3 | cpu_has_mips_4_5_64_r2_r6) #define cpu_has_mips_4_5_64_r2_r6 \ (cpu_has_mips_4_5 | cpu_has_mips64r1 | \ - cpu_has_mips_r2 | cpu_has_mips_r6) + cpu_has_mips_r2 | cpu_has_mips_r5 | \ + cpu_has_mips_r6) -#define cpu_has_mips32 (cpu_has_mips32r1 | cpu_has_mips32r2 | cpu_has_mips32r6) -#define cpu_has_mips64 (cpu_has_mips64r1 | cpu_has_mips64r2 | cpu_has_mips64r6) +#define cpu_has_mips32 (cpu_has_mips32r1 | cpu_has_mips32r2 | \ + cpu_has_mips32r5 | cpu_has_mips32r6) +#define cpu_has_mips64 (cpu_has_mips64r1 | cpu_has_mips64r2 | \ + cpu_has_mips64r5 | cpu_has_mips64r6) #define cpu_has_mips_r1 (cpu_has_mips32r1 | cpu_has_mips64r1) #define cpu_has_mips_r2 (cpu_has_mips32r2 | cpu_has_mips64r2) +#define cpu_has_mips_r5 (cpu_has_mips32r5 | cpu_has_mips64r5) #define cpu_has_mips_r6 (cpu_has_mips32r6 | cpu_has_mips64r6) #define cpu_has_mips_r (cpu_has_mips32r1 | cpu_has_mips32r2 | \ - cpu_has_mips32r6 | cpu_has_mips64r1 | \ - cpu_has_mips64r2 | cpu_has_mips64r6) + cpu_has_mips32r5 | cpu_has_mips32r6 | \ + cpu_has_mips64r1 | cpu_has_mips64r2 | \ + cpu_has_mips64r5 | cpu_has_mips64r6) -/* MIPSR2 and MIPSR6 have a lot of similarities */ -#define cpu_has_mips_r2_r6 (cpu_has_mips_r2 | cpu_has_mips_r6) +/* MIPSR2 - MIPSR6 have a lot of similarities */ +#define cpu_has_mips_r2_r6 (cpu_has_mips_r2 | cpu_has_mips_r5 | \ + cpu_has_mips_r6) /* * cpu_has_mips_r2_exec_hazard - return if IHB is required on current processor diff --git a/arch/mips/include/asm/cpu-info.h b/arch/mips/include/asm/cpu-info.h index ed7ffe4e63a3..bce3ea7fff7c 100644 --- a/arch/mips/include/asm/cpu-info.h +++ b/arch/mips/include/asm/cpu-info.h @@ -142,7 +142,7 @@ struct proc_cpuinfo_notifier_args { static inline unsigned int cpu_cluster(struct cpuinfo_mips *cpuinfo) { /* Optimisation for systems where multiple clusters aren't used */ - if (!IS_ENABLED(CONFIG_CPU_MIPSR6)) + if (!IS_ENABLED(CONFIG_CPU_MIPSR5) && !IS_ENABLED(CONFIG_CPU_MIPSR6)) return 0; return (cpuinfo->globalnumber & MIPS_GLOBALNUMBER_CLUSTER) >> diff --git a/arch/mips/include/asm/cpu-type.h b/arch/mips/include/asm/cpu-type.h index 49f0061a6051..75a7a382da09 100644 --- a/arch/mips/include/asm/cpu-type.h +++ b/arch/mips/include/asm/cpu-type.h @@ -51,13 +51,18 @@ static inline int __pure __get_cpu_type(const int cpu_type) case CPU_M14KEC: case CPU_INTERAPTIV: case CPU_PROAPTIV: - case CPU_P5600: +#endif + +#ifdef CONFIG_SYS_HAS_CPU_MIPS32_R5 case CPU_M5150: + case CPU_P5600: #endif #if defined(CONFIG_SYS_HAS_CPU_MIPS32_R2) || \ + defined(CONFIG_SYS_HAS_CPU_MIPS32_R5) || \ defined(CONFIG_SYS_HAS_CPU_MIPS32_R6) || \ defined(CONFIG_SYS_HAS_CPU_MIPS64_R2) || \ + defined(CONFIG_SYS_HAS_CPU_MIPS64_R5) || \ defined(CONFIG_SYS_HAS_CPU_MIPS64_R6) case CPU_QEMU_GENERIC: #endif diff --git a/arch/mips/include/asm/cpu.h b/arch/mips/include/asm/cpu.h index 46c190e78acf..4b84fd1df0c7 100644 --- a/arch/mips/include/asm/cpu.h +++ b/arch/mips/include/asm/cpu.h @@ -347,14 +347,16 @@ enum cpu_type_enum { #define MIPS_CPU_ISA_M32R2 0x00000020 #define MIPS_CPU_ISA_M64R1 0x00000040 #define MIPS_CPU_ISA_M64R2 0x00000080 -#define MIPS_CPU_ISA_M32R6 0x00000100 -#define MIPS_CPU_ISA_M64R6 0x00000200 +#define MIPS_CPU_ISA_M32R5 0x00000100 +#define MIPS_CPU_ISA_M64R5 0x00000200 +#define MIPS_CPU_ISA_M32R6 0x00000400 +#define MIPS_CPU_ISA_M64R6 0x00000800 #define MIPS_CPU_ISA_32BIT (MIPS_CPU_ISA_II | MIPS_CPU_ISA_M32R1 | \ - MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M32R6) + MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M32R6) #define MIPS_CPU_ISA_64BIT (MIPS_CPU_ISA_III | MIPS_CPU_ISA_IV | \ MIPS_CPU_ISA_V | MIPS_CPU_ISA_M64R1 | MIPS_CPU_ISA_M64R2 | \ - MIPS_CPU_ISA_M64R6) + MIPS_CPU_ISA_M64R5 | MIPS_CPU_ISA_M64R6) /* * CPU Option encodings diff --git a/arch/mips/include/asm/fpu.h b/arch/mips/include/asm/fpu.h index a9d5123e2a2a..08f9dd6903b7 100644 --- a/arch/mips/include/asm/fpu.h +++ b/arch/mips/include/asm/fpu.h @@ -71,8 +71,8 @@ static inline int __enable_fpu(enum fpu_mode mode) goto fr_common; case FPU_64BIT: -#if !(defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6) \ - || defined(CONFIG_64BIT)) +#if !(defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \ + defined(CONFIG_CPU_MIPSR6) || defined(CONFIG_64BIT)) /* we only have a 32-bit FPU */ return SIGFPE; #endif diff --git a/arch/mips/include/asm/hazards.h b/arch/mips/include/asm/hazards.h index a0b92205f933..f855478d12fa 100644 --- a/arch/mips/include/asm/hazards.h +++ b/arch/mips/include/asm/hazards.h @@ -22,8 +22,9 @@ /* * TLB hazards */ -#if (defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6)) && \ - !defined(CONFIG_CPU_CAVIUM_OCTEON) && !defined(CONFIG_CPU_LOONGSON64) +#if (defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \ + defined(CONFIG_CPU_MIPSR6)) && \ + !defined(CONFIG_CPU_CAVIUM_OCTEON) && !defined(CONFIG_CPU_LOONGSON64) /* * MIPSR2 defines ehb for hazard avoidance @@ -278,7 +279,8 @@ do { \ #define __disable_fpu_hazard -#elif defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6) +#elif defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \ + defined(CONFIG_CPU_MIPSR6) #define __enable_fpu_hazard \ ___ehb diff --git a/arch/mips/include/asm/module.h b/arch/mips/include/asm/module.h index 9846047b3d3d..84776e1ec8e5 100644 --- a/arch/mips/include/asm/module.h +++ b/arch/mips/include/asm/module.h @@ -89,12 +89,16 @@ search_module_dbetables(unsigned long addr) #define MODULE_PROC_FAMILY "MIPS32_R1 " #elif defined CONFIG_CPU_MIPS32_R2 #define MODULE_PROC_FAMILY "MIPS32_R2 " +#elif defined CONFIG_CPU_MIPS32_R5 +#define MODULE_PROC_FAMILY "MIPS32_R5 " #elif defined CONFIG_CPU_MIPS32_R6 #define MODULE_PROC_FAMILY "MIPS32_R6 " #elif defined CONFIG_CPU_MIPS64_R1 #define MODULE_PROC_FAMILY "MIPS64_R1 " #elif defined CONFIG_CPU_MIPS64_R2 #define MODULE_PROC_FAMILY "MIPS64_R2 " +#elif defined CONFIG_CPU_MIPS64_R5 +#define MODULE_PROC_FAMILY "MIPS64_R5 " #elif defined CONFIG_CPU_MIPS64_R6 #define MODULE_PROC_FAMILY "MIPS64_R6 " #elif defined CONFIG_CPU_R3000 diff --git a/arch/mips/include/asm/stackframe.h b/arch/mips/include/asm/stackframe.h index 4d6ad907ae54..3e8d2aaf96af 100644 --- a/arch/mips/include/asm/stackframe.h +++ b/arch/mips/include/asm/stackframe.h @@ -424,7 +424,7 @@ .macro RESTORE_SP_AND_RET docfi=0 RESTORE_SP \docfi -#ifdef CONFIG_CPU_MIPSR6 +#if defined(CONFIG_CPU_MIPSR5) || defined(CONFIG_CPU_MIPSR6) eretnc #else .set push diff --git a/arch/mips/include/asm/switch_to.h b/arch/mips/include/asm/switch_to.h index 09cbe9042828..0b0a93bf83cd 100644 --- a/arch/mips/include/asm/switch_to.h +++ b/arch/mips/include/asm/switch_to.h @@ -67,11 +67,11 @@ do { \ #endif /* - * Clear LLBit during context switches on MIPSr6 such that eretnc can be used + * Clear LLBit during context switches on MIPSr5+ such that eretnc can be used * unconditionally when returning to userland in entry.S. */ -#define __clear_r6_hw_ll_bit() do { \ - if (cpu_has_mips_r6) \ +#define __clear_r5_hw_ll_bit() do { \ + if (cpu_has_mips_r5 || cpu_has_mips_r6) \ write_c0_lladdr(0); \ } while (0) @@ -129,7 +129,7 @@ do { \ } \ clear_c0_status(ST0_CU2); \ } \ - __clear_r6_hw_ll_bit(); \ + __clear_r5_hw_ll_bit(); \ __clear_software_ll_bit(); \ if (cpu_has_userlocal) \ write_c0_userlocal(task_thread_info(next)->tp_value); \ diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index a0ef21b2d8b3..da2820991e21 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -92,6 +92,7 @@ static void cpu_set_fpu_2008(struct cpuinfo_mips *c) { if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1 | MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 | + MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 | MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)) { unsigned long sr, fir, fcsr, fcsr0, fcsr1; @@ -172,6 +173,7 @@ static void cpu_set_nofpu_2008(struct cpuinfo_mips *c) case STRICT: if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1 | MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 | + MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 | MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)) { c->options |= MIPS_CPU_NAN_2008 | MIPS_CPU_NAN_LEGACY; } else { @@ -263,9 +265,11 @@ static void cpu_set_nofpu_id(struct cpuinfo_mips *c) value = 0; if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1 | MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 | + MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 | MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)) value |= MIPS_FPIR_D | MIPS_FPIR_S; if (c->isa_level & (MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 | + MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 | MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)) value |= MIPS_FPIR_F64 | MIPS_FPIR_L | MIPS_FPIR_W; if (c->options & MIPS_CPU_NAN_2008) @@ -286,6 +290,7 @@ static void cpu_set_fpu_opts(struct cpuinfo_mips *c) if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1 | MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 | + MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 | MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)) { if (c->fpu_id & MIPS_FPIR_3D) c->ases |= MIPS_ASE_MIPS3D; @@ -532,6 +537,10 @@ static inline void cpu_probe_vmbits(struct cpuinfo_mips *c) static void set_isa(struct cpuinfo_mips *c, unsigned int isa) { switch (isa) { + case MIPS_CPU_ISA_M64R5: + c->isa_level |= MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5; + set_elf_base_platform("mips64r5"); + fallthrough; case MIPS_CPU_ISA_M64R2: c->isa_level |= MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2; set_elf_base_platform("mips64r2"); @@ -563,6 +572,10 @@ static void set_isa(struct cpuinfo_mips *c, unsigned int isa) set_elf_base_platform("mips32r6"); /* Break here so we don't add incompatible ISAs */ break; + case MIPS_CPU_ISA_M32R5: + c->isa_level |= MIPS_CPU_ISA_M32R5; + set_elf_base_platform("mips32r5"); + fallthrough; case MIPS_CPU_ISA_M32R2: c->isa_level |= MIPS_CPU_ISA_M32R2; set_elf_base_platform("mips32r2"); @@ -1751,6 +1764,10 @@ static inline void cpu_probe_mips(struct cpuinfo_mips *c, unsigned int cpu) spram_config(); switch (__get_cpu_type(c->cputype)) { + case CPU_M5150: + case CPU_P5600: + set_isa(c, MIPS_CPU_ISA_M32R5); + break; case CPU_I6500: c->options |= MIPS_CPU_SHARED_FTLB_ENTRIES; fallthrough; diff --git a/arch/mips/kernel/entry.S b/arch/mips/kernel/entry.S index 4849a48afc0f..4b896f5023ff 100644 --- a/arch/mips/kernel/entry.S +++ b/arch/mips/kernel/entry.S @@ -169,8 +169,8 @@ syscall_exit_work: jal syscall_trace_leave b resume_userspace -#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR6) || \ - defined(CONFIG_MIPS_MT) +#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \ + defined(CONFIG_CPU_MIPSR6) || defined(CONFIG_MIPS_MT) /* * MIPS32R2 Instruction Hazard Barrier - must be called @@ -183,4 +183,4 @@ LEAF(mips_ihb) nop END(mips_ihb) -#endif /* CONFIG_CPU_MIPSR2 or CONFIG_CPU_MIPSR6 or CONFIG_MIPS_MT */ +#endif /* CONFIG_CPU_MIPSR2 - CONFIG_CPU_MIPSR6 or CONFIG_MIPS_MT */ diff --git a/arch/mips/kernel/proc.c b/arch/mips/kernel/proc.c index f8d36710cd58..4184d641f05e 100644 --- a/arch/mips/kernel/proc.c +++ b/arch/mips/kernel/proc.c @@ -98,12 +98,16 @@ static int show_cpuinfo(struct seq_file *m, void *v) seq_printf(m, "%s", " mips32r1"); if (cpu_has_mips32r2) seq_printf(m, "%s", " mips32r2"); + if (cpu_has_mips32r5) + seq_printf(m, "%s", " mips32r5"); if (cpu_has_mips32r6) seq_printf(m, "%s", " mips32r6"); if (cpu_has_mips64r1) seq_printf(m, "%s", " mips64r1"); if (cpu_has_mips64r2) seq_printf(m, "%s", " mips64r2"); + if (cpu_has_mips64r5) + seq_printf(m, "%s", " mips64r5"); if (cpu_has_mips64r6) seq_printf(m, "%s", " mips64r6"); seq_printf(m, "\n"); diff --git a/arch/mips/kernel/r4k_fpu.S b/arch/mips/kernel/r4k_fpu.S index 59be5c812aa2..b91e91106475 100644 --- a/arch/mips/kernel/r4k_fpu.S +++ b/arch/mips/kernel/r4k_fpu.S @@ -41,7 +41,7 @@ LEAF(_save_fp) EXPORT_SYMBOL(_save_fp) #if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPSR2) || \ - defined(CONFIG_CPU_MIPSR6) + defined(CONFIG_CPU_MIPSR5) || defined(CONFIG_CPU_MIPSR6) mfc0 t0, CP0_STATUS #endif fpu_save_double a0 t0 t1 # clobbers t1 @@ -53,7 +53,7 @@ EXPORT_SYMBOL(_save_fp) */ LEAF(_restore_fp) #if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPSR2) || \ - defined(CONFIG_CPU_MIPSR6) + defined(CONFIG_CPU_MIPSR5) || defined(CONFIG_CPU_MIPSR6) mfc0 t0, CP0_STATUS #endif fpu_restore_double a0 t0 t1 # clobbers t1 @@ -103,10 +103,10 @@ LEAF(_save_fp_context) .set pop #if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPSR2) || \ - defined(CONFIG_CPU_MIPSR6) + defined(CONFIG_CPU_MIPSR5) || defined(CONFIG_CPU_MIPSR6) .set push SET_HARDFLOAT -#ifdef CONFIG_CPU_MIPSR2 +#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) .set mips32r2 .set fp=64 mfc0 t0, CP0_STATUS @@ -170,11 +170,11 @@ LEAF(_save_fp_context) LEAF(_restore_fp_context) EX lw t1, 0(a1) -#if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPSR2) || \ - defined(CONFIG_CPU_MIPSR6) +#if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPSR2) || \ + defined(CONFIG_CPU_MIPSR5) || defined(CONFIG_CPU_MIPSR6) .set push SET_HARDFLOAT -#ifdef CONFIG_CPU_MIPSR2 +#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) .set mips32r2 .set fp=64 mfc0 t0, CP0_STATUS diff --git a/arch/mips/kvm/vz.c b/arch/mips/kvm/vz.c index 389dd0fbd051..51f51009a53f 100644 --- a/arch/mips/kvm/vz.c +++ b/arch/mips/kvm/vz.c @@ -2980,7 +2980,7 @@ static int kvm_vz_vcpu_setup(struct kvm_vcpu *vcpu) */ /* PageGrain */ - if (cpu_has_mips_r6) + if (cpu_has_mips_r5 || cpu_has_mips_r6) kvm_write_sw_gc0_pagegrain(cop0, PG_RIE | PG_XIE | PG_IEC); /* Wired */ if (cpu_has_mips_r6) @@ -2988,7 +2988,7 @@ static int kvm_vz_vcpu_setup(struct kvm_vcpu *vcpu) read_gc0_wired() & MIPSR6_WIRED_LIMIT); /* Status */ kvm_write_sw_gc0_status(cop0, ST0_BEV | ST0_ERL); - if (cpu_has_mips_r6) + if (cpu_has_mips_r5 || cpu_has_mips_r6) kvm_change_sw_gc0_status(cop0, ST0_FR, read_gc0_status()); /* IntCtl */ kvm_write_sw_gc0_intctl(cop0, read_gc0_intctl() & @@ -3086,7 +3086,7 @@ static int kvm_vz_vcpu_setup(struct kvm_vcpu *vcpu) } /* reset HTW registers */ - if (cpu_guest_has_htw && cpu_has_mips_r6) { + if (cpu_guest_has_htw && (cpu_has_mips_r5 || cpu_has_mips_r6)) { /* PWField */ kvm_write_sw_gc0_pwfield(cop0, 0x0c30c302); /* PWSize */ diff --git a/arch/mips/lib/csum_partial.S b/arch/mips/lib/csum_partial.S index fda7b57b826e..87fda0713b84 100644 --- a/arch/mips/lib/csum_partial.S +++ b/arch/mips/lib/csum_partial.S @@ -279,7 +279,8 @@ EXPORT_SYMBOL(csum_partial) #endif /* odd buffer alignment? */ -#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_LOONGSON64) +#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \ + defined(CONFIG_CPU_LOONGSON64) .set push .set arch=mips32r2 wsbh v1, sum @@ -732,7 +733,8 @@ EXPORT_SYMBOL(csum_partial) addu sum, v1 #endif -#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_LOONGSON64) +#if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_CPU_MIPSR5) || \ + defined(CONFIG_CPU_LOONGSON64) .set push .set arch=mips32r2 wsbh v1, sum diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index 54c18b8a2406..a9f55bf90967 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -1709,9 +1709,10 @@ static void setup_scache(void) return; default: - if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M32R2 | - MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R1 | - MIPS_CPU_ISA_M64R2 | MIPS_CPU_ISA_M64R6)) { + if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1 | + MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 | + MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 | + MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6)) { #ifdef CONFIG_MIPS_CPU_SCACHE if (mips_sc_init ()) { scache_size = c->scache.ways * c->scache.sets * c->scache.linesz; diff --git a/arch/mips/mm/sc-mips.c b/arch/mips/mm/sc-mips.c index dbdbfe5d8408..eedad47df24f 100644 --- a/arch/mips/mm/sc-mips.c +++ b/arch/mips/mm/sc-mips.c @@ -194,9 +194,10 @@ static inline int __init mips_sc_probe(void) return mips_sc_probe_cm3(); /* Ignore anything but MIPSxx processors */ - if (!(c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M32R2 | - MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R1 | - MIPS_CPU_ISA_M64R2 | MIPS_CPU_ISA_M64R6))) + if (!(c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1 | + MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2 | + MIPS_CPU_ISA_M32R5 | MIPS_CPU_ISA_M64R5 | + MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R6))) return 0; /* Does this MIPS32/MIPS64 CPU have a config2 register? */ From 281e3aea35e521a90b0b05face3196da23758092 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 17:07:15 +0300 Subject: [PATCH 0644/1043] mips: Add MIPS Warrior P5600 support This is a MIPS32 Release 5 based IP core with XPA, EVA, dual/quad issue exec pipes, MMU with two-levels TLB, UCA, MSA, MDU core level features and system level features like up to six P5600 calculation cores, CM2 with L2 cache, IOCU/IOMMU (though might be unused depending on the system-specific IP core configuration), GIC, CPC, virtualisation module, eJTAG and PDtrace. As being MIPS32 Release 5 based core it provides all the features available by the CPU_MIPS32_R5 config, while adding a few more like UCA attribute support, availability of CPU-freq (by means of L2/CM clock ratio setting), EI/VI GIC modes detection at runtime. In addition to this if P5600 architecture is enabled modern GNU GCC provides a specific tuning for P5600 processors with respect to the classic MIPS32 Release 5. First of all branch-likely avoidance is activated only when the code is compiled with the speed optimization (avoidance is always enabled for the pure MIPS32 Release 5 architecture). Secondly the madd/msub avoidance is enabled since madd/msub utilization isn't profitable due to overhead of getting the result out of the HI/LO registers. Multiply-accumulate instructions are activated and utilized together with the necessary code reorder when multiply-add/multiply-subtract statements are met. Finally load/store bonding is activated by default. All of these optimizations may make the code relatively faster than if just MIP32 release 5 architecture was requested. Co-developed-by: Alexey Malahov Signed-off-by: Alexey Malahov Signed-off-by: Serge Semin Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Arnd Bergmann Cc: Rob Herring Cc: devicetree@vger.kernel.org Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kconfig | 37 +++++++++++++++++++++++++++++----- arch/mips/Makefile | 1 + arch/mips/include/asm/module.h | 2 ++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 9dc173ff7293..2747b1b2d435 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1618,6 +1618,28 @@ config CPU_MIPS64_R6 family, are based on a MIPS64r6 processor. If you own an older processor, you probably need to select MIPS64r1 or MIPS64r2 instead. +config CPU_P5600 + bool "MIPS Warrior P5600" + depends on SYS_HAS_CPU_P5600 + select CPU_HAS_PREFETCH + select CPU_SUPPORTS_32BIT_KERNEL + select CPU_SUPPORTS_HIGHMEM + select CPU_SUPPORTS_MSA + select CPU_SUPPORTS_UNCACHED_ACCELERATED + select CPU_SUPPORTS_CPUFREQ + select CPU_MIPSR2_IRQ_VI + select CPU_MIPSR2_IRQ_EI + select HAVE_KVM + select MIPS_O32_FP64_SUPPORT + help + Choose this option to build a kernel for MIPS Warrior P5600 CPU. + It's based on MIPS32r5 ISA with XPA, EVA, dual/quad issue exec pipes, + MMU with two-levels TLB, UCA, MSA, MDU core level features and system + level features like up to six P5600 calculation cores, CM2 with L2 + cache, IOCU/IOMMU (though might be unused depending on the system- + specific IP core configuration), GIC, CPC, virtualisation module, + eJTAG and PDtrace. + config CPU_R3000 bool "R3000" depends on SYS_HAS_CPU_R3000 @@ -1794,7 +1816,8 @@ endchoice config CPU_MIPS32_3_5_FEATURES bool "MIPS32 Release 3.5 Features" depends on SYS_HAS_CPU_MIPS32_R3_5 - depends on CPU_MIPS32_R2 || CPU_MIPS32_R5 || CPU_MIPS32_R6 + depends on CPU_MIPS32_R2 || CPU_MIPS32_R5 || CPU_MIPS32_R6 || \ + CPU_P5600 help Choose this option to build a kernel for release 2 or later of the MIPS32 architecture including features from the 3.5 release such as @@ -1814,7 +1837,7 @@ config CPU_MIPS32_3_5_EVA config CPU_MIPS32_R5_FEATURES bool "MIPS32 Release 5 Features" depends on SYS_HAS_CPU_MIPS32_R5 - depends on CPU_MIPS32_R2 || CPU_MIPS32_R5 + depends on CPU_MIPS32_R2 || CPU_MIPS32_R5 || CPU_P5600 help Choose this option to build a kernel for release 2 or later of the MIPS32 architecture including features from release 5 such as @@ -1969,6 +1992,10 @@ config SYS_HAS_CPU_MIPS64_R6 bool select ARCH_HAS_SYNC_DMA_FOR_CPU if DMA_NONCOHERENT +config SYS_HAS_CPU_P5600 + bool + select ARCH_HAS_SYNC_DMA_FOR_CPU if DMA_NONCOHERENT + config SYS_HAS_CPU_R3000 bool @@ -2053,7 +2080,7 @@ endmenu config CPU_MIPS32 bool default y if CPU_MIPS32_R1 || CPU_MIPS32_R2 || CPU_MIPS32_R5 || \ - CPU_MIPS32_R6 + CPU_MIPS32_R6 || CPU_P5600 config CPU_MIPS64 bool @@ -2076,7 +2103,7 @@ config CPU_MIPSR2 config CPU_MIPSR5 bool - default y if CPU_MIPS32_R5 || CPU_MIPS64_R5 + default y if CPU_MIPS32_R5 || CPU_MIPS64_R5 || CPU_P5600 select CPU_HAS_RIXI select CPU_HAS_DIEI if !CPU_DIEI_BROKEN select MIPS_SPRAM @@ -2689,7 +2716,7 @@ config RELOCATABLE depends on CPU_MIPS32_R2 || CPU_MIPS64_R2 || \ CPU_MIPS32_R5 || CPU_MIPS64_R5 || \ CPU_MIPS32_R6 || CPU_MIPS64_R6 || \ - CAVIUM_OCTEON_SOC + CPU_P5600 || CAVIUM_OCTEON_SOC help This builds a kernel image that retains relocation information so it can be loaded someplace besides the default 1MB. diff --git a/arch/mips/Makefile b/arch/mips/Makefile index 5d7a33ae86a4..0d0f29d662c9 100644 --- a/arch/mips/Makefile +++ b/arch/mips/Makefile @@ -152,6 +152,7 @@ cflags-$(CONFIG_CPU_MIPS64_R1) += -march=mips64 -Wa,--trap cflags-$(CONFIG_CPU_MIPS64_R2) += -march=mips64r2 -Wa,--trap cflags-$(CONFIG_CPU_MIPS64_R5) += -march=mips64r5 -Wa,--trap cflags-$(CONFIG_CPU_MIPS64_R6) += -march=mips64r6 -Wa,--trap +cflags-$(CONFIG_CPU_P5600) += -march=p5600 -Wa,--trap -modd-spreg cflags-$(CONFIG_CPU_R5000) += -march=r5000 -Wa,--trap cflags-$(CONFIG_CPU_R5500) += $(call cc-option,-march=r5500,-march=r5000) \ -Wa,--trap diff --git a/arch/mips/include/asm/module.h b/arch/mips/include/asm/module.h index 84776e1ec8e5..7be4c68081a8 100644 --- a/arch/mips/include/asm/module.h +++ b/arch/mips/include/asm/module.h @@ -131,6 +131,8 @@ search_module_dbetables(unsigned long addr) #define MODULE_PROC_FAMILY "LOONGSON64 " #elif defined CONFIG_CPU_CAVIUM_OCTEON #define MODULE_PROC_FAMILY "OCTEON " +#elif defined CONFIG_CPU_P5600 +#define MODULE_PROC_FAMILY "P5600 " #elif defined CONFIG_CPU_XLR #define MODULE_PROC_FAMILY "XLR " #elif defined CONFIG_CPU_XLP From a2ac81c6ef4018ea49c034ce165bb9ea1cf99f3e Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 17:07:16 +0300 Subject: [PATCH 0645/1043] mips: Fix cpu_has_mips64r1/2 activation for MIPS32 CPUs Commit 1aeba347b3a9 ("MIPS: Hardcode cpu_has_mips* where target ISA allows") updated the cpu_has_mips* macro to be replaced with a constant expression where it's possible. By mistake it wasn't done correctly for cpu_has_mips64r1/cpu_has_mips64r2 macro. They are defined to be replaced with conditional expression __isa_range_or_flag(), which means either ISA revision being within the range or the corresponding CPU options flag was set at the probe stage or both being true at the same time. But the ISA level value doesn't indicate whether the ISA is MIPS32 or MIPS64. Due to this if we select MIPS32r1 - MIPS32r5 architectures the __isa_range() macro will activate the cpu_has_mips64rX flags, which is incorrect. In order to fix the problem we make sure the 64bits CPU support is enabled by means of checking the flag cpu_has_64bits aside with proper ISA range and specific Revision flag being set. Fixes: 1aeba347b3a9 ("MIPS: Hardcode cpu_has_mips* where target ISA allows") Signed-off-by: Serge Semin Cc: Alexey Malahov Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Arnd Bergmann Cc: Rob Herring Cc: devicetree@vger.kernel.org Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/cpu-features.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h index 227d7416591c..14532cd25807 100644 --- a/arch/mips/include/asm/cpu-features.h +++ b/arch/mips/include/asm/cpu-features.h @@ -291,10 +291,12 @@ # define cpu_has_mips32r6 __isa_ge_or_flag(6, MIPS_CPU_ISA_M32R6) #endif #ifndef cpu_has_mips64r1 -# define cpu_has_mips64r1 __isa_range_or_flag(1, 6, MIPS_CPU_ISA_M64R1) +# define cpu_has_mips64r1 (cpu_has_64bits && \ + __isa_range_or_flag(1, 6, MIPS_CPU_ISA_M64R1)) #endif #ifndef cpu_has_mips64r2 -# define cpu_has_mips64r2 __isa_range_or_flag(2, 6, MIPS_CPU_ISA_M64R2) +# define cpu_has_mips64r2 (cpu_has_64bits && \ + __isa_range_or_flag(2, 6, MIPS_CPU_ISA_M64R2)) #endif #ifndef cpu_has_mips64r5 # define cpu_has_mips64r5 (cpu_has_64bits && \ From 742318ad5eeecace49e95da5d3cf4571b0b26b36 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 17:07:17 +0300 Subject: [PATCH 0646/1043] mips: Add CP0 Write Merge config support CP0 config register may indicate whether write-through merging is allowed. Currently there are two types of the merging available: SysAD Valid and Full modes. Whether each of them are supported by the core is implementation dependent. Moreover whether the ability to change the mode also depends on the chip family instance. Taking into account all of this we created a dedicated mm_config() method to detect and enable merging if it's supported. It is called for MIPS-type processors at CPU-probe stage and attempts to detect whether the write merging is available. If it's known to be supported and switchable, then switch on the full mode. Otherwise just perform the CP0.Config.MM field analysis. In addition there are platforms like InterAptiv/ProAptiv, which do have the MM bit field set by default, but having write-through cacheing unsupported makes write-merging also unsupported. In this case we just ignore the MM field value. Co-developed-by: Alexey Malahov Signed-off-by: Alexey Malahov Signed-off-by: Serge Semin Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Arnd Bergmann Cc: Rob Herring Cc: devicetree@vger.kernel.org Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/cpu-features.h | 8 +++++ arch/mips/include/asm/cpu.h | 4 ++- arch/mips/include/asm/mipsregs.h | 3 ++ arch/mips/kernel/cpu-probe.c | 48 ++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h index 14532cd25807..caecbae4b599 100644 --- a/arch/mips/include/asm/cpu-features.h +++ b/arch/mips/include/asm/cpu-features.h @@ -629,6 +629,14 @@ # endif #endif +#ifndef cpu_has_mm_sysad +# define cpu_has_mm_sysad __opt(MIPS_CPU_MM_SYSAD) +#endif + +#ifndef cpu_has_mm_full +# define cpu_has_mm_full __opt(MIPS_CPU_MM_FULL) +#endif + /* * Guest capabilities */ diff --git a/arch/mips/include/asm/cpu.h b/arch/mips/include/asm/cpu.h index 4b84fd1df0c7..104a509312b3 100644 --- a/arch/mips/include/asm/cpu.h +++ b/arch/mips/include/asm/cpu.h @@ -422,7 +422,9 @@ enum cpu_type_enum { #define MIPS_CPU_MT_PER_TC_PERF_COUNTERS \ BIT_ULL(56) /* CPU has perf counters implemented per TC (MIPSMT ASE) */ #define MIPS_CPU_MMID BIT_ULL(57) /* CPU supports MemoryMapIDs */ -#define MIPS_CPU_MAC_2008_ONLY BIT_ULL(58) /* CPU Only support MAC2008 Fused multiply-add instruction */ +#define MIPS_CPU_MM_SYSAD BIT_ULL(58) /* CPU supports write-through SysAD Valid merge */ +#define MIPS_CPU_MM_FULL BIT_ULL(59) /* CPU supports write-through full merge */ +#define MIPS_CPU_MAC_2008_ONLY BIT_ULL(60) /* CPU Only support MAC2008 Fused multiply-add instruction */ /* * CPU ASE encodings diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h index 246b8b2ebdf8..a284da54efd5 100644 --- a/arch/mips/include/asm/mipsregs.h +++ b/arch/mips/include/asm/mipsregs.h @@ -563,6 +563,9 @@ #define MIPS_CONF_MT_FTLB (_ULCAST_(4) << 7) #define MIPS_CONF_AR (_ULCAST_(7) << 10) #define MIPS_CONF_AT (_ULCAST_(3) << 13) +#define MIPS_CONF_MM (_ULCAST_(3) << 17) +#define MIPS_CONF_MM_SYSAD (_ULCAST_(1) << 17) +#define MIPS_CONF_MM_FULL (_ULCAST_(2) << 17) #define MIPS_CONF_M (_ULCAST_(1) << 31) /* diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index da2820991e21..b8ec35737606 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -672,6 +672,52 @@ static int set_ftlb_enable(struct cpuinfo_mips *c, enum ftlb_flags flags) return 0; } +static int mm_config(struct cpuinfo_mips *c) +{ + unsigned int config0, update, mm; + + config0 = read_c0_config(); + mm = config0 & MIPS_CONF_MM; + + /* + * It's implementation dependent what type of write-merge is supported + * and whether it can be enabled/disabled. If it is settable lets make + * the merging allowed by default. Some platforms might have + * write-through caching unsupported. In this case just ignore the + * CP0.Config.MM bit field value. + */ + switch (c->cputype) { + case CPU_24K: + case CPU_34K: + case CPU_74K: + case CPU_P5600: + case CPU_P6600: + c->options |= MIPS_CPU_MM_FULL; + update = MIPS_CONF_MM_FULL; + break; + case CPU_1004K: + case CPU_1074K: + case CPU_INTERAPTIV: + case CPU_PROAPTIV: + mm = 0; + fallthrough; + default: + update = 0; + break; + } + + if (update) { + config0 = (config0 & ~MIPS_CONF_MM) | update; + write_c0_config(config0); + } else if (mm == MIPS_CONF_MM_SYSAD) { + c->options |= MIPS_CPU_MM_SYSAD; + } else if (mm == MIPS_CONF_MM_FULL) { + c->options |= MIPS_CPU_MM_FULL; + } + + return 0; +} + static inline unsigned int decode_config0(struct cpuinfo_mips *c) { unsigned int config0; @@ -1763,6 +1809,8 @@ static inline void cpu_probe_mips(struct cpuinfo_mips *c, unsigned int cpu) spram_config(); + mm_config(c); + switch (__get_cpu_type(c->cputype)) { case CPU_M5150: case CPU_P5600: From 999079c851b46fd945a8b074d115f9f08a056ca9 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 17:07:18 +0300 Subject: [PATCH 0647/1043] mips: Add CONFIG/CONFIG6/Cause reg fields macro There are bit fields which persist in the MIPS CONFIG and CONFIG6 registers, but haven't been described in the generic mipsregs.h header so far. In particular, the generic CONFIG bitfields are BE - endian mode, BM - burst mode, SB - SimpleBE, OCP interface mode indicator, UDI - user-defined "CorExtend" instructions, DSP - data scratch pad RAM present, ISP - instruction scratch pad RAM present, etc. The core-specific CONFIG6 bitfields are JRCD - jump register cache prediction disable, R6 - MIPSr6 extensions enable, IFUPerfCtl - IFU performance control, SPCD - sleep state performance counter, DLSB - disable load/store bonding. A new exception code reported in the ExcCode field of the Cause register: 30 - Parity/ECC error exception happened on either fetch, load or cache refill. Lets add them to the mipsregs.h header to be used in future platform code, which have them utilized. Signed-off-by: Serge Semin Cc: Alexey Malahov Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Arnd Bergmann Cc: Rob Herring Cc: devicetree@vger.kernel.org Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mipsregs.h | 19 +++++++++++++++++++ arch/mips/kernel/spram.c | 4 ++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h index a284da54efd5..fe6293f5b939 100644 --- a/arch/mips/include/asm/mipsregs.h +++ b/arch/mips/include/asm/mipsregs.h @@ -468,6 +468,7 @@ #define EXCCODE_THREAD 25 /* Thread exceptions (MT) */ #define EXCCODE_DSPDIS 26 /* DSP disabled exception */ #define EXCCODE_GE 27 /* Virtualized guest exception (VZ) */ +#define EXCCODE_CACHEERR 30 /* Parity/ECC occured on a core */ /* Implementation specific trap codes used by MIPS cores */ #define MIPS_EXCCODE_TLBPAR 16 /* TLB parity error exception */ @@ -563,9 +564,17 @@ #define MIPS_CONF_MT_FTLB (_ULCAST_(4) << 7) #define MIPS_CONF_AR (_ULCAST_(7) << 10) #define MIPS_CONF_AT (_ULCAST_(3) << 13) +#define MIPS_CONF_BE (_ULCAST_(1) << 15) +#define MIPS_CONF_BM (_ULCAST_(1) << 16) #define MIPS_CONF_MM (_ULCAST_(3) << 17) #define MIPS_CONF_MM_SYSAD (_ULCAST_(1) << 17) #define MIPS_CONF_MM_FULL (_ULCAST_(2) << 17) +#define MIPS_CONF_SB (_ULCAST_(1) << 21) +#define MIPS_CONF_UDI (_ULCAST_(1) << 22) +#define MIPS_CONF_DSP (_ULCAST_(1) << 23) +#define MIPS_CONF_ISP (_ULCAST_(1) << 24) +#define MIPS_CONF_KU (_ULCAST_(3) << 25) +#define MIPS_CONF_K23 (_ULCAST_(3) << 28) #define MIPS_CONF_M (_ULCAST_(1) << 31) /* @@ -677,9 +686,19 @@ #define MIPS_CONF5_CV (_ULCAST_(1) << 29) #define MIPS_CONF5_K (_ULCAST_(1) << 30) +/* Jump register cache prediction disable */ +#define MIPS_CONF6_JRCD (_ULCAST_(1) << 0) +/* MIPSr6 extensions enable */ +#define MIPS_CONF6_R6 (_ULCAST_(1) << 2) +/* IFU Performance Control */ +#define MIPS_CONF6_IFUPERFCTL (_ULCAST_(3) << 10) #define MIPS_CONF6_SYND (_ULCAST_(1) << 13) +/* Sleep state performance counter disable */ +#define MIPS_CONF6_SPCD (_ULCAST_(1) << 14) /* proAptiv FTLB on/off bit */ #define MIPS_CONF6_FTLBEN (_ULCAST_(1) << 15) +/* Disable load/store bonding */ +#define MIPS_CONF6_DLSB (_ULCAST_(1) << 21) /* Loongson-3 FTLB on/off bit */ #define MIPS_CONF6_FTLBDIS (_ULCAST_(1) << 22) /* FTLB probability bits */ diff --git a/arch/mips/kernel/spram.c b/arch/mips/kernel/spram.c index 26d355462ace..d5d96214cce5 100644 --- a/arch/mips/kernel/spram.c +++ b/arch/mips/kernel/spram.c @@ -209,11 +209,11 @@ void spram_config(void) case CPU_P6600: config0 = read_c0_config(); /* FIXME: addresses are Malta specific */ - if (config0 & (1<<24)) { + if (config0 & MIPS_CONF_ISP) { probe_spram("ISPRAM", 0x1c000000, &ispram_load_tag, &ispram_store_tag); } - if (config0 & (1<<23)) + if (config0 & MIPS_CONF_DSP) probe_spram("DSPRAM", 0x1c100000, &dspram_load_tag, &dspram_store_tag); } From ad42e0a8d4d5abd7243597b1a4f5ab4ede09f25a Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 17:07:19 +0300 Subject: [PATCH 0648/1043] mips: Add CPS_NS16550_WIDTH config On some platforms IO-memory might require to use a proper load/store instructions (like Baikal-T1 IO-memory). To fix the cps-vec UART debug printout let's add the CONFIG_CPS_NS16550_WIDTH config to determine which instructions lb/sb, lh/sh or lw/sw are required for MMIO operations. Signed-off-by: Serge Semin Cc: Alexey Malahov Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Arnd Bergmann Cc: Rob Herring Cc: devicetree@vger.kernel.org Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kconfig.debug | 10 ++++++++++ arch/mips/kernel/cps-vec-ns16550.S | 18 ++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/arch/mips/Kconfig.debug b/arch/mips/Kconfig.debug index 93a2974d2ab7..7a8d94cdd493 100644 --- a/arch/mips/Kconfig.debug +++ b/arch/mips/Kconfig.debug @@ -148,4 +148,14 @@ config MIPS_CPS_NS16550_SHIFT form their addresses. That is, log base 2 of the span between adjacent ns16550 registers in the system. +config MIPS_CPS_NS16550_WIDTH + int "UART Register Width" + default 1 + help + ns16550 registers width. UART registers IO access methods will be + selected in accordance with this parameter. By setting it to 1, 2 or + 4 UART registers will be accessed by means of lb/sb, lh/sh or lw/sw + instructions respectively. Any value not from that set activates + lb/sb instructions. + endif # MIPS_CPS_NS16550_BOOL diff --git a/arch/mips/kernel/cps-vec-ns16550.S b/arch/mips/kernel/cps-vec-ns16550.S index d5a67b4ce9f6..30725e1df987 100644 --- a/arch/mips/kernel/cps-vec-ns16550.S +++ b/arch/mips/kernel/cps-vec-ns16550.S @@ -14,16 +14,30 @@ #define UART_TX_OFS (UART_TX << CONFIG_MIPS_CPS_NS16550_SHIFT) #define UART_LSR_OFS (UART_LSR << CONFIG_MIPS_CPS_NS16550_SHIFT) +#if CONFIG_MIPS_CPS_NS16550_WIDTH == 1 +# define UART_L lb +# define UART_S sb +#elif CONFIG_MIPS_CPS_NS16550_WIDTH == 2 +# define UART_L lh +# define UART_S sh +#elif CONFIG_MIPS_CPS_NS16550_WIDTH == 4 +# define UART_L lw +# define UART_S sw +#else +# define UART_L lb +# define UART_S sb +#endif + /** * _mips_cps_putc() - write a character to the UART * @a0: ASCII character to write * @t9: UART base address */ LEAF(_mips_cps_putc) -1: lw t0, UART_LSR_OFS(t9) +1: UART_L t0, UART_LSR_OFS(t9) andi t0, t0, UART_LSR_TEMT beqz t0, 1b - sb a0, UART_TX_OFS(t9) + UART_S a0, UART_TX_OFS(t9) jr ra END(_mips_cps_putc) From ed26aacfb5f71eecb20a51c4467da440cb719d66 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 17:07:22 +0300 Subject: [PATCH 0649/1043] mips: Add udelay lpj numbers adjustment Loops-per-jiffies is a special number which represents a number of noop-loop cycles per CPU-scheduler quantum - jiffies. As you understand aside from CPU-specific implementation it depends on the CPU frequency. So when a platform has the CPU frequency fixed, we have no problem and the current udelay interface will work just fine. But as soon as CPU-freq driver is enabled and the cores frequency changes, we'll end up with distorted udelay's. In order to fix this we have to accordinly adjust the per-CPU udelay_val (the same as the global loops_per_jiffy) number. This can be done in the CPU-freq transition event handler. We subscribe to that event in the MIPS arch time-inititalization method. Co-developed-by: Alexey Malahov Signed-off-by: Alexey Malahov Signed-off-by: Serge Semin Reviewed-by: Jiaxun Yang Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Arnd Bergmann Cc: Rob Herring Cc: devicetree@vger.kernel.org Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/time.c | 70 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/arch/mips/kernel/time.c b/arch/mips/kernel/time.c index 37e9413a393d..caa01457dce6 100644 --- a/arch/mips/kernel/time.c +++ b/arch/mips/kernel/time.c @@ -18,12 +18,82 @@ #include #include #include +#include +#include #include #include #include #include +#ifdef CONFIG_CPU_FREQ + +static DEFINE_PER_CPU(unsigned long, pcp_lpj_ref); +static DEFINE_PER_CPU(unsigned long, pcp_lpj_ref_freq); +static unsigned long glb_lpj_ref; +static unsigned long glb_lpj_ref_freq; + +static int cpufreq_callback(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_freqs *freq = data; + struct cpumask *cpus = freq->policy->cpus; + unsigned long lpj; + int cpu; + + /* + * Skip lpj numbers adjustment if the CPU-freq transition is safe for + * the loops delay. (Is this possible?) + */ + if (freq->flags & CPUFREQ_CONST_LOOPS) + return NOTIFY_OK; + + /* Save the initial values of the lpjes for future scaling. */ + if (!glb_lpj_ref) { + glb_lpj_ref = boot_cpu_data.udelay_val; + glb_lpj_ref_freq = freq->old; + + for_each_online_cpu(cpu) { + per_cpu(pcp_lpj_ref, cpu) = + cpu_data[cpu].udelay_val; + per_cpu(pcp_lpj_ref_freq, cpu) = freq->old; + } + } + + /* + * Adjust global lpj variable and per-CPU udelay_val number in + * accordance with the new CPU frequency. + */ + if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || + (val == CPUFREQ_POSTCHANGE && freq->old > freq->new)) { + loops_per_jiffy = cpufreq_scale(glb_lpj_ref, + glb_lpj_ref_freq, + freq->new); + + for_each_cpu(cpu, cpus) { + lpj = cpufreq_scale(per_cpu(pcp_lpj_ref, cpu), + per_cpu(pcp_lpj_ref_freq, cpu), + freq->new); + cpu_data[cpu].udelay_val = (unsigned int)lpj; + } + } + + return NOTIFY_OK; +} + +static struct notifier_block cpufreq_notifier = { + .notifier_call = cpufreq_callback, +}; + +static int __init register_cpufreq_notifier(void) +{ + return cpufreq_register_notifier(&cpufreq_notifier, + CPUFREQ_TRANSITION_NOTIFIER); +} +core_initcall(register_cpufreq_notifier); + +#endif /* CONFIG_CPU_FREQ */ + /* * forward reference */ From 3858642805973d194935c0641d8e825f9192d384 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 17:07:23 +0300 Subject: [PATCH 0650/1043] mips: csrc-r4k: Mark R4K timer as unstable if CPU freq changes Commit 07d69579e7fe ("MIPS: Don't register r4k sched clock when CPUFREQ enabled") disabled the r4k-clock usage for scheduler ticks counting due to the scheduler being non-tolerant for unstable clocks sources. For the same reason the clock should be used in the system clocksource framework with care. As soon as CPU frequency changes the clocksource framework should be notified about this by marking the R4K timer being unstable (which it really is, since the ticks rate has been changed synchronously with the CPU frequency). Signed-off-by: Serge Semin Cc: Alexey Malahov Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Greg Kroah-Hartman Cc: Arnd Bergmann Cc: Rob Herring Cc: devicetree@vger.kernel.org Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kconfig | 1 + arch/mips/kernel/csrc-r4k.c | 40 +++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 2747b1b2d435..f347312ecd74 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1108,6 +1108,7 @@ config CSRC_IOASIC bool config CSRC_R4K + select CLOCKSOURCE_WATCHDOG if CPU_FREQ bool config CSRC_SB1250 diff --git a/arch/mips/kernel/csrc-r4k.c b/arch/mips/kernel/csrc-r4k.c index 437dda64fd7a..edc4afc080fa 100644 --- a/arch/mips/kernel/csrc-r4k.c +++ b/arch/mips/kernel/csrc-r4k.c @@ -6,6 +6,7 @@ * Copyright (C) 2007 by Ralf Baechle */ #include +#include #include #include @@ -65,6 +66,45 @@ static bool rdhwr_count_usable(void) return false; } +#ifdef CONFIG_CPU_FREQ + +static bool __read_mostly r4k_clock_unstable; + +static void r4k_clocksource_unstable(char *reason) +{ + if (r4k_clock_unstable) + return; + + r4k_clock_unstable = true; + + pr_info("R4K timer is unstable due to %s\n", reason); + + clocksource_mark_unstable(&clocksource_mips); +} + +static int r4k_cpufreq_callback(struct notifier_block *nb, + unsigned long val, void *data) +{ + if (val == CPUFREQ_POSTCHANGE) + r4k_clocksource_unstable("CPU frequency change"); + + return 0; +} + +static struct notifier_block r4k_cpufreq_notifier = { + .notifier_call = r4k_cpufreq_callback, +}; + +static int __init r4k_register_cpufreq_notifier(void) +{ + return cpufreq_register_notifier(&r4k_cpufreq_notifier, + CPUFREQ_TRANSITION_NOTIFIER); + +} +core_initcall(r4k_register_cpufreq_notifier); + +#endif /* !CONFIG_CPU_FREQ */ + int __init init_r4k_clocksource(void) { if (!cpu_has_counter || !mips_hpt_frequency) From 21e1a03e1dbed20e03d88aa077163cd6ceaa128f Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 17:07:24 +0300 Subject: [PATCH 0651/1043] mips: cevt-r4k: Update the r4k-clockevent frequency in sync with CPU Due to being embedded into the CPU cores MIPS count/compare timer frequency is changed together with the CPU clocks alteration. In case if frequency really changes the kernel clockevent framework must be notified, otherwise the kernel timers won't work correctly. Fix this by calling clockevents_update_freq() for each r4k clockevent handlers registered per available CPUs. Traditionally MIPS r4k-clock are clocked with CPU frequency divided by 2. But this isn't true for some of the platforms. Due to this we have to save the basic CPU frequency, so then use it to scale the initial timer frequency (mips_hpt_frequency) and pass the updated value further to the clockevent framework. Signed-off-by: Serge Semin Cc: Alexey Malahov Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Greg Kroah-Hartman Cc: Arnd Bergmann Cc: Rob Herring Cc: devicetree@vger.kernel.org Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/cevt-r4k.c | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/arch/mips/kernel/cevt-r4k.c b/arch/mips/kernel/cevt-r4k.c index 17a9cbb8b3df..995ad9e69ded 100644 --- a/arch/mips/kernel/cevt-r4k.c +++ b/arch/mips/kernel/cevt-r4k.c @@ -8,6 +8,7 @@ */ #include #include +#include #include #include #include @@ -250,6 +251,49 @@ unsigned int __weak get_c0_compare_int(void) return MIPS_CPU_IRQ_BASE + cp0_compare_irq; } +#ifdef CONFIG_CPU_FREQ + +static unsigned long mips_ref_freq; + +static int r4k_cpufreq_callback(struct notifier_block *nb, + unsigned long val, void *data) +{ + struct cpufreq_freqs *freq = data; + struct clock_event_device *cd; + unsigned long rate; + int cpu; + + if (!mips_ref_freq) + mips_ref_freq = freq->old; + + if (val == CPUFREQ_POSTCHANGE) { + rate = cpufreq_scale(mips_hpt_frequency, mips_ref_freq, + freq->new); + + for_each_cpu(cpu, freq->policy->cpus) { + cd = &per_cpu(mips_clockevent_device, cpu); + + clockevents_update_freq(cd, rate); + } + } + + return 0; +} + +static struct notifier_block r4k_cpufreq_notifier = { + .notifier_call = r4k_cpufreq_callback, +}; + +static int __init r4k_register_cpufreq_notifier(void) +{ + return cpufreq_register_notifier(&r4k_cpufreq_notifier, + CPUFREQ_TRANSITION_NOTIFIER); + +} +core_initcall(r4k_register_cpufreq_notifier); + +#endif /* !CONFIG_CPU_FREQ */ + int r4k_clockevent_init(void) { unsigned long flags = IRQF_PERCPU | IRQF_TIMER | IRQF_SHARED; From 3ac4a615bdb10ee7df8818ff107f974322e4c190 Mon Sep 17 00:00:00 2001 From: Max Filippov Date: Fri, 22 May 2020 10:54:32 -0700 Subject: [PATCH 0652/1043] xtensa: add missing __user annotations to __{get,put}_user_check __get_user_check and __put_user_check use temporary pointer but don't mark it as __user, resulting in sparse warnings: sparse: warning: incorrect type in initializer (different address spaces) sparse: expected long *__pu_addr sparse: got long [noderef] *ret sparse: warning: incorrect type in argument 1 (different address spaces) sparse: expected void [noderef] *to sparse: got long *__pu_addr Add __user annotation to temporary pointer in __get_user_check and __put_user_check. Reported-by: kbuild test robot Reported-by: Arnd Bergmann Signed-off-by: Max Filippov --- arch/xtensa/include/asm/uaccess.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index 47b7702aaa40..754a7c96b9da 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -84,7 +84,7 @@ extern long __put_user_bad(void); #define __put_user_check(x, ptr, size) \ ({ \ long __pu_err = -EFAULT; \ - __typeof__(*(ptr)) *__pu_addr = (ptr); \ + __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ if (access_ok(__pu_addr, size)) \ __put_user_size((x), __pu_addr, (size), __pu_err); \ __pu_err; \ @@ -180,7 +180,7 @@ __asm__ __volatile__( \ #define __get_user_check(x, ptr, size) \ ({ \ long __gu_err = -EFAULT; \ - const __typeof__(*(ptr)) *__gu_addr = (ptr); \ + const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ if (access_ok(__gu_addr, size)) \ __get_user_size((x), __gu_addr, (size), __gu_err); \ else \ From c22f9075044057f130f6b73a0638a2eb78f459ad Mon Sep 17 00:00:00 2001 From: Max Filippov Date: Fri, 22 May 2020 13:20:05 -0700 Subject: [PATCH 0653/1043] xtensa: fix type conversion in __get_user_size 8-byte access in __get_user_size converts pointer to temporary variable to the type of original user pointer and then dereferences it, resulting in the following sparse warning: sparse: warning: dereference of noderef expression Instead dereference the original user pointer under the __typeof__ and add indirection outside. Signed-off-by: Max Filippov --- arch/xtensa/include/asm/uaccess.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index 754a7c96b9da..445bb4cf3c28 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -204,7 +204,7 @@ do { \ retval = -EFAULT; \ (x) = 0; \ } else { \ - (x) = *(__force __typeof__((ptr)))&__x; \ + (x) = *(__force __typeof__(*(ptr)) *)&__x; \ } \ break; \ } \ From 83cba9536905e4f82b726a98fe404400f0c9eb76 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Wed, 13 May 2020 14:25:48 +0200 Subject: [PATCH 0654/1043] clocksource/drivers/rda: drop redundant Kconfig dependency Since commit 2f8a26c166eb ("clocksource: Improve GENERIC_CLOCKEVENTS dependency") all clocksource drivers depend on GENERIC_CLOCKEVENTS so drop the redundant attribute from the RDA-timer entry which was added later. Signed-off-by: Johan Hovold Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200513122548.16974-1-johan@kernel.org --- drivers/clocksource/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 9c2d72b33e89..c82400273b0a 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -120,7 +120,6 @@ config OWL_TIMER config RDA_TIMER bool "RDA timer driver" if COMPILE_TEST - depends on GENERIC_CLOCKEVENTS select CLKSRC_MMIO select TIMER_OF help From 311fb70aa55174ddebb5c6022b23e58b85e9f116 Mon Sep 17 00:00:00 2001 From: Dejin Zheng Date: Wed, 29 Apr 2020 23:12:23 +0800 Subject: [PATCH 0655/1043] clocksource/drivers/arc_timer: Remove duplicate error message The function arc_get_timer_clk() prints an error message if it fails, remove the second error message if the function fails. Signed-off-by: Dejin Zheng Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200429151223.3120-1-zhengdejin5@gmail.com --- drivers/clocksource/arc_timer.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/clocksource/arc_timer.c b/drivers/clocksource/arc_timer.c index b29b5a75333e..de93dd1a8c7b 100644 --- a/drivers/clocksource/arc_timer.c +++ b/drivers/clocksource/arc_timer.c @@ -334,10 +334,8 @@ static int __init arc_clockevent_setup(struct device_node *node) } ret = arc_get_timer_clk(node); - if (ret) { - pr_err("clockevent: missing clk\n"); + if (ret) return ret; - } /* Needs apriori irq_set_percpu_devid() done in intc map function */ ret = request_percpu_irq(arc_timer_irq, timer_irq_handler, From d1b5e55208fd8e1c73876ab6ad1ce93485e3f5a2 Mon Sep 17 00:00:00 2001 From: Dejin Zheng Date: Wed, 29 Apr 2020 23:35:59 +0800 Subject: [PATCH 0656/1043] drivers/clocksource/arm_arch_timer: Remove duplicate error message The function acpi_gtdt_init() prints a message in case of error. Remove the error message after testing if the function fails, otherwise it is a duplicate message. Signed-off-by: Dejin Zheng Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200429153559.21189-1-zhengdejin5@gmail.com --- drivers/clocksource/arm_arch_timer.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index d53f4c7ccaae..befe54a9af60 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -1577,10 +1577,8 @@ static int __init arch_timer_acpi_init(struct acpi_table_header *table) arch_timers_present |= ARCH_TIMER_TYPE_CP15; ret = acpi_gtdt_init(table, &platform_timer_count); - if (ret) { - pr_err("Failed to init GTDT table.\n"); + if (ret) return ret; - } arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] = acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI); From 46b30515f97ece3da661b251e4a0ad9ac7a338d3 Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Thu, 7 May 2020 10:23:17 -0700 Subject: [PATCH 0657/1043] clocksource/drivers/timer-ti-32k: Add support for initializing directly Let's allow probing the 32k counter directly based on devicetree data to prepare for dropping the related legacy platform code. Let's only do this if the parent node is compatible with ti-sysc to make sure we have the related devicetree data available. Let's also show the 32k counter information before registering the clocksource, now we see it after the clocksource information which is a bit confusing. Cc: linux-kernel@vger.kernel.org Cc: linux-omap@vger.kernel.org Cc: Daniel Lezcano Cc: Grygorii Strashko Cc: Keerthy Cc: Lokesh Vutla Cc: Rob Herring Cc: Tero Kristo Cc: Thomas Gleixner Signed-off-by: Tony Lindgren Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200507172330.18679-2-tony@atomide.com --- drivers/clocksource/timer-ti-32k.c | 48 +++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/drivers/clocksource/timer-ti-32k.c b/drivers/clocksource/timer-ti-32k.c index abd5f158d6e2..ae12bbf3d68c 100644 --- a/drivers/clocksource/timer-ti-32k.c +++ b/drivers/clocksource/timer-ti-32k.c @@ -24,6 +24,7 @@ * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com */ +#include #include #include #include @@ -76,6 +77,49 @@ static u64 notrace omap_32k_read_sched_clock(void) return ti_32k_read_cycles(&ti_32k_timer.cs); } +static void __init ti_32k_timer_enable_clock(struct device_node *np, + const char *name) +{ + struct clk *clock; + int error; + + clock = of_clk_get_by_name(np->parent, name); + if (IS_ERR(clock)) { + /* Only some SoCs have a separate interface clock */ + if (PTR_ERR(clock) == -EINVAL && !strncmp("ick", name, 3)) + return; + + pr_warn("%s: could not get clock %s %li\n", + __func__, name, PTR_ERR(clock)); + return; + } + + error = clk_prepare_enable(clock); + if (error) { + pr_warn("%s: could not enable %s: %i\n", + __func__, name, error); + return; + } +} + +static void __init ti_32k_timer_module_init(struct device_node *np, + void __iomem *base) +{ + void __iomem *sysc = base + 4; + + if (!of_device_is_compatible(np->parent, "ti,sysc")) + return; + + ti_32k_timer_enable_clock(np, "fck"); + ti_32k_timer_enable_clock(np, "ick"); + + /* + * Force idle module as wkup domain is active with MPU. + * No need to tag the module disabled for ti-sysc probe. + */ + writel_relaxed(0, sysc); +} + static int __init ti_32k_timer_init(struct device_node *np) { int ret; @@ -90,6 +134,7 @@ static int __init ti_32k_timer_init(struct device_node *np) ti_32k_timer.cs.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP; ti_32k_timer.counter = ti_32k_timer.base; + ti_32k_timer_module_init(np, ti_32k_timer.base); /* * 32k sync Counter IP register offsets vary between the highlander @@ -104,6 +149,8 @@ static int __init ti_32k_timer_init(struct device_node *np) else ti_32k_timer.counter += OMAP2_32KSYNCNT_CR_OFF_LOW; + pr_info("OMAP clocksource: 32k_counter at 32768 Hz\n"); + ret = clocksource_register_hz(&ti_32k_timer.cs, 32768); if (ret) { pr_err("32k_counter: can't register clocksource\n"); @@ -111,7 +158,6 @@ static int __init ti_32k_timer_init(struct device_node *np) } sched_clock_register(omap_32k_read_sched_clock, 32, 32768); - pr_info("OMAP clocksource: 32k_counter at 32768 Hz\n"); return 0; } From 52762fbd1c4778ac9b173624ca0faacd22ef4724 Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Thu, 7 May 2020 10:23:18 -0700 Subject: [PATCH 0658/1043] clocksource/drivers/timer-ti-dm: Add clockevent and clocksource support We can move the TI dmtimer clockevent and clocksource to live under drivers/clocksource if we rely only on the clock framework, and handle the module configuration directly in the clocksource driver based on the device tree data. This removes the early dependency with system timers to the interconnect related code, and we can probe pretty much everything else later on at the module_init level. Let's first add a new driver for timer-ti-dm-systimer based on existing arch/arm/mach-omap2/timer.c. Then let's start moving SoCs to probe with device tree data while still keeping the old timer.c. And eventually we can just drop the old timer.c. Let's take the opportunity to switch to use readl/writel as pointed out by Daniel Lezcano . This allows further clean-up of the timer-ti-dm code the a lot of the shared helpers can just become static to the non-syster related code. Note the boards can optionally configure different timer source clocks if needed with assigned-clocks and assigned-clock-parents. Cc: linux-kernel@vger.kernel.org Cc: linux-omap@vger.kernel.org Cc: Daniel Lezcano Cc: Grygorii Strashko Cc: Keerthy Cc: Lokesh Vutla Cc: Rob Herring Cc: Tero Kristo Cc: Thomas Gleixner Signed-off-by: Tony Lindgren Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200507172330.18679-3-tony@atomide.com --- drivers/clocksource/Makefile | 1 + drivers/clocksource/timer-ti-dm-systimer.c | 731 +++++++++++++++++++++ 2 files changed, 732 insertions(+) create mode 100644 drivers/clocksource/timer-ti-dm-systimer.c diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index 641ba5383ab5..bdda1a2e4097 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -18,6 +18,7 @@ obj-$(CONFIG_CLKSRC_MMIO) += mmio.o obj-$(CONFIG_DAVINCI_TIMER) += timer-davinci.o obj-$(CONFIG_DIGICOLOR_TIMER) += timer-digicolor.o obj-$(CONFIG_OMAP_DM_TIMER) += timer-ti-dm.o +obj-$(CONFIG_OMAP_DM_TIMER) += timer-ti-dm-systimer.o obj-$(CONFIG_DW_APB_TIMER) += dw_apb_timer.o obj-$(CONFIG_DW_APB_TIMER_OF) += dw_apb_timer_of.o obj-$(CONFIG_FTTMR010_TIMER) += timer-fttmr010.o diff --git a/drivers/clocksource/timer-ti-dm-systimer.c b/drivers/clocksource/timer-ti-dm-systimer.c new file mode 100644 index 000000000000..1495618a5744 --- /dev/null +++ b/drivers/clocksource/timer-ti-dm-systimer.c @@ -0,0 +1,731 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +/* For type1, set SYSC_OMAP2_CLOCKACTIVITY for fck off on idle, l4 clock on */ +#define DMTIMER_TYPE1_ENABLE ((1 << 9) | (SYSC_IDLE_SMART << 3) | \ + SYSC_OMAP2_ENAWAKEUP | SYSC_OMAP2_AUTOIDLE) + +#define DMTIMER_TYPE2_ENABLE (SYSC_IDLE_SMART_WKUP << 2) +#define DMTIMER_RESET_WAIT 100000 + +#define DMTIMER_INST_DONT_CARE ~0U + +static int counter_32k; +static u32 clocksource; +static u32 clockevent; + +/* + * Subset of the timer registers we use. Note that the register offsets + * depend on the timer revision detected. + */ +struct dmtimer_systimer { + void __iomem *base; + u8 sysc; + u8 irq_stat; + u8 irq_ena; + u8 pend; + u8 load; + u8 counter; + u8 ctrl; + u8 wakeup; + u8 ifctrl; + unsigned long rate; +}; + +struct dmtimer_clockevent { + struct clock_event_device dev; + struct dmtimer_systimer t; + u32 period; +}; + +struct dmtimer_clocksource { + struct clocksource dev; + struct dmtimer_systimer t; + unsigned int loadval; +}; + +/* Assumes v1 ip if bits [31:16] are zero */ +static bool dmtimer_systimer_revision1(struct dmtimer_systimer *t) +{ + u32 tidr = readl_relaxed(t->base); + + return !(tidr >> 16); +} + +static int __init dmtimer_systimer_type1_reset(struct dmtimer_systimer *t) +{ + void __iomem *syss = t->base + OMAP_TIMER_V1_SYS_STAT_OFFSET; + int ret; + u32 l; + + writel_relaxed(BIT(1) | BIT(2), t->base + t->ifctrl); + ret = readl_poll_timeout_atomic(syss, l, l & BIT(0), 100, + DMTIMER_RESET_WAIT); + + return ret; +} + +/* Note we must use io_base instead of func_base for type2 OCP regs */ +static int __init dmtimer_systimer_type2_reset(struct dmtimer_systimer *t) +{ + void __iomem *sysc = t->base + t->sysc; + u32 l; + + l = readl_relaxed(sysc); + l |= BIT(0); + writel_relaxed(l, sysc); + + return readl_poll_timeout_atomic(sysc, l, !(l & BIT(0)), 100, + DMTIMER_RESET_WAIT); +} + +static int __init dmtimer_systimer_reset(struct dmtimer_systimer *t) +{ + int ret; + + if (dmtimer_systimer_revision1(t)) + ret = dmtimer_systimer_type1_reset(t); + else + ret = dmtimer_systimer_type2_reset(t); + if (ret < 0) { + pr_err("%s failed with %i\n", __func__, ret); + + return ret; + } + + return 0; +} + +static const struct of_device_id counter_match_table[] = { + { .compatible = "ti,omap-counter32k" }, + { /* Sentinel */ }, +}; + +/* + * Check if the SoC als has a usable working 32 KiHz counter. The 32 KiHz + * counter is handled by timer-ti-32k, but we need to detect it as it + * affects the preferred dmtimer system timer configuration. There is + * typically no use for a dmtimer clocksource if the 32 KiHz counter is + * present, except on am437x as described below. + */ +static void __init dmtimer_systimer_check_counter32k(void) +{ + struct device_node *np; + + if (counter_32k) + return; + + np = of_find_matching_node(NULL, counter_match_table); + if (!np) { + counter_32k = -ENODEV; + + return; + } + + if (of_device_is_available(np)) + counter_32k = 1; + else + counter_32k = -ENODEV; + + of_node_put(np); +} + +static const struct of_device_id dmtimer_match_table[] = { + { .compatible = "ti,omap2420-timer", }, + { .compatible = "ti,omap3430-timer", }, + { .compatible = "ti,omap4430-timer", }, + { .compatible = "ti,omap5430-timer", }, + { .compatible = "ti,am335x-timer", }, + { .compatible = "ti,am335x-timer-1ms", }, + { .compatible = "ti,dm814-timer", }, + { .compatible = "ti,dm816-timer", }, + { /* Sentinel */ }, +}; + +/* + * Checks that system timers are configured to not reset and idle during + * the generic timer-ti-dm device driver probe. And that the system timer + * source clocks are properly configured. Also, let's not hog any DSP and + * PWM capable timers unnecessarily as system timers. + */ +static bool __init dmtimer_is_preferred(struct device_node *np) +{ + if (!of_device_is_available(np)) + return false; + + if (!of_property_read_bool(np->parent, + "ti,no-reset-on-init")) + return false; + + if (!of_property_read_bool(np->parent, "ti,no-idle")) + return false; + + /* Secure gptimer12 is always clocked with a fixed source */ + if (!of_property_read_bool(np, "ti,timer-secure")) { + if (!of_property_read_bool(np, "assigned-clocks")) + return false; + + if (!of_property_read_bool(np, "assigned-clock-parents")) + return false; + } + + if (of_property_read_bool(np, "ti,timer-dsp")) + return false; + + if (of_property_read_bool(np, "ti,timer-pwm")) + return false; + + return true; +} + +/* + * Finds the first available usable always-on timer, and assigns it to either + * clockevent or clocksource depending if the counter_32k is available on the + * SoC or not. + * + * Some omap3 boards with unreliable oscillator must not use the counter_32k + * or dmtimer1 with 32 KiHz source. Additionally, the boards with unreliable + * oscillator should really set counter_32k as disabled, and delete dmtimer1 + * ti,always-on property, but let's not count on it. For these quirky cases, + * we prefer using the always-on secure dmtimer12 with the internal 32 KiHz + * clock as the clocksource, and any available dmtimer as clockevent. + * + * For am437x, we are using am335x style dmtimer clocksource. It is unclear + * if this quirk handling is really needed, but let's change it separately + * based on testing as it might cause side effects. + */ +static void __init dmtimer_systimer_assign_alwon(void) +{ + struct device_node *np; + u32 pa = 0; + bool quirk_unreliable_oscillator = false; + + /* Quirk unreliable 32 KiHz oscillator with incomplete dts */ + if (of_machine_is_compatible("ti,omap3-beagle") || + of_machine_is_compatible("timll,omap3-devkit8000")) { + quirk_unreliable_oscillator = true; + counter_32k = -ENODEV; + } + + /* Quirk am437x using am335x style dmtimer clocksource */ + if (of_machine_is_compatible("ti,am43")) + counter_32k = -ENODEV; + + for_each_matching_node(np, dmtimer_match_table) { + if (!dmtimer_is_preferred(np)) + continue; + + if (of_property_read_bool(np, "ti,timer-alwon")) { + const __be32 *addr; + + addr = of_get_address(np, 0, NULL, NULL); + pa = of_translate_address(np, addr); + if (pa) { + /* Quirky omap3 boards must use dmtimer12 */ + if (quirk_unreliable_oscillator && + pa == 0x48318000) + continue; + + of_node_put(np); + break; + } + } + } + + /* Usually no need for dmtimer clocksource if we have counter32 */ + if (counter_32k >= 0) { + clockevent = pa; + clocksource = 0; + } else { + clocksource = pa; + clockevent = DMTIMER_INST_DONT_CARE; + } +} + +/* Finds the first usable dmtimer, used for the don't care case */ +static u32 __init dmtimer_systimer_find_first_available(void) +{ + struct device_node *np; + const __be32 *addr; + u32 pa = 0; + + for_each_matching_node(np, dmtimer_match_table) { + if (!dmtimer_is_preferred(np)) + continue; + + addr = of_get_address(np, 0, NULL, NULL); + pa = of_translate_address(np, addr); + if (pa) { + if (pa == clocksource || pa == clockevent) { + pa = 0; + continue; + } + + of_node_put(np); + break; + } + } + + return pa; +} + +/* Selects the best clocksource and clockevent to use */ +static void __init dmtimer_systimer_select_best(void) +{ + dmtimer_systimer_check_counter32k(); + dmtimer_systimer_assign_alwon(); + + if (clockevent == DMTIMER_INST_DONT_CARE) + clockevent = dmtimer_systimer_find_first_available(); + + pr_debug("%s: counter_32k: %i clocksource: %08x clockevent: %08x\n", + __func__, counter_32k, clocksource, clockevent); +} + +/* Interface clocks are only available on some SoCs variants */ +static int __init dmtimer_systimer_init_clock(struct device_node *np, + const char *name, + unsigned long *rate) +{ + struct clk *clock; + unsigned long r; + int error; + + clock = of_clk_get_by_name(np, name); + if ((PTR_ERR(clock) == -EINVAL) && !strncmp(name, "ick", 3)) + return 0; + else if (IS_ERR(clock)) + return PTR_ERR(clock); + + error = clk_prepare_enable(clock); + if (error) + return error; + + r = clk_get_rate(clock); + if (!r) + return -ENODEV; + + *rate = r; + + return 0; +} + +static void dmtimer_systimer_enable(struct dmtimer_systimer *t) +{ + u32 val; + + if (dmtimer_systimer_revision1(t)) + val = DMTIMER_TYPE1_ENABLE; + else + val = DMTIMER_TYPE2_ENABLE; + + writel_relaxed(val, t->base + t->sysc); +} + +static void dmtimer_systimer_disable(struct dmtimer_systimer *t) +{ + writel_relaxed(0, t->base + t->sysc); +} + +static int __init dmtimer_systimer_setup(struct device_node *np, + struct dmtimer_systimer *t) +{ + unsigned long rate; + u8 regbase; + int error; + + if (!of_device_is_compatible(np->parent, "ti,sysc")) + return -EINVAL; + + t->base = of_iomap(np, 0); + if (!t->base) + return -ENXIO; + + /* + * Enable optional assigned-clock-parents configured at the timer + * node level. For regular device drivers, this is done automatically + * by bus related code such as platform_drv_probe(). + */ + error = of_clk_set_defaults(np, false); + if (error < 0) + pr_err("%s: clock source init failed: %i\n", __func__, error); + + /* For ti-sysc, we have timer clocks at the parent module level */ + error = dmtimer_systimer_init_clock(np->parent, "fck", &rate); + if (error) + goto err_unmap; + + t->rate = rate; + + error = dmtimer_systimer_init_clock(np->parent, "ick", &rate); + if (error) + goto err_unmap; + + if (dmtimer_systimer_revision1(t)) { + t->irq_stat = OMAP_TIMER_V1_STAT_OFFSET; + t->irq_ena = OMAP_TIMER_V1_INT_EN_OFFSET; + t->pend = _OMAP_TIMER_WRITE_PEND_OFFSET; + regbase = 0; + } else { + t->irq_stat = OMAP_TIMER_V2_IRQSTATUS; + t->irq_ena = OMAP_TIMER_V2_IRQENABLE_SET; + regbase = OMAP_TIMER_V2_FUNC_OFFSET; + t->pend = regbase + _OMAP_TIMER_WRITE_PEND_OFFSET; + } + + t->sysc = OMAP_TIMER_OCP_CFG_OFFSET; + t->load = regbase + _OMAP_TIMER_LOAD_OFFSET; + t->counter = regbase + _OMAP_TIMER_COUNTER_OFFSET; + t->ctrl = regbase + _OMAP_TIMER_CTRL_OFFSET; + t->wakeup = regbase + _OMAP_TIMER_WAKEUP_EN_OFFSET; + t->ifctrl = regbase + _OMAP_TIMER_IF_CTRL_OFFSET; + + dmtimer_systimer_enable(t); + dmtimer_systimer_reset(t); + pr_debug("dmtimer rev %08x sysc %08x\n", readl_relaxed(t->base), + readl_relaxed(t->base + t->sysc)); + + return 0; + +err_unmap: + iounmap(t->base); + + return error; +} + +/* Clockevent */ +static struct dmtimer_clockevent * +to_dmtimer_clockevent(struct clock_event_device *clockevent) +{ + return container_of(clockevent, struct dmtimer_clockevent, dev); +} + +static irqreturn_t dmtimer_clockevent_interrupt(int irq, void *data) +{ + struct dmtimer_clockevent *clkevt = data; + struct dmtimer_systimer *t = &clkevt->t; + + writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_stat); + clkevt->dev.event_handler(&clkevt->dev); + + return IRQ_HANDLED; +} + +static int dmtimer_set_next_event(unsigned long cycles, + struct clock_event_device *evt) +{ + struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt); + struct dmtimer_systimer *t = &clkevt->t; + void __iomem *pend = t->base + t->pend; + + writel_relaxed(0xffffffff - cycles, t->base + t->counter); + while (readl_relaxed(pend) & WP_TCRR) + cpu_relax(); + + writel_relaxed(OMAP_TIMER_CTRL_ST, t->base + t->ctrl); + while (readl_relaxed(pend) & WP_TCLR) + cpu_relax(); + + return 0; +} + +static int dmtimer_clockevent_shutdown(struct clock_event_device *evt) +{ + struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt); + struct dmtimer_systimer *t = &clkevt->t; + void __iomem *ctrl = t->base + t->ctrl; + u32 l; + + l = readl_relaxed(ctrl); + if (l & OMAP_TIMER_CTRL_ST) { + l &= ~BIT(0); + writel_relaxed(l, ctrl); + /* Flush posted write */ + l = readl_relaxed(ctrl); + /* Wait for functional clock period x 3.5 */ + udelay(3500000 / t->rate + 1); + } + writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_stat); + + return 0; +} + +static int dmtimer_set_periodic(struct clock_event_device *evt) +{ + struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt); + struct dmtimer_systimer *t = &clkevt->t; + void __iomem *pend = t->base + t->pend; + + dmtimer_clockevent_shutdown(evt); + + /* Looks like we need to first set the load value separately */ + writel_relaxed(clkevt->period, t->base + t->load); + while (readl_relaxed(pend) & WP_TLDR) + cpu_relax(); + + writel_relaxed(clkevt->period, t->base + t->counter); + while (readl_relaxed(pend) & WP_TCRR) + cpu_relax(); + + writel_relaxed(OMAP_TIMER_CTRL_AR | OMAP_TIMER_CTRL_ST, + t->base + t->ctrl); + while (readl_relaxed(pend) & WP_TCLR) + cpu_relax(); + + return 0; +} + +static void omap_clockevent_idle(struct clock_event_device *evt) +{ + struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt); + struct dmtimer_systimer *t = &clkevt->t; + + dmtimer_systimer_disable(t); +} + +static void omap_clockevent_unidle(struct clock_event_device *evt) +{ + struct dmtimer_clockevent *clkevt = to_dmtimer_clockevent(evt); + struct dmtimer_systimer *t = &clkevt->t; + + dmtimer_systimer_enable(t); + writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_ena); + writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->wakeup); +} + +static int __init dmtimer_clockevent_init(struct device_node *np) +{ + struct dmtimer_clockevent *clkevt; + struct clock_event_device *dev; + struct dmtimer_systimer *t; + int error; + u32 pa; + + clkevt = kzalloc(sizeof(*clkevt), GFP_KERNEL); + if (!clkevt) + return -ENOMEM; + + t = &clkevt->t; + dev = &clkevt->dev; + + /* + * We mostly use cpuidle_coupled with ARM local timers for runtime, + * so there's probably no use for CLOCK_EVT_FEAT_DYNIRQ here. + */ + dev->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT; + dev->rating = 300; + dev->set_next_event = dmtimer_set_next_event; + dev->set_state_shutdown = dmtimer_clockevent_shutdown; + dev->set_state_periodic = dmtimer_set_periodic; + dev->set_state_oneshot = dmtimer_clockevent_shutdown; + dev->tick_resume = dmtimer_clockevent_shutdown; + dev->cpumask = cpu_possible_mask; + + dev->irq = irq_of_parse_and_map(np, 0); + if (!dev->irq) { + error = -ENXIO; + goto err_out_free; + } + + error = dmtimer_systimer_setup(np, &clkevt->t); + if (error) + goto err_out_free; + + clkevt->period = 0xffffffff - DIV_ROUND_CLOSEST(t->rate, HZ); + + /* + * For clock-event timers we never read the timer counter and + * so we are not impacted by errata i103 and i767. Therefore, + * we can safely ignore this errata for clock-event timers. + */ + writel_relaxed(OMAP_TIMER_CTRL_POSTED, t->base + t->ifctrl); + + error = request_irq(dev->irq, dmtimer_clockevent_interrupt, + IRQF_TIMER, "clockevent", clkevt); + if (error) + goto err_out_unmap; + + writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_ena); + writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->wakeup); + + pa = of_translate_address(np, of_get_address(np, 0, NULL, NULL)); + pr_info("TI gptimer clockevent: %s%lu Hz at %pOF\n", + of_find_property(np, "ti,timer-alwon", NULL) ? + "always-on " : "", t->rate, np->parent); + + clockevents_config_and_register(dev, t->rate, + 3, /* Timer internal resynch latency */ + 0xffffffff); + + if (of_device_is_compatible(np, "ti,am33xx") || + of_device_is_compatible(np, "ti,am43")) { + dev->suspend = omap_clockevent_idle; + dev->resume = omap_clockevent_unidle; + } + + return 0; + +err_out_unmap: + iounmap(t->base); + +err_out_free: + kfree(clkevt); + + return error; +} + +/* Clocksource */ +static struct dmtimer_clocksource * +to_dmtimer_clocksource(struct clocksource *cs) +{ + return container_of(cs, struct dmtimer_clocksource, dev); +} + +static u64 dmtimer_clocksource_read_cycles(struct clocksource *cs) +{ + struct dmtimer_clocksource *clksrc = to_dmtimer_clocksource(cs); + struct dmtimer_systimer *t = &clksrc->t; + + return (u64)readl_relaxed(t->base + t->counter); +} + +static void __iomem *dmtimer_sched_clock_counter; + +static u64 notrace dmtimer_read_sched_clock(void) +{ + return readl_relaxed(dmtimer_sched_clock_counter); +} + +static void dmtimer_clocksource_suspend(struct clocksource *cs) +{ + struct dmtimer_clocksource *clksrc = to_dmtimer_clocksource(cs); + struct dmtimer_systimer *t = &clksrc->t; + + clksrc->loadval = readl_relaxed(t->base + t->counter); + dmtimer_systimer_disable(t); +} + +static void dmtimer_clocksource_resume(struct clocksource *cs) +{ + struct dmtimer_clocksource *clksrc = to_dmtimer_clocksource(cs); + struct dmtimer_systimer *t = &clksrc->t; + + dmtimer_systimer_enable(t); + writel_relaxed(clksrc->loadval, t->base + t->counter); + writel_relaxed(OMAP_TIMER_CTRL_ST | OMAP_TIMER_CTRL_AR, + t->base + t->ctrl); +} + +static int __init dmtimer_clocksource_init(struct device_node *np) +{ + struct dmtimer_clocksource *clksrc; + struct dmtimer_systimer *t; + struct clocksource *dev; + int error; + u32 pa; + + clksrc = kzalloc(sizeof(*clksrc), GFP_KERNEL); + if (!clksrc) + return -ENOMEM; + + dev = &clksrc->dev; + t = &clksrc->t; + + error = dmtimer_systimer_setup(np, t); + if (error) + goto err_out_free; + + dev->name = "dmtimer"; + dev->rating = 300; + dev->read = dmtimer_clocksource_read_cycles; + dev->mask = CLOCKSOURCE_MASK(32); + dev->flags = CLOCK_SOURCE_IS_CONTINUOUS; + + if (of_device_is_compatible(np, "ti,am33xx") || + of_device_is_compatible(np, "ti,am43")) { + dev->suspend = dmtimer_clocksource_suspend; + dev->resume = dmtimer_clocksource_resume; + } + + writel_relaxed(0, t->base + t->counter); + writel_relaxed(OMAP_TIMER_CTRL_ST | OMAP_TIMER_CTRL_AR, + t->base + t->ctrl); + + pa = of_translate_address(np, of_get_address(np, 0, NULL, NULL)); + pr_info("TI gptimer clocksource: %s%pOF\n", + of_find_property(np, "ti,timer-alwon", NULL) ? + "always-on " : "", np->parent); + + if (!dmtimer_sched_clock_counter) { + dmtimer_sched_clock_counter = t->base + t->counter; + sched_clock_register(dmtimer_read_sched_clock, 32, t->rate); + } + + if (clocksource_register_hz(dev, t->rate)) + pr_err("Could not register clocksource %pOF\n", np); + + return 0; + +err_out_free: + kfree(clksrc); + + return -ENODEV; +} + +/* + * To detect between a clocksource and clockevent, we assume the device tree + * has no interrupts configured for a clocksource timer. + */ +static int __init dmtimer_systimer_init(struct device_node *np) +{ + const __be32 *addr; + u32 pa; + + /* One time init for the preferred timer configuration */ + if (!clocksource && !clockevent) + dmtimer_systimer_select_best(); + + if (!clocksource && !clockevent) { + pr_err("%s: unable to detectt system timers, update dtb?\n", + __func__); + + return -EINVAL; + } + + addr = of_get_address(np, 0, NULL, NULL); + pa = of_translate_address(np, addr); + if (!pa) + return -EINVAL; + + if (counter_32k <= 0 && clocksource == pa) + return dmtimer_clocksource_init(np); + + if (clockevent == pa) + return dmtimer_clockevent_init(np); + + return 0; +} + +TIMER_OF_DECLARE(systimer_omap2, "ti,omap2420-timer", dmtimer_systimer_init); +TIMER_OF_DECLARE(systimer_omap3, "ti,omap3430-timer", dmtimer_systimer_init); +TIMER_OF_DECLARE(systimer_omap4, "ti,omap4430-timer", dmtimer_systimer_init); +TIMER_OF_DECLARE(systimer_omap5, "ti,omap5430-timer", dmtimer_systimer_init); +TIMER_OF_DECLARE(systimer_am33x, "ti,am335x-timer", dmtimer_systimer_init); +TIMER_OF_DECLARE(systimer_am3ms, "ti,am335x-timer-1ms", dmtimer_systimer_init); +TIMER_OF_DECLARE(systimer_dm814, "ti,dm814-timer", dmtimer_systimer_init); +TIMER_OF_DECLARE(systimer_dm816, "ti,dm816-timer", dmtimer_systimer_init); From c177e2975430cec296aa52a0d413e447417d6cf9 Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Tue, 19 May 2020 08:51:57 -0700 Subject: [PATCH 0659/1043] clocksource/drivers/timer-ti-dm: Fix warning for set but not used We can get a warning for dmtimer_clocksource_init() with 'pa' set but not used. This was used in the earlier revisions of the code but no longer needed, so let's remove the unused pa and of_translate_address(). Let's also do it for dmtimer_clockevent_init() that has a similar issue. Reported-by: kbuild test robot Signed-off-by: Tony Lindgren Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200519155157.12804-1-tony@atomide.com --- drivers/clocksource/timer-ti-dm-systimer.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/clocksource/timer-ti-dm-systimer.c b/drivers/clocksource/timer-ti-dm-systimer.c index 1495618a5744..7da998d0dd58 100644 --- a/drivers/clocksource/timer-ti-dm-systimer.c +++ b/drivers/clocksource/timer-ti-dm-systimer.c @@ -514,7 +514,6 @@ static int __init dmtimer_clockevent_init(struct device_node *np) struct clock_event_device *dev; struct dmtimer_systimer *t; int error; - u32 pa; clkevt = kzalloc(sizeof(*clkevt), GFP_KERNEL); if (!clkevt) @@ -563,7 +562,6 @@ static int __init dmtimer_clockevent_init(struct device_node *np) writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->irq_ena); writel_relaxed(OMAP_TIMER_INT_OVERFLOW, t->base + t->wakeup); - pa = of_translate_address(np, of_get_address(np, 0, NULL, NULL)); pr_info("TI gptimer clockevent: %s%lu Hz at %pOF\n", of_find_property(np, "ti,timer-alwon", NULL) ? "always-on " : "", t->rate, np->parent); @@ -637,7 +635,6 @@ static int __init dmtimer_clocksource_init(struct device_node *np) struct dmtimer_systimer *t; struct clocksource *dev; int error; - u32 pa; clksrc = kzalloc(sizeof(*clksrc), GFP_KERNEL); if (!clksrc) @@ -666,7 +663,6 @@ static int __init dmtimer_clocksource_init(struct device_node *np) writel_relaxed(OMAP_TIMER_CTRL_ST | OMAP_TIMER_CTRL_AR, t->base + t->ctrl); - pa = of_translate_address(np, of_get_address(np, 0, NULL, NULL)); pr_info("TI gptimer clocksource: %s%pOF\n", of_find_property(np, "ti,timer-alwon", NULL) ? "always-on " : "", np->parent); From ac593e62b0cfcbc53502be8b6c7e40fed8baff8c Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 19 May 2020 23:44:28 +0100 Subject: [PATCH 0660/1043] clocksource/drivers/timer-ti-dm: Fix spelling mistake "detectt" -> "detect" There is a spelling mistake in a pr_err message. Fix it. Signed-off-by: Colin Ian King Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200519224428.6195-1-colin.king@canonical.com --- drivers/clocksource/timer-ti-dm-systimer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clocksource/timer-ti-dm-systimer.c b/drivers/clocksource/timer-ti-dm-systimer.c index 7da998d0dd58..6fd1f219a512 100644 --- a/drivers/clocksource/timer-ti-dm-systimer.c +++ b/drivers/clocksource/timer-ti-dm-systimer.c @@ -697,7 +697,7 @@ static int __init dmtimer_systimer_init(struct device_node *np) dmtimer_systimer_select_best(); if (!clocksource && !clockevent) { - pr_err("%s: unable to detectt system timers, update dtb?\n", + pr_err("%s: unable to detect system timers, update dtb?\n", __func__); return -EINVAL; From 264418e20d1fedbed8ad79683b63caa3d72c3b2e Mon Sep 17 00:00:00 2001 From: Lokesh Vutla Date: Mon, 27 Apr 2020 22:58:31 +0530 Subject: [PATCH 0661/1043] clocksource/drivers/timer-ti-dm: Do one override clock parent in prepare() omap_dm_timer_prepare() is setting up the parent 32KHz clock. This prepare() gets called by request_timer in the client's driver. Because of this, the timer clock parent that is set with assigned-clock-parent is being overwritten. So drop this default setting of parent in prepare(). Signed-off-by: Lokesh Vutla Reviewed-by: Suman Anna Acked-by: Tony Lindgren Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200427172831.16546-1-lokeshvutla@ti.com --- drivers/clocksource/timer-ti-dm.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/clocksource/timer-ti-dm.c b/drivers/clocksource/timer-ti-dm.c index 2531eab3d6d7..60aff087947a 100644 --- a/drivers/clocksource/timer-ti-dm.c +++ b/drivers/clocksource/timer-ti-dm.c @@ -258,9 +258,7 @@ static int omap_dm_timer_prepare(struct omap_dm_timer *timer) __omap_dm_timer_enable_posted(timer); omap_dm_timer_disable(timer); - rc = omap_dm_timer_set_source(timer, OMAP_TIMER_SRC_32_KHZ); - - return rc; + return 0; } static inline u32 omap_dm_timer_reserved_systimer(int id) From b33aaf5cd68d0fa0f0d6aa15831a1e82e2ef98e1 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 23:48:10 +0300 Subject: [PATCH 0662/1043] dt-bindings: rtc: Convert snps,dw-apb-timer to DT schema Modern device tree bindings are supposed to be created as YAML-files in accordance with DT schema. This commit replaces Synopsys DW Timer legacy bare text binding with YAML file. As before the binding file states that the corresponding dts node is supposed to be compatible with generic DW APB Timer indicated by the "snps,dw-apb-timer" compatible string and to provide a mandatory registers memory range, one timer interrupt, either reference clock source or a fixed clock rate value. It may also have an optional APB bus reference clock phandle specified. Signed-off-by: Serge Semin Reviewed-by: Rob Herring Cc: Alexey Malahov Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Thomas Gleixner Cc: Daniel Lezcano Cc: Arnd Bergmann Cc: linux-mips@vger.kernel.org Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200521204818.25436-2-Sergey.Semin@baikalelectronics.ru --- .../devicetree/bindings/rtc/dw-apb.txt | 32 ------- .../bindings/rtc/snps,dw-apb-timer.yaml | 88 +++++++++++++++++++ 2 files changed, 88 insertions(+), 32 deletions(-) delete mode 100644 Documentation/devicetree/bindings/rtc/dw-apb.txt create mode 100644 Documentation/devicetree/bindings/rtc/snps,dw-apb-timer.yaml diff --git a/Documentation/devicetree/bindings/rtc/dw-apb.txt b/Documentation/devicetree/bindings/rtc/dw-apb.txt deleted file mode 100644 index c703d51abb6c..000000000000 --- a/Documentation/devicetree/bindings/rtc/dw-apb.txt +++ /dev/null @@ -1,32 +0,0 @@ -* Designware APB timer - -Required properties: -- compatible: One of: - "snps,dw-apb-timer" - "snps,dw-apb-timer-sp" - "snps,dw-apb-timer-osc" -- reg: physical base address of the controller and length of memory mapped - region. -- interrupts: IRQ line for the timer. -- either clocks+clock-names or clock-frequency properties - -Optional properties: -- clocks : list of clock specifiers, corresponding to entries in - the clock-names property; -- clock-names : should contain "timer" and "pclk" entries, matching entries - in the clocks property. -- clock-frequency: The frequency in HZ of the timer. -- clock-freq: For backwards compatibility with picoxcell - -If using the clock specifiers, the pclk clock is optional, as not all -systems may use one. - - -Example: - timer@ffe00000 { - compatible = "snps,dw-apb-timer"; - interrupts = <0 170 4>; - reg = <0xffe00000 0x1000>; - clocks = <&timer_clk>, <&timer_pclk>; - clock-names = "timer", "pclk"; - }; diff --git a/Documentation/devicetree/bindings/rtc/snps,dw-apb-timer.yaml b/Documentation/devicetree/bindings/rtc/snps,dw-apb-timer.yaml new file mode 100644 index 000000000000..002fe1ee709b --- /dev/null +++ b/Documentation/devicetree/bindings/rtc/snps,dw-apb-timer.yaml @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rtc/snps,dw-apb-timer.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Synopsys DesignWare APB Timer + +maintainers: + - Daniel Lezcano + +properties: + compatible: + oneOf: + - const: snps,dw-apb-timer + - enum: + - snps,dw-apb-timer-sp + - snps,dw-apb-timer-osc + deprecated: true + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + minItems: 1 + items: + - description: Timer ticks reference clock source + - description: APB interface clock source + + clock-names: + minItems: 1 + items: + - const: timer + - const: pclk + + clock-frequency: true + + clock-freq: + $ref: "/schemas/types.yaml#/definitions/uint32" + description: | + Has the same meaning as the 'clock-frequency' property - timer clock + frequency in HZ, but is defined only for the backwards compatibility + with the picoxcell platform. + +unevaluatedProperties: false + +required: + - compatible + - reg + - interrupts + +oneOf: + - required: + - clocks + - clock-names + - required: + - clock-frequency + - required: + - clock-freq + +examples: + - | + timer@ffe00000 { + compatible = "snps,dw-apb-timer"; + interrupts = <0 170 4>; + reg = <0xffe00000 0x1000>; + clocks = <&timer_clk>, <&timer_pclk>; + clock-names = "timer", "pclk"; + }; + - | + timer@ffe00000 { + compatible = "snps,dw-apb-timer"; + interrupts = <0 170 4>; + reg = <0xffe00000 0x1000>; + clocks = <&timer_clk>; + clock-names = "timer"; + }; + - | + timer@ffe00000 { + compatible = "snps,dw-apb-timer"; + interrupts = <0 170 4>; + reg = <0xffe00000 0x1000>; + clock-frequency = <25000000>; + }; +... From e69bc8999662a3fa6d856820dd09717afff1cbb0 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 23:48:11 +0300 Subject: [PATCH 0663/1043] dt-bindings: timer: Move snps,dw-apb-timer DT schema from rtc This binding file doesn't belong to the rtc seeing it's a pure timer with no rtc facilities like days/months/years counting and alarms. So move the YAML-file to the Documentation/devicetree/bindings/timer/ directory. Signed-off-by: Serge Semin Acked-by: Alexandre Belloni Acked-by: Rob Herring Cc: Alexey Malahov Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Arnd Bergmann Cc: linux-mips@vger.kernel.org Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200521204818.25436-3-Sergey.Semin@baikalelectronics.ru --- .../devicetree/bindings/{rtc => timer}/snps,dw-apb-timer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename Documentation/devicetree/bindings/{rtc => timer}/snps,dw-apb-timer.yaml (96%) diff --git a/Documentation/devicetree/bindings/rtc/snps,dw-apb-timer.yaml b/Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml similarity index 96% rename from Documentation/devicetree/bindings/rtc/snps,dw-apb-timer.yaml rename to Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml index 002fe1ee709b..5d300efdf0ca 100644 --- a/Documentation/devicetree/bindings/rtc/snps,dw-apb-timer.yaml +++ b/Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only %YAML 1.2 --- -$id: http://devicetree.org/schemas/rtc/snps,dw-apb-timer.yaml# +$id: http://devicetree.org/schemas/timer/snps,dw-apb-timer.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# title: Synopsys DesignWare APB Timer From cee43dbf2ee3f430434e2b66994eff8a1aeda889 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 23:48:13 +0300 Subject: [PATCH 0664/1043] clocksource: dw_apb_timer: Make CPU-affiliation being optional Currently the DW APB Timer driver binds each clockevent timers to a particular CPU. This isn't good for multiple reasons. First of all seeing the device is placed on APB bus (which makes it accessible from any CPU core), accessible over MMIO and having the DYNIRQ flag set we can be sure that manually binding the timer to any CPU just isn't correct. By doing so we just set an extra limitation on device usage. This also doesn't reflect the device actual capability, since by setting the IRQ affinity we can make it virtually local to any CPU. Secondly imagine if you had a real CPU-local timer with the same rating and the same CPU-affinity. In this case if DW APB timer was registered first, then due to the clockevent framework tick-timer selection procedure we'll end up with the real CPU-local timer being left unselected for clock-events tracking. But on most of the platforms (MIPS/ARM/etc) such timers are normally embedded into the CPU core and are accessible with much better performance then devices placed on APB. For instance in MIPS architectures there is r4k-timer, which is CPU-local, assigned with the same rating, and normally its clockevent device is registered after the platform-specific one. So in order to fix all of these issues let's make the DW APB Timer CPU affinity being optional and deactivated by passing a negative CPU id, which will effectively set the DW APB clockevent timer cpumask to 'cpu_possible_mask'. Signed-off-by: Serge Semin Cc: Alexey Malahov Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Alessandro Zummo Cc: Alexandre Belloni Cc: Arnd Bergmann Cc: Rob Herring Cc: linux-mips@vger.kernel.org Cc: linux-rtc@vger.kernel.org Cc: devicetree@vger.kernel.org Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200521204818.25436-5-Sergey.Semin@baikalelectronics.ru --- drivers/clocksource/dw_apb_timer.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/clocksource/dw_apb_timer.c b/drivers/clocksource/dw_apb_timer.c index b207a77b0831..f5f24a95ee82 100644 --- a/drivers/clocksource/dw_apb_timer.c +++ b/drivers/clocksource/dw_apb_timer.c @@ -222,7 +222,8 @@ static int apbt_next_event(unsigned long delta, /** * dw_apb_clockevent_init() - use an APB timer as a clock_event_device * - * @cpu: The CPU the events will be targeted at. + * @cpu: The CPU the events will be targeted at or -1 if CPU affiliation + * isn't required. * @name: The name used for the timer and the IRQ for it. * @rating: The rating to give the timer. * @base: I/O base for the timer registers. @@ -257,7 +258,7 @@ dw_apb_clockevent_init(int cpu, const char *name, unsigned rating, dw_ced->ced.max_delta_ticks = 0x7fffffff; dw_ced->ced.min_delta_ns = clockevent_delta2ns(5000, &dw_ced->ced); dw_ced->ced.min_delta_ticks = 5000; - dw_ced->ced.cpumask = cpumask_of(cpu); + dw_ced->ced.cpumask = cpu < 0 ? cpu_possible_mask : cpumask_of(cpu); dw_ced->ced.features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_DYNIRQ; dw_ced->ced.set_state_shutdown = apbt_shutdown; From 65e0f876405ef4f0ff25eb1c5ff3e9b536d68805 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 23:48:14 +0300 Subject: [PATCH 0665/1043] clocksource: dw_apb_timer: Affiliate of-based timer with any CPU Currently any DW APB Timer device detected in OF is bound to CPU #0. Doing so is redundant since DW APB Timer isn't CPU-local timer, but as having APB interface is normally accessible from any CPU in the system. By artificially affiliating the DW timer to the very first CPU we may and in our case will make the clockevent subsystem to decline the more performant real CPU-local timers selection in favor of in fact non-local and accessible over a slow bus - DW APB Timers. Let's not affiliate the of-detected DW APB Timers to any CPU. By doing so the clockevent framework would prefer to select the real CPU-local timer instead of DW APB one. Otherwise if there is no other than DW APB device for clockevents tracking then it will be selected. Signed-off-by: Serge Semin Cc: Alexey Malahov Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Alessandro Zummo Cc: Alexandre Belloni Cc: Arnd Bergmann Cc: Rob Herring Cc: linux-mips@vger.kernel.org Cc: linux-rtc@vger.kernel.org Cc: devicetree@vger.kernel.org Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200521204818.25436-6-Sergey.Semin@baikalelectronics.ru --- drivers/clocksource/dw_apb_timer_of.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clocksource/dw_apb_timer_of.c b/drivers/clocksource/dw_apb_timer_of.c index 8c28b127759f..2db490f35c20 100644 --- a/drivers/clocksource/dw_apb_timer_of.c +++ b/drivers/clocksource/dw_apb_timer_of.c @@ -73,7 +73,7 @@ static void __init add_clockevent(struct device_node *event_timer) timer_get_base_and_rate(event_timer, &iobase, &rate); - ced = dw_apb_clockevent_init(0, event_timer->name, 300, iobase, irq, + ced = dw_apb_clockevent_init(-1, event_timer->name, 300, iobase, irq, rate); if (!ced) panic("Unable to initialise clockevent device"); From 6d2e16a3181bafb77b535095c39ad1c8b9558c8c Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 23:48:15 +0300 Subject: [PATCH 0666/1043] clocksource: dw_apb_timer_of: Fix missing clockevent timers Commit 100214889973 ("clocksource: dw_apb_timer_of: use clocksource_of_init") replaced a publicly available driver initialization method with one called by the timer_probe() method available after CLKSRC_OF. In current implementation it traverses all the timers available in the system and calls their initialization methods if corresponding devices were either in dtb or in acpi. But if before the commit any number of available timers would be installed as clockevent and clocksource devices, after that there would be at most two. The rest are just ignored since default case branch doesn't do anything. I don't see a reason of such behaviour, neither the commit message explains it. Moreover this might be wrong if on some platforms these timers might be used for different purpose, as virtually CPU-local clockevent timers and as an independent broadcast timer. So in order to keep the compatibility with the platforms where the order of the timers detection has some meaning, lets add the secondly discovered timer to be of clocksource/sched_clock type, while the very first and the others would provide the clockevents service. Fixes: 100214889973 ("clocksource: dw_apb_timer_of: use clocksource_of_init") Signed-off-by: Serge Semin Cc: Alexey Malahov Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Alessandro Zummo Cc: Alexandre Belloni Cc: Arnd Bergmann Cc: Rob Herring Cc: linux-mips@vger.kernel.org Cc: linux-rtc@vger.kernel.org Cc: devicetree@vger.kernel.org Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200521204818.25436-7-Sergey.Semin@baikalelectronics.ru --- drivers/clocksource/dw_apb_timer_of.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/clocksource/dw_apb_timer_of.c b/drivers/clocksource/dw_apb_timer_of.c index 2db490f35c20..ab3ddebe8344 100644 --- a/drivers/clocksource/dw_apb_timer_of.c +++ b/drivers/clocksource/dw_apb_timer_of.c @@ -147,10 +147,6 @@ static int num_called; static int __init dw_apb_timer_init(struct device_node *timer) { switch (num_called) { - case 0: - pr_debug("%s: found clockevent timer\n", __func__); - add_clockevent(timer); - break; case 1: pr_debug("%s: found clocksource timer\n", __func__); add_clocksource(timer); @@ -161,6 +157,8 @@ static int __init dw_apb_timer_init(struct device_node *timer) #endif break; default: + pr_debug("%s: found clockevent timer\n", __func__); + add_clockevent(timer); break; } From 48016e78d328998b1f00bcfb639adeabca51abe5 Mon Sep 17 00:00:00 2001 From: Paul Burton Date: Thu, 21 May 2020 23:48:16 +0300 Subject: [PATCH 0667/1043] clocksource: mips-gic-timer: Register as sched_clock The MIPS GIC timer is well suited for use as sched_clock, so register it as such. Whilst the existing gic_read_count() function matches the prototype needed by sched_clock_register() already, we split it into 2 functions in order to remove the need to evaluate the mips_cm_is64 condition within each call since sched_clock should be as fast as possible. Note the sched clock framework needs the clock source being stable in order to rely on it. So we register the MIPS GIC timer as schedule clocks only if it's, if either the system doesn't have CPU-frequency enabled or the CPU frequency is changed by means of the CPC core clock divider available on the platforms with CM3 or newer. Signed-off-by: Paul Burton Co-developed-by: Serge Semin [Sergey.Semin@baikalelectronics.ru: Register sched-clock if CM3 or !CPU-freq] Signed-off-by: Serge Semin Cc: Alexey Malahov Cc: Thomas Bogendoerfer Cc: Ralf Baechle Cc: Alessandro Zummo Cc: Alexandre Belloni Cc: Arnd Bergmann Cc: Rob Herring Cc: linux-mips@vger.kernel.org Cc: linux-rtc@vger.kernel.org Cc: devicetree@vger.kernel.org Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200521204818.25436-8-Sergey.Semin@baikalelectronics.ru --- drivers/clocksource/mips-gic-timer.c | 31 ++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c index 8b5f8ae723cb..ef12c12c2432 100644 --- a/drivers/clocksource/mips-gic-timer.c +++ b/drivers/clocksource/mips-gic-timer.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -24,13 +25,10 @@ static DEFINE_PER_CPU(struct clock_event_device, gic_clockevent_device); static int gic_timer_irq; static unsigned int gic_frequency; -static u64 notrace gic_read_count(void) +static u64 notrace gic_read_count_2x32(void) { unsigned int hi, hi2, lo; - if (mips_cm_is64) - return read_gic_counter(); - do { hi = read_gic_counter_32h(); lo = read_gic_counter_32l(); @@ -40,6 +38,19 @@ static u64 notrace gic_read_count(void) return (((u64) hi) << 32) + lo; } +static u64 notrace gic_read_count_64(void) +{ + return read_gic_counter(); +} + +static u64 notrace gic_read_count(void) +{ + if (mips_cm_is64) + return gic_read_count_64(); + + return gic_read_count_2x32(); +} + static int gic_next_event(unsigned long delta, struct clock_event_device *evt) { int cpu = cpumask_first(evt->cpumask); @@ -228,6 +239,18 @@ static int __init gic_clocksource_of_init(struct device_node *node) /* And finally start the counter */ clear_gic_config(GIC_CONFIG_COUNTSTOP); + /* + * It's safe to use the MIPS GIC timer as a sched clock source only if + * its ticks are stable, which is true on either the platforms with + * stable CPU frequency or on the platforms with CM3 and CPU frequency + * change performed by the CPC core clocks divider. + */ + if (mips_cm_revision() >= CM_REV_CM3 || !IS_ENABLED(CONFIG_CPU_FREQ)) { + sched_clock_register(mips_cm_is64 ? + gic_read_count_64 : gic_read_count_2x32, + 64, gic_frequency); + } + return 0; } TIMER_OF_DECLARE(mips_gic_timer, "mti,gic-timer", From 7d7de1a65349811b24971c5e8e040e6aac192dd4 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Thu, 21 May 2020 23:48:17 +0300 Subject: [PATCH 0668/1043] clocksource: mips-gic-timer: Mark GIC timer as unstable if ref clock changes Currently clocksource framework doesn't support the clocks with variable frequency. Since MIPS GIC timer ticks rate might be unstable on some platforms, we must make sure that it justifies the clocksource requirements. MIPS GIC timer is incremented with the CPU cluster reference clocks rate. So in case if CPU frequency changes, the MIPS GIC tick rate changes synchronously. Due to this the clocksource subsystem can't rely on the timer to measure system clocks anymore. This commit marks the MIPS GIC based clocksource as unstable if reference clock (normally it's a CPU reference clocks) rate changes. The clocksource will execute a watchdog thread, which lowers the MIPS GIC timer rating to zero and fallbacks to a new stable one. Note we don't need to set the CLOCK_SOURCE_MUST_VERIFY flag to the MIPS GIC clocksource since normally the timer is stable. The only reason why it gets unstable is due to the ref clock rate change, which event we detect here in the driver by means of the clocks event notifier. Signed-off-by: Serge Semin Cc: Alexey Malahov Cc: Thomas Bogendoerfer Cc: Paul Burton Cc: Ralf Baechle Cc: Alessandro Zummo Cc: Alexandre Belloni Cc: Arnd Bergmann Cc: Rob Herring Cc: linux-mips@vger.kernel.org Cc: linux-rtc@vger.kernel.org Cc: devicetree@vger.kernel.org Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200521204818.25436-9-Sergey.Semin@baikalelectronics.ru --- drivers/clocksource/Kconfig | 1 + drivers/clocksource/mips-gic-timer.c | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index c82400273b0a..91418381fcd4 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -570,6 +570,7 @@ config CLKSRC_VERSATILE config CLKSRC_MIPS_GIC bool depends on MIPS_GIC + select CLOCKSOURCE_WATCHDOG select TIMER_OF config CLKSRC_TANGO_XTAL diff --git a/drivers/clocksource/mips-gic-timer.c b/drivers/clocksource/mips-gic-timer.c index ef12c12c2432..be4175f415ba 100644 --- a/drivers/clocksource/mips-gic-timer.c +++ b/drivers/clocksource/mips-gic-timer.c @@ -24,6 +24,9 @@ static DEFINE_PER_CPU(struct clock_event_device, gic_clockevent_device); static int gic_timer_irq; static unsigned int gic_frequency; +static bool __read_mostly gic_clock_unstable; + +static void gic_clocksource_unstable(char *reason); static u64 notrace gic_read_count_2x32(void) { @@ -125,8 +128,10 @@ static int gic_clk_notifier(struct notifier_block *nb, unsigned long action, { struct clk_notifier_data *cnd = data; - if (action == POST_RATE_CHANGE) + if (action == POST_RATE_CHANGE) { + gic_clocksource_unstable("ref clock rate change"); on_each_cpu(gic_update_frequency, (void *)cnd->new_rate, 1); + } return NOTIFY_OK; } @@ -172,6 +177,18 @@ static struct clocksource gic_clocksource = { .vdso_clock_mode = VDSO_CLOCKMODE_GIC, }; +static void gic_clocksource_unstable(char *reason) +{ + if (gic_clock_unstable) + return; + + gic_clock_unstable = true; + + pr_info("GIC timer is unstable due to %s\n", reason); + + clocksource_mark_unstable(&gic_clocksource); +} + static int __init __gic_clocksource_init(void) { unsigned int count_width; From 7a3768c206a006525afc090f92d4d618d8356b92 Mon Sep 17 00:00:00 2001 From: Saravana Kannan Date: Tue, 24 Mar 2020 12:53:02 -0700 Subject: [PATCH 0669/1043] clocksource/drivers/timer-versatile: Clear OF_POPULATED flag The commit 4f41fe386a94 ("clocksource/drivers/timer-probe: Avoid creating dead devices") broke the handling of arm,vexpress-sysreg [1]. The arm,vexpress-sysreg device is handled by both timer-versatile.c and drivers/mfd/vexpress-sysreg.c. While the timer driver doesn't use the device, the mfd driver still needs a device to probe. So, this patch clears the OF_POPULATED flag to continue creating the device. [1] - https://lore.kernel.org/lkml/20200324175955.GA16972@arm.com/ Fixes: 4f41fe386a94 ("clocksource/drivers/timer-probe: Avoid creating dead devices") Signed-off-by: Saravana Kannan Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200324195302.203115-1-saravanak@google.com --- drivers/clocksource/timer-versatile.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/clocksource/timer-versatile.c b/drivers/clocksource/timer-versatile.c index e4ebb656d005..f5d017b31afa 100644 --- a/drivers/clocksource/timer-versatile.c +++ b/drivers/clocksource/timer-versatile.c @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -22,6 +23,8 @@ static int __init versatile_sched_clock_init(struct device_node *node) { void __iomem *base = of_iomap(node, 0); + of_node_clear_flag(node, OF_POPULATED); + if (!base) return -ENXIO; From 809eb4e9bf9d84eb5b703358afd0d564d514f6d2 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Tue, 19 May 2020 10:11:01 +0200 Subject: [PATCH 0670/1043] dt-bindings: timer: Add renesas,em-sti bindings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document Device Tree bindings for the Renesas EMMA Mobile System Timer. Signed-off-by: Geert Uytterhoeven Reviewed-by: Niklas Söderlund Signed-off-by: Daniel Lezcano Link: https://lore.kernel.org/r/20200519081101.28973-1-geert+renesas@glider.be --- .../bindings/timer/renesas,em-sti.yaml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 Documentation/devicetree/bindings/timer/renesas,em-sti.yaml diff --git a/Documentation/devicetree/bindings/timer/renesas,em-sti.yaml b/Documentation/devicetree/bindings/timer/renesas,em-sti.yaml new file mode 100644 index 000000000000..233d74d5402c --- /dev/null +++ b/Documentation/devicetree/bindings/timer/renesas,em-sti.yaml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/timer/renesas,em-sti.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Renesas EMMA Mobile System Timer + +maintainers: + - Magnus Damm + +properties: + compatible: + const: renesas,em-sti + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + const: sclk + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + +additionalProperties: false + +examples: + - | + #include + timer@e0180000 { + compatible = "renesas,em-sti"; + reg = <0xe0180000 0x54>; + interrupts = ; + clocks = <&sti_sclk>; + clock-names = "sclk"; + }; From 9afcc71b4f85ee9c9604c9b8349bac0eed44aa63 Mon Sep 17 00:00:00 2001 From: Max Filippov Date: Fri, 22 May 2020 15:52:03 -0700 Subject: [PATCH 0671/1043] xtensa: fix error paths in __get_user_{check,size} Error paths in __get_user_check and __get_user_size directly assing 0 to the result. It causes the following sparse warnings: sparse: warning: Using plain integer as NULL pointer Convert 0 to the type pointed to by the user pointer before assigning it. Signed-off-by: Max Filippov --- arch/xtensa/include/asm/uaccess.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index 445bb4cf3c28..0fd9b4086ae2 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -184,7 +184,7 @@ __asm__ __volatile__( \ if (access_ok(__gu_addr, size)) \ __get_user_size((x), __gu_addr, (size), __gu_err); \ else \ - (x) = 0; \ + (x) = (__typeof__(*(ptr)))0; \ __gu_err; \ }) @@ -202,13 +202,15 @@ do { \ u64 __x; \ if (unlikely(__copy_from_user(&__x, ptr, 8))) { \ retval = -EFAULT; \ - (x) = 0; \ + (x) = (__typeof__(*(ptr)))0; \ } else { \ (x) = *(__force __typeof__(*(ptr)) *)&__x; \ } \ break; \ } \ - default: (x) = 0; __get_user_bad(); \ + default: \ + (x) = (__typeof__(*(ptr)))0; \ + __get_user_bad(); \ } \ } while (0) From 2adf5352a34ac65a776d709607939bc82cf9819e Mon Sep 17 00:00:00 2001 From: Max Filippov Date: Fri, 22 May 2020 11:40:20 -0700 Subject: [PATCH 0672/1043] xtensa: add missing __user annotations to asm/uaccess.h clear_user, strncpy_user, strnlen_user and their helpers operate on user pointers, but don't have their arguments marked as __user. Add __user annotation to userspace pointers of those functions. Fix open-coded access check in the strnlen_user while at it. Signed-off-by: Max Filippov --- arch/xtensa/include/asm/uaccess.h | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index 0fd9b4086ae2..e57f0d0a88d8 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -272,15 +272,15 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) */ static inline unsigned long -__xtensa_clear_user(void *addr, unsigned long size) +__xtensa_clear_user(void __user *addr, unsigned long size) { - if (!__memset(addr, 0, size)) + if (!__memset((void __force *)addr, 0, size)) return size; return 0; } static inline unsigned long -clear_user(void *addr, unsigned long size) +clear_user(void __user *addr, unsigned long size) { if (access_ok(addr, size)) return __xtensa_clear_user(addr, size); @@ -292,10 +292,10 @@ clear_user(void *addr, unsigned long size) #ifndef CONFIG_GENERIC_STRNCPY_FROM_USER -extern long __strncpy_user(char *, const char *, long); +extern long __strncpy_user(char *dst, const char __user *src, long count); static inline long -strncpy_from_user(char *dst, const char *src, long count) +strncpy_from_user(char *dst, const char __user *src, long count) { if (access_ok(src, 1)) return __strncpy_user(dst, src, count); @@ -308,13 +308,11 @@ long strncpy_from_user(char *dst, const char *src, long count); /* * Return the size of a string (including the ending 0!) */ -extern long __strnlen_user(const char *, long); +extern long __strnlen_user(const char __user *str, long len); -static inline long strnlen_user(const char *str, long len) +static inline long strnlen_user(const char __user *str, long len) { - unsigned long top = __kernel_ok ? ~0UL : TASK_SIZE - 1; - - if ((unsigned long)str > top) + if (!access_ok(str, 1)) return 0; return __strnlen_user(str, len); } From a949e86c0d7802c05b2ae726a84fae89ddb5be7d Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Fri, 22 May 2020 19:19:00 +0200 Subject: [PATCH 0673/1043] Drivers: hv: vmbus: Resolve race between init_vp_index() and CPU hotplug vmbus_process_offer() does two things (among others): 1) first, it sets the channel's target CPU with cpu_hotplug_lock; 2) it then adds the channel to the channel list(s) with channel_mutex. Since cpu_hotplug_lock is released before (2), the channel's target CPU (as designated in (1)) can be deemed "free" by hv_synic_cleanup() and go offline before the channel is added to the list. Fix the race condition by "extending" the cpu_hotplug_lock critical section to include (2) (and (1)), nesting the channel_mutex critical section within the cpu_hotplug_lock critical section as done elsewhere (hv_synic_cleanup(), target_cpu_store()) in the hyperv drivers code. Move even further by extending the channel_mutex critical section to include (1) (and (2)): this change allows to remove (the now redundant) bind_channel_to_cpu_lock, and generally simplifies the handling of the target CPUs (that are now always modified with channel_mutex held). Fixes: d570aec0f2154e ("Drivers: hv: vmbus: Synchronize init_vp_index() vs. CPU hotplug") Signed-off-by: Andrea Parri (Microsoft) Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200522171901.204127-2-parri.andrea@gmail.com Signed-off-by: Wei Liu --- drivers/hv/channel_mgmt.c | 46 +++++++++++++++------------------------ 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index fde806d6525b..89eaacf069a8 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -554,26 +554,34 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel) bool fnew = true; /* - * Initialize the target_CPU before inserting the channel in - * the chn_list and sc_list lists, within the channel_mutex - * critical section: + * Synchronize vmbus_process_offer() and CPU hotplugging: * * CPU1 CPU2 * - * [vmbus_process_offer()] [hv_syninc_cleanup()] + * [vmbus_process_offer()] [Hot removal of the CPU] * - * STORE target_cpu LOCK channel_mutex - * LOCK channel_mutex SEARCH chn_list - * INSERT chn_list LOAD target_cpu - * UNLOCK channel_mutex UNLOCK channel_mutex + * CPU_READ_LOCK CPUS_WRITE_LOCK + * LOAD cpu_online_mask SEARCH chn_list + * STORE target_cpu LOAD target_cpu + * INSERT chn_list STORE cpu_online_mask + * CPUS_READ_UNLOCK CPUS_WRITE_UNLOCK + * + * Forbids: CPU1's LOAD from *not* seing CPU2's STORE && + * CPU2's SEARCH from *not* seeing CPU1's INSERT * * Forbids: CPU2's SEARCH from seeing CPU1's INSERT && * CPU2's LOAD from *not* seing CPU1's STORE */ - init_vp_index(newchannel, hv_get_dev_type(newchannel)); + cpus_read_lock(); + /* + * Serializes the modifications of the chn_list list as well as + * the accesses to next_numa_node_id in init_vp_index(). + */ mutex_lock(&vmbus_connection.channel_mutex); + init_vp_index(newchannel, hv_get_dev_type(newchannel)); + /* Remember the channels that should be cleaned up upon suspend. */ if (is_hvsock_channel(newchannel) || is_sub_channel(newchannel)) atomic_inc(&vmbus_connection.nr_chan_close_on_suspend); @@ -623,6 +631,7 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel) vmbus_channel_map_relid(newchannel); mutex_unlock(&vmbus_connection.channel_mutex); + cpus_read_unlock(); /* * vmbus_process_offer() mustn't call channel->sc_creation_callback() @@ -655,13 +664,6 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel) * We use this state to statically distribute the channel interrupt load. */ static int next_numa_node_id; -/* - * init_vp_index() accesses global variables like next_numa_node_id, and - * it can run concurrently for primary channels and sub-channels: see - * vmbus_process_offer(), so we need the lock to protect the global - * variables. - */ -static DEFINE_SPINLOCK(bind_channel_to_cpu_lock); /* * Starting with Win8, we can statically distribute the incoming @@ -700,15 +702,6 @@ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type) return; } - /* No CPUs can come up or down during this. */ - cpus_read_lock(); - - /* - * Serializes the accesses to the global variable next_numa_node_id. - * See also the header comment of the spin lock declaration. - */ - spin_lock(&bind_channel_to_cpu_lock); - while (true) { numa_node = next_numa_node_id++; if (numa_node == nr_node_ids) { @@ -739,9 +732,6 @@ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type) channel->target_cpu = target_cpu; channel->target_vp = hv_cpu_number_to_vp_number(target_cpu); - spin_unlock(&bind_channel_to_cpu_lock); - cpus_read_unlock(); - free_cpumask_var(available_mask); } From afaa33da08abd10be8978781d7c99a9e67d2bbff Mon Sep 17 00:00:00 2001 From: "Andrea Parri (Microsoft)" Date: Fri, 22 May 2020 19:19:01 +0200 Subject: [PATCH 0674/1043] Drivers: hv: vmbus: Resolve more races involving init_vp_index() init_vp_index() uses the (per-node) hv_numa_map[] masks to record the CPUs allocated for channel interrupts at a given time, and distribute the performance-critical channels across the available CPUs: in part., the mask of "candidate" target CPUs in a given NUMA node, for a newly offered channel, is determined by XOR-ing the node's CPU mask and the node's hv_numa_map. This operation/mechanism assumes that no offline CPUs is set in the hv_numa_map mask, an assumption that does not hold since such mask is currently not updated when a channel is removed or assigned to a different CPU. To address the issues described above, this adds hooks in the channel removal path (hv_process_channel_removal()) and in target_cpu_store() in order to clear, resp. to update, the hv_numa_map[] masks as needed. This also adds a (missed) update of the masks in init_vp_index() (cf., e.g., the memory-allocation failure path in this function). Like in the case of init_vp_index(), such hooks require to determine if the given channel is performance critical. init_vp_index() does this by parsing the channel's offer, it can not rely on the device data structure (device_obj) to retrieve such information because the device data structure has not been allocated/linked with the channel by the time that init_vp_index() executes. A similar situation may hold in hv_is_alloced_cpu() (defined below); the adopted approach is to "cache" the device type of the channel, as computed by parsing the channel's offer, in the channel structure itself. Fixes: 7527810573436f ("Drivers: hv: vmbus: Introduce the CHANNELMSG_MODIFYCHANNEL message type") Signed-off-by: Andrea Parri (Microsoft) Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20200522171901.204127-3-parri.andrea@gmail.com Signed-off-by: Wei Liu --- drivers/hv/channel_mgmt.c | 22 +++++++++++++----- drivers/hv/hyperv_vmbus.h | 48 +++++++++++++++++++++++++++++++++++++++ drivers/hv/vmbus_drv.c | 19 +++++++++++----- include/linux/hyperv.h | 7 ++++++ 4 files changed, 84 insertions(+), 12 deletions(-) diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index 89eaacf069a8..417a95e5094d 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -24,9 +24,9 @@ #include "hyperv_vmbus.h" -static void init_vp_index(struct vmbus_channel *channel, u16 dev_type); +static void init_vp_index(struct vmbus_channel *channel); -static const struct vmbus_device vmbus_devs[] = { +const struct vmbus_device vmbus_devs[] = { /* IDE */ { .dev_type = HV_IDE, HV_IDE_GUID, @@ -431,6 +431,13 @@ void hv_process_channel_removal(struct vmbus_channel *channel) spin_unlock_irqrestore(&primary_channel->lock, flags); } + /* + * If this is a "perf" channel, updates the hv_numa_map[] masks so that + * init_vp_index() can (re-)use the CPU. + */ + if (hv_is_perf_channel(channel)) + hv_clear_alloced_cpu(channel->target_cpu); + /* * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and * the relid is invalidated; after hibernation, when the user-space app @@ -497,7 +504,7 @@ static void vmbus_add_channel_work(struct work_struct *work) if (!newchannel->device_obj) goto err_deq_chan; - newchannel->device_obj->device_id = hv_get_dev_type(newchannel); + newchannel->device_obj->device_id = newchannel->device_id; /* * Add the new device to the bus. This will kick off device-driver * binding which eventually invokes the device driver's AddDevice() @@ -580,7 +587,7 @@ static void vmbus_process_offer(struct vmbus_channel *newchannel) */ mutex_lock(&vmbus_connection.channel_mutex); - init_vp_index(newchannel, hv_get_dev_type(newchannel)); + init_vp_index(newchannel); /* Remember the channels that should be cleaned up upon suspend. */ if (is_hvsock_channel(newchannel) || is_sub_channel(newchannel)) @@ -676,9 +683,9 @@ static int next_numa_node_id; * evenly among all the available NUMA nodes. Once the node is assigned, * we will assign the CPU based on a simple round robin scheme. */ -static void init_vp_index(struct vmbus_channel *channel, u16 dev_type) +static void init_vp_index(struct vmbus_channel *channel) { - bool perf_chn = vmbus_devs[dev_type].perf_device; + bool perf_chn = hv_is_perf_channel(channel); cpumask_var_t available_mask; struct cpumask *alloced_mask; u32 target_cpu; @@ -699,6 +706,8 @@ static void init_vp_index(struct vmbus_channel *channel, u16 dev_type) channel->target_cpu = VMBUS_CONNECT_CPU; channel->target_vp = hv_cpu_number_to_vp_number(VMBUS_CONNECT_CPU); + if (perf_chn) + hv_set_alloced_cpu(VMBUS_CONNECT_CPU); return; } @@ -862,6 +871,7 @@ static void vmbus_setup_channel_state(struct vmbus_channel *channel, sizeof(struct vmbus_channel_offer_channel)); channel->monitor_grp = (u8)offer->monitorid / 32; channel->monitor_bit = (u8)offer->monitorid % 32; + channel->device_id = hv_get_dev_type(channel); } /* diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h index 5e5cebe5d048..40e2b9f91163 100644 --- a/drivers/hv/hyperv_vmbus.h +++ b/drivers/hv/hyperv_vmbus.h @@ -395,6 +395,54 @@ enum delay { MESSAGE_DELAY = 1, }; +extern const struct vmbus_device vmbus_devs[]; + +static inline bool hv_is_perf_channel(struct vmbus_channel *channel) +{ + return vmbus_devs[channel->device_id].perf_device; +} + +static inline bool hv_is_alloced_cpu(unsigned int cpu) +{ + struct vmbus_channel *channel, *sc; + + lockdep_assert_held(&vmbus_connection.channel_mutex); + /* + * List additions/deletions as well as updates of the target CPUs are + * protected by channel_mutex. + */ + list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { + if (!hv_is_perf_channel(channel)) + continue; + if (channel->target_cpu == cpu) + return true; + list_for_each_entry(sc, &channel->sc_list, sc_list) { + if (sc->target_cpu == cpu) + return true; + } + } + return false; +} + +static inline void hv_set_alloced_cpu(unsigned int cpu) +{ + cpumask_set_cpu(cpu, &hv_context.hv_numa_map[cpu_to_node(cpu)]); +} + +static inline void hv_clear_alloced_cpu(unsigned int cpu) +{ + if (hv_is_alloced_cpu(cpu)) + return; + cpumask_clear_cpu(cpu, &hv_context.hv_numa_map[cpu_to_node(cpu)]); +} + +static inline void hv_update_alloced_cpus(unsigned int old_cpu, + unsigned int new_cpu) +{ + hv_set_alloced_cpu(new_cpu); + hv_clear_alloced_cpu(old_cpu); +} + #ifdef CONFIG_HYPERV_TESTING int hv_debug_add_dev_dir(struct hv_device *dev); diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index c2a4a7c0b99a..47747755d2e1 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1687,8 +1687,8 @@ static ssize_t target_cpu_show(struct vmbus_channel *channel, char *buf) static ssize_t target_cpu_store(struct vmbus_channel *channel, const char *buf, size_t count) { + u32 target_cpu, origin_cpu; ssize_t ret = count; - u32 target_cpu; if (vmbus_proto_version < VERSION_WIN10_V4_1) return -EIO; @@ -1741,7 +1741,8 @@ static ssize_t target_cpu_store(struct vmbus_channel *channel, goto cpu_store_unlock; } - if (channel->target_cpu == target_cpu) + origin_cpu = channel->target_cpu; + if (target_cpu == origin_cpu) goto cpu_store_unlock; if (vmbus_send_modifychannel(channel->offermsg.child_relid, @@ -1763,14 +1764,20 @@ static ssize_t target_cpu_store(struct vmbus_channel *channel, * in on a CPU that is different from the channel target_cpu value. */ - if (channel->change_target_cpu_callback) - (*channel->change_target_cpu_callback)(channel, - channel->target_cpu, target_cpu); - channel->target_cpu = target_cpu; channel->target_vp = hv_cpu_number_to_vp_number(target_cpu); channel->numa_node = cpu_to_node(target_cpu); + /* See init_vp_index(). */ + if (hv_is_perf_channel(channel)) + hv_update_alloced_cpus(origin_cpu, target_cpu); + + /* Currently set only for storvsc channels. */ + if (channel->change_target_cpu_callback) { + (*channel->change_target_cpu_callback)(channel, + origin_cpu, target_cpu); + } + cpu_store_unlock: mutex_unlock(&vmbus_connection.channel_mutex); cpus_read_unlock(); diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index d783847d8cb4..40df3103e890 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -901,6 +901,13 @@ struct vmbus_channel { bool probe_done; + /* + * Cache the device ID here for easy access; this is useful, in + * particular, in situations where the channel's device_obj has + * not been allocated/initialized yet. + */ + u16 device_id; + /* * We must offload the handling of the primary/sub channels * from the single-threaded vmbus_connection.work_queue to From f39293fd37fff74c531b7a52d0459cc77db85e7f Mon Sep 17 00:00:00 2001 From: Ben Hutchings Date: Sat, 23 May 2020 23:50:34 +0800 Subject: [PATCH 0675/1043] MIPS: Fix exception handler memcpy() The exception handler subroutines are declared as a single char, but when copied to the required addresses the copy length is 0x80. When range checks are enabled for memcpy() this results in a build failure, with error messages such as: In file included from arch/mips/mti-malta/malta-init.c:15: In function 'memcpy', inlined from 'mips_nmi_setup' at arch/mips/mti-malta/malta-init.c:98:2: include/linux/string.h:376:4: error: call to '__read_overflow2' declared with attribute error: detected read beyond size of object passed as 2nd parameter 376 | __read_overflow2(); | ^~~~~~~~~~~~~~~~~~ Change the declarations to use type char[]. Signed-off-by: Ben Hutchings Signed-off-by: YunQiang Su Signed-off-by: Thomas Bogendoerfer --- arch/mips/loongson2ef/common/init.c | 4 ++-- arch/mips/loongson64/init.c | 4 ++-- arch/mips/mti-malta/malta-init.c | 8 ++++---- arch/mips/pistachio/init.c | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/arch/mips/loongson2ef/common/init.c b/arch/mips/loongson2ef/common/init.c index 45512178be77..ce3f02f75e2a 100644 --- a/arch/mips/loongson2ef/common/init.c +++ b/arch/mips/loongson2ef/common/init.c @@ -19,10 +19,10 @@ unsigned long __maybe_unused _loongson_addrwincfg_base; static void __init mips_nmi_setup(void) { void *base; - extern char except_vec_nmi; + extern char except_vec_nmi[]; base = (void *)(CAC_BASE + 0x380); - memcpy(base, &except_vec_nmi, 0x80); + memcpy(base, except_vec_nmi, 0x80); flush_icache_range((unsigned long)base, (unsigned long)base + 0x80); } diff --git a/arch/mips/loongson64/init.c b/arch/mips/loongson64/init.c index 2b45ca6ca98d..23eeb85b1abf 100644 --- a/arch/mips/loongson64/init.c +++ b/arch/mips/loongson64/init.c @@ -22,10 +22,10 @@ u32 node_id_offset; static void __init mips_nmi_setup(void) { void *base; - extern char except_vec_nmi; + extern char except_vec_nmi[]; base = (void *)(CAC_BASE + 0x380); - memcpy(base, &except_vec_nmi, 0x80); + memcpy(base, except_vec_nmi, 0x80); flush_icache_range((unsigned long)base, (unsigned long)base + 0x80); } diff --git a/arch/mips/mti-malta/malta-init.c b/arch/mips/mti-malta/malta-init.c index ff2c1d809538..893af377aacc 100644 --- a/arch/mips/mti-malta/malta-init.c +++ b/arch/mips/mti-malta/malta-init.c @@ -90,24 +90,24 @@ static void __init console_config(void) static void __init mips_nmi_setup(void) { void *base; - extern char except_vec_nmi; + extern char except_vec_nmi[]; base = cpu_has_veic ? (void *)(CAC_BASE + 0xa80) : (void *)(CAC_BASE + 0x380); - memcpy(base, &except_vec_nmi, 0x80); + memcpy(base, except_vec_nmi, 0x80); flush_icache_range((unsigned long)base, (unsigned long)base + 0x80); } static void __init mips_ejtag_setup(void) { void *base; - extern char except_vec_ejtag_debug; + extern char except_vec_ejtag_debug[]; base = cpu_has_veic ? (void *)(CAC_BASE + 0xa00) : (void *)(CAC_BASE + 0x300); - memcpy(base, &except_vec_ejtag_debug, 0x80); + memcpy(base, except_vec_ejtag_debug, 0x80); flush_icache_range((unsigned long)base, (unsigned long)base + 0x80); } diff --git a/arch/mips/pistachio/init.c b/arch/mips/pistachio/init.c index a09a5da38e6b..558995ed6fe8 100644 --- a/arch/mips/pistachio/init.c +++ b/arch/mips/pistachio/init.c @@ -83,12 +83,12 @@ phys_addr_t mips_cdmm_phys_base(void) static void __init mips_nmi_setup(void) { void *base; - extern char except_vec_nmi; + extern char except_vec_nmi[]; base = cpu_has_veic ? (void *)(CAC_BASE + 0xa80) : (void *)(CAC_BASE + 0x380); - memcpy(base, &except_vec_nmi, 0x80); + memcpy(base, except_vec_nmi, 0x80); flush_icache_range((unsigned long)base, (unsigned long)base + 0x80); } @@ -96,12 +96,12 @@ static void __init mips_nmi_setup(void) static void __init mips_ejtag_setup(void) { void *base; - extern char except_vec_ejtag_debug; + extern char except_vec_ejtag_debug[]; base = cpu_has_veic ? (void *)(CAC_BASE + 0xa00) : (void *)(CAC_BASE + 0x300); - memcpy(base, &except_vec_ejtag_debug, 0x80); + memcpy(base, except_vec_ejtag_debug, 0x80); flush_icache_range((unsigned long)base, (unsigned long)base + 0x80); } From 41528ba6afe62d472a729b223f8542ccc1156df1 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Thu, 21 May 2020 10:15:06 +0800 Subject: [PATCH 0676/1043] MIPS: DTS: Only build subdir of current platform Add config check in Makefile to only build the subdir of current platform. E.g. without this patch: AR arch/mips/built-in.a AR arch/mips/boot/dts/brcm/built-in.a AR arch/mips/boot/dts/cavium-octeon/built-in.a AR arch/mips/boot/dts/img/built-in.a AR arch/mips/boot/dts/ingenic/built-in.a AR arch/mips/boot/dts/lantiq/built-in.a DTC arch/mips/boot/dts/loongson/loongson3_4core_rs780e.dtb DTB arch/mips/boot/dts/loongson/loongson3_4core_rs780e.dtb.S AS arch/mips/boot/dts/loongson/loongson3_4core_rs780e.dtb.o DTC arch/mips/boot/dts/loongson/loongson3_8core_rs780e.dtb DTB arch/mips/boot/dts/loongson/loongson3_8core_rs780e.dtb.S AS arch/mips/boot/dts/loongson/loongson3_8core_rs780e.dtb.o AR arch/mips/boot/dts/loongson/built-in.a AR arch/mips/boot/dts/mscc/built-in.a AR arch/mips/boot/dts/mti/built-in.a AR arch/mips/boot/dts/netlogic/built-in.a AR arch/mips/boot/dts/ni/built-in.a AR arch/mips/boot/dts/pic32/built-in.a AR arch/mips/boot/dts/qca/built-in.a AR arch/mips/boot/dts/ralink/built-in.a AR arch/mips/boot/dts/xilfpga/built-in.a AR arch/mips/boot/dts/built-in.a With this patch: AR arch/mips/built-in.a DTC arch/mips/boot/dts/loongson/loongson3_4core_rs780e.dtb DTB arch/mips/boot/dts/loongson/loongson3_4core_rs780e.dtb.S AS arch/mips/boot/dts/loongson/loongson3_4core_rs780e.dtb.o DTC arch/mips/boot/dts/loongson/loongson3_8core_rs780e.dtb DTB arch/mips/boot/dts/loongson/loongson3_8core_rs780e.dtb.S AS arch/mips/boot/dts/loongson/loongson3_8core_rs780e.dtb.o AR arch/mips/boot/dts/loongson/built-in.a AR arch/mips/boot/dts/built-in.a Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/Makefile | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/arch/mips/boot/dts/Makefile b/arch/mips/boot/dts/Makefile index d429a69bfe30..dce32d16be73 100644 --- a/arch/mips/boot/dts/Makefile +++ b/arch/mips/boot/dts/Makefile @@ -1,17 +1,17 @@ # SPDX-License-Identifier: GPL-2.0 -subdir-y += brcm -subdir-y += cavium-octeon -subdir-y += img -subdir-y += ingenic -subdir-y += lantiq -subdir-y += loongson -subdir-y += mscc -subdir-y += mti -subdir-y += netlogic -subdir-y += ni -subdir-y += pic32 -subdir-y += qca -subdir-y += ralink -subdir-y += xilfpga +subdir-$(CONFIG_BMIPS_GENERIC) += brcm +subdir-$(CONFIG_CAVIUM_OCTEON_SOC) += cavium-octeon +subdir-$(CONFIG_MACH_PISTACHIO) += img +subdir-$(CONFIG_MACH_INGENIC) += ingenic +subdir-$(CONFIG_LANTIQ) += lantiq +subdir-$(CONFIG_MACH_LOONGSON64) += loongson +subdir-$(CONFIG_MSCC_OCELOT) += mscc +subdir-$(CONFIG_MIPS_MALTA) += mti +subdir-$(CONFIG_NLM_XLP_BOARD) += netlogic +subdir-$(CONFIG_FIT_IMAGE_FDT_NI169445) += ni +subdir-$(CONFIG_MACH_PIC32) += pic32 +subdir-$(CONFIG_ATH79) += qca +subdir-$(CONFIG_RALINK) += ralink +subdir-$(CONFIG_FIT_IMAGE_FDT_XILFPGA) += xilfpga obj-$(CONFIG_BUILTIN_DTB) := $(addsuffix /, $(subdir-y)) From 8267e78f020a8de2752754c42ec1d56e92431477 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Sat, 23 May 2020 15:51:45 +0800 Subject: [PATCH 0677/1043] MIPS: Tidy up CP0.Config6 bits definition CP0.Config6 is a Vendor-defined register whose bits definitions are different from one to another. Recently, Xuerui's Loongson-3 patch and Serge's P5600 patch make the definitions inconsistency and unclear. To make life easy, this patch tidy the definition up: 1, Add a _MTI_ infix for proAptiv/P5600 feature bits; 2, Add a _LOONGSON_ infix for Loongson-3 feature bits; 3, Add bit6/bit7 definition for Loongson-3 which will be used later. All existing users of these macros are updated. Cc: WANG Xuerui Cc: Serge Semin Signed-off-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mipsregs.h | 37 +++++++++++++++++++++----------- arch/mips/kernel/cpu-probe.c | 12 +++++------ arch/mips/mm/c-r4k.c | 4 ++-- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h index fe6293f5b939..796dbb86575b 100644 --- a/arch/mips/include/asm/mipsregs.h +++ b/arch/mips/include/asm/mipsregs.h @@ -686,27 +686,38 @@ #define MIPS_CONF5_CV (_ULCAST_(1) << 29) #define MIPS_CONF5_K (_ULCAST_(1) << 30) +/* Config6 feature bits for proAptiv/P5600 */ + /* Jump register cache prediction disable */ -#define MIPS_CONF6_JRCD (_ULCAST_(1) << 0) +#define MIPS_CONF6_MTI_JRCD (_ULCAST_(1) << 0) /* MIPSr6 extensions enable */ -#define MIPS_CONF6_R6 (_ULCAST_(1) << 2) +#define MIPS_CONF6_MTI_R6 (_ULCAST_(1) << 2) /* IFU Performance Control */ -#define MIPS_CONF6_IFUPERFCTL (_ULCAST_(3) << 10) -#define MIPS_CONF6_SYND (_ULCAST_(1) << 13) +#define MIPS_CONF6_MTI_IFUPERFCTL (_ULCAST_(3) << 10) +#define MIPS_CONF6_MTI_SYND (_ULCAST_(1) << 13) /* Sleep state performance counter disable */ -#define MIPS_CONF6_SPCD (_ULCAST_(1) << 14) +#define MIPS_CONF6_MTI_SPCD (_ULCAST_(1) << 14) /* proAptiv FTLB on/off bit */ -#define MIPS_CONF6_FTLBEN (_ULCAST_(1) << 15) +#define MIPS_CONF6_MTI_FTLBEN (_ULCAST_(1) << 15) /* Disable load/store bonding */ -#define MIPS_CONF6_DLSB (_ULCAST_(1) << 21) -/* Loongson-3 FTLB on/off bit */ -#define MIPS_CONF6_FTLBDIS (_ULCAST_(1) << 22) +#define MIPS_CONF6_MTI_DLSB (_ULCAST_(1) << 21) /* FTLB probability bits */ -#define MIPS_CONF6_FTLBP_SHIFT (16) -/* Loongson-3 feature bits */ -#define MIPS_CONF6_LOONGSON_SCRAND (_ULCAST_(1) << 17) +#define MIPS_CONF6_MTI_FTLBP_SHIFT (16) + +/* Config6 feature bits for Loongson-3 */ + +/* Loongson-3 internal timer bit */ +#define MIPS_CONF6_LOONGSON_INTIMER (_ULCAST_(1) << 6) +/* Loongson-3 external timer bit */ +#define MIPS_CONF6_LOONGSON_EXTIMER (_ULCAST_(1) << 7) +/* Loongson-3 SFB on/off bit, STFill in manual */ +#define MIPS_CONF6_LOONGSON_SFBEN (_ULCAST_(1) << 8) +/* Loongson-3's LL on exclusive cacheline */ #define MIPS_CONF6_LOONGSON_LLEXC (_ULCAST_(1) << 16) -#define MIPS_CONF6_LOONGSON_STFILL (_ULCAST_(1) << 8) +/* Loongson-3's SC has a random delay */ +#define MIPS_CONF6_LOONGSON_SCRAND (_ULCAST_(1) << 17) +/* Loongson-3 FTLB on/off bit, VTLBOnly in manual */ +#define MIPS_CONF6_LOONGSON_FTLBDIS (_ULCAST_(1) << 22) #define MIPS_CONF7_WII (_ULCAST_(1) << 31) diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index b8ec35737606..f7c4b1d86274 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -633,14 +633,14 @@ static int set_ftlb_enable(struct cpuinfo_mips *c, enum ftlb_flags flags) config = read_c0_config6(); if (flags & FTLB_EN) - config |= MIPS_CONF6_FTLBEN; + config |= MIPS_CONF6_MTI_FTLBEN; else - config &= ~MIPS_CONF6_FTLBEN; + config &= ~MIPS_CONF6_MTI_FTLBEN; if (flags & FTLB_SET_PROB) { - config &= ~(3 << MIPS_CONF6_FTLBP_SHIFT); + config &= ~(3 << MIPS_CONF6_MTI_FTLBP_SHIFT); config |= calculate_ftlb_probability(c) - << MIPS_CONF6_FTLBP_SHIFT; + << MIPS_CONF6_MTI_FTLBP_SHIFT; } write_c0_config6(config); @@ -660,10 +660,10 @@ static int set_ftlb_enable(struct cpuinfo_mips *c, enum ftlb_flags flags) config = read_c0_config6(); if (flags & FTLB_EN) /* Enable FTLB */ - write_c0_config6(config & ~MIPS_CONF6_FTLBDIS); + write_c0_config6(config & ~MIPS_CONF6_LOONGSON_FTLBDIS); else /* Disable FTLB */ - write_c0_config6(config | MIPS_CONF6_FTLBDIS); + write_c0_config6(config | MIPS_CONF6_LOONGSON_FTLBDIS); break; default: return 1; diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index a9f55bf90967..6fb83ac7c475 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -1073,12 +1073,12 @@ static inline int alias_74k_erratum(struct cpuinfo_mips *c) if (rev <= PRID_REV_ENCODE_332(2, 4, 0)) present = 1; if (rev == PRID_REV_ENCODE_332(2, 4, 0)) - write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND); + write_c0_config6(read_c0_config6() | MIPS_CONF6_MTI_SYND); break; case PRID_IMP_1074K: if (rev <= PRID_REV_ENCODE_332(1, 1, 0)) { present = 1; - write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND); + write_c0_config6(read_c0_config6() | MIPS_CONF6_MTI_SYND); } break; default: From ec7a93188a75b57b9f704db6862e7137f01aa80b Mon Sep 17 00:00:00 2001 From: WANG Xuerui Date: Sat, 23 May 2020 21:37:01 +0800 Subject: [PATCH 0678/1043] MIPS: emulate CPUCFG instruction on older Loongson64 cores CPUCFG is the instruction for querying processor characteristics on newer Loongson processors, much like CPUID of x86. Since the instruction is supposedly designed to provide a unified way to do feature detection (without having to, for example, parse /proc/cpuinfo which is too heavyweight), it is important to provide compatibility for older cores without native support. Fortunately, most of the fields can be synthesized without changes to semantics. Performance is not really big a concern, because feature detection logic is not expected to be invoked very often in typical userland applications. The instruction can't be emulated on LOONGSON_2EF cores, according to FlyGoat's experiments. Because the LWC2 opcode is assigned to other valid instructions on 2E and 2F, no RI exception is raised for us to intercept. So compatibility is only extended back furthest to Loongson-3A1000. Loongson-2K is covered too, as it is basically a remix of various blocks from the 3A/3B models from a kernel perspective. This is lightly based on Loongson's work on their Linux 3.10 fork, for being the authority on the right feature flags to fill in, where things aren't otherwise discoverable. Signed-off-by: WANG Xuerui Reviewed-by: Jiaxun Yang Cc: Huacai Chen Cc: Jiaxun Yang Cc: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kconfig | 12 + arch/mips/include/asm/cpu-info.h | 9 + .../include/asm/mach-loongson64/cpucfg-emul.h | 63 +++++ arch/mips/kernel/cpu-probe.c | 9 + arch/mips/kernel/traps.c | 45 ++++ arch/mips/loongson64/Makefile | 1 + arch/mips/loongson64/cpucfg-emul.c | 217 ++++++++++++++++++ 7 files changed, 356 insertions(+) create mode 100644 arch/mips/include/asm/mach-loongson64/cpucfg-emul.h create mode 100644 arch/mips/loongson64/cpucfg-emul.c diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index f347312ecd74..c5fe30c78402 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1441,6 +1441,18 @@ config CPU_LOONGSON3_WORKAROUNDS If unsure, please say Y. +config CPU_LOONGSON3_CPUCFG_EMULATION + bool "Emulate the CPUCFG instruction on older Loongson cores" + default y + depends on CPU_LOONGSON64 + help + Loongson-3A R4 and newer have the CPUCFG instruction available for + userland to query CPU capabilities, much like CPUID on x86. This + option provides emulation of the instruction on older Loongson + cores, back to Loongson-3A1000. + + If unsure, please say Y. + config CPU_LOONGSON2E bool "Loongson 2E" depends on SYS_HAS_CPU_LOONGSON2E diff --git a/arch/mips/include/asm/cpu-info.h b/arch/mips/include/asm/cpu-info.h index bce3ea7fff7c..a600670d00e9 100644 --- a/arch/mips/include/asm/cpu-info.h +++ b/arch/mips/include/asm/cpu-info.h @@ -105,6 +105,15 @@ struct cpuinfo_mips { unsigned int gtoffset_mask; unsigned int guestid_mask; unsigned int guestid_cache; + +#ifdef CONFIG_CPU_LOONGSON3_CPUCFG_EMULATION + /* CPUCFG data for this CPU, synthesized at probe time. + * + * CPUCFG select 0 is PRId, 4 and above are unimplemented for now. + * So the only stored values are for CPUCFG selects 1-3 inclusive. + */ + u32 loongson3_cpucfg_data[3]; +#endif } __attribute__((aligned(SMP_CACHE_BYTES))); extern struct cpuinfo_mips cpu_data[]; diff --git a/arch/mips/include/asm/mach-loongson64/cpucfg-emul.h b/arch/mips/include/asm/mach-loongson64/cpucfg-emul.h new file mode 100644 index 000000000000..01dc308df7b2 --- /dev/null +++ b/arch/mips/include/asm/mach-loongson64/cpucfg-emul.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_MACH_LOONGSON64_CPUCFG_EMUL_H_ +#define _ASM_MACH_LOONGSON64_CPUCFG_EMUL_H_ + +#include + +#ifdef CONFIG_CPU_LOONGSON3_CPUCFG_EMULATION + +#include + +#define LOONGSON_FPREV_MASK 0x7 + +void loongson3_cpucfg_synthesize_data(struct cpuinfo_mips *c); + +static inline u32 loongson3_cpucfg_read_synthesized(struct cpuinfo_mips *c, + __u64 sel) +{ + switch (sel) { + case LOONGSON_CFG0: + return c->processor_id; + case LOONGSON_CFG1: + case LOONGSON_CFG2: + case LOONGSON_CFG3: + return c->loongson3_cpucfg_data[sel - 1]; + case LOONGSON_CFG4: + case LOONGSON_CFG5: + /* CPUCFG selects 4 and 5 are related to the input clock + * signal. + * + * Unimplemented for now. + */ + return 0; + case LOONGSON_CFG6: + /* CPUCFG select 6 is for the undocumented Safe Extension. */ + return 0; + case LOONGSON_CFG7: + /* CPUCFG select 7 is for the virtualization extension. + * We don't know if the two currently known features are + * supported on older cores according to the public + * documentation, so leave this at zero. + */ + return 0; + } + + /* + * Return 0 for unrecognized CPUCFG selects, which is real hardware + * behavior observed on Loongson 3A R4. + */ + return 0; +} +#else +static inline void loongson3_cpucfg_synthesize_data(struct cpuinfo_mips *c) +{ +} + +static inline u32 loongson3_cpucfg_read_synthesized(struct cpuinfo_mips *c, + __u64 sel) +{ + return 0; +} +#endif + +#endif /* _ASM_MACH_LOONGSON64_CPUCFG_EMUL_H_ */ diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index f7c4b1d86274..6b93162d7c5a 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -28,6 +28,8 @@ #include #include +#include + /* Hardware capabilities */ unsigned int elf_hwcap __read_mostly; EXPORT_SYMBOL_GPL(elf_hwcap); @@ -2400,6 +2402,13 @@ void cpu_probe(void) cpu_probe_vmbits(c); + /* Synthesize CPUCFG data if running on Loongson processors; + * no-op otherwise. + * + * This looks at previously probed features, so keep this at bottom. + */ + loongson3_cpucfg_synthesize_data(c); + #ifdef CONFIG_64BIT if (cpu == 0) __ua_limit = ~((1ull << cpu_vmbits) - 1); diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 89eb82f6c837..2d5b16daf331 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -71,6 +71,8 @@ #include #include +#include + extern void check_wait(void); extern asmlinkage void rollback_handle_int(void); extern asmlinkage void handle_int(void); @@ -693,6 +695,44 @@ static int simulate_sync(struct pt_regs *regs, unsigned int opcode) return -1; /* Must be something else ... */ } +/* + * Loongson-3 CSR instructions emulation + */ + +#ifdef CONFIG_CPU_LOONGSON3_CPUCFG_EMULATION + +#define LWC2 0xc8000000 +#define RS BASE +#define CSR_OPCODE2 0x00000118 +#define CSR_OPCODE2_MASK 0x000007ff +#define CSR_FUNC_MASK RT +#define CSR_FUNC_CPUCFG 0x8 + +static int simulate_loongson3_cpucfg(struct pt_regs *regs, + unsigned int opcode) +{ + int op = opcode & OPCODE; + int op2 = opcode & CSR_OPCODE2_MASK; + int csr_func = (opcode & CSR_FUNC_MASK) >> 16; + + if (op == LWC2 && op2 == CSR_OPCODE2 && csr_func == CSR_FUNC_CPUCFG) { + int rd = (opcode & RD) >> 11; + int rs = (opcode & RS) >> 21; + __u64 sel = regs->regs[rs]; + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); + + regs->regs[rd] = loongson3_cpucfg_read_synthesized( + ¤t_cpu_data, sel); + + return 0; + } + + /* Not ours. */ + return -1; +} +#endif /* CONFIG_CPU_LOONGSON3_CPUCFG_EMULATION */ + asmlinkage void do_ov(struct pt_regs *regs) { enum ctx_state prev_state; @@ -1166,6 +1206,11 @@ no_r2_instr: if (status < 0) status = simulate_fp(regs, opcode, old_epc, old31); + +#ifdef CONFIG_CPU_LOONGSON3_CPUCFG_EMULATION + if (status < 0) + status = simulate_loongson3_cpucfg(regs, opcode); +#endif } else if (cpu_has_mmips) { unsigned short mmop[2] = { 0 }; diff --git a/arch/mips/loongson64/Makefile b/arch/mips/loongson64/Makefile index 6f3c2b47f66f..61f6add20530 100644 --- a/arch/mips/loongson64/Makefile +++ b/arch/mips/loongson64/Makefile @@ -10,3 +10,4 @@ obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_RS780_HPET) += hpet.o obj-$(CONFIG_PCI) += pci.o obj-$(CONFIG_SUSPEND) += pm.o +obj-$(CONFIG_CPU_LOONGSON3_CPUCFG_EMULATION) += cpucfg-emul.o diff --git a/arch/mips/loongson64/cpucfg-emul.c b/arch/mips/loongson64/cpucfg-emul.c new file mode 100644 index 000000000000..fdd52b21c1df --- /dev/null +++ b/arch/mips/loongson64/cpucfg-emul.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include + +#include +#include + +static bool is_loongson(struct cpuinfo_mips *c) +{ + switch (c->processor_id & PRID_COMP_MASK) { + case PRID_COMP_LEGACY: + return ((c->processor_id & PRID_IMP_MASK) == + PRID_IMP_LOONGSON_64C); + + case PRID_COMP_LOONGSON: + return true; + + default: + return false; + } +} + +static u32 get_loongson_fprev(struct cpuinfo_mips *c) +{ + return c->fpu_id & LOONGSON_FPREV_MASK; +} + +static bool cpu_has_uca(void) +{ + u32 diag = read_c0_diag(); + u32 new_diag; + + if (diag & LOONGSON_DIAG_UCAC) + /* UCA is already enabled. */ + return true; + + /* See if UCAC bit can be flipped on. This should be safe. */ + new_diag = diag | LOONGSON_DIAG_UCAC; + write_c0_diag(new_diag); + new_diag = read_c0_diag(); + write_c0_diag(diag); + + return (new_diag & LOONGSON_DIAG_UCAC) != 0; +} + +static void probe_uca(struct cpuinfo_mips *c) +{ + if (cpu_has_uca()) + c->loongson3_cpucfg_data[0] |= LOONGSON_CFG1_LSUCA; +} + +static void decode_loongson_config6(struct cpuinfo_mips *c) +{ + u32 config6 = read_c0_config6(); + + if (config6 & MIPS_CONF6_LOONGSON_SFBEN) + c->loongson3_cpucfg_data[0] |= LOONGSON_CFG1_SFBP; + if (config6 & MIPS_CONF6_LOONGSON_LLEXC) + c->loongson3_cpucfg_data[0] |= LOONGSON_CFG1_LLEXC; + if (config6 & MIPS_CONF6_LOONGSON_SCRAND) + c->loongson3_cpucfg_data[0] |= LOONGSON_CFG1_SCRAND; +} + +static void patch_cpucfg_sel1(struct cpuinfo_mips *c) +{ + u64 ases = c->ases; + u64 options = c->options; + u32 data = c->loongson3_cpucfg_data[0]; + + if (options & MIPS_CPU_FPU) { + data |= LOONGSON_CFG1_FP; + data |= get_loongson_fprev(c) << LOONGSON_CFG1_FPREV_OFFSET; + } + if (ases & MIPS_ASE_LOONGSON_MMI) + data |= LOONGSON_CFG1_MMI; + if (ases & MIPS_ASE_MSA) + data |= LOONGSON_CFG1_MSA1; + + c->loongson3_cpucfg_data[0] = data; +} + +static void patch_cpucfg_sel2(struct cpuinfo_mips *c) +{ + u64 ases = c->ases; + u64 options = c->options; + u32 data = c->loongson3_cpucfg_data[1]; + + if (ases & MIPS_ASE_LOONGSON_EXT) + data |= LOONGSON_CFG2_LEXT1; + if (ases & MIPS_ASE_LOONGSON_EXT2) + data |= LOONGSON_CFG2_LEXT2; + if (options & MIPS_CPU_LDPTE) + data |= LOONGSON_CFG2_LSPW; + + if (ases & MIPS_ASE_VZ) + data |= LOONGSON_CFG2_LVZP; + else + data &= ~LOONGSON_CFG2_LVZREV; + + c->loongson3_cpucfg_data[1] = data; +} + +static void patch_cpucfg_sel3(struct cpuinfo_mips *c) +{ + u64 ases = c->ases; + u32 data = c->loongson3_cpucfg_data[2]; + + if (ases & MIPS_ASE_LOONGSON_CAM) { + data |= LOONGSON_CFG3_LCAMP; + } else { + data &= ~LOONGSON_CFG3_LCAMREV; + data &= ~LOONGSON_CFG3_LCAMNUM; + data &= ~LOONGSON_CFG3_LCAMKW; + data &= ~LOONGSON_CFG3_LCAMVW; + } + + c->loongson3_cpucfg_data[2] = data; +} + +void loongson3_cpucfg_synthesize_data(struct cpuinfo_mips *c) +{ + /* Only engage the logic on Loongson processors. */ + if (!is_loongson(c)) + return; + + /* CPUs with CPUCFG support don't need to synthesize anything. */ + if (cpu_has_cfg()) + return; + + c->loongson3_cpucfg_data[0] = 0; + c->loongson3_cpucfg_data[1] = 0; + c->loongson3_cpucfg_data[2] = 0; + + /* Add CPUCFG features non-discoverable otherwise. + * + * All Loongson processors covered by CPUCFG emulation have distinct + * PRID_REV, so take advantage of this. + */ + switch (c->processor_id & PRID_REV_MASK) { + case PRID_REV_LOONGSON3A_R1: + c->loongson3_cpucfg_data[0] |= (LOONGSON_CFG1_LSLDR0 | + LOONGSON_CFG1_LSSYNCI | LOONGSON_CFG1_LSUCA | + LOONGSON_CFG1_LLSYNC | LOONGSON_CFG1_TGTSYNC); + c->loongson3_cpucfg_data[1] |= (LOONGSON_CFG2_LBT1 | + LOONGSON_CFG2_LPMP | LOONGSON_CFG2_LPM_REV1); + c->loongson3_cpucfg_data[2] |= ( + LOONGSON_CFG3_LCAM_REV1 | + LOONGSON_CFG3_LCAMNUM_REV1 | + LOONGSON_CFG3_LCAMKW_REV1 | + LOONGSON_CFG3_LCAMVW_REV1); + break; + + case PRID_REV_LOONGSON3B_R1: + case PRID_REV_LOONGSON3B_R2: + c->loongson3_cpucfg_data[0] |= (LOONGSON_CFG1_LSLDR0 | + LOONGSON_CFG1_LSSYNCI | LOONGSON_CFG1_LSUCA | + LOONGSON_CFG1_LLSYNC | LOONGSON_CFG1_TGTSYNC); + c->loongson3_cpucfg_data[1] |= (LOONGSON_CFG2_LBT1 | + LOONGSON_CFG2_LPMP | LOONGSON_CFG2_LPM_REV1); + c->loongson3_cpucfg_data[2] |= ( + LOONGSON_CFG3_LCAM_REV1 | + LOONGSON_CFG3_LCAMNUM_REV1 | + LOONGSON_CFG3_LCAMKW_REV1 | + LOONGSON_CFG3_LCAMVW_REV1); + break; + + case PRID_REV_LOONGSON2K_R1_0: + case PRID_REV_LOONGSON2K_R1_1: + case PRID_REV_LOONGSON2K_R1_2: + case PRID_REV_LOONGSON2K_R1_3: + decode_loongson_config6(c); + probe_uca(c); + + c->loongson3_cpucfg_data[0] |= (LOONGSON_CFG1_LSLDR0 | + LOONGSON_CFG1_LSSYNCI | LOONGSON_CFG1_LLSYNC | + LOONGSON_CFG1_TGTSYNC); + c->loongson3_cpucfg_data[1] |= (LOONGSON_CFG2_LBT1 | + LOONGSON_CFG2_LBT2 | LOONGSON_CFG2_LPMP | + LOONGSON_CFG2_LPM_REV2); + c->loongson3_cpucfg_data[2] = 0; + break; + + case PRID_REV_LOONGSON3A_R2_0: + case PRID_REV_LOONGSON3A_R2_1: + case PRID_REV_LOONGSON3A_R3_0: + case PRID_REV_LOONGSON3A_R3_1: + decode_loongson_config6(c); + probe_uca(c); + + c->loongson3_cpucfg_data[0] |= (LOONGSON_CFG1_CNT64 | + LOONGSON_CFG1_LSLDR0 | LOONGSON_CFG1_LSPREF | + LOONGSON_CFG1_LSPREFX | LOONGSON_CFG1_LSSYNCI | + LOONGSON_CFG1_LLSYNC | LOONGSON_CFG1_TGTSYNC); + c->loongson3_cpucfg_data[1] |= (LOONGSON_CFG2_LBT1 | + LOONGSON_CFG2_LBT2 | LOONGSON_CFG2_LBTMMU | + LOONGSON_CFG2_LPMP | LOONGSON_CFG2_LPM_REV1 | + LOONGSON_CFG2_LVZ_REV1); + c->loongson3_cpucfg_data[2] |= (LOONGSON_CFG3_LCAM_REV1 | + LOONGSON_CFG3_LCAMNUM_REV1 | + LOONGSON_CFG3_LCAMKW_REV1 | + LOONGSON_CFG3_LCAMVW_REV1); + break; + } + + /* This feature is set by firmware, but all known Loongson-64 systems + * are configured this way. + */ + c->loongson3_cpucfg_data[0] |= LOONGSON_CFG1_CDMAP; + + /* Patch in dynamically probed bits. */ + patch_cpucfg_sel1(c); + patch_cpucfg_sel2(c); + patch_cpucfg_sel3(c); +} From b34a1a712024cd1cf50e405102eb9f71961fd6cd Mon Sep 17 00:00:00 2001 From: Joshua Kinard Date: Sun, 17 May 2020 19:24:39 -0400 Subject: [PATCH 0679/1043] MIPS: SGI-IP30: Reorder the macros in war.h Fix the ordering of the macros in arch/mips/mach-ip30/war.h to match those in arch/mips/mach-ip27/war.h. Signed-off-by: Joshua Kinard Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mach-ip30/war.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/mips/include/asm/mach-ip30/war.h b/arch/mips/include/asm/mach-ip30/war.h index ad3352d3d203..a1fa0c1f5300 100644 --- a/arch/mips/include/asm/mach-ip30/war.h +++ b/arch/mips/include/asm/mach-ip30/war.h @@ -8,18 +8,17 @@ #define R4600_V1_INDEX_ICACHEOP_WAR 0 #define R4600_V1_HIT_CACHEOP_WAR 0 #define R4600_V2_HIT_CACHEOP_WAR 0 -#define MIPS_CACHE_SYNC_WAR 0 #define BCM1250_M3_WAR 0 #define SIBYTE_1956_WAR 0 #define MIPS4K_ICACHE_REFILL_WAR 0 -#define MIPS34K_MISSED_ITLB_WAR 0 +#define MIPS_CACHE_SYNC_WAR 0 #define TX49XX_ICACHE_INDEX_INV_WAR 0 #define ICACHE_REFILLS_WORKAROUND_WAR 0 - #ifdef CONFIG_CPU_R10000 #define R10000_LLSC_WAR 1 #else #define R10000_LLSC_WAR 0 #endif +#define MIPS34K_MISSED_ITLB_WAR 0 #endif /* __ASM_MIPS_MACH_IP30_WAR_H */ From f33a0b941017b9cb5a4e975af198b855b2f2b455 Mon Sep 17 00:00:00 2001 From: Kaige Li Date: Thu, 14 May 2020 20:59:41 +0800 Subject: [PATCH 0680/1043] MIPS: tools: Fix resource leak in elf-entry.c There is a file descriptor resource leak in elf-entry.c, fix this by adding fclose() before return and die. Signed-off-by: Kaige Li Signed-off-by: Thomas Bogendoerfer --- arch/mips/tools/elf-entry.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/mips/tools/elf-entry.c b/arch/mips/tools/elf-entry.c index adde79ce7fc0..dbd14ff05b4c 100644 --- a/arch/mips/tools/elf-entry.c +++ b/arch/mips/tools/elf-entry.c @@ -51,11 +51,14 @@ int main(int argc, const char *argv[]) nread = fread(&hdr, 1, sizeof(hdr), file); if (nread != sizeof(hdr)) { perror("Unable to read input file"); + fclose(file); return EXIT_FAILURE; } - if (memcmp(hdr.ehdr32.e_ident, ELFMAG, SELFMAG)) + if (memcmp(hdr.ehdr32.e_ident, ELFMAG, SELFMAG)) { + fclose(file); die("Input is not an ELF\n"); + } switch (hdr.ehdr32.e_ident[EI_CLASS]) { case ELFCLASS32: @@ -67,6 +70,7 @@ int main(int argc, const char *argv[]) entry = be32toh(hdr.ehdr32.e_entry); break; default: + fclose(file); die("Invalid ELF encoding\n"); } @@ -83,14 +87,17 @@ int main(int argc, const char *argv[]) entry = be64toh(hdr.ehdr64.e_entry); break; default: + fclose(file); die("Invalid ELF encoding\n"); } break; default: + fclose(file); die("Invalid ELF class\n"); } printf("0x%016" PRIx64 "\n", entry); + fclose(file); return EXIT_SUCCESS; } From d29b92f57ecee125a86587919a22152a702a6411 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Sun, 24 May 2020 17:10:43 +0100 Subject: [PATCH 0681/1043] loop: remove redundant assignment to variable error The variable error is being assigned a value that is never read so the assignment is redundant and can be removed. Addresses-Coverity: ("Unused value") Signed-off-by: Colin Ian King Signed-off-by: Jens Axboe --- drivers/block/loop.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index a565c5aafa52..8462ada86e91 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -1136,8 +1136,6 @@ static int loop_configure(struct loop_device *lo, fmode_t mode, if (error) goto out_unlock; - error = 0; - set_device_ro(bdev, (lo->lo_flags & LO_FLAGS_READ_ONLY) != 0); lo->use_dio = lo->lo_flags & LO_FLAGS_DIRECT_IO; From c6e7bd7afaeb3af55ffac122828035f1c01d1d7b Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Sun, 24 May 2020 21:29:55 +0100 Subject: [PATCH 0682/1043] sched/core: Optimize ttwu() spinning on p->on_cpu Both Rik and Mel reported seeing ttwu() spend significant time on: smp_cond_load_acquire(&p->on_cpu, !VAL); Attempt to avoid this by queueing the wakeup on the CPU that owns the p->on_cpu value. This will then allow the ttwu() to complete without further waiting. Since we run schedule() with interrupts disabled, the IPI is guaranteed to happen after p->on_cpu is cleared, this is what makes it safe to queue early. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Mel Gorman Signed-off-by: Ingo Molnar Cc: Jirka Hladky Cc: Vincent Guittot Cc: valentin.schneider@arm.com Cc: Hillf Danton Cc: Rik van Riel Link: https://lore.kernel.org/r/20200524202956.27665-2-mgorman@techsingularity.net --- kernel/sched/core.c | 52 +++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index fa905b6daafa..903c9ee1db45 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2312,7 +2312,7 @@ static void wake_csd_func(void *info) sched_ttwu_pending(); } -static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags) +static void __ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags) { struct rq *rq = cpu_rq(cpu); @@ -2354,6 +2354,17 @@ bool cpus_share_cache(int this_cpu, int that_cpu) { return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); } + +static bool ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags) +{ + if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { + sched_clock_cpu(cpu); /* Sync clocks across CPUs */ + __ttwu_queue_remote(p, cpu, wake_flags); + return true; + } + + return false; +} #endif /* CONFIG_SMP */ static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) @@ -2362,11 +2373,8 @@ static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) struct rq_flags rf; #if defined(CONFIG_SMP) - if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { - sched_clock_cpu(cpu); /* Sync clocks across CPUs */ - ttwu_queue_remote(p, cpu, wake_flags); + if (ttwu_queue_remote(p, cpu, wake_flags)) return; - } #endif rq_lock(rq, &rf); @@ -2548,7 +2556,15 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) if (p->on_rq && ttwu_remote(p, wake_flags)) goto unlock; + if (p->in_iowait) { + delayacct_blkio_end(p); + atomic_dec(&task_rq(p)->nr_iowait); + } + #ifdef CONFIG_SMP + p->sched_contributes_to_load = !!task_contributes_to_load(p); + p->state = TASK_WAKING; + /* * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be * possible to, falsely, observe p->on_cpu == 0. @@ -2570,6 +2586,16 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ smp_rmb(); + /* + * If the owning (remote) CPU is still in the middle of schedule() with + * this task as prev, considering queueing p on the remote CPUs wake_list + * which potentially sends an IPI instead of spinning on p->on_cpu to + * let the waker make forward progress. This is safe because IRQs are + * disabled and the IPI will deliver after on_cpu is cleared. + */ + if (READ_ONCE(p->on_cpu) && ttwu_queue_remote(p, cpu, wake_flags)) + goto unlock; + /* * If the owning (remote) CPU is still in the middle of schedule() with * this task as prev, wait until its done referencing the task. @@ -2581,28 +2607,12 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ smp_cond_load_acquire(&p->on_cpu, !VAL); - p->sched_contributes_to_load = !!task_contributes_to_load(p); - p->state = TASK_WAKING; - - if (p->in_iowait) { - delayacct_blkio_end(p); - atomic_dec(&task_rq(p)->nr_iowait); - } - cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags); if (task_cpu(p) != cpu) { wake_flags |= WF_MIGRATED; psi_ttwu_dequeue(p); set_task_cpu(p, cpu); } - -#else /* CONFIG_SMP */ - - if (p->in_iowait) { - delayacct_blkio_end(p); - atomic_dec(&task_rq(p)->nr_iowait); - } - #endif /* CONFIG_SMP */ ttwu_queue(p, cpu, wake_flags); From 2ebb17717550607bcd85fb8cf7d24ac870e9d762 Mon Sep 17 00:00:00 2001 From: Mel Gorman Date: Sun, 24 May 2020 21:29:56 +0100 Subject: [PATCH 0683/1043] sched/core: Offload wakee task activation if it the wakee is descheduling The previous commit: c6e7bd7afaeb: ("sched/core: Optimize ttwu() spinning on p->on_cpu") avoids spinning on p->on_rq when the task is descheduling, but only if the wakee is on a CPU that does not share cache with the waker. This patch offloads the activation of the wakee to the CPU that is about to go idle if the task is the only one on the runqueue. This potentially allows the waker task to continue making progress when the wakeup is not strictly synchronous. This is very obvious with netperf UDP_STREAM running on localhost. The waker is sending packets as quickly as possible without waiting for any reply. It frequently wakes the server for the processing of packets and when netserver is using local memory, it quickly completes the processing and goes back to idle. The waker often observes that netserver is on_rq and spins excessively leading to a drop in throughput. This is a comparison of 5.7-rc6 against "sched: Optimize ttwu() spinning on p->on_cpu" and against this patch labeled vanilla, optttwu-v1r1 and localwakelist-v1r2 respectively. 5.7.0-rc6 5.7.0-rc6 5.7.0-rc6 vanilla optttwu-v1r1 localwakelist-v1r2 Hmean send-64 251.49 ( 0.00%) 258.05 * 2.61%* 305.59 * 21.51%* Hmean send-128 497.86 ( 0.00%) 519.89 * 4.43%* 600.25 * 20.57%* Hmean send-256 944.90 ( 0.00%) 997.45 * 5.56%* 1140.19 * 20.67%* Hmean send-1024 3779.03 ( 0.00%) 3859.18 * 2.12%* 4518.19 * 19.56%* Hmean send-2048 7030.81 ( 0.00%) 7315.99 * 4.06%* 8683.01 * 23.50%* Hmean send-3312 10847.44 ( 0.00%) 11149.43 * 2.78%* 12896.71 * 18.89%* Hmean send-4096 13436.19 ( 0.00%) 13614.09 ( 1.32%) 15041.09 * 11.94%* Hmean send-8192 22624.49 ( 0.00%) 23265.32 * 2.83%* 24534.96 * 8.44%* Hmean send-16384 34441.87 ( 0.00%) 36457.15 * 5.85%* 35986.21 * 4.48%* Note that this benefit is not universal to all wakeups, it only applies to the case where the waker often spins on p->on_rq. The impact can be seen from a "perf sched latency" report generated from a single iteration of one packet size: ----------------------------------------------------------------------------------------------------------------- Task | Runtime ms | Switches | Average delay ms | Maximum delay ms | Maximum delay at | ----------------------------------------------------------------------------------------------------------------- vanilla netperf:4337 | 21709.193 ms | 2932 | avg: 0.002 ms | max: 0.041 ms | max at: 112.154512 s netserver:4338 | 14629.459 ms | 5146990 | avg: 0.001 ms | max: 1615.864 ms | max at: 140.134496 s localwakelist-v1r2 netperf:4339 | 29789.717 ms | 2460 | avg: 0.002 ms | max: 0.059 ms | max at: 138.205389 s netserver:4340 | 18858.767 ms | 7279005 | avg: 0.001 ms | max: 0.362 ms | max at: 135.709683 s ----------------------------------------------------------------------------------------------------------------- Note that the average wakeup delay is quite small on both the vanilla kernel and with the two patches applied. However, there are significant outliers with the vanilla kernel with the maximum one measured as 1615 milliseconds with a vanilla kernel but never worse than 0.362 ms with both patches applied and a much higher rate of context switching. Similarly a separate profile of cycles showed that 2.83% of all cycles were spent in try_to_wake_up() with almost half of the cycles spent on spinning on p->on_rq. With the two patches, the percentage of cycles spent in try_to_wake_up() drops to 1.13% Signed-off-by: Mel Gorman Signed-off-by: Ingo Molnar Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Jirka Hladky Cc: Vincent Guittot Cc: valentin.schneider@arm.com Cc: Hillf Danton Cc: Rik van Riel Link: https://lore.kernel.org/r/20200524202956.27665-3-mgorman@techsingularity.net --- kernel/sched/core.c | 39 +++++++++++++++++++++++++++++++++------ kernel/sched/sched.h | 3 ++- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 903c9ee1db45..6130ab170e93 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2312,7 +2312,13 @@ static void wake_csd_func(void *info) sched_ttwu_pending(); } -static void __ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags) +/* + * Queue a task on the target CPUs wake_list and wake the CPU via IPI if + * necessary. The wakee CPU on receipt of the IPI will queue the task + * via sched_ttwu_wakeup() for activation so the wakee incurs the cost + * of the wakeup instead of the waker. + */ +static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) { struct rq *rq = cpu_rq(cpu); @@ -2355,11 +2361,32 @@ bool cpus_share_cache(int this_cpu, int that_cpu) return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); } -static bool ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags) +static inline bool ttwu_queue_cond(int cpu, int wake_flags) { - if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { + /* + * If the CPU does not share cache, then queue the task on the + * remote rqs wakelist to avoid accessing remote data. + */ + if (!cpus_share_cache(smp_processor_id(), cpu)) + return true; + + /* + * If the task is descheduling and the only running task on the + * CPU then use the wakelist to offload the task activation to + * the soon-to-be-idle CPU as the current CPU is likely busy. + * nr_running is checked to avoid unnecessary task stacking. + */ + if ((wake_flags & WF_ON_RQ) && cpu_rq(cpu)->nr_running <= 1) + return true; + + return false; +} + +static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags) +{ + if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(cpu, wake_flags)) { sched_clock_cpu(cpu); /* Sync clocks across CPUs */ - __ttwu_queue_remote(p, cpu, wake_flags); + __ttwu_queue_wakelist(p, cpu, wake_flags); return true; } @@ -2373,7 +2400,7 @@ static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags) struct rq_flags rf; #if defined(CONFIG_SMP) - if (ttwu_queue_remote(p, cpu, wake_flags)) + if (ttwu_queue_wakelist(p, cpu, wake_flags)) return; #endif @@ -2593,7 +2620,7 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) * let the waker make forward progress. This is safe because IRQs are * disabled and the IPI will deliver after on_cpu is cleared. */ - if (READ_ONCE(p->on_cpu) && ttwu_queue_remote(p, cpu, wake_flags)) + if (READ_ONCE(p->on_cpu) && ttwu_queue_wakelist(p, cpu, wake_flags | WF_ON_RQ)) goto unlock; /* diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f7ab6334e992..4b32cff0dcbe 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1685,7 +1685,8 @@ static inline int task_on_rq_migrating(struct task_struct *p) */ #define WF_SYNC 0x01 /* Waker goes to sleep after wakeup */ #define WF_FORK 0x02 /* Child wakeup after fork */ -#define WF_MIGRATED 0x4 /* Internal use, task got migrated */ +#define WF_MIGRATED 0x04 /* Internal use, task got migrated */ +#define WF_ON_RQ 0x08 /* Wakee is on_rq */ /* * To aid in avoiding the subversion of "niceness" due to uneven distribution From c142c6a449c7765b0d3b6ecce0be8d783a19abcb Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Tue, 31 Mar 2020 21:46:41 +0100 Subject: [PATCH 0684/1043] btrfs: add missing annotation for btrfs_lock_cluster() Sparse reports a warning at btrfs_lock_cluster() warning: context imbalance in btrfs_lock_cluster() - wrong count The root cause is the missing annotation at btrfs_lock_cluster() Add the missing __acquires(&cluster->refill_lock) annotation. Signed-off-by: Jules Irenge Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent-tree.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 54a64d1e18c6..752b229cbb13 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -3369,6 +3369,7 @@ static struct btrfs_block_group *btrfs_lock_cluster( struct btrfs_block_group *block_group, struct btrfs_free_cluster *cluster, int delalloc) + __acquires(&cluster->refill_lock) { struct btrfs_block_group *used_bg = NULL; From 78d933c79cb649906577715af15400c7724ca633 Mon Sep 17 00:00:00 2001 From: Jules Irenge Date: Tue, 31 Mar 2020 21:46:42 +0100 Subject: [PATCH 0685/1043] btrfs: add missing annotation for btrfs_tree_lock() Sparse reports a warning at btrfs_tree_lock() warning: context imbalance in btrfs_tree_lock() - wrong count at exit The root cause is the missing annotation at btrfs_tree_lock() Add the missing __acquires(&eb->lock) annotation Signed-off-by: Jules Irenge Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/locking.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/btrfs/locking.c b/fs/btrfs/locking.c index fb647d8cf527..f75612e18a82 100644 --- a/fs/btrfs/locking.c +++ b/fs/btrfs/locking.c @@ -410,6 +410,7 @@ void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb) * The rwlock is held for write upon exit. */ void btrfs_tree_lock(struct extent_buffer *eb) + __acquires(&eb->lock) { u64 start_ns = 0; From a37f232b7b65789cadc9834d389f6390de11b583 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 13 Feb 2020 14:11:04 +0800 Subject: [PATCH 0686/1043] btrfs: backref: introduce the skeleton of btrfs_backref_iter Due to the complex nature of btrfs extent tree, when we want to iterate all backrefs of one extent, this involves quite a lot of work, like searching the EXTENT_ITEM/METADATA_ITEM, iteration through inline and keyed backrefs. Normally this would result in a complex code, something like: btrfs_search_slot() /* Ensure we are at EXTENT_ITEM/METADATA_ITEM */ while (1) { /* Loop for extent tree items */ while (ptr < end) { /* Loop for inlined items */ /* Real work here */ } next: ret = btrfs_next_item() /* Ensure we're still at keyed item for specified bytenr */ } The idea of btrfs_backref_iter is to avoid such complex and hard to read code structure, but something like the following: iter = btrfs_backref_iter_alloc(); ret = btrfs_backref_iter_start(iter, bytenr); if (ret < 0) goto out; for (; ; ret = btrfs_backref_iter_next(iter)) { /* Real work here */ } out: btrfs_backref_iter_free(iter); This patch is just the skeleton + btrfs_backref_iter_start() code. Reviewed-by: Johannes Thumshirn Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 110 +++++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/backref.h | 38 ++++++++++++++++ 2 files changed, 148 insertions(+) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 0cc02577577b..a1bac1b87d17 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -2295,3 +2295,113 @@ void free_ipath(struct inode_fs_paths *ipath) kvfree(ipath->fspath); kfree(ipath); } + +struct btrfs_backref_iter *btrfs_backref_iter_alloc( + struct btrfs_fs_info *fs_info, gfp_t gfp_flag) +{ + struct btrfs_backref_iter *ret; + + ret = kzalloc(sizeof(*ret), gfp_flag); + if (!ret) + return NULL; + + ret->path = btrfs_alloc_path(); + if (!ret) { + kfree(ret); + return NULL; + } + + /* Current backref iterator only supports iteration in commit root */ + ret->path->search_commit_root = 1; + ret->path->skip_locking = 1; + ret->fs_info = fs_info; + + return ret; +} + +int btrfs_backref_iter_start(struct btrfs_backref_iter *iter, u64 bytenr) +{ + struct btrfs_fs_info *fs_info = iter->fs_info; + struct btrfs_path *path = iter->path; + struct btrfs_extent_item *ei; + struct btrfs_key key; + int ret; + + key.objectid = bytenr; + key.type = BTRFS_METADATA_ITEM_KEY; + key.offset = (u64)-1; + iter->bytenr = bytenr; + + ret = btrfs_search_slot(NULL, fs_info->extent_root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (ret == 0) { + ret = -EUCLEAN; + goto release; + } + if (path->slots[0] == 0) { + WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG)); + ret = -EUCLEAN; + goto release; + } + path->slots[0]--; + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if ((key.type != BTRFS_EXTENT_ITEM_KEY && + key.type != BTRFS_METADATA_ITEM_KEY) || key.objectid != bytenr) { + ret = -ENOENT; + goto release; + } + memcpy(&iter->cur_key, &key, sizeof(key)); + iter->item_ptr = (u32)btrfs_item_ptr_offset(path->nodes[0], + path->slots[0]); + iter->end_ptr = (u32)(iter->item_ptr + + btrfs_item_size_nr(path->nodes[0], path->slots[0])); + ei = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_extent_item); + + /* + * Only support iteration on tree backref yet. + * + * This is an extra precaution for non skinny-metadata, where + * EXTENT_ITEM is also used for tree blocks, that we can only use + * extent flags to determine if it's a tree block. + */ + if (btrfs_extent_flags(path->nodes[0], ei) & BTRFS_EXTENT_FLAG_DATA) { + ret = -ENOTSUPP; + goto release; + } + iter->cur_ptr = (u32)(iter->item_ptr + sizeof(*ei)); + + /* If there is no inline backref, go search for keyed backref */ + if (iter->cur_ptr >= iter->end_ptr) { + ret = btrfs_next_item(fs_info->extent_root, path); + + /* No inline nor keyed ref */ + if (ret > 0) { + ret = -ENOENT; + goto release; + } + if (ret < 0) + goto release; + + btrfs_item_key_to_cpu(path->nodes[0], &iter->cur_key, + path->slots[0]); + if (iter->cur_key.objectid != bytenr || + (iter->cur_key.type != BTRFS_SHARED_BLOCK_REF_KEY && + iter->cur_key.type != BTRFS_TREE_BLOCK_REF_KEY)) { + ret = -ENOENT; + goto release; + } + iter->cur_ptr = (u32)btrfs_item_ptr_offset(path->nodes[0], + path->slots[0]); + iter->item_ptr = iter->cur_ptr; + iter->end_ptr = (u32)(iter->item_ptr + btrfs_item_size_nr( + path->nodes[0], path->slots[0])); + } + + return 0; +release: + btrfs_backref_iter_release(iter); + return ret; +} diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 723d6da99114..4ae37fad819a 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -78,4 +78,42 @@ struct prelim_ref { u64 wanted_disk_byte; }; +/* + * Iterate backrefs of one extent. + * + * Now it only supports iteration of tree block in commit root. + */ +struct btrfs_backref_iter { + u64 bytenr; + struct btrfs_path *path; + struct btrfs_fs_info *fs_info; + struct btrfs_key cur_key; + u32 item_ptr; + u32 cur_ptr; + u32 end_ptr; +}; + +struct btrfs_backref_iter *btrfs_backref_iter_alloc( + struct btrfs_fs_info *fs_info, gfp_t gfp_flag); + +static inline void btrfs_backref_iter_free(struct btrfs_backref_iter *iter) +{ + if (!iter) + return; + btrfs_free_path(iter->path); + kfree(iter); +} + +int btrfs_backref_iter_start(struct btrfs_backref_iter *iter, u64 bytenr); + +static inline void btrfs_backref_iter_release(struct btrfs_backref_iter *iter) +{ + iter->bytenr = 0; + iter->item_ptr = 0; + iter->cur_ptr = 0; + iter->end_ptr = 0; + btrfs_release_path(iter->path); + memset(&iter->cur_key, 0, sizeof(iter->cur_key)); +} + #endif From c39c2ddc67024979915c4df4e30da96a3e7619b0 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 13 Feb 2020 15:04:04 +0800 Subject: [PATCH 0687/1043] btrfs: backref: implement btrfs_backref_iter_next() This function will go to the next inline/keyed backref for btrfs_backref_iter infrastructure. Reviewed-by: Johannes Thumshirn Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/backref.h | 34 ++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index a1bac1b87d17..27f9a5923796 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -2405,3 +2405,63 @@ release: btrfs_backref_iter_release(iter); return ret; } + +/* + * Go to the next backref item of current bytenr, can be either inlined or + * keyed. + * + * Caller needs to check whether it's inline ref or not by iter->cur_key. + * + * Return 0 if we get next backref without problem. + * Return >0 if there is no extra backref for this bytenr. + * Return <0 if there is something wrong happened. + */ +int btrfs_backref_iter_next(struct btrfs_backref_iter *iter) +{ + struct extent_buffer *eb = btrfs_backref_get_eb(iter); + struct btrfs_path *path = iter->path; + struct btrfs_extent_inline_ref *iref; + int ret; + u32 size; + + if (btrfs_backref_iter_is_inline_ref(iter)) { + /* We're still inside the inline refs */ + ASSERT(iter->cur_ptr < iter->end_ptr); + + if (btrfs_backref_has_tree_block_info(iter)) { + /* First tree block info */ + size = sizeof(struct btrfs_tree_block_info); + } else { + /* Use inline ref type to determine the size */ + int type; + + iref = (struct btrfs_extent_inline_ref *) + ((unsigned long)iter->cur_ptr); + type = btrfs_extent_inline_ref_type(eb, iref); + + size = btrfs_extent_inline_ref_size(type); + } + iter->cur_ptr += size; + if (iter->cur_ptr < iter->end_ptr) + return 0; + + /* All inline items iterated, fall through */ + } + + /* We're at keyed items, there is no inline item, go to the next one */ + ret = btrfs_next_item(iter->fs_info->extent_root, iter->path); + if (ret) + return ret; + + btrfs_item_key_to_cpu(path->nodes[0], &iter->cur_key, path->slots[0]); + if (iter->cur_key.objectid != iter->bytenr || + (iter->cur_key.type != BTRFS_TREE_BLOCK_REF_KEY && + iter->cur_key.type != BTRFS_SHARED_BLOCK_REF_KEY)) + return 1; + iter->item_ptr = (u32)btrfs_item_ptr_offset(path->nodes[0], + path->slots[0]); + iter->cur_ptr = iter->item_ptr; + iter->end_ptr = iter->item_ptr + (u32)btrfs_item_size_nr(path->nodes[0], + path->slots[0]); + return 0; +} diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 4ae37fad819a..46faf9b93576 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -104,8 +104,42 @@ static inline void btrfs_backref_iter_free(struct btrfs_backref_iter *iter) kfree(iter); } +static inline struct extent_buffer *btrfs_backref_get_eb( + struct btrfs_backref_iter *iter) +{ + if (!iter) + return NULL; + return iter->path->nodes[0]; +} + +/* + * For metadata with EXTENT_ITEM key (non-skinny) case, the first inline data + * is btrfs_tree_block_info, without a btrfs_extent_inline_ref header. + * + * This helper determines if that's the case. + */ +static inline bool btrfs_backref_has_tree_block_info( + struct btrfs_backref_iter *iter) +{ + if (iter->cur_key.type == BTRFS_EXTENT_ITEM_KEY && + iter->cur_ptr - iter->item_ptr == sizeof(struct btrfs_extent_item)) + return true; + return false; +} + int btrfs_backref_iter_start(struct btrfs_backref_iter *iter, u64 bytenr); +int btrfs_backref_iter_next(struct btrfs_backref_iter *iter); + +static inline bool btrfs_backref_iter_is_inline_ref( + struct btrfs_backref_iter *iter) +{ + if (iter->cur_key.type == BTRFS_EXTENT_ITEM_KEY || + iter->cur_key.type == BTRFS_METADATA_ITEM_KEY) + return true; + return false; +} + static inline void btrfs_backref_iter_release(struct btrfs_backref_iter *iter) { iter->bytenr = 0; From 71f572a9e82fb7ed6f8c625e2682160f3a498db3 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Fri, 14 Feb 2020 13:48:20 +0800 Subject: [PATCH 0688/1043] btrfs: reloc: use btrfs_backref_iter infrastructure In the core function of relocation, build_backref_tree, it needs to iterate all backref items of one tree block. Use btrfs_backref_iter infrastructure to do the loop and make the code more readable. The backref items look would be much more easier to read: ret = btrfs_backref_iter_start(iter, cur->bytenr); for (; ret == 0; ret = btrfs_backref_iter_next(iter)) { /* The really important work */ } Reviewed-by: Johannes Thumshirn Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 193 ++++++++++++++---------------------------- 1 file changed, 63 insertions(+), 130 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 03bc7134e8cb..97a29bf14fe0 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -654,48 +654,6 @@ static struct btrfs_root *read_fs_root(struct btrfs_fs_info *fs_info, return btrfs_get_fs_root(fs_info, &key, false); } -static noinline_for_stack -int find_inline_backref(struct extent_buffer *leaf, int slot, - unsigned long *ptr, unsigned long *end) -{ - struct btrfs_key key; - struct btrfs_extent_item *ei; - struct btrfs_tree_block_info *bi; - u32 item_size; - - btrfs_item_key_to_cpu(leaf, &key, slot); - - item_size = btrfs_item_size_nr(leaf, slot); - if (item_size < sizeof(*ei)) { - btrfs_print_v0_err(leaf->fs_info); - btrfs_handle_fs_error(leaf->fs_info, -EINVAL, NULL); - return 1; - } - ei = btrfs_item_ptr(leaf, slot, struct btrfs_extent_item); - WARN_ON(!(btrfs_extent_flags(leaf, ei) & - BTRFS_EXTENT_FLAG_TREE_BLOCK)); - - if (key.type == BTRFS_EXTENT_ITEM_KEY && - item_size <= sizeof(*ei) + sizeof(*bi)) { - WARN_ON(item_size < sizeof(*ei) + sizeof(*bi)); - return 1; - } - if (key.type == BTRFS_METADATA_ITEM_KEY && - item_size <= sizeof(*ei)) { - WARN_ON(item_size < sizeof(*ei)); - return 1; - } - - if (key.type == BTRFS_EXTENT_ITEM_KEY) { - bi = (struct btrfs_tree_block_info *)(ei + 1); - *ptr = (unsigned long)(bi + 1); - } else { - *ptr = (unsigned long)(ei + 1); - } - *end = (unsigned long)ei + item_size; - return 0; -} - /* * build backref tree for a given tree block. root of the backref tree * corresponds the tree block, leaves of the backref tree correspond @@ -715,10 +673,10 @@ struct backref_node *build_backref_tree(struct reloc_control *rc, struct btrfs_key *node_key, int level, u64 bytenr) { + struct btrfs_backref_iter *iter; struct backref_cache *cache = &rc->backref_cache; - struct btrfs_path *path1; /* For searching extent root */ - struct btrfs_path *path2; /* For searching parent of TREE_BLOCK_REF */ - struct extent_buffer *eb; + /* For searching parent of TREE_BLOCK_REF */ + struct btrfs_path *path; struct btrfs_root *root; struct backref_node *cur; struct backref_node *upper; @@ -727,9 +685,6 @@ struct backref_node *build_backref_tree(struct reloc_control *rc, struct backref_node *exist = NULL; struct backref_edge *edge; struct rb_node *rb_node; - struct btrfs_key key; - unsigned long end; - unsigned long ptr; LIST_HEAD(list); /* Pending edge list, upper node needs to be checked */ LIST_HEAD(useless); int cowonly; @@ -737,9 +692,11 @@ struct backref_node *build_backref_tree(struct reloc_control *rc, int err = 0; bool need_check = true; - path1 = btrfs_alloc_path(); - path2 = btrfs_alloc_path(); - if (!path1 || !path2) { + iter = btrfs_backref_iter_alloc(rc->extent_root->fs_info, GFP_NOFS); + if (!iter) + return ERR_PTR(-ENOMEM); + path = btrfs_alloc_path(); + if (!path) { err = -ENOMEM; goto out; } @@ -755,25 +712,28 @@ struct backref_node *build_backref_tree(struct reloc_control *rc, node->lowest = 1; cur = node; again: - end = 0; - ptr = 0; - key.objectid = cur->bytenr; - key.type = BTRFS_METADATA_ITEM_KEY; - key.offset = (u64)-1; - - path1->search_commit_root = 1; - path1->skip_locking = 1; - ret = btrfs_search_slot(NULL, rc->extent_root, &key, path1, - 0, 0); + ret = btrfs_backref_iter_start(iter, cur->bytenr); if (ret < 0) { err = ret; goto out; } - ASSERT(ret); - ASSERT(path1->slots[0]); - - path1->slots[0]--; + /* + * We skip the first btrfs_tree_block_info, as we don't use the key + * stored in it, but fetch it from the tree block + */ + if (btrfs_backref_has_tree_block_info(iter)) { + ret = btrfs_backref_iter_next(iter); + if (ret < 0) { + err = ret; + goto out; + } + /* No extra backref? This means the tree block is corrupted */ + if (ret > 0) { + err = -EUCLEAN; + goto out; + } + } WARN_ON(cur->checked); if (!list_empty(&cur->upper)) { /* @@ -795,42 +755,21 @@ again: exist = NULL; } - while (1) { + for (; ret == 0; ret = btrfs_backref_iter_next(iter)) { + struct extent_buffer *eb; + struct btrfs_key key; + int type; + cond_resched(); - eb = path1->nodes[0]; + eb = btrfs_backref_get_eb(iter); - if (ptr >= end) { - if (path1->slots[0] >= btrfs_header_nritems(eb)) { - ret = btrfs_next_leaf(rc->extent_root, path1); - if (ret < 0) { - err = ret; - goto out; - } - if (ret > 0) - break; - eb = path1->nodes[0]; - } - - btrfs_item_key_to_cpu(eb, &key, path1->slots[0]); - if (key.objectid != cur->bytenr) { - WARN_ON(exist); - break; - } - - if (key.type == BTRFS_EXTENT_ITEM_KEY || - key.type == BTRFS_METADATA_ITEM_KEY) { - ret = find_inline_backref(eb, path1->slots[0], - &ptr, &end); - if (ret) - goto next; - } - } - - if (ptr < end) { - /* update key for inline back ref */ + key.objectid = iter->bytenr; + if (btrfs_backref_iter_is_inline_ref(iter)) { struct btrfs_extent_inline_ref *iref; - int type; - iref = (struct btrfs_extent_inline_ref *)ptr; + + /* update key for inline back ref */ + iref = (struct btrfs_extent_inline_ref *) + ((unsigned long)iter->cur_ptr); type = btrfs_get_extent_inline_ref_type(eb, iref, BTRFS_REF_TYPE_BLOCK); if (type == BTRFS_REF_TYPE_INVALID) { @@ -839,9 +778,9 @@ again: } key.type = type; key.offset = btrfs_extent_inline_ref_offset(eb, iref); - - WARN_ON(key.type != BTRFS_TREE_BLOCK_REF_KEY && - key.type != BTRFS_SHARED_BLOCK_REF_KEY); + } else { + key.type = iter->cur_key.type; + key.offset = iter->cur_key.offset; } /* @@ -854,7 +793,7 @@ again: (key.type == BTRFS_SHARED_BLOCK_REF_KEY && exist->bytenr == key.offset))) { exist = NULL; - goto next; + continue; } /* SHARED_BLOCK_REF means key.offset is the parent bytenr */ @@ -900,7 +839,7 @@ again: edge->node[LOWER] = cur; edge->node[UPPER] = upper; - goto next; + continue; } else if (unlikely(key.type == BTRFS_EXTENT_REF_V0_KEY)) { err = -EINVAL; btrfs_print_v0_err(rc->extent_root->fs_info); @@ -908,7 +847,7 @@ again: NULL); goto out; } else if (key.type != BTRFS_TREE_BLOCK_REF_KEY) { - goto next; + continue; } /* @@ -941,21 +880,21 @@ again: level = cur->level + 1; /* Search the tree to find parent blocks referring the block. */ - path2->search_commit_root = 1; - path2->skip_locking = 1; - path2->lowest_level = level; - ret = btrfs_search_slot(NULL, root, node_key, path2, 0, 0); - path2->lowest_level = 0; + path->search_commit_root = 1; + path->skip_locking = 1; + path->lowest_level = level; + ret = btrfs_search_slot(NULL, root, node_key, path, 0, 0); + path->lowest_level = 0; if (ret < 0) { btrfs_put_root(root); err = ret; goto out; } - if (ret > 0 && path2->slots[level] > 0) - path2->slots[level]--; + if (ret > 0 && path->slots[level] > 0) + path->slots[level]--; - eb = path2->nodes[level]; - if (btrfs_node_blockptr(eb, path2->slots[level]) != + eb = path->nodes[level]; + if (btrfs_node_blockptr(eb, path->slots[level]) != cur->bytenr) { btrfs_err(root->fs_info, "couldn't find block (%llu) (level %d) in tree (%llu) with key (%llu %u %llu)", @@ -972,7 +911,7 @@ again: /* Add all nodes and edges in the path */ for (; level < BTRFS_MAX_LEVEL; level++) { - if (!path2->nodes[level]) { + if (!path->nodes[level]) { ASSERT(btrfs_root_bytenr(&root->root_item) == lower->bytenr); if (should_ignore_root(root)) { @@ -991,7 +930,7 @@ again: goto out; } - eb = path2->nodes[level]; + eb = path->nodes[level]; rb_node = tree_search(&cache->rb_root, eb->start); if (!rb_node) { upper = alloc_backref_node(cache); @@ -1051,20 +990,14 @@ again: lower = upper; upper = NULL; } - btrfs_release_path(path2); -next: - if (ptr < end) { - ptr += btrfs_extent_inline_ref_size(key.type); - if (ptr >= end) { - WARN_ON(ptr > end); - ptr = 0; - end = 0; - } - } - if (ptr >= end) - path1->slots[0]++; + btrfs_release_path(path); } - btrfs_release_path(path1); + if (ret < 0) { + err = ret; + goto out; + } + ret = 0; + btrfs_backref_iter_release(iter); cur->checked = 1; WARN_ON(exist); @@ -1182,8 +1115,8 @@ next: } } out: - btrfs_free_path(path1); - btrfs_free_path(path2); + btrfs_backref_iter_free(iter); + btrfs_free_path(path); if (err) { while (!list_empty(&useless)) { lower = list_entry(useless.next, From 9569cc203d23ddaed7f7f2ca986a7cda7f1c33c0 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 20 Feb 2020 15:16:16 +0800 Subject: [PATCH 0689/1043] btrfs: reloc: rename mark_block_processed and __mark_block_processed These two functions are weirdly named, mark_block_processed() in fact just marks a range dirty unconditionally, while __mark_block_processed() does extra check before doing the marking. This patch will open code old mark_block_processed, and rename __mark_block_processed() to remove the "__" prefix. Since we're here, also kill the forward declaration, which could also kill in_block_group() with in_range() macro. Reviewed-by: Nikolay Borisov Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 56 +++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 97a29bf14fe0..a8b5ea53e962 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -237,8 +237,22 @@ struct reloc_control { static void remove_backref_node(struct backref_cache *cache, struct backref_node *node); -static void __mark_block_processed(struct reloc_control *rc, - struct backref_node *node); + +static void mark_block_processed(struct reloc_control *rc, + struct backref_node *node) +{ + u32 blocksize; + + if (node->level == 0 || + in_range(node->bytenr, rc->block_group->start, + rc->block_group->length)) { + blocksize = rc->extent_root->fs_info->nodesize; + set_extent_bits(&rc->processed_blocks, node->bytenr, + node->bytenr + blocksize - 1, EXTENT_DIRTY); + } + node->processed = 1; +} + static void mapping_tree_init(struct mapping_tree *tree) { @@ -1105,7 +1119,7 @@ again: if (list_empty(&lower->upper)) list_add(&lower->list, &useless); } - __mark_block_processed(rc, upper); + mark_block_processed(rc, upper); if (upper->level > 0) { list_add(&upper->list, &cache->detached); upper->detached = 1; @@ -1612,14 +1626,6 @@ again: return NULL; } -static int in_block_group(u64 bytenr, struct btrfs_block_group *block_group) -{ - if (bytenr >= block_group->start && - bytenr < block_group->start + block_group->length) - return 1; - return 0; -} - /* * get new location of data */ @@ -1717,7 +1723,8 @@ int replace_file_extents(struct btrfs_trans_handle *trans, num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi); if (bytenr == 0) continue; - if (!in_block_group(bytenr, rc->block_group)) + if (!in_range(bytenr, rc->block_group->start, + rc->block_group->length)) continue; /* @@ -2679,7 +2686,7 @@ struct btrfs_root *select_reloc_root(struct btrfs_trans_handle *trans, ASSERT(next->root); list_add_tail(&next->list, &rc->backref_cache.changed); - __mark_block_processed(rc, next); + mark_block_processed(rc, next); break; } @@ -3029,25 +3036,6 @@ static int finish_pending_nodes(struct btrfs_trans_handle *trans, return err; } -static void mark_block_processed(struct reloc_control *rc, - u64 bytenr, u32 blocksize) -{ - set_extent_bits(&rc->processed_blocks, bytenr, bytenr + blocksize - 1, - EXTENT_DIRTY); -} - -static void __mark_block_processed(struct reloc_control *rc, - struct backref_node *node) -{ - u32 blocksize; - if (node->level == 0 || - in_block_group(node->bytenr, rc->block_group)) { - blocksize = rc->extent_root->fs_info->nodesize; - mark_block_processed(rc, node->bytenr, blocksize); - } - node->processed = 1; -} - /* * mark a block and all blocks directly/indirectly reference the block * as processed. @@ -3066,7 +3054,7 @@ static void update_processed_blocks(struct reloc_control *rc, if (next->processed) break; - __mark_block_processed(rc, next); + mark_block_processed(rc, next); if (list_empty(&next->upper)) break; @@ -4636,7 +4624,7 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, } if (first_cow) - __mark_block_processed(rc, node); + mark_block_processed(rc, node); if (first_cow && level > 0) rc->nodes_relocated += buf->len; From 84780289335fe614057e5ddf796050ce15751f4a Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 5 Mar 2020 13:48:31 +0800 Subject: [PATCH 0690/1043] btrfs: reloc: add backref_cache::pending_edge and backref_cache::useless_node These two new members will act the same as the existing local lists, @useless and @list in build_backref_tree(). Currently build_backref_tree() is only executed serially, thus moving such local list into backref_cache is still safe. Also since we're here, use list_first_entry() to replace a lot of list_entry() calls after !list_empty(). Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 74 +++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index a8b5ea53e962..39294c595c66 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -158,6 +158,12 @@ struct backref_cache { int nr_nodes; int nr_edges; + + /* The list of unchecked backref edges during backref cache build */ + struct list_head pending_edge; + + /* The list of useless backref nodes during backref cache build */ + struct list_head useless_node; }; /* @@ -269,6 +275,8 @@ static void backref_cache_init(struct backref_cache *cache) INIT_LIST_HEAD(&cache->changed); INIT_LIST_HEAD(&cache->detached); INIT_LIST_HEAD(&cache->leaves); + INIT_LIST_HEAD(&cache->pending_edge); + INIT_LIST_HEAD(&cache->useless_node); } static void backref_cache_cleanup(struct backref_cache *cache) @@ -292,6 +300,8 @@ static void backref_cache_cleanup(struct backref_cache *cache) for (i = 0; i < BTRFS_MAX_LEVEL; i++) ASSERT(list_empty(&cache->pending[i])); + ASSERT(list_empty(&cache->pending_edge)); + ASSERT(list_empty(&cache->useless_node)); ASSERT(list_empty(&cache->changed)); ASSERT(list_empty(&cache->detached)); ASSERT(RB_EMPTY_ROOT(&cache->rb_root)); @@ -699,8 +709,6 @@ struct backref_node *build_backref_tree(struct reloc_control *rc, struct backref_node *exist = NULL; struct backref_edge *edge; struct rb_node *rb_node; - LIST_HEAD(list); /* Pending edge list, upper node needs to be checked */ - LIST_HEAD(useless); int cowonly; int ret; int err = 0; @@ -764,7 +772,7 @@ again: * check its backrefs */ if (!exist->checked) - list_add_tail(&edge->list[UPPER], &list); + list_add_tail(&edge->list[UPPER], &cache->pending_edge); } else { exist = NULL; } @@ -842,7 +850,8 @@ again: * backrefs for the upper level block isn't * cached, add the block to pending list */ - list_add_tail(&edge->list[UPPER], &list); + list_add_tail(&edge->list[UPPER], + &cache->pending_edge); } else { upper = rb_entry(rb_node, struct backref_node, rb_node); @@ -884,7 +893,7 @@ again: cur->bytenr); if (should_ignore_root(root)) { btrfs_put_root(root); - list_add(&cur->list, &useless); + list_add(&cur->list, &cache->useless_node); } else { cur->root = root; } @@ -930,7 +939,8 @@ again: lower->bytenr); if (should_ignore_root(root)) { btrfs_put_root(root); - list_add(&lower->list, &useless); + list_add(&lower->list, + &cache->useless_node); } else { lower->root = root; } @@ -979,7 +989,7 @@ again: if (!upper->checked && need_check) { need_check = false; list_add_tail(&edge->list[UPPER], - &list); + &cache->pending_edge); } else { if (upper->checked) need_check = true; @@ -1017,8 +1027,9 @@ again: WARN_ON(exist); /* the pending list isn't empty, take the first block to process */ - if (!list_empty(&list)) { - edge = list_entry(list.next, struct backref_edge, list[UPPER]); + if (!list_empty(&cache->pending_edge)) { + edge = list_first_entry(&cache->pending_edge, + struct backref_edge, list[UPPER]); list_del_init(&edge->list[UPPER]); cur = edge->node[UPPER]; goto again; @@ -1039,10 +1050,11 @@ again: } list_for_each_entry(edge, &node->upper, list[LOWER]) - list_add_tail(&edge->list[UPPER], &list); + list_add_tail(&edge->list[UPPER], &cache->pending_edge); - while (!list_empty(&list)) { - edge = list_entry(list.next, struct backref_edge, list[UPPER]); + while (!list_empty(&cache->pending_edge)) { + edge = list_first_entry(&cache->pending_edge, + struct backref_edge, list[UPPER]); list_del_init(&edge->list[UPPER]); upper = edge->node[UPPER]; if (upper->detached) { @@ -1050,7 +1062,7 @@ again: lower = edge->node[LOWER]; free_backref_edge(cache, edge); if (list_empty(&lower->upper)) - list_add(&lower->list, &useless); + list_add(&lower->list, &cache->useless_node); continue; } @@ -1090,7 +1102,7 @@ again: list_add_tail(&edge->list[UPPER], &upper->lower); list_for_each_entry(edge, &upper->upper, list[LOWER]) - list_add_tail(&edge->list[UPPER], &list); + list_add_tail(&edge->list[UPPER], &cache->pending_edge); } /* * process useless backref nodes. backref nodes for tree leaves @@ -1098,8 +1110,9 @@ again: * tree blocks are left in the cache to avoid unnecessary backref * lookup. */ - while (!list_empty(&useless)) { - upper = list_entry(useless.next, struct backref_node, list); + while (!list_empty(&cache->useless_node)) { + upper = list_first_entry(&cache->useless_node, + struct backref_node, list); list_del_init(&upper->list); ASSERT(list_empty(&upper->upper)); if (upper == node) @@ -1109,7 +1122,7 @@ again: upper->lowest = 0; } while (!list_empty(&upper->lower)) { - edge = list_entry(upper->lower.next, + edge = list_first_entry(&upper->lower, struct backref_edge, list[UPPER]); list_del(&edge->list[UPPER]); list_del(&edge->list[LOWER]); @@ -1117,7 +1130,7 @@ again: free_backref_edge(cache, edge); if (list_empty(&lower->upper)) - list_add(&lower->list, &useless); + list_add(&lower->list, &cache->useless_node); } mark_block_processed(rc, upper); if (upper->level > 0) { @@ -1132,14 +1145,14 @@ out: btrfs_backref_iter_free(iter); btrfs_free_path(path); if (err) { - while (!list_empty(&useless)) { - lower = list_entry(useless.next, + while (!list_empty(&cache->useless_node)) { + lower = list_first_entry(&cache->useless_node, struct backref_node, list); list_del_init(&lower->list); } - while (!list_empty(&list)) { - edge = list_first_entry(&list, struct backref_edge, - list[UPPER]); + while (!list_empty(&cache->pending_edge)) { + edge = list_first_entry(&cache->pending_edge, + struct backref_edge, list[UPPER]); list_del(&edge->list[UPPER]); list_del(&edge->list[LOWER]); lower = edge->node[LOWER]; @@ -1152,20 +1165,21 @@ out: */ if (list_empty(&lower->upper) && RB_EMPTY_NODE(&lower->rb_node)) - list_add(&lower->list, &useless); + list_add(&lower->list, &cache->useless_node); if (!RB_EMPTY_NODE(&upper->rb_node)) continue; /* Add this guy's upper edges to the list to process */ list_for_each_entry(edge, &upper->upper, list[LOWER]) - list_add_tail(&edge->list[UPPER], &list); + list_add_tail(&edge->list[UPPER], + &cache->pending_edge); if (list_empty(&upper->upper)) - list_add(&upper->list, &useless); + list_add(&upper->list, &cache->useless_node); } - while (!list_empty(&useless)) { - lower = list_entry(useless.next, + while (!list_empty(&cache->useless_node)) { + lower = list_first_entry(&cache->useless_node, struct backref_node, list); list_del_init(&lower->list); if (lower == node) @@ -1174,9 +1188,13 @@ out: } remove_backref_node(cache, node); + ASSERT(list_empty(&cache->useless_node) && + list_empty(&cache->pending_edge)); return ERR_PTR(err); } ASSERT(!node || !node->detached); + ASSERT(list_empty(&cache->useless_node) && + list_empty(&cache->pending_edge)); return node; } From 33a0f1f716973ae87d94e03e597d0c46fd032541 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 5 Mar 2020 13:54:11 +0800 Subject: [PATCH 0691/1043] btrfs: reloc: add backref_cache::fs_info member Add this member so that we can grab fs_info without the help from reloc_control. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 39294c595c66..0833cfa82da9 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -164,6 +164,8 @@ struct backref_cache { /* The list of useless backref nodes during backref cache build */ struct list_head useless_node; + + struct btrfs_fs_info *fs_info; }; /* @@ -266,7 +268,8 @@ static void mapping_tree_init(struct mapping_tree *tree) spin_lock_init(&tree->lock); } -static void backref_cache_init(struct backref_cache *cache) +static void backref_cache_init(struct btrfs_fs_info *fs_info, + struct backref_cache *cache) { int i; cache->rb_root = RB_ROOT; @@ -277,6 +280,7 @@ static void backref_cache_init(struct backref_cache *cache) INIT_LIST_HEAD(&cache->leaves); INIT_LIST_HEAD(&cache->pending_edge); INIT_LIST_HEAD(&cache->useless_node); + cache->fs_info = fs_info; } static void backref_cache_cleanup(struct backref_cache *cache) @@ -4188,7 +4192,7 @@ static struct reloc_control *alloc_reloc_control(struct btrfs_fs_info *fs_info) INIT_LIST_HEAD(&rc->reloc_roots); INIT_LIST_HEAD(&rc->dirty_subvol_roots); - backref_cache_init(&rc->backref_cache); + backref_cache_init(fs_info, &rc->backref_cache); mapping_tree_init(&rc->reloc_root_tree); extent_io_tree_init(fs_info, &rc->processed_blocks, IO_TREE_RELOC_BLOCKS, NULL); From 2433bea592d26daf6bd15492ce4262b598a7f804 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Fri, 6 Mar 2020 14:04:12 +0800 Subject: [PATCH 0692/1043] btrfs: reloc: make reloc root search-specific for relocation backref cache find_reloc_root() searches reloc_control::reloc_root_tree to find the reloc root. This behavior is only useful for relocation backref cache. For the incoming more generic purpose backref cache, we don't care about who owns the reloc root, but only care if it's a reloc root. So this patch makes the following modifications to make the reloc root search more specific to relocation backref: - Add backref_node::is_reloc_root This will be an extra indicator for generic purposed backref cache. User doesn't need to read root key from backref_node::root to determine if it's a reloc root. Also for reloc tree root, it's useless and will be queued to useless list. - Add backref_cache::is_reloc This will allow backref cache code to do different behavior for generic purpose backref cache and relocation backref cache. - Pass fs_info to find_reloc_root() - Export find_reloc_root() So backref.c can utilize this function. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 2 ++ fs/btrfs/relocation.c | 50 +++++++++++++++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 8aa7b9dac405..1e8a0a513e73 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -3381,6 +3381,8 @@ void btrfs_reloc_pre_snapshot(struct btrfs_pending_snapshot *pending, int btrfs_reloc_post_snapshot(struct btrfs_trans_handle *trans, struct btrfs_pending_snapshot *pending); int btrfs_should_cancel_balance(struct btrfs_fs_info *fs_info); +struct btrfs_root *find_reloc_root(struct btrfs_fs_info *fs_info, + u64 bytenr); /* scrub.c */ int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start, diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 0833cfa82da9..a46f82744b25 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -122,6 +122,12 @@ struct backref_node { * backref node. */ unsigned int detached:1; + + /* + * For generic purpose backref cache, where we only care if it's a reloc + * root, doesn't care the source subvolid. + */ + unsigned int is_reloc_root:1; }; /* @@ -166,6 +172,14 @@ struct backref_cache { struct list_head useless_node; struct btrfs_fs_info *fs_info; + + /* + * Whether this cache is for relocation + * + * Reloction backref cache require more info for reloc root compared + * to generic backref cache. + */ + unsigned int is_reloc; }; /* @@ -269,7 +283,7 @@ static void mapping_tree_init(struct mapping_tree *tree) } static void backref_cache_init(struct btrfs_fs_info *fs_info, - struct backref_cache *cache) + struct backref_cache *cache, int is_reloc) { int i; cache->rb_root = RB_ROOT; @@ -281,6 +295,7 @@ static void backref_cache_init(struct btrfs_fs_info *fs_info, INIT_LIST_HEAD(&cache->pending_edge); INIT_LIST_HEAD(&cache->useless_node); cache->fs_info = fs_info; + cache->is_reloc = is_reloc; } static void backref_cache_cleanup(struct backref_cache *cache) @@ -653,13 +668,14 @@ static int should_ignore_root(struct btrfs_root *root) /* * find reloc tree by address of tree root */ -static struct btrfs_root *find_reloc_root(struct reloc_control *rc, - u64 bytenr) +struct btrfs_root *find_reloc_root(struct btrfs_fs_info *fs_info, u64 bytenr) { + struct reloc_control *rc = fs_info->reloc_ctl; struct rb_node *rb_node; struct mapping_node *node; struct btrfs_root *root = NULL; + ASSERT(rc); spin_lock(&rc->reloc_root_tree.lock); rb_node = tree_search(&rc->reloc_root_tree.rb_root, bytenr); if (rb_node) { @@ -703,6 +719,7 @@ struct backref_node *build_backref_tree(struct reloc_control *rc, { struct btrfs_backref_iter *iter; struct backref_cache *cache = &rc->backref_cache; + struct btrfs_fs_info *fs_info = cache->fs_info; /* For searching parent of TREE_BLOCK_REF */ struct btrfs_path *path; struct btrfs_root *root; @@ -825,13 +842,24 @@ again: /* SHARED_BLOCK_REF means key.offset is the parent bytenr */ if (key.type == BTRFS_SHARED_BLOCK_REF_KEY) { if (key.objectid == key.offset) { - /* - * Only root blocks of reloc trees use backref - * pointing to itself. - */ - root = find_reloc_root(rc, cur->bytenr); - ASSERT(root); - cur->root = root; + cur->is_reloc_root = 1; + /* Only reloc backref cache cares exact root */ + if (cache->is_reloc) { + root = find_reloc_root(fs_info, + cur->bytenr); + if (WARN_ON(!root)) { + err = -ENOENT; + goto out; + } + cur->root = root; + } else { + /* + * For generic purpose backref cache, + * reloc root node is useless. + */ + list_add(&cur->list, + &cache->useless_node); + } break; } @@ -4192,7 +4220,7 @@ static struct reloc_control *alloc_reloc_control(struct btrfs_fs_info *fs_info) INIT_LIST_HEAD(&rc->reloc_roots); INIT_LIST_HEAD(&rc->dirty_subvol_roots); - backref_cache_init(fs_info, &rc->backref_cache); + backref_cache_init(fs_info, &rc->backref_cache, 1); mapping_tree_init(&rc->reloc_root_tree); extent_io_tree_init(fs_info, &rc->processed_blocks, IO_TREE_RELOC_BLOCKS, NULL); From 4007ea87d900e20a7986cdcdcfdc866e8f8bd473 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 5 Mar 2020 14:06:29 +0800 Subject: [PATCH 0693/1043] btrfs: reloc: refactor direct tree backref processing into its own function For BTRFS_SHARED_BLOCK_REF_KEY, its processing is straightforward, as we now the parent node bytenr directly. If the parent is already cached, or a root, call it a day. If the parent is not cached, add it pending list. This patch will just refactor this part into its own function, handle_direct_tree_backref() and add some comment explaining the @ref_key parameter. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 130 +++++++++++++++++++++++++----------------- 1 file changed, 78 insertions(+), 52 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index a46f82744b25..ff0b2f9b68eb 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -698,6 +698,81 @@ static struct btrfs_root *read_fs_root(struct btrfs_fs_info *fs_info, return btrfs_get_fs_root(fs_info, &key, false); } +/* + * Handle direct tree backref + * + * Direct tree backref means, the backref item shows its parent bytenr + * directly. This is for SHARED_BLOCK_REF backref (keyed or inlined). + * + * @ref_key: The converted backref key. + * For keyed backref, it's the item key. + * For inlined backref, objectid is the bytenr, + * type is btrfs_inline_ref_type, offset is + * btrfs_inline_ref_offset. + */ +static int handle_direct_tree_backref(struct backref_cache *cache, + struct btrfs_key *ref_key, + struct backref_node *cur) +{ + struct backref_edge *edge; + struct backref_node *upper; + struct rb_node *rb_node; + + ASSERT(ref_key->type == BTRFS_SHARED_BLOCK_REF_KEY); + + /* Only reloc root uses backref pointing to itself */ + if (ref_key->objectid == ref_key->offset) { + struct btrfs_root *root; + + cur->is_reloc_root = 1; + /* Only reloc backref cache cares about a specific root */ + if (cache->is_reloc) { + root = find_reloc_root(cache->fs_info, cur->bytenr); + if (WARN_ON(!root)) + return -ENOENT; + cur->root = root; + } else { + /* + * For generic purpose backref cache, reloc root node + * is useless. + */ + list_add(&cur->list, &cache->useless_node); + } + return 0; + } + + edge = alloc_backref_edge(cache); + if (!edge) + return -ENOMEM; + + rb_node = tree_search(&cache->rb_root, ref_key->offset); + if (!rb_node) { + /* Parent node not yet cached */ + upper = alloc_backref_node(cache); + if (!upper) { + free_backref_edge(cache, edge); + return -ENOMEM; + } + upper->bytenr = ref_key->offset; + upper->level = cur->level + 1; + + /* + * Backrefs for the upper level block isn't cached, add the + * block to pending list + */ + list_add_tail(&edge->list[UPPER], &cache->pending_edge); + } else { + /* Parent node already cached */ + upper = rb_entry(rb_node, struct backref_node, rb_node); + ASSERT(upper->checked); + INIT_LIST_HEAD(&edge->list[UPPER]); + } + list_add_tail(&edge->list[LOWER], &cur->upper); + edge->node[LOWER] = cur; + edge->node[UPPER] = upper; + return 0; +} + /* * build backref tree for a given tree block. root of the backref tree * corresponds the tree block, leaves of the backref tree correspond @@ -719,7 +794,6 @@ struct backref_node *build_backref_tree(struct reloc_control *rc, { struct btrfs_backref_iter *iter; struct backref_cache *cache = &rc->backref_cache; - struct btrfs_fs_info *fs_info = cache->fs_info; /* For searching parent of TREE_BLOCK_REF */ struct btrfs_path *path; struct btrfs_root *root; @@ -841,59 +915,11 @@ again: /* SHARED_BLOCK_REF means key.offset is the parent bytenr */ if (key.type == BTRFS_SHARED_BLOCK_REF_KEY) { - if (key.objectid == key.offset) { - cur->is_reloc_root = 1; - /* Only reloc backref cache cares exact root */ - if (cache->is_reloc) { - root = find_reloc_root(fs_info, - cur->bytenr); - if (WARN_ON(!root)) { - err = -ENOENT; - goto out; - } - cur->root = root; - } else { - /* - * For generic purpose backref cache, - * reloc root node is useless. - */ - list_add(&cur->list, - &cache->useless_node); - } - break; - } - - edge = alloc_backref_edge(cache); - if (!edge) { - err = -ENOMEM; + ret = handle_direct_tree_backref(cache, &key, cur); + if (ret < 0) { + err = ret; goto out; } - rb_node = tree_search(&cache->rb_root, key.offset); - if (!rb_node) { - upper = alloc_backref_node(cache); - if (!upper) { - free_backref_edge(cache, edge); - err = -ENOMEM; - goto out; - } - upper->bytenr = key.offset; - upper->level = cur->level + 1; - /* - * backrefs for the upper level block isn't - * cached, add the block to pending list - */ - list_add_tail(&edge->list[UPPER], - &cache->pending_edge); - } else { - upper = rb_entry(rb_node, struct backref_node, - rb_node); - ASSERT(upper->checked); - INIT_LIST_HEAD(&edge->list[UPPER]); - } - list_add_tail(&edge->list[LOWER], &cur->upper); - edge->node[LOWER] = cur; - edge->node[UPPER] = upper; - continue; } else if (unlikely(key.type == BTRFS_EXTENT_REF_V0_KEY)) { err = -EINVAL; From 4d81ea8bb4fc40903f4950b082955c709d785467 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 5 Mar 2020 14:22:43 +0800 Subject: [PATCH 0694/1043] btrfs: reloc: refactor indirect tree backref processing into its own function The processing of indirect tree backref (TREE_BLOCK_REF) is the most complex work. We need to grab the fs root, do a tree search to locate all its parent nodes, link all needed edges, and put all uncached edges to pending edge list. This is definitely worth a helper function. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 294 +++++++++++++++++++++++------------------- 1 file changed, 159 insertions(+), 135 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index ff0b2f9b68eb..18a4b3f57b14 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -773,6 +773,163 @@ static int handle_direct_tree_backref(struct backref_cache *cache, return 0; } +/* + * Handle indirect tree backref + * + * Indirect tree backref means, we only know which tree the node belongs to. + * We still need to do a tree search to find out the parents. This is for + * TREE_BLOCK_REF backref (keyed or inlined). + * + * @ref_key: The same as @ref_key in handle_direct_tree_backref() + * @tree_key: The first key of this tree block. + * @path: A clean (released) path, to avoid allocating path everytime + * the function get called. + */ +static int handle_indirect_tree_backref(struct backref_cache *cache, + struct btrfs_path *path, + struct btrfs_key *ref_key, + struct btrfs_key *tree_key, + struct backref_node *cur) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct backref_node *upper; + struct backref_node *lower; + struct backref_edge *edge; + struct extent_buffer *eb; + struct btrfs_root *root; + struct rb_node *rb_node; + int level; + bool need_check = true; + int ret; + + root = read_fs_root(fs_info, ref_key->offset); + if (IS_ERR(root)) + return PTR_ERR(root); + if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) + cur->cowonly = 1; + + if (btrfs_root_level(&root->root_item) == cur->level) { + /* Tree root */ + ASSERT(btrfs_root_bytenr(&root->root_item) == cur->bytenr); + if (should_ignore_root(root)) { + btrfs_put_root(root); + list_add(&cur->list, &cache->useless_node); + } else { + cur->root = root; + } + return 0; + } + + level = cur->level + 1; + + /* Search the tree to find parent blocks referring to the block */ + path->search_commit_root = 1; + path->skip_locking = 1; + path->lowest_level = level; + ret = btrfs_search_slot(NULL, root, tree_key, path, 0, 0); + path->lowest_level = 0; + if (ret < 0) { + btrfs_put_root(root); + return ret; + } + if (ret > 0 && path->slots[level] > 0) + path->slots[level]--; + + eb = path->nodes[level]; + if (btrfs_node_blockptr(eb, path->slots[level]) != cur->bytenr) { + btrfs_err(fs_info, +"couldn't find block (%llu) (level %d) in tree (%llu) with key (%llu %u %llu)", + cur->bytenr, level - 1, root->root_key.objectid, + tree_key->objectid, tree_key->type, tree_key->offset); + btrfs_put_root(root); + ret = -ENOENT; + goto out; + } + lower = cur; + + /* Add all nodes and edges in the path */ + for (; level < BTRFS_MAX_LEVEL; level++) { + if (!path->nodes[level]) { + ASSERT(btrfs_root_bytenr(&root->root_item) == + lower->bytenr); + if (should_ignore_root(root)) { + btrfs_put_root(root); + list_add(&lower->list, &cache->useless_node); + } else { + lower->root = root; + } + break; + } + + edge = alloc_backref_edge(cache); + if (!edge) { + btrfs_put_root(root); + ret = -ENOMEM; + goto out; + } + + eb = path->nodes[level]; + rb_node = tree_search(&cache->rb_root, eb->start); + if (!rb_node) { + upper = alloc_backref_node(cache); + if (!upper) { + btrfs_put_root(root); + free_backref_edge(cache, edge); + ret = -ENOMEM; + goto out; + } + upper->bytenr = eb->start; + upper->owner = btrfs_header_owner(eb); + upper->level = lower->level + 1; + if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) + upper->cowonly = 1; + + /* + * If we know the block isn't shared we can avoid + * checking its backrefs. + */ + if (btrfs_block_can_be_shared(root, eb)) + upper->checked = 0; + else + upper->checked = 1; + + /* + * Add the block to pending list if we need to check its + * backrefs, we only do this once while walking up a + * tree as we will catch anything else later on. + */ + if (!upper->checked && need_check) { + need_check = false; + list_add_tail(&edge->list[UPPER], + &cache->pending_edge); + } else { + if (upper->checked) + need_check = true; + INIT_LIST_HEAD(&edge->list[UPPER]); + } + } else { + upper = rb_entry(rb_node, struct backref_node, rb_node); + ASSERT(upper->checked); + INIT_LIST_HEAD(&edge->list[UPPER]); + if (!upper->owner) + upper->owner = btrfs_header_owner(eb); + } + list_add_tail(&edge->list[LOWER], &lower->upper); + edge->node[LOWER] = lower; + edge->node[UPPER] = upper; + + if (rb_node) { + btrfs_put_root(root); + break; + } + lower = upper; + upper = NULL; + } +out: + btrfs_release_path(path); + return ret; +} + /* * build backref tree for a given tree block. root of the backref tree * corresponds the tree block, leaves of the backref tree correspond @@ -796,7 +953,6 @@ struct backref_node *build_backref_tree(struct reloc_control *rc, struct backref_cache *cache = &rc->backref_cache; /* For searching parent of TREE_BLOCK_REF */ struct btrfs_path *path; - struct btrfs_root *root; struct backref_node *cur; struct backref_node *upper; struct backref_node *lower; @@ -807,7 +963,6 @@ struct backref_node *build_backref_tree(struct reloc_control *rc, int cowonly; int ret; int err = 0; - bool need_check = true; iter = btrfs_backref_iter_alloc(rc->extent_root->fs_info, GFP_NOFS); if (!iter) @@ -936,143 +1091,12 @@ again: * means the root objectid. We need to search the tree to get * its parent bytenr. */ - root = read_fs_root(rc->extent_root->fs_info, key.offset); - if (IS_ERR(root)) { - err = PTR_ERR(root); - goto out; - } - - if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) - cur->cowonly = 1; - - if (btrfs_root_level(&root->root_item) == cur->level) { - /* tree root */ - ASSERT(btrfs_root_bytenr(&root->root_item) == - cur->bytenr); - if (should_ignore_root(root)) { - btrfs_put_root(root); - list_add(&cur->list, &cache->useless_node); - } else { - cur->root = root; - } - break; - } - - level = cur->level + 1; - - /* Search the tree to find parent blocks referring the block. */ - path->search_commit_root = 1; - path->skip_locking = 1; - path->lowest_level = level; - ret = btrfs_search_slot(NULL, root, node_key, path, 0, 0); - path->lowest_level = 0; + ret = handle_indirect_tree_backref(cache, path, &key, node_key, + cur); if (ret < 0) { - btrfs_put_root(root); err = ret; goto out; } - if (ret > 0 && path->slots[level] > 0) - path->slots[level]--; - - eb = path->nodes[level]; - if (btrfs_node_blockptr(eb, path->slots[level]) != - cur->bytenr) { - btrfs_err(root->fs_info, - "couldn't find block (%llu) (level %d) in tree (%llu) with key (%llu %u %llu)", - cur->bytenr, level - 1, - root->root_key.objectid, - node_key->objectid, node_key->type, - node_key->offset); - btrfs_put_root(root); - err = -ENOENT; - goto out; - } - lower = cur; - need_check = true; - - /* Add all nodes and edges in the path */ - for (; level < BTRFS_MAX_LEVEL; level++) { - if (!path->nodes[level]) { - ASSERT(btrfs_root_bytenr(&root->root_item) == - lower->bytenr); - if (should_ignore_root(root)) { - btrfs_put_root(root); - list_add(&lower->list, - &cache->useless_node); - } else { - lower->root = root; - } - break; - } - - edge = alloc_backref_edge(cache); - if (!edge) { - btrfs_put_root(root); - err = -ENOMEM; - goto out; - } - - eb = path->nodes[level]; - rb_node = tree_search(&cache->rb_root, eb->start); - if (!rb_node) { - upper = alloc_backref_node(cache); - if (!upper) { - btrfs_put_root(root); - free_backref_edge(cache, edge); - err = -ENOMEM; - goto out; - } - upper->bytenr = eb->start; - upper->owner = btrfs_header_owner(eb); - upper->level = lower->level + 1; - if (!test_bit(BTRFS_ROOT_REF_COWS, - &root->state)) - upper->cowonly = 1; - - /* - * if we know the block isn't shared - * we can void checking its backrefs. - */ - if (btrfs_block_can_be_shared(root, eb)) - upper->checked = 0; - else - upper->checked = 1; - - /* - * add the block to pending list if we - * need check its backrefs, we only do this once - * while walking up a tree as we will catch - * anything else later on. - */ - if (!upper->checked && need_check) { - need_check = false; - list_add_tail(&edge->list[UPPER], - &cache->pending_edge); - } else { - if (upper->checked) - need_check = true; - INIT_LIST_HEAD(&edge->list[UPPER]); - } - } else { - upper = rb_entry(rb_node, struct backref_node, - rb_node); - ASSERT(upper->checked); - INIT_LIST_HEAD(&edge->list[UPPER]); - if (!upper->owner) - upper->owner = btrfs_header_owner(eb); - } - list_add_tail(&edge->list[LOWER], &lower->upper); - edge->node[LOWER] = lower; - edge->node[UPPER] = upper; - - if (rb_node) { - btrfs_put_root(root); - break; - } - lower = upper; - upper = NULL; - } - btrfs_release_path(path); } if (ret < 0) { err = ret; From 2a979612d594c16953ca512f2de7f0bb385ffb65 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 24 Feb 2020 09:19:02 +0800 Subject: [PATCH 0695/1043] btrfs: reloc: use wrapper to replace open-coded edge linking Since backref_edge is used to connect upper and lower backref nodes, and needs to access both nodes, some code can look pretty nasty: list_add_tail(&edge->list[LOWER], &cur->upper); The above code will link @cur to the LOWER side of the edge, while both "LOWER" and "upper" words show up. This can sometimes be very confusing for reader to grasp. This patch introduces a new wrapper, link_backref_edge(), to handle the linking behavior. Which also has extra ASSERT() to ensure caller won't pass wrong nodes. Also, this updates the comment of related lists of backref_node and backref_edge, to make it more clear that each list points to what. Reviewed-by: Josef Bacik Reviewed-by: Nikolay Borisov Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 61 +++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 18a4b3f57b14..d197f44eadee 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -92,10 +92,12 @@ struct backref_node { u64 owner; /* link to pending, changed or detached list */ struct list_head list; - /* list of upper level blocks reference this block */ + + /* List of upper level edges, which link this node to its parents */ struct list_head upper; - /* list of child blocks in the cache */ + /* List of lower level edges, which link this node to its children */ struct list_head lower; + /* NULL if this node is not tree root */ struct btrfs_root *root; /* extent buffer got by COW the block */ @@ -130,17 +132,26 @@ struct backref_node { unsigned int is_reloc_root:1; }; -/* - * present a block pointer in the backref cache - */ -struct backref_edge { - struct list_head list[2]; - struct backref_node *node[2]; -}; - #define LOWER 0 #define UPPER 1 #define RELOCATION_RESERVED_NODES 256 +/* + * present an edge connecting upper and lower backref nodes. + */ +struct backref_edge { + /* + * list[LOWER] is linked to backref_node::upper of lower level node, + * and list[UPPER] is linked to backref_node::lower of upper level node. + * + * Also, build_backref_tree() uses list[UPPER] for pending edges, before + * linking list[UPPER] to its upper level nodes. + */ + struct list_head list[2]; + + /* Two related nodes */ + struct backref_node *node[2]; +}; + struct backref_cache { /* red black tree of all backref nodes in the cache */ @@ -363,6 +374,22 @@ static struct backref_edge *alloc_backref_edge(struct backref_cache *cache) return edge; } +#define LINK_LOWER (1 << 0) +#define LINK_UPPER (1 << 1) +static void link_backref_edge(struct backref_edge *edge, + struct backref_node *lower, + struct backref_node *upper, + int link_which) +{ + ASSERT(upper && lower && upper->level == lower->level + 1); + edge->node[LOWER] = lower; + edge->node[UPPER] = upper; + if (link_which & LINK_LOWER) + list_add_tail(&edge->list[LOWER], &lower->upper); + if (link_which & LINK_UPPER) + list_add_tail(&edge->list[UPPER], &upper->lower); +} + static void free_backref_edge(struct backref_cache *cache, struct backref_edge *edge) { @@ -767,9 +794,7 @@ static int handle_direct_tree_backref(struct backref_cache *cache, ASSERT(upper->checked); INIT_LIST_HEAD(&edge->list[UPPER]); } - list_add_tail(&edge->list[LOWER], &cur->upper); - edge->node[LOWER] = cur; - edge->node[UPPER] = upper; + link_backref_edge(edge, cur, upper, LINK_LOWER); return 0; } @@ -914,9 +939,7 @@ static int handle_indirect_tree_backref(struct backref_cache *cache, if (!upper->owner) upper->owner = btrfs_header_owner(eb); } - list_add_tail(&edge->list[LOWER], &lower->upper); - edge->node[LOWER] = lower; - edge->node[UPPER] = upper; + link_backref_edge(edge, lower, upper, LINK_LOWER); if (rb_node) { btrfs_put_root(root); @@ -1340,10 +1363,8 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, if (!new_edge) goto fail; - new_edge->node[UPPER] = new_node; - new_edge->node[LOWER] = edge->node[LOWER]; - list_add_tail(&new_edge->list[UPPER], - &new_node->lower); + link_backref_edge(new_edge, edge->node[LOWER], new_node, + LINK_UPPER); } } else { list_add_tail(&new_node->lower, &cache->leaves); From 0304f2d8cce7fc23baf9e005c095beff7a29847d Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 24 Feb 2020 09:34:55 +0800 Subject: [PATCH 0696/1043] btrfs: reloc: pass essential members for alloc_backref_node() Bytenr and level are essential parameters for backref_node, thus it makes sense to initialize them at allocation time. Reviewed-by: Josef Bacik Reviewed-by: Nikolay Borisov Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index d197f44eadee..be66d035d0fb 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -339,18 +339,24 @@ static void backref_cache_cleanup(struct backref_cache *cache) ASSERT(!cache->nr_edges); } -static struct backref_node *alloc_backref_node(struct backref_cache *cache) +static struct backref_node *alloc_backref_node(struct backref_cache *cache, + u64 bytenr, int level) { struct backref_node *node; + ASSERT(level >= 0 && level < BTRFS_MAX_LEVEL); node = kzalloc(sizeof(*node), GFP_NOFS); - if (node) { - INIT_LIST_HEAD(&node->list); - INIT_LIST_HEAD(&node->upper); - INIT_LIST_HEAD(&node->lower); - RB_CLEAR_NODE(&node->rb_node); - cache->nr_nodes++; - } + if (!node) + return node; + + INIT_LIST_HEAD(&node->list); + INIT_LIST_HEAD(&node->upper); + INIT_LIST_HEAD(&node->lower); + RB_CLEAR_NODE(&node->rb_node); + cache->nr_nodes++; + node->level = level; + node->bytenr = bytenr; + return node; } @@ -775,13 +781,12 @@ static int handle_direct_tree_backref(struct backref_cache *cache, rb_node = tree_search(&cache->rb_root, ref_key->offset); if (!rb_node) { /* Parent node not yet cached */ - upper = alloc_backref_node(cache); + upper = alloc_backref_node(cache, ref_key->offset, + cur->level + 1); if (!upper) { free_backref_edge(cache, edge); return -ENOMEM; } - upper->bytenr = ref_key->offset; - upper->level = cur->level + 1; /* * Backrefs for the upper level block isn't cached, add the @@ -896,16 +901,15 @@ static int handle_indirect_tree_backref(struct backref_cache *cache, eb = path->nodes[level]; rb_node = tree_search(&cache->rb_root, eb->start); if (!rb_node) { - upper = alloc_backref_node(cache); + upper = alloc_backref_node(cache, eb->start, + lower->level + 1); if (!upper) { btrfs_put_root(root); free_backref_edge(cache, edge); ret = -ENOMEM; goto out; } - upper->bytenr = eb->start; upper->owner = btrfs_header_owner(eb); - upper->level = lower->level + 1; if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) upper->cowonly = 1; @@ -996,14 +1000,12 @@ struct backref_node *build_backref_tree(struct reloc_control *rc, goto out; } - node = alloc_backref_node(cache); + node = alloc_backref_node(cache, bytenr, level); if (!node) { err = -ENOMEM; goto out; } - node->bytenr = bytenr; - node->level = level; node->lowest = 1; cur = node; again: @@ -1346,12 +1348,10 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, if (!node) return 0; - new_node = alloc_backref_node(cache); + new_node = alloc_backref_node(cache, dest->node->start, node->level); if (!new_node) return -ENOMEM; - new_node->bytenr = dest->node->start; - new_node->level = node->level; new_node->lowest = node->lowest; new_node->checked = 1; new_node->root = btrfs_grab_root(dest); From e7d571c7b004dc20f385d53d0c89e99d078e0415 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 5 Mar 2020 14:49:29 +0800 Subject: [PATCH 0697/1043] btrfs: reloc: remove the open-coded goto loop for breadth-first search build_backref_tree() uses "goto again;" to implement a breadth-first search to build backref cache. This patch will extract most of its work into a wrapper, handle_one_tree_block(), and use a do {} while() loop to implement the same thing. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 175 ++++++++++++++++++++++-------------------- 1 file changed, 91 insertions(+), 84 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index be66d035d0fb..ad94482a2e65 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -957,77 +957,31 @@ out: return ret; } -/* - * build backref tree for a given tree block. root of the backref tree - * corresponds the tree block, leaves of the backref tree correspond - * roots of b-trees that reference the tree block. - * - * the basic idea of this function is check backrefs of a given block - * to find upper level blocks that reference the block, and then check - * backrefs of these upper level blocks recursively. the recursion stop - * when tree root is reached or backrefs for the block is cached. - * - * NOTE: if we find backrefs for a block are cached, we know backrefs - * for all upper level blocks that directly/indirectly reference the - * block are also cached. - */ -static noinline_for_stack -struct backref_node *build_backref_tree(struct reloc_control *rc, - struct btrfs_key *node_key, - int level, u64 bytenr) +static int handle_one_tree_block(struct backref_cache *cache, + struct btrfs_path *path, + struct btrfs_backref_iter *iter, + struct btrfs_key *node_key, + struct backref_node *cur) { - struct btrfs_backref_iter *iter; - struct backref_cache *cache = &rc->backref_cache; - /* For searching parent of TREE_BLOCK_REF */ - struct btrfs_path *path; - struct backref_node *cur; - struct backref_node *upper; - struct backref_node *lower; - struct backref_node *node = NULL; - struct backref_node *exist = NULL; + struct btrfs_fs_info *fs_info = cache->fs_info; struct backref_edge *edge; - struct rb_node *rb_node; - int cowonly; + struct backref_node *exist; int ret; - int err = 0; - iter = btrfs_backref_iter_alloc(rc->extent_root->fs_info, GFP_NOFS); - if (!iter) - return ERR_PTR(-ENOMEM); - path = btrfs_alloc_path(); - if (!path) { - err = -ENOMEM; - goto out; - } - - node = alloc_backref_node(cache, bytenr, level); - if (!node) { - err = -ENOMEM; - goto out; - } - - node->lowest = 1; - cur = node; -again: ret = btrfs_backref_iter_start(iter, cur->bytenr); - if (ret < 0) { - err = ret; - goto out; - } - + if (ret < 0) + return ret; /* * We skip the first btrfs_tree_block_info, as we don't use the key * stored in it, but fetch it from the tree block */ if (btrfs_backref_has_tree_block_info(iter)) { ret = btrfs_backref_iter_next(iter); - if (ret < 0) { - err = ret; + if (ret < 0) goto out; - } /* No extra backref? This means the tree block is corrupted */ if (ret > 0) { - err = -EUCLEAN; + ret = -EUCLEAN; goto out; } } @@ -1070,7 +1024,7 @@ again: type = btrfs_get_extent_inline_ref_type(eb, iref, BTRFS_REF_TYPE_BLOCK); if (type == BTRFS_REF_TYPE_INVALID) { - err = -EUCLEAN; + ret = -EUCLEAN; goto out; } key.type = type; @@ -1096,16 +1050,13 @@ again: /* SHARED_BLOCK_REF means key.offset is the parent bytenr */ if (key.type == BTRFS_SHARED_BLOCK_REF_KEY) { ret = handle_direct_tree_backref(cache, &key, cur); - if (ret < 0) { - err = ret; + if (ret < 0) goto out; - } continue; } else if (unlikely(key.type == BTRFS_EXTENT_REF_V0_KEY)) { - err = -EINVAL; - btrfs_print_v0_err(rc->extent_root->fs_info); - btrfs_handle_fs_error(rc->extent_root->fs_info, err, - NULL); + ret = -EINVAL; + btrfs_print_v0_err(fs_info); + btrfs_handle_fs_error(fs_info, ret, NULL); goto out; } else if (key.type != BTRFS_TREE_BLOCK_REF_KEY) { continue; @@ -1118,29 +1069,85 @@ again: */ ret = handle_indirect_tree_backref(cache, path, &key, node_key, cur); + if (ret < 0) + goto out; + } + ret = 0; + cur->checked = 1; + WARN_ON(exist); +out: + btrfs_backref_iter_release(iter); + return ret; +} + +/* + * Build backref tree for a given tree block. Root of the backref tree + * corresponds the tree block, leaves of the backref tree correspond roots of + * b-trees that reference the tree block. + * + * The basic idea of this function is check backrefs of a given block to find + * upper level blocks that reference the block, and then check backrefs of + * these upper level blocks recursively. The recursion stops when tree root is + * reached or backrefs for the block is cached. + * + * NOTE: if we find that backrefs for a block are cached, we know backrefs for + * all upper level blocks that directly/indirectly reference the block are also + * cached. + */ +static noinline_for_stack struct backref_node *build_backref_tree( + struct reloc_control *rc, struct btrfs_key *node_key, + int level, u64 bytenr) +{ + struct btrfs_backref_iter *iter; + struct backref_cache *cache = &rc->backref_cache; + /* For searching parent of TREE_BLOCK_REF */ + struct btrfs_path *path; + struct backref_node *cur; + struct backref_node *upper; + struct backref_node *lower; + struct backref_node *node = NULL; + struct backref_edge *edge; + struct rb_node *rb_node; + int cowonly; + int ret; + int err = 0; + + iter = btrfs_backref_iter_alloc(rc->extent_root->fs_info, GFP_NOFS); + if (!iter) + return ERR_PTR(-ENOMEM); + path = btrfs_alloc_path(); + if (!path) { + err = -ENOMEM; + goto out; + } + + node = alloc_backref_node(cache, bytenr, level); + if (!node) { + err = -ENOMEM; + goto out; + } + + node->lowest = 1; + cur = node; + + /* Breadth-first search to build backref cache */ + do { + ret = handle_one_tree_block(cache, path, iter, node_key, cur); if (ret < 0) { err = ret; goto out; } - } - if (ret < 0) { - err = ret; - goto out; - } - ret = 0; - btrfs_backref_iter_release(iter); - - cur->checked = 1; - WARN_ON(exist); - - /* the pending list isn't empty, take the first block to process */ - if (!list_empty(&cache->pending_edge)) { - edge = list_first_entry(&cache->pending_edge, - struct backref_edge, list[UPPER]); - list_del_init(&edge->list[UPPER]); - cur = edge->node[UPPER]; - goto again; - } + edge = list_first_entry_or_null(&cache->pending_edge, + struct backref_edge, list[UPPER]); + /* + * The pending list isn't empty, take the first block to + * process + */ + if (edge) { + list_del_init(&edge->list[UPPER]); + cur = edge->node[UPPER]; + } + } while (edge); /* * everything goes well, connect backref nodes and insert backref nodes From 1f872924663f9a15924cc7169932608c1d697ee1 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 25 Feb 2020 14:20:13 +0800 Subject: [PATCH 0698/1043] btrfs: reloc: refactor finishing part of upper linkage into finish_upper_links() After handle_one_tree_backref(), all newly added (not cached) edges and nodes have the following features: - Only backref_edge::list[LOWER] is linked. This means, we can only iterate from botton to top, not the other direction. - Newly added nodes are not added to cache rb_tree yet So to finish the backref cache, we still need to finish the links and add all nodes into backref cache rb_tree. This patch will refactor the existing code into finish_upper_links(), add more comments of each branch, and why we need to do all the work. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 186 ++++++++++++++++++++++++++---------------- 1 file changed, 117 insertions(+), 69 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index ad94482a2e65..29d53400c64c 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -1080,6 +1080,118 @@ out: return ret; } +/* + * In handle_one_tree_backref(), we have only linked the lower node to the edge, + * but the upper node hasn't been linked to the edge. + * This means we can only iterate through backref_node::upper to reach parent + * edges, but not through backref_node::lower to reach children edges. + * + * This function will finish the backref_node::lower to related edges, so that + * backref cache can be bi-directionally iterated. + * + * Also, this will add the nodes to backref cache for the next run. + */ +static int finish_upper_links(struct backref_cache *cache, + struct backref_node *start) +{ + struct list_head *useless_node = &cache->useless_node; + struct backref_edge *edge; + struct rb_node *rb_node; + LIST_HEAD(pending_edge); + + ASSERT(start->checked); + + /* Insert this node to cache if it's not COW-only */ + if (!start->cowonly) { + rb_node = tree_insert(&cache->rb_root, start->bytenr, + &start->rb_node); + if (rb_node) + backref_tree_panic(rb_node, -EEXIST, start->bytenr); + list_add_tail(&start->lower, &cache->leaves); + } + + /* + * Use breadth first search to iterate all related edges. + * + * The starting points are all the edges of this node + */ + list_for_each_entry(edge, &start->upper, list[LOWER]) + list_add_tail(&edge->list[UPPER], &pending_edge); + + while (!list_empty(&pending_edge)) { + struct backref_node *upper; + struct backref_node *lower; + struct rb_node *rb_node; + + edge = list_first_entry(&pending_edge, struct backref_edge, + list[UPPER]); + list_del_init(&edge->list[UPPER]); + upper = edge->node[UPPER]; + lower = edge->node[LOWER]; + + /* Parent is detached, no need to keep any edges */ + if (upper->detached) { + list_del(&edge->list[LOWER]); + free_backref_edge(cache, edge); + + /* Lower node is orphan, queue for cleanup */ + if (list_empty(&lower->upper)) + list_add(&lower->list, useless_node); + continue; + } + + /* + * All new nodes added in current build_backref_tree() haven't + * been linked to the cache rb tree. + * So if we have upper->rb_node populated, this means a cache + * hit. We only need to link the edge, as @upper and all its + * parent have already been linked. + */ + if (!RB_EMPTY_NODE(&upper->rb_node)) { + if (upper->lowest) { + list_del_init(&upper->lower); + upper->lowest = 0; + } + + list_add_tail(&edge->list[UPPER], &upper->lower); + continue; + } + + /* Sanity check, we shouldn't have any unchecked nodes */ + if (!upper->checked) { + ASSERT(0); + return -EUCLEAN; + } + + /* Sanity check, COW-only node has non-COW-only parent */ + if (start->cowonly != upper->cowonly) { + ASSERT(0); + return -EUCLEAN; + } + + /* Only cache non-COW-only (subvolume trees) tree blocks */ + if (!upper->cowonly) { + rb_node = tree_insert(&cache->rb_root, upper->bytenr, + &upper->rb_node); + if (rb_node) { + backref_tree_panic(rb_node, -EEXIST, + upper->bytenr); + return -EUCLEAN; + } + } + + list_add_tail(&edge->list[UPPER], &upper->lower); + + /* + * Also queue all the parent edges of this uncached node to + * finish the upper linkage + */ + list_for_each_entry(edge, &upper->upper, list[LOWER]) + list_add_tail(&edge->list[UPPER], &pending_edge); + } + return 0; +} + /* * Build backref tree for a given tree block. Root of the backref tree * corresponds the tree block, leaves of the backref tree correspond roots of @@ -1107,8 +1219,6 @@ static noinline_for_stack struct backref_node *build_backref_tree( struct backref_node *lower; struct backref_node *node = NULL; struct backref_edge *edge; - struct rb_node *rb_node; - int cowonly; int ret; int err = 0; @@ -1149,75 +1259,13 @@ static noinline_for_stack struct backref_node *build_backref_tree( } } while (edge); - /* - * everything goes well, connect backref nodes and insert backref nodes - * into the cache. - */ - ASSERT(node->checked); - cowonly = node->cowonly; - if (!cowonly) { - rb_node = tree_insert(&cache->rb_root, node->bytenr, - &node->rb_node); - if (rb_node) - backref_tree_panic(rb_node, -EEXIST, node->bytenr); - list_add_tail(&node->lower, &cache->leaves); + /* Finish the upper linkage of newly added edges/nodes */ + ret = finish_upper_links(cache, node); + if (ret < 0) { + err = ret; + goto out; } - list_for_each_entry(edge, &node->upper, list[LOWER]) - list_add_tail(&edge->list[UPPER], &cache->pending_edge); - - while (!list_empty(&cache->pending_edge)) { - edge = list_first_entry(&cache->pending_edge, - struct backref_edge, list[UPPER]); - list_del_init(&edge->list[UPPER]); - upper = edge->node[UPPER]; - if (upper->detached) { - list_del(&edge->list[LOWER]); - lower = edge->node[LOWER]; - free_backref_edge(cache, edge); - if (list_empty(&lower->upper)) - list_add(&lower->list, &cache->useless_node); - continue; - } - - if (!RB_EMPTY_NODE(&upper->rb_node)) { - if (upper->lowest) { - list_del_init(&upper->lower); - upper->lowest = 0; - } - - list_add_tail(&edge->list[UPPER], &upper->lower); - continue; - } - - if (!upper->checked) { - /* - * Still want to blow up for developers since this is a - * logic bug. - */ - ASSERT(0); - err = -EINVAL; - goto out; - } - if (cowonly != upper->cowonly) { - ASSERT(0); - err = -EINVAL; - goto out; - } - - if (!cowonly) { - rb_node = tree_insert(&cache->rb_root, upper->bytenr, - &upper->rb_node); - if (rb_node) - backref_tree_panic(rb_node, -EEXIST, - upper->bytenr); - } - - list_add_tail(&edge->list[UPPER], &upper->lower); - - list_for_each_entry(edge, &upper->upper, list[LOWER]) - list_add_tail(&edge->list[UPPER], &cache->pending_edge); - } /* * process useless backref nodes. backref nodes for tree leaves * are deleted from the cache. backref nodes for upper level From 29db137b6bb2f79851d86fa267ad8d6e6540a855 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Wed, 26 Feb 2020 13:08:36 +0800 Subject: [PATCH 0699/1043] btrfs: reloc: refactor useless nodes handling into its own function This patch will also add some comment for the cleanup. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 113 ++++++++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 37 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 29d53400c64c..96da33a9b692 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -1192,6 +1192,80 @@ static int finish_upper_links(struct backref_cache *cache, return 0; } +/* + * For useless nodes, do two major clean ups: + * + * - Cleanup the children edges and nodes + * If child node is also orphan (no parent) during cleanup, then the child + * node will also be cleaned up. + * + * - Freeing up leaves (level 0), keeps nodes detached + * For nodes, the node is still cached as "detached" + * + * Return false if @node is not in the @useless_nodes list. + * Return true if @node is in the @useless_nodes list. + */ +static bool handle_useless_nodes(struct reloc_control *rc, + struct backref_node *node) +{ + struct backref_cache *cache = &rc->backref_cache; + struct list_head *useless_node = &cache->useless_node; + bool ret = false; + + while (!list_empty(useless_node)) { + struct backref_node *cur; + + cur = list_first_entry(useless_node, struct backref_node, + list); + list_del_init(&cur->list); + + /* Only tree root nodes can be added to @useless_nodes */ + ASSERT(list_empty(&cur->upper)); + + if (cur == node) + ret = true; + + /* The node is the lowest node */ + if (cur->lowest) { + list_del_init(&cur->lower); + cur->lowest = 0; + } + + /* Cleanup the lower edges */ + while (!list_empty(&cur->lower)) { + struct backref_edge *edge; + struct backref_node *lower; + + edge = list_entry(cur->lower.next, + struct backref_edge, list[UPPER]); + list_del(&edge->list[UPPER]); + list_del(&edge->list[LOWER]); + lower = edge->node[LOWER]; + free_backref_edge(cache, edge); + + /* Child node is also orphan, queue for cleanup */ + if (list_empty(&lower->upper)) + list_add(&lower->list, useless_node); + } + /* Mark this block processed for relocation */ + mark_block_processed(rc, cur); + + /* + * Backref nodes for tree leaves are deleted from the cache. + * Backref nodes for upper level tree blocks are left in the + * cache to avoid unnecessary backref lookup. + */ + if (cur->level > 0) { + list_add(&cur->list, &cache->detached); + cur->detached = 1; + } else { + rb_erase(&cur->rb_node, &cache->rb_root); + free_backref_node(cache, cur); + } + } + return ret; +} + /* * Build backref tree for a given tree block. Root of the backref tree * corresponds the tree block, leaves of the backref tree correspond roots of @@ -1266,43 +1340,8 @@ static noinline_for_stack struct backref_node *build_backref_tree( goto out; } - /* - * process useless backref nodes. backref nodes for tree leaves - * are deleted from the cache. backref nodes for upper level - * tree blocks are left in the cache to avoid unnecessary backref - * lookup. - */ - while (!list_empty(&cache->useless_node)) { - upper = list_first_entry(&cache->useless_node, - struct backref_node, list); - list_del_init(&upper->list); - ASSERT(list_empty(&upper->upper)); - if (upper == node) - node = NULL; - if (upper->lowest) { - list_del_init(&upper->lower); - upper->lowest = 0; - } - while (!list_empty(&upper->lower)) { - edge = list_first_entry(&upper->lower, - struct backref_edge, list[UPPER]); - list_del(&edge->list[UPPER]); - list_del(&edge->list[LOWER]); - lower = edge->node[LOWER]; - free_backref_edge(cache, edge); - - if (list_empty(&lower->upper)) - list_add(&lower->list, &cache->useless_node); - } - mark_block_processed(rc, upper); - if (upper->level > 0) { - list_add(&upper->list, &cache->detached); - upper->detached = 1; - } else { - rb_erase(&upper->rb_node, &cache->rb_root); - free_backref_node(cache, upper); - } - } + if (handle_useless_nodes(rc, node)) + node = NULL; out: btrfs_backref_iter_free(iter); btrfs_free_path(path); From a26195a5230d61e4c214b17cc611e3ac7cc64c33 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 23 Mar 2020 14:59:06 +0800 Subject: [PATCH 0700/1043] btrfs: reloc: add btrfs_ prefix for backref_node/edge/cache Those three structures are the main elements of backref cache. Add the "btrfs_" prefix for later export. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 278 +++++++++++++++++++++--------------------- 1 file changed, 141 insertions(+), 137 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 96da33a9b692..453711bec3ea 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -73,7 +73,7 @@ */ /* - * backref_node, mapping_node and tree_block start with this + * btrfs_backref_node, mapping_node and tree_block start with this */ struct tree_entry { struct rb_node rb_node; @@ -83,7 +83,7 @@ struct tree_entry { /* * present a tree block in the backref cache */ -struct backref_node { +struct btrfs_backref_node { struct rb_node rb_node; u64 bytenr; @@ -138,10 +138,11 @@ struct backref_node { /* * present an edge connecting upper and lower backref nodes. */ -struct backref_edge { +struct btrfs_backref_edge { /* - * list[LOWER] is linked to backref_node::upper of lower level node, - * and list[UPPER] is linked to backref_node::lower of upper level node. + * list[LOWER] is linked to btrfs_backref_node::upper of lower level + * node, and list[UPPER] is linked to btrfs_backref_node::lower of + * upper level node. * * Also, build_backref_tree() uses list[UPPER] for pending edges, before * linking list[UPPER] to its upper level nodes. @@ -149,15 +150,15 @@ struct backref_edge { struct list_head list[2]; /* Two related nodes */ - struct backref_node *node[2]; + struct btrfs_backref_node *node[2]; }; -struct backref_cache { +struct btrfs_backref_cache { /* red black tree of all backref nodes in the cache */ struct rb_root rb_root; /* for passing backref nodes to btrfs_reloc_cow_block */ - struct backref_node *path[BTRFS_MAX_LEVEL]; + struct btrfs_backref_node *path[BTRFS_MAX_LEVEL]; /* * list of blocks that have been cowed but some block * pointers in upper level blocks may not reflect the @@ -237,7 +238,7 @@ struct reloc_control { struct btrfs_block_rsv *block_rsv; - struct backref_cache backref_cache; + struct btrfs_backref_cache backref_cache; struct file_extent_cluster cluster; /* tree blocks have been processed */ @@ -268,11 +269,11 @@ struct reloc_control { #define MOVE_DATA_EXTENTS 0 #define UPDATE_DATA_PTRS 1 -static void remove_backref_node(struct backref_cache *cache, - struct backref_node *node); +static void remove_backref_node(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node); static void mark_block_processed(struct reloc_control *rc, - struct backref_node *node) + struct btrfs_backref_node *node) { u32 blocksize; @@ -294,7 +295,7 @@ static void mapping_tree_init(struct mapping_tree *tree) } static void backref_cache_init(struct btrfs_fs_info *fs_info, - struct backref_cache *cache, int is_reloc) + struct btrfs_backref_cache *cache, int is_reloc) { int i; cache->rb_root = RB_ROOT; @@ -309,20 +310,20 @@ static void backref_cache_init(struct btrfs_fs_info *fs_info, cache->is_reloc = is_reloc; } -static void backref_cache_cleanup(struct backref_cache *cache) +static void backref_cache_cleanup(struct btrfs_backref_cache *cache) { - struct backref_node *node; + struct btrfs_backref_node *node; int i; while (!list_empty(&cache->detached)) { node = list_entry(cache->detached.next, - struct backref_node, list); + struct btrfs_backref_node, list); remove_backref_node(cache, node); } while (!list_empty(&cache->leaves)) { node = list_entry(cache->leaves.next, - struct backref_node, lower); + struct btrfs_backref_node, lower); remove_backref_node(cache, node); } @@ -339,10 +340,10 @@ static void backref_cache_cleanup(struct backref_cache *cache) ASSERT(!cache->nr_edges); } -static struct backref_node *alloc_backref_node(struct backref_cache *cache, - u64 bytenr, int level) +static struct btrfs_backref_node *alloc_backref_node( + struct btrfs_backref_cache *cache, u64 bytenr, int level) { - struct backref_node *node; + struct btrfs_backref_node *node; ASSERT(level >= 0 && level < BTRFS_MAX_LEVEL); node = kzalloc(sizeof(*node), GFP_NOFS); @@ -360,8 +361,8 @@ static struct backref_node *alloc_backref_node(struct backref_cache *cache, return node; } -static void free_backref_node(struct backref_cache *cache, - struct backref_node *node) +static void free_backref_node(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node) { if (node) { cache->nr_nodes--; @@ -370,9 +371,10 @@ static void free_backref_node(struct backref_cache *cache, } } -static struct backref_edge *alloc_backref_edge(struct backref_cache *cache) +static struct btrfs_backref_edge *alloc_backref_edge( + struct btrfs_backref_cache *cache) { - struct backref_edge *edge; + struct btrfs_backref_edge *edge; edge = kzalloc(sizeof(*edge), GFP_NOFS); if (edge) @@ -382,9 +384,9 @@ static struct backref_edge *alloc_backref_edge(struct backref_cache *cache) #define LINK_LOWER (1 << 0) #define LINK_UPPER (1 << 1) -static void link_backref_edge(struct backref_edge *edge, - struct backref_node *lower, - struct backref_node *upper, +static void link_backref_edge(struct btrfs_backref_edge *edge, + struct btrfs_backref_node *lower, + struct btrfs_backref_node *upper, int link_which) { ASSERT(upper && lower && upper->level == lower->level + 1); @@ -396,8 +398,8 @@ static void link_backref_edge(struct backref_edge *edge, list_add_tail(&edge->list[UPPER], &upper->lower); } -static void free_backref_edge(struct backref_cache *cache, - struct backref_edge *edge) +static void free_backref_edge(struct btrfs_backref_cache *cache, + struct btrfs_backref_edge *edge) { if (edge) { cache->nr_edges--; @@ -451,8 +453,8 @@ static void backref_tree_panic(struct rb_node *rb_node, int errno, u64 bytenr) { struct btrfs_fs_info *fs_info = NULL; - struct backref_node *bnode = rb_entry(rb_node, struct backref_node, - rb_node); + struct btrfs_backref_node *bnode = rb_entry(rb_node, + struct btrfs_backref_node, rb_node); if (bnode->root) fs_info = bnode->root->fs_info; btrfs_panic(fs_info, errno, @@ -463,16 +465,16 @@ static void backref_tree_panic(struct rb_node *rb_node, int errno, u64 bytenr) /* * walk up backref nodes until reach node presents tree root */ -static struct backref_node *walk_up_backref(struct backref_node *node, - struct backref_edge *edges[], - int *index) +static struct btrfs_backref_node *walk_up_backref( + struct btrfs_backref_node *node, + struct btrfs_backref_edge *edges[], int *index) { - struct backref_edge *edge; + struct btrfs_backref_edge *edge; int idx = *index; while (!list_empty(&node->upper)) { edge = list_entry(node->upper.next, - struct backref_edge, list[LOWER]); + struct btrfs_backref_edge, list[LOWER]); edges[idx++] = edge; node = edge->node[UPPER]; } @@ -484,11 +486,11 @@ static struct backref_node *walk_up_backref(struct backref_node *node, /* * walk down backref nodes to find start of next reference path */ -static struct backref_node *walk_down_backref(struct backref_edge *edges[], - int *index) +static struct btrfs_backref_node *walk_down_backref( + struct btrfs_backref_edge *edges[], int *index) { - struct backref_edge *edge; - struct backref_node *lower; + struct btrfs_backref_edge *edge; + struct btrfs_backref_node *lower; int idx = *index; while (idx > 0) { @@ -499,7 +501,7 @@ static struct backref_node *walk_down_backref(struct backref_edge *edges[], continue; } edge = list_entry(edge->list[LOWER].next, - struct backref_edge, list[LOWER]); + struct btrfs_backref_edge, list[LOWER]); edges[idx - 1] = edge; *index = idx; return edge->node[UPPER]; @@ -508,7 +510,7 @@ static struct backref_node *walk_down_backref(struct backref_edge *edges[], return NULL; } -static void unlock_node_buffer(struct backref_node *node) +static void unlock_node_buffer(struct btrfs_backref_node *node) { if (node->locked) { btrfs_tree_unlock(node->eb); @@ -516,7 +518,7 @@ static void unlock_node_buffer(struct backref_node *node) } } -static void drop_node_buffer(struct backref_node *node) +static void drop_node_buffer(struct btrfs_backref_node *node) { if (node->eb) { unlock_node_buffer(node); @@ -525,8 +527,8 @@ static void drop_node_buffer(struct backref_node *node) } } -static void drop_backref_node(struct backref_cache *tree, - struct backref_node *node) +static void drop_backref_node(struct btrfs_backref_cache *tree, + struct btrfs_backref_node *node) { BUG_ON(!list_empty(&node->upper)); @@ -541,18 +543,18 @@ static void drop_backref_node(struct backref_cache *tree, /* * remove a backref node from the backref cache */ -static void remove_backref_node(struct backref_cache *cache, - struct backref_node *node) +static void remove_backref_node(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node) { - struct backref_node *upper; - struct backref_edge *edge; + struct btrfs_backref_node *upper; + struct btrfs_backref_edge *edge; if (!node) return; BUG_ON(!node->lowest && !node->detached); while (!list_empty(&node->upper)) { - edge = list_entry(node->upper.next, struct backref_edge, + edge = list_entry(node->upper.next, struct btrfs_backref_edge, list[LOWER]); upper = edge->node[UPPER]; list_del(&edge->list[LOWER]); @@ -579,8 +581,8 @@ static void remove_backref_node(struct backref_cache *cache, drop_backref_node(cache, node); } -static void update_backref_node(struct backref_cache *cache, - struct backref_node *node, u64 bytenr) +static void update_backref_node(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node, u64 bytenr) { struct rb_node *rb_node; rb_erase(&node->rb_node, &cache->rb_root); @@ -594,9 +596,9 @@ static void update_backref_node(struct backref_cache *cache, * update backref cache after a transaction commit */ static int update_backref_cache(struct btrfs_trans_handle *trans, - struct backref_cache *cache) + struct btrfs_backref_cache *cache) { - struct backref_node *node; + struct btrfs_backref_node *node; int level = 0; if (cache->last_trans == 0) { @@ -614,13 +616,13 @@ static int update_backref_cache(struct btrfs_trans_handle *trans, */ while (!list_empty(&cache->detached)) { node = list_entry(cache->detached.next, - struct backref_node, list); + struct btrfs_backref_node, list); remove_backref_node(cache, node); } while (!list_empty(&cache->changed)) { node = list_entry(cache->changed.next, - struct backref_node, list); + struct btrfs_backref_node, list); list_del_init(&node->list); BUG_ON(node->pending); update_backref_node(cache, node, node->new_bytenr); @@ -743,12 +745,12 @@ static struct btrfs_root *read_fs_root(struct btrfs_fs_info *fs_info, * type is btrfs_inline_ref_type, offset is * btrfs_inline_ref_offset. */ -static int handle_direct_tree_backref(struct backref_cache *cache, +static int handle_direct_tree_backref(struct btrfs_backref_cache *cache, struct btrfs_key *ref_key, - struct backref_node *cur) + struct btrfs_backref_node *cur) { - struct backref_edge *edge; - struct backref_node *upper; + struct btrfs_backref_edge *edge; + struct btrfs_backref_node *upper; struct rb_node *rb_node; ASSERT(ref_key->type == BTRFS_SHARED_BLOCK_REF_KEY); @@ -795,7 +797,7 @@ static int handle_direct_tree_backref(struct backref_cache *cache, list_add_tail(&edge->list[UPPER], &cache->pending_edge); } else { /* Parent node already cached */ - upper = rb_entry(rb_node, struct backref_node, rb_node); + upper = rb_entry(rb_node, struct btrfs_backref_node, rb_node); ASSERT(upper->checked); INIT_LIST_HEAD(&edge->list[UPPER]); } @@ -815,16 +817,16 @@ static int handle_direct_tree_backref(struct backref_cache *cache, * @path: A clean (released) path, to avoid allocating path everytime * the function get called. */ -static int handle_indirect_tree_backref(struct backref_cache *cache, +static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, struct btrfs_path *path, struct btrfs_key *ref_key, struct btrfs_key *tree_key, - struct backref_node *cur) + struct btrfs_backref_node *cur) { struct btrfs_fs_info *fs_info = cache->fs_info; - struct backref_node *upper; - struct backref_node *lower; - struct backref_edge *edge; + struct btrfs_backref_node *upper; + struct btrfs_backref_node *lower; + struct btrfs_backref_edge *edge; struct extent_buffer *eb; struct btrfs_root *root; struct rb_node *rb_node; @@ -937,7 +939,8 @@ static int handle_indirect_tree_backref(struct backref_cache *cache, INIT_LIST_HEAD(&edge->list[UPPER]); } } else { - upper = rb_entry(rb_node, struct backref_node, rb_node); + upper = rb_entry(rb_node, struct btrfs_backref_node, + rb_node); ASSERT(upper->checked); INIT_LIST_HEAD(&edge->list[UPPER]); if (!upper->owner) @@ -957,15 +960,15 @@ out: return ret; } -static int handle_one_tree_block(struct backref_cache *cache, +static int handle_one_tree_block(struct btrfs_backref_cache *cache, struct btrfs_path *path, struct btrfs_backref_iter *iter, struct btrfs_key *node_key, - struct backref_node *cur) + struct btrfs_backref_node *cur) { struct btrfs_fs_info *fs_info = cache->fs_info; - struct backref_edge *edge; - struct backref_node *exist; + struct btrfs_backref_edge *edge; + struct btrfs_backref_node *exist; int ret; ret = btrfs_backref_iter_start(iter, cur->bytenr); @@ -992,7 +995,7 @@ static int handle_one_tree_block(struct backref_cache *cache, * backref of type BTRFS_TREE_BLOCK_REF_KEY */ ASSERT(list_is_singular(&cur->upper)); - edge = list_entry(cur->upper.next, struct backref_edge, + edge = list_entry(cur->upper.next, struct btrfs_backref_edge, list[LOWER]); ASSERT(list_empty(&edge->list[UPPER])); exist = edge->node[UPPER]; @@ -1083,19 +1086,20 @@ out: /* * In handle_one_tree_backref(), we have only linked the lower node to the edge, * but the upper node hasn't been linked to the edge. - * This means we can only iterate through backref_node::upper to reach parent - * edges, but not through backref_node::lower to reach children edges. + * This means we can only iterate through btrfs_backref_node::upper to reach + * parent edges, but not through btrfs_backref_node::lower to reach children + * edges. * - * This function will finish the backref_node::lower to related edges, so that - * backref cache can be bi-directionally iterated. + * This function will finish the btrfs_backref_node::lower to related edges, + * so that backref cache can be bi-directionally iterated. * * Also, this will add the nodes to backref cache for the next run. */ -static int finish_upper_links(struct backref_cache *cache, - struct backref_node *start) +static int finish_upper_links(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *start) { struct list_head *useless_node = &cache->useless_node; - struct backref_edge *edge; + struct btrfs_backref_edge *edge; struct rb_node *rb_node; LIST_HEAD(pending_edge); @@ -1119,12 +1123,12 @@ static int finish_upper_links(struct backref_cache *cache, list_add_tail(&edge->list[UPPER], &pending_edge); while (!list_empty(&pending_edge)) { - struct backref_node *upper; - struct backref_node *lower; + struct btrfs_backref_node *upper; + struct btrfs_backref_node *lower; struct rb_node *rb_node; - edge = list_first_entry(&pending_edge, struct backref_edge, - list[UPPER]); + edge = list_first_entry(&pending_edge, + struct btrfs_backref_edge, list[UPPER]); list_del_init(&edge->list[UPPER]); upper = edge->node[UPPER]; lower = edge->node[LOWER]; @@ -1206,16 +1210,16 @@ static int finish_upper_links(struct backref_cache *cache, * Return true if @node is in the @useless_nodes list. */ static bool handle_useless_nodes(struct reloc_control *rc, - struct backref_node *node) + struct btrfs_backref_node *node) { - struct backref_cache *cache = &rc->backref_cache; + struct btrfs_backref_cache *cache = &rc->backref_cache; struct list_head *useless_node = &cache->useless_node; bool ret = false; while (!list_empty(useless_node)) { - struct backref_node *cur; + struct btrfs_backref_node *cur; - cur = list_first_entry(useless_node, struct backref_node, + cur = list_first_entry(useless_node, struct btrfs_backref_node, list); list_del_init(&cur->list); @@ -1233,11 +1237,11 @@ static bool handle_useless_nodes(struct reloc_control *rc, /* Cleanup the lower edges */ while (!list_empty(&cur->lower)) { - struct backref_edge *edge; - struct backref_node *lower; + struct btrfs_backref_edge *edge; + struct btrfs_backref_node *lower; edge = list_entry(cur->lower.next, - struct backref_edge, list[UPPER]); + struct btrfs_backref_edge, list[UPPER]); list_del(&edge->list[UPPER]); list_del(&edge->list[LOWER]); lower = edge->node[LOWER]; @@ -1280,19 +1284,19 @@ static bool handle_useless_nodes(struct reloc_control *rc, * all upper level blocks that directly/indirectly reference the block are also * cached. */ -static noinline_for_stack struct backref_node *build_backref_tree( +static noinline_for_stack struct btrfs_backref_node *build_backref_tree( struct reloc_control *rc, struct btrfs_key *node_key, int level, u64 bytenr) { struct btrfs_backref_iter *iter; - struct backref_cache *cache = &rc->backref_cache; + struct btrfs_backref_cache *cache = &rc->backref_cache; /* For searching parent of TREE_BLOCK_REF */ struct btrfs_path *path; - struct backref_node *cur; - struct backref_node *upper; - struct backref_node *lower; - struct backref_node *node = NULL; - struct backref_edge *edge; + struct btrfs_backref_node *cur; + struct btrfs_backref_node *upper; + struct btrfs_backref_node *lower; + struct btrfs_backref_node *node = NULL; + struct btrfs_backref_edge *edge; int ret; int err = 0; @@ -1322,7 +1326,7 @@ static noinline_for_stack struct backref_node *build_backref_tree( goto out; } edge = list_first_entry_or_null(&cache->pending_edge, - struct backref_edge, list[UPPER]); + struct btrfs_backref_edge, list[UPPER]); /* * The pending list isn't empty, take the first block to * process @@ -1348,12 +1352,12 @@ out: if (err) { while (!list_empty(&cache->useless_node)) { lower = list_first_entry(&cache->useless_node, - struct backref_node, list); + struct btrfs_backref_node, list); list_del_init(&lower->list); } while (!list_empty(&cache->pending_edge)) { edge = list_first_entry(&cache->pending_edge, - struct backref_edge, list[UPPER]); + struct btrfs_backref_edge, list[UPPER]); list_del(&edge->list[UPPER]); list_del(&edge->list[LOWER]); lower = edge->node[LOWER]; @@ -1381,7 +1385,7 @@ out: while (!list_empty(&cache->useless_node)) { lower = list_first_entry(&cache->useless_node, - struct backref_node, list); + struct btrfs_backref_node, list); list_del_init(&lower->list); if (lower == node) node = NULL; @@ -1410,11 +1414,11 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, struct btrfs_root *dest) { struct btrfs_root *reloc_root = src->reloc_root; - struct backref_cache *cache = &rc->backref_cache; - struct backref_node *node = NULL; - struct backref_node *new_node; - struct backref_edge *edge; - struct backref_edge *new_edge; + struct btrfs_backref_cache *cache = &rc->backref_cache; + struct btrfs_backref_node *node = NULL; + struct btrfs_backref_node *new_node; + struct btrfs_backref_edge *edge; + struct btrfs_backref_edge *new_edge; struct rb_node *rb_node; if (cache->last_trans > 0) @@ -1422,7 +1426,7 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, rb_node = tree_search(&cache->rb_root, src->commit_root->start); if (rb_node) { - node = rb_entry(rb_node, struct backref_node, rb_node); + node = rb_entry(rb_node, struct btrfs_backref_node, rb_node); if (node->detached) node = NULL; else @@ -1433,7 +1437,7 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, rb_node = tree_search(&cache->rb_root, reloc_root->commit_root->start); if (rb_node) { - node = rb_entry(rb_node, struct backref_node, + node = rb_entry(rb_node, struct btrfs_backref_node, rb_node); BUG_ON(node->detached); } @@ -1479,7 +1483,7 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, fail: while (!list_empty(&new_node->lower)) { new_edge = list_entry(new_node->lower.next, - struct backref_edge, list[UPPER]); + struct btrfs_backref_edge, list[UPPER]); list_del(&new_edge->list[UPPER]); free_backref_edge(cache, new_edge); } @@ -2869,10 +2873,10 @@ static int record_reloc_root_in_trans(struct btrfs_trans_handle *trans, static noinline_for_stack struct btrfs_root *select_reloc_root(struct btrfs_trans_handle *trans, struct reloc_control *rc, - struct backref_node *node, - struct backref_edge *edges[]) + struct btrfs_backref_node *node, + struct btrfs_backref_edge *edges[]) { - struct backref_node *next; + struct btrfs_backref_node *next; struct btrfs_root *root; int index = 0; @@ -2932,12 +2936,12 @@ struct btrfs_root *select_reloc_root(struct btrfs_trans_handle *trans, * counted. return -ENOENT if the block is root of reloc tree. */ static noinline_for_stack -struct btrfs_root *select_one_root(struct backref_node *node) +struct btrfs_root *select_one_root(struct btrfs_backref_node *node) { - struct backref_node *next; + struct btrfs_backref_node *next; struct btrfs_root *root; struct btrfs_root *fs_root = NULL; - struct backref_edge *edges[BTRFS_MAX_LEVEL - 1]; + struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1]; int index = 0; next = node; @@ -2969,12 +2973,12 @@ struct btrfs_root *select_one_root(struct backref_node *node) static noinline_for_stack u64 calcu_metadata_size(struct reloc_control *rc, - struct backref_node *node, int reserve) + struct btrfs_backref_node *node, int reserve) { struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; - struct backref_node *next = node; - struct backref_edge *edge; - struct backref_edge *edges[BTRFS_MAX_LEVEL - 1]; + struct btrfs_backref_node *next = node; + struct btrfs_backref_edge *edge; + struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1]; u64 num_bytes = 0; int index = 0; @@ -2992,7 +2996,7 @@ u64 calcu_metadata_size(struct reloc_control *rc, break; edge = list_entry(next->upper.next, - struct backref_edge, list[LOWER]); + struct btrfs_backref_edge, list[LOWER]); edges[index++] = edge; next = edge->node[UPPER]; } @@ -3003,7 +3007,7 @@ u64 calcu_metadata_size(struct reloc_control *rc, static int reserve_metadata_space(struct btrfs_trans_handle *trans, struct reloc_control *rc, - struct backref_node *node) + struct btrfs_backref_node *node) { struct btrfs_root *root = rc->extent_root; struct btrfs_fs_info *fs_info = root->fs_info; @@ -3051,14 +3055,14 @@ static int reserve_metadata_space(struct btrfs_trans_handle *trans, */ static int do_relocation(struct btrfs_trans_handle *trans, struct reloc_control *rc, - struct backref_node *node, + struct btrfs_backref_node *node, struct btrfs_key *key, struct btrfs_path *path, int lowest) { struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; - struct backref_node *upper; - struct backref_edge *edge; - struct backref_edge *edges[BTRFS_MAX_LEVEL - 1]; + struct btrfs_backref_node *upper; + struct btrfs_backref_edge *edge; + struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1]; struct btrfs_root *root; struct extent_buffer *eb; u32 blocksize; @@ -3214,7 +3218,7 @@ next: static int link_to_upper(struct btrfs_trans_handle *trans, struct reloc_control *rc, - struct backref_node *node, + struct btrfs_backref_node *node, struct btrfs_path *path) { struct btrfs_key key; @@ -3228,15 +3232,15 @@ static int finish_pending_nodes(struct btrfs_trans_handle *trans, struct btrfs_path *path, int err) { LIST_HEAD(list); - struct backref_cache *cache = &rc->backref_cache; - struct backref_node *node; + struct btrfs_backref_cache *cache = &rc->backref_cache; + struct btrfs_backref_node *node; int level; int ret; for (level = 0; level < BTRFS_MAX_LEVEL; level++) { while (!list_empty(&cache->pending[level])) { node = list_entry(cache->pending[level].next, - struct backref_node, list); + struct btrfs_backref_node, list); list_move_tail(&node->list, &list); BUG_ON(!node->pending); @@ -3256,11 +3260,11 @@ static int finish_pending_nodes(struct btrfs_trans_handle *trans, * as processed. */ static void update_processed_blocks(struct reloc_control *rc, - struct backref_node *node) + struct btrfs_backref_node *node) { - struct backref_node *next = node; - struct backref_edge *edge; - struct backref_edge *edges[BTRFS_MAX_LEVEL - 1]; + struct btrfs_backref_node *next = node; + struct btrfs_backref_edge *edge; + struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1]; int index = 0; while (next) { @@ -3275,7 +3279,7 @@ static void update_processed_blocks(struct reloc_control *rc, break; edge = list_entry(next->upper.next, - struct backref_edge, list[LOWER]); + struct btrfs_backref_edge, list[LOWER]); edges[index++] = edge; next = edge->node[UPPER]; } @@ -3320,7 +3324,7 @@ static int get_tree_block_key(struct btrfs_fs_info *fs_info, */ static int relocate_tree_block(struct btrfs_trans_handle *trans, struct reloc_control *rc, - struct backref_node *node, + struct btrfs_backref_node *node, struct btrfs_key *key, struct btrfs_path *path) { @@ -3382,7 +3386,7 @@ int relocate_tree_blocks(struct btrfs_trans_handle *trans, struct reloc_control *rc, struct rb_root *blocks) { struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; - struct backref_node *node; + struct btrfs_backref_node *node; struct btrfs_path *path; struct tree_block *block; struct tree_block *next; @@ -4802,7 +4806,7 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, { struct btrfs_fs_info *fs_info = root->fs_info; struct reloc_control *rc; - struct backref_node *node; + struct btrfs_backref_node *node; int first_cow = 0; int level; int ret = 0; From 7053544146ac7eb71de6cee1ffda678714f905d8 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 23 Mar 2020 15:03:56 +0800 Subject: [PATCH 0701/1043] btrfs: backref: move btrfs_backref_(node|edge|cache) structures to backref.h These 3 structures are the main part of btrfs backref cache, move them to backref.h to build the basis for later reuse. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.h | 116 ++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/relocation.c | 113 ---------------------------------------- 2 files changed, 116 insertions(+), 113 deletions(-) diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 46faf9b93576..55f1f56b378e 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -150,4 +150,120 @@ static inline void btrfs_backref_iter_release(struct btrfs_backref_iter *iter) memset(&iter->cur_key, 0, sizeof(iter->cur_key)); } +/* + * Backref cache related structures + * + * The whole objective of backref_cache is to build a bi-directional map + * of tree blocks (represented by backref_node) and all their parents. + */ + +/* + * Represent a tree block in the backref cache + */ +struct btrfs_backref_node { + struct rb_node rb_node; + u64 bytenr; + + u64 new_bytenr; + /* Objectid of tree block owner, can be not uptodate */ + u64 owner; + /* Link to pending, changed or detached list */ + struct list_head list; + + /* List of upper level edges, which link this node to its parents */ + struct list_head upper; + /* List of lower level edges, which link this node to its children */ + struct list_head lower; + + /* NULL if this node is not tree root */ + struct btrfs_root *root; + /* Extent buffer got by COWing the block */ + struct extent_buffer *eb; + /* Level of the tree block */ + unsigned int level:8; + /* Is the block in non-reference counted tree */ + unsigned int cowonly:1; + /* 1 if no child node is in the cache */ + unsigned int lowest:1; + /* Is the extent buffer locked */ + unsigned int locked:1; + /* Has the block been processed */ + unsigned int processed:1; + /* Have backrefs of this block been checked */ + unsigned int checked:1; + /* + * 1 if corresponding block has been COWed but some upper level block + * pointers may not point to the new location + */ + unsigned int pending:1; + /* 1 if the backref node isn't connected to any other backref node */ + unsigned int detached:1; + + /* + * For generic purpose backref cache, where we only care if it's a reloc + * root, doesn't care the source subvolid. + */ + unsigned int is_reloc_root:1; +}; + +#define LOWER 0 +#define UPPER 1 + +/* + * Represent an edge connecting upper and lower backref nodes. + */ +struct btrfs_backref_edge { + /* + * list[LOWER] is linked to btrfs_backref_node::upper of lower level + * node, and list[UPPER] is linked to btrfs_backref_node::lower of + * upper level node. + * + * Also, build_backref_tree() uses list[UPPER] for pending edges, before + * linking list[UPPER] to its upper level nodes. + */ + struct list_head list[2]; + + /* Two related nodes */ + struct btrfs_backref_node *node[2]; +}; + +struct btrfs_backref_cache { + /* Red black tree of all backref nodes in the cache */ + struct rb_root rb_root; + /* For passing backref nodes to btrfs_reloc_cow_block */ + struct btrfs_backref_node *path[BTRFS_MAX_LEVEL]; + /* + * List of blocks that have been COWed but some block pointers in upper + * level blocks may not reflect the new location + */ + struct list_head pending[BTRFS_MAX_LEVEL]; + /* List of backref nodes with no child node */ + struct list_head leaves; + /* List of blocks that have been COWed in current transaction */ + struct list_head changed; + /* List of detached backref node. */ + struct list_head detached; + + u64 last_trans; + + int nr_nodes; + int nr_edges; + + /* List of unchecked backref edges during backref cache build */ + struct list_head pending_edge; + + /* List of useless backref nodes during backref cache build */ + struct list_head useless_node; + + struct btrfs_fs_info *fs_info; + + /* + * Whether this cache is for relocation + * + * Reloction backref cache require more info for reloc root compared + * to generic backref cache. + */ + unsigned int is_reloc; +}; + #endif diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 453711bec3ea..8fa10d8306c2 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -80,120 +80,7 @@ struct tree_entry { u64 bytenr; }; -/* - * present a tree block in the backref cache - */ -struct btrfs_backref_node { - struct rb_node rb_node; - u64 bytenr; - - u64 new_bytenr; - /* objectid of tree block owner, can be not uptodate */ - u64 owner; - /* link to pending, changed or detached list */ - struct list_head list; - - /* List of upper level edges, which link this node to its parents */ - struct list_head upper; - /* List of lower level edges, which link this node to its children */ - struct list_head lower; - - /* NULL if this node is not tree root */ - struct btrfs_root *root; - /* extent buffer got by COW the block */ - struct extent_buffer *eb; - /* level of tree block */ - unsigned int level:8; - /* is the block in non-reference counted tree */ - unsigned int cowonly:1; - /* 1 if no child node in the cache */ - unsigned int lowest:1; - /* is the extent buffer locked */ - unsigned int locked:1; - /* has the block been processed */ - unsigned int processed:1; - /* have backrefs of this block been checked */ - unsigned int checked:1; - /* - * 1 if corresponding block has been cowed but some upper - * level block pointers may not point to the new location - */ - unsigned int pending:1; - /* - * 1 if the backref node isn't connected to any other - * backref node. - */ - unsigned int detached:1; - - /* - * For generic purpose backref cache, where we only care if it's a reloc - * root, doesn't care the source subvolid. - */ - unsigned int is_reloc_root:1; -}; - -#define LOWER 0 -#define UPPER 1 #define RELOCATION_RESERVED_NODES 256 -/* - * present an edge connecting upper and lower backref nodes. - */ -struct btrfs_backref_edge { - /* - * list[LOWER] is linked to btrfs_backref_node::upper of lower level - * node, and list[UPPER] is linked to btrfs_backref_node::lower of - * upper level node. - * - * Also, build_backref_tree() uses list[UPPER] for pending edges, before - * linking list[UPPER] to its upper level nodes. - */ - struct list_head list[2]; - - /* Two related nodes */ - struct btrfs_backref_node *node[2]; -}; - - -struct btrfs_backref_cache { - /* red black tree of all backref nodes in the cache */ - struct rb_root rb_root; - /* for passing backref nodes to btrfs_reloc_cow_block */ - struct btrfs_backref_node *path[BTRFS_MAX_LEVEL]; - /* - * list of blocks that have been cowed but some block - * pointers in upper level blocks may not reflect the - * new location - */ - struct list_head pending[BTRFS_MAX_LEVEL]; - /* list of backref nodes with no child node */ - struct list_head leaves; - /* list of blocks that have been cowed in current transaction */ - struct list_head changed; - /* list of detached backref node. */ - struct list_head detached; - - u64 last_trans; - - int nr_nodes; - int nr_edges; - - /* The list of unchecked backref edges during backref cache build */ - struct list_head pending_edge; - - /* The list of useless backref nodes during backref cache build */ - struct list_head useless_node; - - struct btrfs_fs_info *fs_info; - - /* - * Whether this cache is for relocation - * - * Reloction backref cache require more info for reloc root compared - * to generic backref cache. - */ - unsigned int is_reloc; -}; - /* * map address of tree root to tree */ From e9a28dc52af31d8af1883afe08e724a303b3c4eb Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 26 Mar 2020 14:11:09 +0800 Subject: [PATCH 0702/1043] btrfs: rename tree_entry to rb_simple_node and export it Structure tree_entry provides a very simple rb_tree which only uses bytenr as search index. That tree_entry is used in 3 structures: backref_node, mapping_node and tree_block. Since we're going to make backref_node independnt from relocation, it's a good time to extract the tree_entry into rb_simple_node, and export it into misc.h. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.h | 6 ++- fs/btrfs/misc.h | 54 +++++++++++++++++++++ fs/btrfs/relocation.c | 109 +++++++++++++----------------------------- 3 files changed, 90 insertions(+), 79 deletions(-) diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 55f1f56b378e..0d3fb76364c6 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -161,8 +161,10 @@ static inline void btrfs_backref_iter_release(struct btrfs_backref_iter *iter) * Represent a tree block in the backref cache */ struct btrfs_backref_node { - struct rb_node rb_node; - u64 bytenr; + struct { + struct rb_node rb_node; + u64 bytenr; + }; /* Use rb_simple_node for search/insert */ u64 new_bytenr; /* Objectid of tree block owner, can be not uptodate */ diff --git a/fs/btrfs/misc.h b/fs/btrfs/misc.h index 72bab64ecf60..6461ebc3a1c1 100644 --- a/fs/btrfs/misc.h +++ b/fs/btrfs/misc.h @@ -6,6 +6,7 @@ #include #include #include +#include #define in_range(b, first, len) ((b) >= (first) && (b) < (first) + (len)) @@ -58,4 +59,57 @@ static inline bool has_single_bit_set(u64 n) return is_power_of_two_u64(n); } +/* + * Simple bytenr based rb_tree relate structures + * + * Any structure wants to use bytenr as single search index should have their + * structure start with these members. + */ +struct rb_simple_node { + struct rb_node rb_node; + u64 bytenr; +}; + +static inline struct rb_node *rb_simple_search(struct rb_root *root, u64 bytenr) +{ + struct rb_node *node = root->rb_node; + struct rb_simple_node *entry; + + while (node) { + entry = rb_entry(node, struct rb_simple_node, rb_node); + + if (bytenr < entry->bytenr) + node = node->rb_left; + else if (bytenr > entry->bytenr) + node = node->rb_right; + else + return node; + } + return NULL; +} + +static inline struct rb_node *rb_simple_insert(struct rb_root *root, u64 bytenr, + struct rb_node *node) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct rb_simple_node *entry; + + while (*p) { + parent = *p; + entry = rb_entry(parent, struct rb_simple_node, rb_node); + + if (bytenr < entry->bytenr) + p = &(*p)->rb_left; + else if (bytenr > entry->bytenr) + p = &(*p)->rb_right; + else + return parent; + } + + rb_link_node(node, parent, p); + rb_insert_color(node, root); + return NULL; +} + #endif diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 8fa10d8306c2..09076ac21590 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -24,6 +24,7 @@ #include "delalloc-space.h" #include "block-group.h" #include "backref.h" +#include "misc.h" /* * Relocation overview @@ -72,21 +73,15 @@ * The entry point of relocation is relocate_block_group() function. */ -/* - * btrfs_backref_node, mapping_node and tree_block start with this - */ -struct tree_entry { - struct rb_node rb_node; - u64 bytenr; -}; - #define RELOCATION_RESERVED_NODES 256 /* * map address of tree root to tree */ struct mapping_node { - struct rb_node rb_node; - u64 bytenr; + struct { + struct rb_node rb_node; + u64 bytenr; + }; /* Use rb_simle_node for search/insert */ void *data; }; @@ -99,8 +94,10 @@ struct mapping_tree { * present a tree block to process */ struct tree_block { - struct rb_node rb_node; - u64 bytenr; + struct { + struct rb_node rb_node; + u64 bytenr; + }; /* Use rb_simple_node for search/insert */ struct btrfs_key key; unsigned int level:8; unsigned int key_ready:1; @@ -294,48 +291,6 @@ static void free_backref_edge(struct btrfs_backref_cache *cache, } } -static struct rb_node *tree_insert(struct rb_root *root, u64 bytenr, - struct rb_node *node) -{ - struct rb_node **p = &root->rb_node; - struct rb_node *parent = NULL; - struct tree_entry *entry; - - while (*p) { - parent = *p; - entry = rb_entry(parent, struct tree_entry, rb_node); - - if (bytenr < entry->bytenr) - p = &(*p)->rb_left; - else if (bytenr > entry->bytenr) - p = &(*p)->rb_right; - else - return parent; - } - - rb_link_node(node, parent, p); - rb_insert_color(node, root); - return NULL; -} - -static struct rb_node *tree_search(struct rb_root *root, u64 bytenr) -{ - struct rb_node *n = root->rb_node; - struct tree_entry *entry; - - while (n) { - entry = rb_entry(n, struct tree_entry, rb_node); - - if (bytenr < entry->bytenr) - n = n->rb_left; - else if (bytenr > entry->bytenr) - n = n->rb_right; - else - return n; - } - return NULL; -} - static void backref_tree_panic(struct rb_node *rb_node, int errno, u64 bytenr) { @@ -474,7 +429,7 @@ static void update_backref_node(struct btrfs_backref_cache *cache, struct rb_node *rb_node; rb_erase(&node->rb_node, &cache->rb_root); node->bytenr = bytenr; - rb_node = tree_insert(&cache->rb_root, node->bytenr, &node->rb_node); + rb_node = rb_simple_insert(&cache->rb_root, node->bytenr, &node->rb_node); if (rb_node) backref_tree_panic(rb_node, -EEXIST, bytenr); } @@ -599,7 +554,7 @@ struct btrfs_root *find_reloc_root(struct btrfs_fs_info *fs_info, u64 bytenr) ASSERT(rc); spin_lock(&rc->reloc_root_tree.lock); - rb_node = tree_search(&rc->reloc_root_tree.rb_root, bytenr); + rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, bytenr); if (rb_node) { node = rb_entry(rb_node, struct mapping_node, rb_node); root = (struct btrfs_root *)node->data; @@ -667,7 +622,7 @@ static int handle_direct_tree_backref(struct btrfs_backref_cache *cache, if (!edge) return -ENOMEM; - rb_node = tree_search(&cache->rb_root, ref_key->offset); + rb_node = rb_simple_search(&cache->rb_root, ref_key->offset); if (!rb_node) { /* Parent node not yet cached */ upper = alloc_backref_node(cache, ref_key->offset, @@ -788,7 +743,7 @@ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, } eb = path->nodes[level]; - rb_node = tree_search(&cache->rb_root, eb->start); + rb_node = rb_simple_search(&cache->rb_root, eb->start); if (!rb_node) { upper = alloc_backref_node(cache, eb->start, lower->level + 1); @@ -994,8 +949,8 @@ static int finish_upper_links(struct btrfs_backref_cache *cache, /* Insert this node to cache if it's not COW-only */ if (!start->cowonly) { - rb_node = tree_insert(&cache->rb_root, start->bytenr, - &start->rb_node); + rb_node = rb_simple_insert(&cache->rb_root, start->bytenr, + &start->rb_node); if (rb_node) backref_tree_panic(rb_node, -EEXIST, start->bytenr); list_add_tail(&start->lower, &cache->leaves); @@ -1062,8 +1017,8 @@ static int finish_upper_links(struct btrfs_backref_cache *cache, /* Only cache non-COW-only (subvolume trees) tree blocks */ if (!upper->cowonly) { - rb_node = tree_insert(&cache->rb_root, upper->bytenr, - &upper->rb_node); + rb_node = rb_simple_insert(&cache->rb_root, upper->bytenr, + &upper->rb_node); if (rb_node) { backref_tree_panic(rb_node, -EEXIST, upper->bytenr); @@ -1311,7 +1266,7 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, if (cache->last_trans > 0) update_backref_cache(trans, cache); - rb_node = tree_search(&cache->rb_root, src->commit_root->start); + rb_node = rb_simple_search(&cache->rb_root, src->commit_root->start); if (rb_node) { node = rb_entry(rb_node, struct btrfs_backref_node, rb_node); if (node->detached) @@ -1321,8 +1276,8 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, } if (!node) { - rb_node = tree_search(&cache->rb_root, - reloc_root->commit_root->start); + rb_node = rb_simple_search(&cache->rb_root, + reloc_root->commit_root->start); if (rb_node) { node = rb_entry(rb_node, struct btrfs_backref_node, rb_node); @@ -1355,8 +1310,8 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, list_add_tail(&new_node->lower, &cache->leaves); } - rb_node = tree_insert(&cache->rb_root, new_node->bytenr, - &new_node->rb_node); + rb_node = rb_simple_insert(&cache->rb_root, new_node->bytenr, + &new_node->rb_node); if (rb_node) backref_tree_panic(rb_node, -EEXIST, new_node->bytenr); @@ -1396,8 +1351,8 @@ static int __must_check __add_reloc_root(struct btrfs_root *root) node->data = root; spin_lock(&rc->reloc_root_tree.lock); - rb_node = tree_insert(&rc->reloc_root_tree.rb_root, - node->bytenr, &node->rb_node); + rb_node = rb_simple_insert(&rc->reloc_root_tree.rb_root, + node->bytenr, &node->rb_node); spin_unlock(&rc->reloc_root_tree.lock); if (rb_node) { btrfs_panic(fs_info, -EEXIST, @@ -1423,8 +1378,8 @@ static void __del_reloc_root(struct btrfs_root *root) if (rc && root->node) { spin_lock(&rc->reloc_root_tree.lock); - rb_node = tree_search(&rc->reloc_root_tree.rb_root, - root->commit_root->start); + rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, + root->commit_root->start); if (rb_node) { node = rb_entry(rb_node, struct mapping_node, rb_node); rb_erase(&node->rb_node, &rc->reloc_root_tree.rb_root); @@ -1467,8 +1422,8 @@ static int __update_reloc_root(struct btrfs_root *root) struct reloc_control *rc = fs_info->reloc_ctl; spin_lock(&rc->reloc_root_tree.lock); - rb_node = tree_search(&rc->reloc_root_tree.rb_root, - root->commit_root->start); + rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, + root->commit_root->start); if (rb_node) { node = rb_entry(rb_node, struct mapping_node, rb_node); rb_erase(&node->rb_node, &rc->reloc_root_tree.rb_root); @@ -1481,8 +1436,8 @@ static int __update_reloc_root(struct btrfs_root *root) spin_lock(&rc->reloc_root_tree.lock); node->bytenr = root->node->start; - rb_node = tree_insert(&rc->reloc_root_tree.rb_root, - node->bytenr, &node->rb_node); + rb_node = rb_simple_insert(&rc->reloc_root_tree.rb_root, + node->bytenr, &node->rb_node); spin_unlock(&rc->reloc_root_tree.lock); if (rb_node) backref_tree_panic(rb_node, -EEXIST, node->bytenr); @@ -3640,7 +3595,7 @@ static int add_tree_block(struct reloc_control *rc, block->level = level; block->key_ready = 0; - rb_node = tree_insert(blocks, block->bytenr, &block->rb_node); + rb_node = rb_simple_insert(blocks, block->bytenr, &block->rb_node); if (rb_node) backref_tree_panic(rb_node, -EEXIST, block->bytenr); @@ -3663,7 +3618,7 @@ static int __add_tree_block(struct reloc_control *rc, if (tree_block_processed(bytenr, rc)) return 0; - if (tree_search(blocks, bytenr)) + if (rb_simple_search(blocks, bytenr)) return 0; path = btrfs_alloc_path(); From 584fb12187f087f4909f74fa91dbb61856107684 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 3 Mar 2020 13:14:41 +0800 Subject: [PATCH 0703/1043] btrfs: backref: rename and move backref_cache_init() Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 17 +++++++++++++++++ fs/btrfs/backref.h | 3 +++ fs/btrfs/relocation.c | 18 +----------------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 27f9a5923796..9c5097dcd5a3 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -2465,3 +2465,20 @@ int btrfs_backref_iter_next(struct btrfs_backref_iter *iter) path->slots[0]); return 0; } + +void btrfs_backref_init_cache(struct btrfs_fs_info *fs_info, + struct btrfs_backref_cache *cache, int is_reloc) +{ + int i; + + cache->rb_root = RB_ROOT; + for (i = 0; i < BTRFS_MAX_LEVEL; i++) + INIT_LIST_HEAD(&cache->pending[i]); + INIT_LIST_HEAD(&cache->changed); + INIT_LIST_HEAD(&cache->detached); + INIT_LIST_HEAD(&cache->leaves); + INIT_LIST_HEAD(&cache->pending_edge); + INIT_LIST_HEAD(&cache->useless_node); + cache->fs_info = fs_info; + cache->is_reloc = is_reloc; +} diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 0d3fb76364c6..70d5a1fdc30e 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -268,4 +268,7 @@ struct btrfs_backref_cache { unsigned int is_reloc; }; +void btrfs_backref_init_cache(struct btrfs_fs_info *fs_info, + struct btrfs_backref_cache *cache, int is_reloc); + #endif diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 09076ac21590..f0726b212c2f 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -178,22 +178,6 @@ static void mapping_tree_init(struct mapping_tree *tree) spin_lock_init(&tree->lock); } -static void backref_cache_init(struct btrfs_fs_info *fs_info, - struct btrfs_backref_cache *cache, int is_reloc) -{ - int i; - cache->rb_root = RB_ROOT; - for (i = 0; i < BTRFS_MAX_LEVEL; i++) - INIT_LIST_HEAD(&cache->pending[i]); - INIT_LIST_HEAD(&cache->changed); - INIT_LIST_HEAD(&cache->detached); - INIT_LIST_HEAD(&cache->leaves); - INIT_LIST_HEAD(&cache->pending_edge); - INIT_LIST_HEAD(&cache->useless_node); - cache->fs_info = fs_info; - cache->is_reloc = is_reloc; -} - static void backref_cache_cleanup(struct btrfs_backref_cache *cache) { struct btrfs_backref_node *node; @@ -4231,7 +4215,7 @@ static struct reloc_control *alloc_reloc_control(struct btrfs_fs_info *fs_info) INIT_LIST_HEAD(&rc->reloc_roots); INIT_LIST_HEAD(&rc->dirty_subvol_roots); - backref_cache_init(fs_info, &rc->backref_cache, 1); + btrfs_backref_init_cache(fs_info, &rc->backref_cache, 1); mapping_tree_init(&rc->reloc_root_tree); extent_io_tree_init(fs_info, &rc->processed_blocks, IO_TREE_RELOC_BLOCKS, NULL); From b1818dab9bda1da8f3ea5a13230b5d91ae964f00 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 3 Mar 2020 13:21:30 +0800 Subject: [PATCH 0704/1043] btrfs: backref: rename and move alloc_backref_node() Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 21 +++++++++++++++++++++ fs/btrfs/backref.h | 2 ++ fs/btrfs/relocation.c | 32 ++++++-------------------------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 9c5097dcd5a3..dbe170de3516 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -2482,3 +2482,24 @@ void btrfs_backref_init_cache(struct btrfs_fs_info *fs_info, cache->fs_info = fs_info; cache->is_reloc = is_reloc; } + +struct btrfs_backref_node *btrfs_backref_alloc_node( + struct btrfs_backref_cache *cache, u64 bytenr, int level) +{ + struct btrfs_backref_node *node; + + ASSERT(level >= 0 && level < BTRFS_MAX_LEVEL); + node = kzalloc(sizeof(*node), GFP_NOFS); + if (!node) + return node; + + INIT_LIST_HEAD(&node->list); + INIT_LIST_HEAD(&node->upper); + INIT_LIST_HEAD(&node->lower); + RB_CLEAR_NODE(&node->rb_node); + cache->nr_nodes++; + node->level = level; + node->bytenr = bytenr; + + return node; +} diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 70d5a1fdc30e..94d49f2177d9 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -270,5 +270,7 @@ struct btrfs_backref_cache { void btrfs_backref_init_cache(struct btrfs_fs_info *fs_info, struct btrfs_backref_cache *cache, int is_reloc); +struct btrfs_backref_node *btrfs_backref_alloc_node( + struct btrfs_backref_cache *cache, u64 bytenr, int level); #endif diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index f0726b212c2f..da0b7f7dc062 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -208,27 +208,6 @@ static void backref_cache_cleanup(struct btrfs_backref_cache *cache) ASSERT(!cache->nr_edges); } -static struct btrfs_backref_node *alloc_backref_node( - struct btrfs_backref_cache *cache, u64 bytenr, int level) -{ - struct btrfs_backref_node *node; - - ASSERT(level >= 0 && level < BTRFS_MAX_LEVEL); - node = kzalloc(sizeof(*node), GFP_NOFS); - if (!node) - return node; - - INIT_LIST_HEAD(&node->list); - INIT_LIST_HEAD(&node->upper); - INIT_LIST_HEAD(&node->lower); - RB_CLEAR_NODE(&node->rb_node); - cache->nr_nodes++; - node->level = level; - node->bytenr = bytenr; - - return node; -} - static void free_backref_node(struct btrfs_backref_cache *cache, struct btrfs_backref_node *node) { @@ -609,7 +588,7 @@ static int handle_direct_tree_backref(struct btrfs_backref_cache *cache, rb_node = rb_simple_search(&cache->rb_root, ref_key->offset); if (!rb_node) { /* Parent node not yet cached */ - upper = alloc_backref_node(cache, ref_key->offset, + upper = btrfs_backref_alloc_node(cache, ref_key->offset, cur->level + 1); if (!upper) { free_backref_edge(cache, edge); @@ -729,8 +708,8 @@ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, eb = path->nodes[level]; rb_node = rb_simple_search(&cache->rb_root, eb->start); if (!rb_node) { - upper = alloc_backref_node(cache, eb->start, - lower->level + 1); + upper = btrfs_backref_alloc_node(cache, eb->start, + lower->level + 1); if (!upper) { btrfs_put_root(root); free_backref_edge(cache, edge); @@ -1135,7 +1114,7 @@ static noinline_for_stack struct btrfs_backref_node *build_backref_tree( goto out; } - node = alloc_backref_node(cache, bytenr, level); + node = btrfs_backref_alloc_node(cache, bytenr, level); if (!node) { err = -ENOMEM; goto out; @@ -1272,7 +1251,8 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, if (!node) return 0; - new_node = alloc_backref_node(cache, dest->node->start, node->level); + new_node = btrfs_backref_alloc_node(cache, dest->node->start, + node->level); if (!new_node) return -ENOMEM; From 47254d07f37590fddc1516006e3e79453e755424 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 3 Mar 2020 13:22:57 +0800 Subject: [PATCH 0705/1043] btrfs: backref: rename and move alloc_backref_edge() Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 11 +++++++++++ fs/btrfs/backref.h | 2 ++ fs/btrfs/relocation.c | 17 +++-------------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index dbe170de3516..4837dcf06ba4 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -2503,3 +2503,14 @@ struct btrfs_backref_node *btrfs_backref_alloc_node( return node; } + +struct btrfs_backref_edge *btrfs_backref_alloc_edge( + struct btrfs_backref_cache *cache) +{ + struct btrfs_backref_edge *edge; + + edge = kzalloc(sizeof(*edge), GFP_NOFS); + if (edge) + cache->nr_edges++; + return edge; +} diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 94d49f2177d9..4f140a0ebe1c 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -272,5 +272,7 @@ void btrfs_backref_init_cache(struct btrfs_fs_info *fs_info, struct btrfs_backref_cache *cache, int is_reloc); struct btrfs_backref_node *btrfs_backref_alloc_node( struct btrfs_backref_cache *cache, u64 bytenr, int level); +struct btrfs_backref_edge *btrfs_backref_alloc_edge( + struct btrfs_backref_cache *cache); #endif diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index da0b7f7dc062..8b22b5966039 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -218,17 +218,6 @@ static void free_backref_node(struct btrfs_backref_cache *cache, } } -static struct btrfs_backref_edge *alloc_backref_edge( - struct btrfs_backref_cache *cache) -{ - struct btrfs_backref_edge *edge; - - edge = kzalloc(sizeof(*edge), GFP_NOFS); - if (edge) - cache->nr_edges++; - return edge; -} - #define LINK_LOWER (1 << 0) #define LINK_UPPER (1 << 1) static void link_backref_edge(struct btrfs_backref_edge *edge, @@ -581,7 +570,7 @@ static int handle_direct_tree_backref(struct btrfs_backref_cache *cache, return 0; } - edge = alloc_backref_edge(cache); + edge = btrfs_backref_alloc_edge(cache); if (!edge) return -ENOMEM; @@ -698,7 +687,7 @@ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, break; } - edge = alloc_backref_edge(cache); + edge = btrfs_backref_alloc_edge(cache); if (!edge) { btrfs_put_root(root); ret = -ENOMEM; @@ -1263,7 +1252,7 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, if (!node->lowest) { list_for_each_entry(edge, &node->lower, list[UPPER]) { - new_edge = alloc_backref_edge(cache); + new_edge = btrfs_backref_alloc_edge(cache); if (!new_edge) goto fail; From f39911e5527e8be7ab56cadde306237092bcc78f Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 3 Mar 2020 13:24:06 +0800 Subject: [PATCH 0706/1043] btrfs: backref: rename and move link_backref_edge() Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.h | 16 ++++++++++++++++ fs/btrfs/relocation.c | 23 ++++------------------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 4f140a0ebe1c..01c559543133 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -275,4 +275,20 @@ struct btrfs_backref_node *btrfs_backref_alloc_node( struct btrfs_backref_edge *btrfs_backref_alloc_edge( struct btrfs_backref_cache *cache); +#define LINK_LOWER (1 << 0) +#define LINK_UPPER (1 << 1) +static inline void btrfs_backref_link_edge(struct btrfs_backref_edge *edge, + struct btrfs_backref_node *lower, + struct btrfs_backref_node *upper, + int link_which) +{ + ASSERT(upper && lower && upper->level == lower->level + 1); + edge->node[LOWER] = lower; + edge->node[UPPER] = upper; + if (link_which & LINK_LOWER) + list_add_tail(&edge->list[LOWER], &lower->upper); + if (link_which & LINK_UPPER) + list_add_tail(&edge->list[UPPER], &upper->lower); +} + #endif diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 8b22b5966039..fd6a6e0b8849 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -218,21 +218,6 @@ static void free_backref_node(struct btrfs_backref_cache *cache, } } -#define LINK_LOWER (1 << 0) -#define LINK_UPPER (1 << 1) -static void link_backref_edge(struct btrfs_backref_edge *edge, - struct btrfs_backref_node *lower, - struct btrfs_backref_node *upper, - int link_which) -{ - ASSERT(upper && lower && upper->level == lower->level + 1); - edge->node[LOWER] = lower; - edge->node[UPPER] = upper; - if (link_which & LINK_LOWER) - list_add_tail(&edge->list[LOWER], &lower->upper); - if (link_which & LINK_UPPER) - list_add_tail(&edge->list[UPPER], &upper->lower); -} static void free_backref_edge(struct btrfs_backref_cache *cache, struct btrfs_backref_edge *edge) @@ -595,7 +580,7 @@ static int handle_direct_tree_backref(struct btrfs_backref_cache *cache, ASSERT(upper->checked); INIT_LIST_HEAD(&edge->list[UPPER]); } - link_backref_edge(edge, cur, upper, LINK_LOWER); + btrfs_backref_link_edge(edge, cur, upper, LINK_LOWER); return 0; } @@ -740,7 +725,7 @@ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, if (!upper->owner) upper->owner = btrfs_header_owner(eb); } - link_backref_edge(edge, lower, upper, LINK_LOWER); + btrfs_backref_link_edge(edge, lower, upper, LINK_LOWER); if (rb_node) { btrfs_put_root(root); @@ -1256,8 +1241,8 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, if (!new_edge) goto fail; - link_backref_edge(new_edge, edge->node[LOWER], new_node, - LINK_UPPER); + btrfs_backref_link_edge(new_edge, edge->node[LOWER], + new_node, LINK_UPPER); } } else { list_add_tail(&new_node->lower, &cache->leaves); From 741188d3a549af328cc7946ce9650dd33a25087b Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 3 Mar 2020 13:26:12 +0800 Subject: [PATCH 0707/1043] btrfs: backref: rename and move free_backref_(node|edge) Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.h | 20 ++++++++++++++++++++ fs/btrfs/relocation.c | 42 +++++++++++------------------------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 01c559543133..0a1f296b1f26 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -8,6 +8,7 @@ #include #include "ulist.h" +#include "disk-io.h" #include "extent_io.h" struct inode_fs_paths { @@ -291,4 +292,23 @@ static inline void btrfs_backref_link_edge(struct btrfs_backref_edge *edge, list_add_tail(&edge->list[UPPER], &upper->lower); } +static inline void btrfs_backref_free_node(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node) +{ + if (node) { + cache->nr_nodes--; + btrfs_put_root(node->root); + kfree(node); + } +} + +static inline void btrfs_backref_free_edge(struct btrfs_backref_cache *cache, + struct btrfs_backref_edge *edge) +{ + if (edge) { + cache->nr_edges--; + kfree(edge); + } +} + #endif diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index fd6a6e0b8849..ec35348c0fae 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -208,26 +208,6 @@ static void backref_cache_cleanup(struct btrfs_backref_cache *cache) ASSERT(!cache->nr_edges); } -static void free_backref_node(struct btrfs_backref_cache *cache, - struct btrfs_backref_node *node) -{ - if (node) { - cache->nr_nodes--; - btrfs_put_root(node->root); - kfree(node); - } -} - - -static void free_backref_edge(struct btrfs_backref_cache *cache, - struct btrfs_backref_edge *edge) -{ - if (edge) { - cache->nr_edges--; - kfree(edge); - } -} - static void backref_tree_panic(struct rb_node *rb_node, int errno, u64 bytenr) { @@ -316,7 +296,7 @@ static void drop_backref_node(struct btrfs_backref_cache *tree, list_del(&node->lower); if (!RB_EMPTY_NODE(&node->rb_node)) rb_erase(&node->rb_node, &tree->rb_root); - free_backref_node(tree, node); + btrfs_backref_free_node(tree, node); } /* @@ -338,7 +318,7 @@ static void remove_backref_node(struct btrfs_backref_cache *cache, upper = edge->node[UPPER]; list_del(&edge->list[LOWER]); list_del(&edge->list[UPPER]); - free_backref_edge(cache, edge); + btrfs_backref_free_edge(cache, edge); if (RB_EMPTY_NODE(&upper->rb_node)) { BUG_ON(!list_empty(&node->upper)); @@ -565,7 +545,7 @@ static int handle_direct_tree_backref(struct btrfs_backref_cache *cache, upper = btrfs_backref_alloc_node(cache, ref_key->offset, cur->level + 1); if (!upper) { - free_backref_edge(cache, edge); + btrfs_backref_free_edge(cache, edge); return -ENOMEM; } @@ -686,7 +666,7 @@ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, lower->level + 1); if (!upper) { btrfs_put_root(root); - free_backref_edge(cache, edge); + btrfs_backref_free_edge(cache, edge); ret = -ENOMEM; goto out; } @@ -915,7 +895,7 @@ static int finish_upper_links(struct btrfs_backref_cache *cache, /* Parent is detached, no need to keep any edges */ if (upper->detached) { list_del(&edge->list[LOWER]); - free_backref_edge(cache, edge); + btrfs_backref_free_edge(cache, edge); /* Lower node is orphan, queue for cleanup */ if (list_empty(&lower->upper)) @@ -1024,7 +1004,7 @@ static bool handle_useless_nodes(struct reloc_control *rc, list_del(&edge->list[UPPER]); list_del(&edge->list[LOWER]); lower = edge->node[LOWER]; - free_backref_edge(cache, edge); + btrfs_backref_free_edge(cache, edge); /* Child node is also orphan, queue for cleanup */ if (list_empty(&lower->upper)) @@ -1043,7 +1023,7 @@ static bool handle_useless_nodes(struct reloc_control *rc, cur->detached = 1; } else { rb_erase(&cur->rb_node, &cache->rb_root); - free_backref_node(cache, cur); + btrfs_backref_free_node(cache, cur); } } return ret; @@ -1141,7 +1121,7 @@ out: list_del(&edge->list[LOWER]); lower = edge->node[LOWER]; upper = edge->node[UPPER]; - free_backref_edge(cache, edge); + btrfs_backref_free_edge(cache, edge); /* * Lower is no longer linked to any upper backref nodes @@ -1168,7 +1148,7 @@ out: list_del_init(&lower->list); if (lower == node) node = NULL; - free_backref_node(cache, lower); + btrfs_backref_free_node(cache, lower); } remove_backref_node(cache, node); @@ -1265,9 +1245,9 @@ fail: new_edge = list_entry(new_node->lower.next, struct btrfs_backref_edge, list[UPPER]); list_del(&new_edge->list[UPPER]); - free_backref_edge(cache, new_edge); + btrfs_backref_free_edge(cache, new_edge); } - free_backref_node(cache, new_node); + btrfs_backref_free_node(cache, new_node); return -ENOMEM; } From b0fe7078d62c23bc94d0203887d9ad7b128f684b Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 3 Mar 2020 13:35:27 +0800 Subject: [PATCH 0708/1043] btrfs: backref: rename and move drop_backref_node() With extra comment for drop_backref_node() as it has some similarity with remove_backref_node(), thus we need extra comment explaining the difference. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.h | 39 +++++++++++++++++++++++++++++++++++++ fs/btrfs/relocation.c | 45 +++++++------------------------------------ 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 0a1f296b1f26..a14c79bbbeb9 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -311,4 +311,43 @@ static inline void btrfs_backref_free_edge(struct btrfs_backref_cache *cache, } } +static inline void btrfs_backref_unlock_node_buffer( + struct btrfs_backref_node *node) +{ + if (node->locked) { + btrfs_tree_unlock(node->eb); + node->locked = 0; + } +} + +static inline void btrfs_backref_drop_node_buffer( + struct btrfs_backref_node *node) +{ + if (node->eb) { + btrfs_backref_unlock_node_buffer(node); + free_extent_buffer(node->eb); + node->eb = NULL; + } +} + +/* + * Drop the backref node from cache without cleaning up its children + * edges. + * + * This can only be called on node without parent edges. + * The children edges are still kept as is. + */ +static inline void btrfs_backref_drop_node(struct btrfs_backref_cache *tree, + struct btrfs_backref_node *node) +{ + BUG_ON(!list_empty(&node->upper)); + + btrfs_backref_drop_node_buffer(node); + list_del(&node->list); + list_del(&node->lower); + if (!RB_EMPTY_NODE(&node->rb_node)) + rb_erase(&node->rb_node, &tree->rb_root); + btrfs_backref_free_node(tree, node); +} + #endif diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index ec35348c0fae..d23b9a54e002 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -268,37 +268,6 @@ static struct btrfs_backref_node *walk_down_backref( *index = 0; return NULL; } - -static void unlock_node_buffer(struct btrfs_backref_node *node) -{ - if (node->locked) { - btrfs_tree_unlock(node->eb); - node->locked = 0; - } -} - -static void drop_node_buffer(struct btrfs_backref_node *node) -{ - if (node->eb) { - unlock_node_buffer(node); - free_extent_buffer(node->eb); - node->eb = NULL; - } -} - -static void drop_backref_node(struct btrfs_backref_cache *tree, - struct btrfs_backref_node *node) -{ - BUG_ON(!list_empty(&node->upper)); - - drop_node_buffer(node); - list_del(&node->list); - list_del(&node->lower); - if (!RB_EMPTY_NODE(&node->rb_node)) - rb_erase(&node->rb_node, &tree->rb_root); - btrfs_backref_free_node(tree, node); -} - /* * remove a backref node from the backref cache */ @@ -322,7 +291,7 @@ static void remove_backref_node(struct btrfs_backref_cache *cache, if (RB_EMPTY_NODE(&upper->rb_node)) { BUG_ON(!list_empty(&node->upper)); - drop_backref_node(cache, node); + btrfs_backref_drop_node(cache, node); node = upper; node->lowest = 1; continue; @@ -337,7 +306,7 @@ static void remove_backref_node(struct btrfs_backref_cache *cache, } } - drop_backref_node(cache, node); + btrfs_backref_drop_node(cache, node); } static void update_backref_node(struct btrfs_backref_cache *cache, @@ -2859,7 +2828,7 @@ static int do_relocation(struct btrfs_trans_handle *trans, if (node->eb->start == bytenr) goto next; } - drop_node_buffer(upper); + btrfs_backref_drop_node_buffer(upper); } if (!upper->eb) { @@ -2958,15 +2927,15 @@ static int do_relocation(struct btrfs_trans_handle *trans, } next: if (!upper->pending) - drop_node_buffer(upper); + btrfs_backref_drop_node_buffer(upper); else - unlock_node_buffer(upper); + btrfs_backref_unlock_node_buffer(upper); if (err) break; } if (!err && node->pending) { - drop_node_buffer(node); + btrfs_backref_drop_node_buffer(node); list_move_tail(&node->list, &rc->backref_cache.changed); node->pending = 0; } @@ -4591,7 +4560,7 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, BUG_ON(node->bytenr != buf->start && node->new_bytenr != buf->start); - drop_node_buffer(node); + btrfs_backref_drop_node_buffer(node); atomic_inc(&cow->refs); node->eb = cow; node->new_bytenr = cow->start; From 023acb07bc996636b39c70e07966f70e70c0a008 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 23 Mar 2020 15:42:25 +0800 Subject: [PATCH 0709/1043] btrfs: backref: rename and move remove_backref_node() Also add comment explaining the cleanup progress, to differ it from btrfs_backref_drop_node(). Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 45 ++++++++++++++++++++++++++++++++++++ fs/btrfs/backref.h | 3 +++ fs/btrfs/relocation.c | 53 ++++--------------------------------------- 3 files changed, 53 insertions(+), 48 deletions(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 4837dcf06ba4..daac20c1a417 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -2514,3 +2514,48 @@ struct btrfs_backref_edge *btrfs_backref_alloc_edge( cache->nr_edges++; return edge; } + +/* + * Drop the backref node from cache, also cleaning up all its + * upper edges and any uncached nodes in the path. + * + * This cleanup happens bottom up, thus the node should either + * be the lowest node in the cache or a detached node. + */ +void btrfs_backref_cleanup_node(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node) +{ + struct btrfs_backref_node *upper; + struct btrfs_backref_edge *edge; + + if (!node) + return; + + BUG_ON(!node->lowest && !node->detached); + while (!list_empty(&node->upper)) { + edge = list_entry(node->upper.next, struct btrfs_backref_edge, + list[LOWER]); + upper = edge->node[UPPER]; + list_del(&edge->list[LOWER]); + list_del(&edge->list[UPPER]); + btrfs_backref_free_edge(cache, edge); + + if (RB_EMPTY_NODE(&upper->rb_node)) { + BUG_ON(!list_empty(&node->upper)); + btrfs_backref_drop_node(cache, node); + node = upper; + node->lowest = 1; + continue; + } + /* + * Add the node to leaf node list if no other child block + * cached. + */ + if (list_empty(&upper->lower)) { + list_add_tail(&upper->lower, &cache->leaves); + upper->lowest = 1; + } + } + + btrfs_backref_drop_node(cache, node); +} diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index a14c79bbbeb9..1710cf994f4c 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -350,4 +350,7 @@ static inline void btrfs_backref_drop_node(struct btrfs_backref_cache *tree, btrfs_backref_free_node(tree, node); } +void btrfs_backref_cleanup_node(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node); + #endif diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index d23b9a54e002..4b7d7d54c64b 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -153,9 +153,6 @@ struct reloc_control { #define MOVE_DATA_EXTENTS 0 #define UPDATE_DATA_PTRS 1 -static void remove_backref_node(struct btrfs_backref_cache *cache, - struct btrfs_backref_node *node); - static void mark_block_processed(struct reloc_control *rc, struct btrfs_backref_node *node) { @@ -186,13 +183,13 @@ static void backref_cache_cleanup(struct btrfs_backref_cache *cache) while (!list_empty(&cache->detached)) { node = list_entry(cache->detached.next, struct btrfs_backref_node, list); - remove_backref_node(cache, node); + btrfs_backref_cleanup_node(cache, node); } while (!list_empty(&cache->leaves)) { node = list_entry(cache->leaves.next, struct btrfs_backref_node, lower); - remove_backref_node(cache, node); + btrfs_backref_cleanup_node(cache, node); } cache->last_trans = 0; @@ -268,46 +265,6 @@ static struct btrfs_backref_node *walk_down_backref( *index = 0; return NULL; } -/* - * remove a backref node from the backref cache - */ -static void remove_backref_node(struct btrfs_backref_cache *cache, - struct btrfs_backref_node *node) -{ - struct btrfs_backref_node *upper; - struct btrfs_backref_edge *edge; - - if (!node) - return; - - BUG_ON(!node->lowest && !node->detached); - while (!list_empty(&node->upper)) { - edge = list_entry(node->upper.next, struct btrfs_backref_edge, - list[LOWER]); - upper = edge->node[UPPER]; - list_del(&edge->list[LOWER]); - list_del(&edge->list[UPPER]); - btrfs_backref_free_edge(cache, edge); - - if (RB_EMPTY_NODE(&upper->rb_node)) { - BUG_ON(!list_empty(&node->upper)); - btrfs_backref_drop_node(cache, node); - node = upper; - node->lowest = 1; - continue; - } - /* - * add the node to leaf node list if no other - * child block cached. - */ - if (list_empty(&upper->lower)) { - list_add_tail(&upper->lower, &cache->leaves); - upper->lowest = 1; - } - } - - btrfs_backref_drop_node(cache, node); -} static void update_backref_node(struct btrfs_backref_cache *cache, struct btrfs_backref_node *node, u64 bytenr) @@ -345,7 +302,7 @@ static int update_backref_cache(struct btrfs_trans_handle *trans, while (!list_empty(&cache->detached)) { node = list_entry(cache->detached.next, struct btrfs_backref_node, list); - remove_backref_node(cache, node); + btrfs_backref_cleanup_node(cache, node); } while (!list_empty(&cache->changed)) { @@ -1120,7 +1077,7 @@ out: btrfs_backref_free_node(cache, lower); } - remove_backref_node(cache, node); + btrfs_backref_cleanup_node(cache, node); ASSERT(list_empty(&cache->useless_node) && list_empty(&cache->pending_edge)); return ERR_PTR(err); @@ -3103,7 +3060,7 @@ static int relocate_tree_block(struct btrfs_trans_handle *trans, } out: if (ret || node->level == 0 || node->cowonly) - remove_backref_node(&rc->backref_cache, node); + btrfs_backref_cleanup_node(&rc->backref_cache, node); return ret; } From 13fe1bdb22f8302afa5e588e89a7a94606ef35fb Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 3 Mar 2020 13:55:12 +0800 Subject: [PATCH 0710/1043] btrfs: backref: rename and move backref_cache_cleanup() Since we're releasing all existing nodes/edges, other than cleanup the mess after error, "release" is a more proper naming here. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 33 +++++++++++++++++++++++++++++++++ fs/btrfs/backref.h | 2 ++ fs/btrfs/relocation.c | 32 +------------------------------- 3 files changed, 36 insertions(+), 31 deletions(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index daac20c1a417..44808a0b480f 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -2559,3 +2559,36 @@ void btrfs_backref_cleanup_node(struct btrfs_backref_cache *cache, btrfs_backref_drop_node(cache, node); } + +/* + * Release all nodes/edges from current cache + */ +void btrfs_backref_release_cache(struct btrfs_backref_cache *cache) +{ + struct btrfs_backref_node *node; + int i; + + while (!list_empty(&cache->detached)) { + node = list_entry(cache->detached.next, + struct btrfs_backref_node, list); + btrfs_backref_cleanup_node(cache, node); + } + + while (!list_empty(&cache->leaves)) { + node = list_entry(cache->leaves.next, + struct btrfs_backref_node, lower); + btrfs_backref_cleanup_node(cache, node); + } + + cache->last_trans = 0; + + for (i = 0; i < BTRFS_MAX_LEVEL; i++) + ASSERT(list_empty(&cache->pending[i])); + ASSERT(list_empty(&cache->pending_edge)); + ASSERT(list_empty(&cache->useless_node)); + ASSERT(list_empty(&cache->changed)); + ASSERT(list_empty(&cache->detached)); + ASSERT(RB_EMPTY_ROOT(&cache->rb_root)); + ASSERT(!cache->nr_nodes); + ASSERT(!cache->nr_edges); +} diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 1710cf994f4c..5fb38dfb6440 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -353,4 +353,6 @@ static inline void btrfs_backref_drop_node(struct btrfs_backref_cache *tree, void btrfs_backref_cleanup_node(struct btrfs_backref_cache *cache, struct btrfs_backref_node *node); +void btrfs_backref_release_cache(struct btrfs_backref_cache *cache); + #endif diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 4b7d7d54c64b..19481fd4afe6 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -175,36 +175,6 @@ static void mapping_tree_init(struct mapping_tree *tree) spin_lock_init(&tree->lock); } -static void backref_cache_cleanup(struct btrfs_backref_cache *cache) -{ - struct btrfs_backref_node *node; - int i; - - while (!list_empty(&cache->detached)) { - node = list_entry(cache->detached.next, - struct btrfs_backref_node, list); - btrfs_backref_cleanup_node(cache, node); - } - - while (!list_empty(&cache->leaves)) { - node = list_entry(cache->leaves.next, - struct btrfs_backref_node, lower); - btrfs_backref_cleanup_node(cache, node); - } - - cache->last_trans = 0; - - for (i = 0; i < BTRFS_MAX_LEVEL; i++) - ASSERT(list_empty(&cache->pending[i])); - ASSERT(list_empty(&cache->pending_edge)); - ASSERT(list_empty(&cache->useless_node)); - ASSERT(list_empty(&cache->changed)); - ASSERT(list_empty(&cache->detached)); - ASSERT(RB_EMPTY_ROOT(&cache->rb_root)); - ASSERT(!cache->nr_nodes); - ASSERT(!cache->nr_edges); -} - static void backref_tree_panic(struct rb_node *rb_node, int errno, u64 bytenr) { @@ -3948,7 +3918,7 @@ restart: rc->create_reloc_tree = 0; set_reloc_control(rc); - backref_cache_cleanup(&rc->backref_cache); + btrfs_backref_release_cache(&rc->backref_cache); btrfs_block_rsv_release(fs_info, rc->block_rsv, (u64)-1, NULL); /* From 982c92cbd51cf6ab1319e1e94dfd4902d3678d9a Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 26 Mar 2020 14:21:36 +0800 Subject: [PATCH 0711/1043] btrfs: backref: rename and move backref_tree_panic() Also change the parameter, since all callers can easily grab an fs_info, there is no need for all the pointer chasing. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.h | 8 ++++++++ fs/btrfs/relocation.c | 29 +++++++++-------------------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 5fb38dfb6440..a7697de925d5 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -355,4 +355,12 @@ void btrfs_backref_cleanup_node(struct btrfs_backref_cache *cache, void btrfs_backref_release_cache(struct btrfs_backref_cache *cache); +static inline void btrfs_backref_panic(struct btrfs_fs_info *fs_info, + u64 bytenr, int errno) +{ + btrfs_panic(fs_info, errno, + "Inconsistency in backref cache found at offset %llu", + bytenr); +} + #endif diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 19481fd4afe6..6f712b22c66f 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -175,19 +175,6 @@ static void mapping_tree_init(struct mapping_tree *tree) spin_lock_init(&tree->lock); } -static void backref_tree_panic(struct rb_node *rb_node, int errno, u64 bytenr) -{ - - struct btrfs_fs_info *fs_info = NULL; - struct btrfs_backref_node *bnode = rb_entry(rb_node, - struct btrfs_backref_node, rb_node); - if (bnode->root) - fs_info = bnode->root->fs_info; - btrfs_panic(fs_info, errno, - "Inconsistency in backref cache found at offset %llu", - bytenr); -} - /* * walk up backref nodes until reach node presents tree root */ @@ -244,7 +231,7 @@ static void update_backref_node(struct btrfs_backref_cache *cache, node->bytenr = bytenr; rb_node = rb_simple_insert(&cache->rb_root, node->bytenr, &node->rb_node); if (rb_node) - backref_tree_panic(rb_node, -EEXIST, bytenr); + btrfs_backref_panic(cache->fs_info, bytenr, -EEXIST); } /* @@ -765,7 +752,8 @@ static int finish_upper_links(struct btrfs_backref_cache *cache, rb_node = rb_simple_insert(&cache->rb_root, start->bytenr, &start->rb_node); if (rb_node) - backref_tree_panic(rb_node, -EEXIST, start->bytenr); + btrfs_backref_panic(cache->fs_info, start->bytenr, + -EEXIST); list_add_tail(&start->lower, &cache->leaves); } @@ -833,8 +821,8 @@ static int finish_upper_links(struct btrfs_backref_cache *cache, rb_node = rb_simple_insert(&cache->rb_root, upper->bytenr, &upper->rb_node); if (rb_node) { - backref_tree_panic(rb_node, -EEXIST, - upper->bytenr); + btrfs_backref_panic(cache->fs_info, + upper->bytenr, -EEXIST); return -EUCLEAN; } } @@ -1127,7 +1115,7 @@ static int clone_backref_node(struct btrfs_trans_handle *trans, rb_node = rb_simple_insert(&cache->rb_root, new_node->bytenr, &new_node->rb_node); if (rb_node) - backref_tree_panic(rb_node, -EEXIST, new_node->bytenr); + btrfs_backref_panic(trans->fs_info, new_node->bytenr, -EEXIST); if (!new_node->lowest) { list_for_each_entry(new_edge, &new_node->lower, list[UPPER]) { @@ -1254,7 +1242,7 @@ static int __update_reloc_root(struct btrfs_root *root) node->bytenr, &node->rb_node); spin_unlock(&rc->reloc_root_tree.lock); if (rb_node) - backref_tree_panic(rb_node, -EEXIST, node->bytenr); + btrfs_backref_panic(fs_info, node->bytenr, -EEXIST); return 0; } @@ -3411,7 +3399,8 @@ static int add_tree_block(struct reloc_control *rc, rb_node = rb_simple_insert(blocks, block->bytenr, &block->rb_node); if (rb_node) - backref_tree_panic(rb_node, -EEXIST, block->bytenr); + btrfs_backref_panic(rc->extent_root->fs_info, block->bytenr, + -EEXIST); return 0; } From 55465730bcea75606c2c281ca55701c7fc20a000 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 3 Mar 2020 14:26:02 +0800 Subject: [PATCH 0712/1043] btrfs: backref: rename and move should_ignore_root() This function is mostly single purpose to relocation backref cache, but since we're moving the main part of backref cache to backref.c, we need to export such function. And to avoid confusion, rename the function to btrfs_should_ignore_reloc_root() make the name a little more clear. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 1 + fs/btrfs/relocation.c | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 1e8a0a513e73..01b03e8a671f 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -3383,6 +3383,7 @@ int btrfs_reloc_post_snapshot(struct btrfs_trans_handle *trans, int btrfs_should_cancel_balance(struct btrfs_fs_info *fs_info); struct btrfs_root *find_reloc_root(struct btrfs_fs_info *fs_info, u64 bytenr); +int btrfs_should_ignore_reloc_root(struct btrfs_root *root); /* scrub.c */ int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start, diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 6f712b22c66f..2a0a734a5d88 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -305,7 +305,8 @@ static bool reloc_root_is_dead(struct btrfs_root *root) * * Reloc tree after swap is considered dead, thus not considered as valid. * This is enough for most callers, as they don't distinguish dead reloc root - * from no reloc root. But should_ignore_root() below is a special case. + * from no reloc root. But btrfs_should_ignore_reloc_root() below is a + * special case. */ static bool have_reloc_root(struct btrfs_root *root) { @@ -316,7 +317,7 @@ static bool have_reloc_root(struct btrfs_root *root) return true; } -static int should_ignore_root(struct btrfs_root *root) +int btrfs_should_ignore_reloc_root(struct btrfs_root *root) { struct btrfs_root *reloc_root; @@ -342,6 +343,7 @@ static int should_ignore_root(struct btrfs_root *root) */ return 1; } + /* * find reloc tree by address of tree root */ @@ -485,7 +487,7 @@ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, if (btrfs_root_level(&root->root_item) == cur->level) { /* Tree root */ ASSERT(btrfs_root_bytenr(&root->root_item) == cur->bytenr); - if (should_ignore_root(root)) { + if (btrfs_should_ignore_reloc_root(root)) { btrfs_put_root(root); list_add(&cur->list, &cache->useless_node); } else { @@ -526,7 +528,7 @@ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, if (!path->nodes[level]) { ASSERT(btrfs_root_bytenr(&root->root_item) == lower->bytenr); - if (should_ignore_root(root)) { + if (btrfs_should_ignore_reloc_root(root)) { btrfs_put_root(root); list_add(&lower->list, &cache->useless_node); } else { From d36e7f0e8fedd0675789b4fc5869d8d48d33e18a Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 5 Mar 2020 15:16:09 +0800 Subject: [PATCH 0713/1043] btrfs: reloc: open code read_fs_root() for handle_indirect_tree_backref() The backref code is going to be moved to backref.c, and read_fs_root() is just a simple wrapper, open-code it to prepare to the incoming code move. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 2a0a734a5d88..b8fc07ad0a64 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -473,12 +473,16 @@ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, struct btrfs_backref_edge *edge; struct extent_buffer *eb; struct btrfs_root *root; + struct btrfs_key root_key; struct rb_node *rb_node; int level; bool need_check = true; int ret; - root = read_fs_root(fs_info, ref_key->offset); + root_key.objectid = ref_key->offset; + root_key.type = BTRFS_ROOT_ITEM_KEY; + root_key.offset = (u64)-1; + root = btrfs_get_fs_root(fs_info, &root_key, false); if (IS_ERR(root)) return PTR_ERR(root); if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) From 1b60d2ec982a35c2953d81d035e1d7fc7c89f42a Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 23 Mar 2020 16:08:34 +0800 Subject: [PATCH 0714/1043] btrfs: backref: rename and move handle_one_tree_block() This function is the major part of backref cache build process, move it to backref.c so we can reuse it later. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 365 ++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/backref.h | 6 + fs/btrfs/relocation.c | 357 +---------------------------------------- 3 files changed, 373 insertions(+), 355 deletions(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 44808a0b480f..832071bbb8c9 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -13,6 +13,7 @@ #include "transaction.h" #include "delayed-ref.h" #include "locking.h" +#include "misc.h" /* Just an arbitrary number so we can be sure this happened */ #define BACKREF_FOUND_SHARED 6 @@ -2592,3 +2593,367 @@ void btrfs_backref_release_cache(struct btrfs_backref_cache *cache) ASSERT(!cache->nr_nodes); ASSERT(!cache->nr_edges); } + +/* + * Handle direct tree backref + * + * Direct tree backref means, the backref item shows its parent bytenr + * directly. This is for SHARED_BLOCK_REF backref (keyed or inlined). + * + * @ref_key: The converted backref key. + * For keyed backref, it's the item key. + * For inlined backref, objectid is the bytenr, + * type is btrfs_inline_ref_type, offset is + * btrfs_inline_ref_offset. + */ +static int handle_direct_tree_backref(struct btrfs_backref_cache *cache, + struct btrfs_key *ref_key, + struct btrfs_backref_node *cur) +{ + struct btrfs_backref_edge *edge; + struct btrfs_backref_node *upper; + struct rb_node *rb_node; + + ASSERT(ref_key->type == BTRFS_SHARED_BLOCK_REF_KEY); + + /* Only reloc root uses backref pointing to itself */ + if (ref_key->objectid == ref_key->offset) { + struct btrfs_root *root; + + cur->is_reloc_root = 1; + /* Only reloc backref cache cares about a specific root */ + if (cache->is_reloc) { + root = find_reloc_root(cache->fs_info, cur->bytenr); + if (WARN_ON(!root)) + return -ENOENT; + cur->root = root; + } else { + /* + * For generic purpose backref cache, reloc root node + * is useless. + */ + list_add(&cur->list, &cache->useless_node); + } + return 0; + } + + edge = btrfs_backref_alloc_edge(cache); + if (!edge) + return -ENOMEM; + + rb_node = rb_simple_search(&cache->rb_root, ref_key->offset); + if (!rb_node) { + /* Parent node not yet cached */ + upper = btrfs_backref_alloc_node(cache, ref_key->offset, + cur->level + 1); + if (!upper) { + btrfs_backref_free_edge(cache, edge); + return -ENOMEM; + } + + /* + * Backrefs for the upper level block isn't cached, add the + * block to pending list + */ + list_add_tail(&edge->list[UPPER], &cache->pending_edge); + } else { + /* Parent node already cached */ + upper = rb_entry(rb_node, struct btrfs_backref_node, rb_node); + ASSERT(upper->checked); + INIT_LIST_HEAD(&edge->list[UPPER]); + } + btrfs_backref_link_edge(edge, cur, upper, LINK_LOWER); + return 0; +} + +/* + * Handle indirect tree backref + * + * Indirect tree backref means, we only know which tree the node belongs to. + * We still need to do a tree search to find out the parents. This is for + * TREE_BLOCK_REF backref (keyed or inlined). + * + * @ref_key: The same as @ref_key in handle_direct_tree_backref() + * @tree_key: The first key of this tree block. + * @path: A clean (released) path, to avoid allocating path everytime + * the function get called. + */ +static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, + struct btrfs_path *path, + struct btrfs_key *ref_key, + struct btrfs_key *tree_key, + struct btrfs_backref_node *cur) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct btrfs_backref_node *upper; + struct btrfs_backref_node *lower; + struct btrfs_backref_edge *edge; + struct extent_buffer *eb; + struct btrfs_root *root; + struct btrfs_key root_key; + struct rb_node *rb_node; + int level; + bool need_check = true; + int ret; + + root_key.objectid = ref_key->offset; + root_key.type = BTRFS_ROOT_ITEM_KEY; + root_key.offset = (u64)-1; + root = btrfs_get_fs_root(fs_info, &root_key, false); + if (IS_ERR(root)) + return PTR_ERR(root); + if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) + cur->cowonly = 1; + + if (btrfs_root_level(&root->root_item) == cur->level) { + /* Tree root */ + ASSERT(btrfs_root_bytenr(&root->root_item) == cur->bytenr); + if (btrfs_should_ignore_reloc_root(root)) { + btrfs_put_root(root); + list_add(&cur->list, &cache->useless_node); + } else { + cur->root = root; + } + return 0; + } + + level = cur->level + 1; + + /* Search the tree to find parent blocks referring to the block */ + path->search_commit_root = 1; + path->skip_locking = 1; + path->lowest_level = level; + ret = btrfs_search_slot(NULL, root, tree_key, path, 0, 0); + path->lowest_level = 0; + if (ret < 0) { + btrfs_put_root(root); + return ret; + } + if (ret > 0 && path->slots[level] > 0) + path->slots[level]--; + + eb = path->nodes[level]; + if (btrfs_node_blockptr(eb, path->slots[level]) != cur->bytenr) { + btrfs_err(fs_info, +"couldn't find block (%llu) (level %d) in tree (%llu) with key (%llu %u %llu)", + cur->bytenr, level - 1, root->root_key.objectid, + tree_key->objectid, tree_key->type, tree_key->offset); + btrfs_put_root(root); + ret = -ENOENT; + goto out; + } + lower = cur; + + /* Add all nodes and edges in the path */ + for (; level < BTRFS_MAX_LEVEL; level++) { + if (!path->nodes[level]) { + ASSERT(btrfs_root_bytenr(&root->root_item) == + lower->bytenr); + if (btrfs_should_ignore_reloc_root(root)) { + btrfs_put_root(root); + list_add(&lower->list, &cache->useless_node); + } else { + lower->root = root; + } + break; + } + + edge = btrfs_backref_alloc_edge(cache); + if (!edge) { + btrfs_put_root(root); + ret = -ENOMEM; + goto out; + } + + eb = path->nodes[level]; + rb_node = rb_simple_search(&cache->rb_root, eb->start); + if (!rb_node) { + upper = btrfs_backref_alloc_node(cache, eb->start, + lower->level + 1); + if (!upper) { + btrfs_put_root(root); + btrfs_backref_free_edge(cache, edge); + ret = -ENOMEM; + goto out; + } + upper->owner = btrfs_header_owner(eb); + if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) + upper->cowonly = 1; + + /* + * If we know the block isn't shared we can avoid + * checking its backrefs. + */ + if (btrfs_block_can_be_shared(root, eb)) + upper->checked = 0; + else + upper->checked = 1; + + /* + * Add the block to pending list if we need to check its + * backrefs, we only do this once while walking up a + * tree as we will catch anything else later on. + */ + if (!upper->checked && need_check) { + need_check = false; + list_add_tail(&edge->list[UPPER], + &cache->pending_edge); + } else { + if (upper->checked) + need_check = true; + INIT_LIST_HEAD(&edge->list[UPPER]); + } + } else { + upper = rb_entry(rb_node, struct btrfs_backref_node, + rb_node); + ASSERT(upper->checked); + INIT_LIST_HEAD(&edge->list[UPPER]); + if (!upper->owner) + upper->owner = btrfs_header_owner(eb); + } + btrfs_backref_link_edge(edge, lower, upper, LINK_LOWER); + + if (rb_node) { + btrfs_put_root(root); + break; + } + lower = upper; + upper = NULL; + } +out: + btrfs_release_path(path); + return ret; +} + +/* + * Add backref node @cur into @cache. + * + * NOTE: Even if the function returned 0, @cur is not yet cached as its upper + * links aren't yet bi-directional. Needs to finish such links. + * + * @path: Released path for indirect tree backref lookup + * @iter: Released backref iter for extent tree search + * @node_key: The first key of the tree block + */ +int btrfs_backref_add_tree_node(struct btrfs_backref_cache *cache, + struct btrfs_path *path, + struct btrfs_backref_iter *iter, + struct btrfs_key *node_key, + struct btrfs_backref_node *cur) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct btrfs_backref_edge *edge; + struct btrfs_backref_node *exist; + int ret; + + ret = btrfs_backref_iter_start(iter, cur->bytenr); + if (ret < 0) + return ret; + /* + * We skip the first btrfs_tree_block_info, as we don't use the key + * stored in it, but fetch it from the tree block + */ + if (btrfs_backref_has_tree_block_info(iter)) { + ret = btrfs_backref_iter_next(iter); + if (ret < 0) + goto out; + /* No extra backref? This means the tree block is corrupted */ + if (ret > 0) { + ret = -EUCLEAN; + goto out; + } + } + WARN_ON(cur->checked); + if (!list_empty(&cur->upper)) { + /* + * The backref was added previously when processing backref of + * type BTRFS_TREE_BLOCK_REF_KEY + */ + ASSERT(list_is_singular(&cur->upper)); + edge = list_entry(cur->upper.next, struct btrfs_backref_edge, + list[LOWER]); + ASSERT(list_empty(&edge->list[UPPER])); + exist = edge->node[UPPER]; + /* + * Add the upper level block to pending list if we need check + * its backrefs + */ + if (!exist->checked) + list_add_tail(&edge->list[UPPER], &cache->pending_edge); + } else { + exist = NULL; + } + + for (; ret == 0; ret = btrfs_backref_iter_next(iter)) { + struct extent_buffer *eb; + struct btrfs_key key; + int type; + + cond_resched(); + eb = btrfs_backref_get_eb(iter); + + key.objectid = iter->bytenr; + if (btrfs_backref_iter_is_inline_ref(iter)) { + struct btrfs_extent_inline_ref *iref; + + /* Update key for inline backref */ + iref = (struct btrfs_extent_inline_ref *) + ((unsigned long)iter->cur_ptr); + type = btrfs_get_extent_inline_ref_type(eb, iref, + BTRFS_REF_TYPE_BLOCK); + if (type == BTRFS_REF_TYPE_INVALID) { + ret = -EUCLEAN; + goto out; + } + key.type = type; + key.offset = btrfs_extent_inline_ref_offset(eb, iref); + } else { + key.type = iter->cur_key.type; + key.offset = iter->cur_key.offset; + } + + /* + * Parent node found and matches current inline ref, no need to + * rebuild this node for this inline ref + */ + if (exist && + ((key.type == BTRFS_TREE_BLOCK_REF_KEY && + exist->owner == key.offset) || + (key.type == BTRFS_SHARED_BLOCK_REF_KEY && + exist->bytenr == key.offset))) { + exist = NULL; + continue; + } + + /* SHARED_BLOCK_REF means key.offset is the parent bytenr */ + if (key.type == BTRFS_SHARED_BLOCK_REF_KEY) { + ret = handle_direct_tree_backref(cache, &key, cur); + if (ret < 0) + goto out; + continue; + } else if (unlikely(key.type == BTRFS_EXTENT_REF_V0_KEY)) { + ret = -EINVAL; + btrfs_print_v0_err(fs_info); + btrfs_handle_fs_error(fs_info, ret, NULL); + goto out; + } else if (key.type != BTRFS_TREE_BLOCK_REF_KEY) { + continue; + } + + /* + * key.type == BTRFS_TREE_BLOCK_REF_KEY, inline ref offset + * means the root objectid. We need to search the tree to get + * its parent bytenr. + */ + ret = handle_indirect_tree_backref(cache, path, &key, node_key, + cur); + if (ret < 0) + goto out; + } + ret = 0; + cur->checked = 1; + WARN_ON(exist); +out: + btrfs_backref_iter_release(iter); + return ret; +} diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index a7697de925d5..5839da215ff6 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -363,4 +363,10 @@ static inline void btrfs_backref_panic(struct btrfs_fs_info *fs_info, bytenr); } +int btrfs_backref_add_tree_node(struct btrfs_backref_cache *cache, + struct btrfs_path *path, + struct btrfs_backref_iter *iter, + struct btrfs_key *node_key, + struct btrfs_backref_node *cur); + #endif diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index b8fc07ad0a64..275e4c9e9685 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -377,360 +377,6 @@ static struct btrfs_root *read_fs_root(struct btrfs_fs_info *fs_info, return btrfs_get_fs_root(fs_info, &key, false); } -/* - * Handle direct tree backref - * - * Direct tree backref means, the backref item shows its parent bytenr - * directly. This is for SHARED_BLOCK_REF backref (keyed or inlined). - * - * @ref_key: The converted backref key. - * For keyed backref, it's the item key. - * For inlined backref, objectid is the bytenr, - * type is btrfs_inline_ref_type, offset is - * btrfs_inline_ref_offset. - */ -static int handle_direct_tree_backref(struct btrfs_backref_cache *cache, - struct btrfs_key *ref_key, - struct btrfs_backref_node *cur) -{ - struct btrfs_backref_edge *edge; - struct btrfs_backref_node *upper; - struct rb_node *rb_node; - - ASSERT(ref_key->type == BTRFS_SHARED_BLOCK_REF_KEY); - - /* Only reloc root uses backref pointing to itself */ - if (ref_key->objectid == ref_key->offset) { - struct btrfs_root *root; - - cur->is_reloc_root = 1; - /* Only reloc backref cache cares about a specific root */ - if (cache->is_reloc) { - root = find_reloc_root(cache->fs_info, cur->bytenr); - if (WARN_ON(!root)) - return -ENOENT; - cur->root = root; - } else { - /* - * For generic purpose backref cache, reloc root node - * is useless. - */ - list_add(&cur->list, &cache->useless_node); - } - return 0; - } - - edge = btrfs_backref_alloc_edge(cache); - if (!edge) - return -ENOMEM; - - rb_node = rb_simple_search(&cache->rb_root, ref_key->offset); - if (!rb_node) { - /* Parent node not yet cached */ - upper = btrfs_backref_alloc_node(cache, ref_key->offset, - cur->level + 1); - if (!upper) { - btrfs_backref_free_edge(cache, edge); - return -ENOMEM; - } - - /* - * Backrefs for the upper level block isn't cached, add the - * block to pending list - */ - list_add_tail(&edge->list[UPPER], &cache->pending_edge); - } else { - /* Parent node already cached */ - upper = rb_entry(rb_node, struct btrfs_backref_node, rb_node); - ASSERT(upper->checked); - INIT_LIST_HEAD(&edge->list[UPPER]); - } - btrfs_backref_link_edge(edge, cur, upper, LINK_LOWER); - return 0; -} - -/* - * Handle indirect tree backref - * - * Indirect tree backref means, we only know which tree the node belongs to. - * We still need to do a tree search to find out the parents. This is for - * TREE_BLOCK_REF backref (keyed or inlined). - * - * @ref_key: The same as @ref_key in handle_direct_tree_backref() - * @tree_key: The first key of this tree block. - * @path: A clean (released) path, to avoid allocating path everytime - * the function get called. - */ -static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, - struct btrfs_path *path, - struct btrfs_key *ref_key, - struct btrfs_key *tree_key, - struct btrfs_backref_node *cur) -{ - struct btrfs_fs_info *fs_info = cache->fs_info; - struct btrfs_backref_node *upper; - struct btrfs_backref_node *lower; - struct btrfs_backref_edge *edge; - struct extent_buffer *eb; - struct btrfs_root *root; - struct btrfs_key root_key; - struct rb_node *rb_node; - int level; - bool need_check = true; - int ret; - - root_key.objectid = ref_key->offset; - root_key.type = BTRFS_ROOT_ITEM_KEY; - root_key.offset = (u64)-1; - root = btrfs_get_fs_root(fs_info, &root_key, false); - if (IS_ERR(root)) - return PTR_ERR(root); - if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) - cur->cowonly = 1; - - if (btrfs_root_level(&root->root_item) == cur->level) { - /* Tree root */ - ASSERT(btrfs_root_bytenr(&root->root_item) == cur->bytenr); - if (btrfs_should_ignore_reloc_root(root)) { - btrfs_put_root(root); - list_add(&cur->list, &cache->useless_node); - } else { - cur->root = root; - } - return 0; - } - - level = cur->level + 1; - - /* Search the tree to find parent blocks referring to the block */ - path->search_commit_root = 1; - path->skip_locking = 1; - path->lowest_level = level; - ret = btrfs_search_slot(NULL, root, tree_key, path, 0, 0); - path->lowest_level = 0; - if (ret < 0) { - btrfs_put_root(root); - return ret; - } - if (ret > 0 && path->slots[level] > 0) - path->slots[level]--; - - eb = path->nodes[level]; - if (btrfs_node_blockptr(eb, path->slots[level]) != cur->bytenr) { - btrfs_err(fs_info, -"couldn't find block (%llu) (level %d) in tree (%llu) with key (%llu %u %llu)", - cur->bytenr, level - 1, root->root_key.objectid, - tree_key->objectid, tree_key->type, tree_key->offset); - btrfs_put_root(root); - ret = -ENOENT; - goto out; - } - lower = cur; - - /* Add all nodes and edges in the path */ - for (; level < BTRFS_MAX_LEVEL; level++) { - if (!path->nodes[level]) { - ASSERT(btrfs_root_bytenr(&root->root_item) == - lower->bytenr); - if (btrfs_should_ignore_reloc_root(root)) { - btrfs_put_root(root); - list_add(&lower->list, &cache->useless_node); - } else { - lower->root = root; - } - break; - } - - edge = btrfs_backref_alloc_edge(cache); - if (!edge) { - btrfs_put_root(root); - ret = -ENOMEM; - goto out; - } - - eb = path->nodes[level]; - rb_node = rb_simple_search(&cache->rb_root, eb->start); - if (!rb_node) { - upper = btrfs_backref_alloc_node(cache, eb->start, - lower->level + 1); - if (!upper) { - btrfs_put_root(root); - btrfs_backref_free_edge(cache, edge); - ret = -ENOMEM; - goto out; - } - upper->owner = btrfs_header_owner(eb); - if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) - upper->cowonly = 1; - - /* - * If we know the block isn't shared we can avoid - * checking its backrefs. - */ - if (btrfs_block_can_be_shared(root, eb)) - upper->checked = 0; - else - upper->checked = 1; - - /* - * Add the block to pending list if we need to check its - * backrefs, we only do this once while walking up a - * tree as we will catch anything else later on. - */ - if (!upper->checked && need_check) { - need_check = false; - list_add_tail(&edge->list[UPPER], - &cache->pending_edge); - } else { - if (upper->checked) - need_check = true; - INIT_LIST_HEAD(&edge->list[UPPER]); - } - } else { - upper = rb_entry(rb_node, struct btrfs_backref_node, - rb_node); - ASSERT(upper->checked); - INIT_LIST_HEAD(&edge->list[UPPER]); - if (!upper->owner) - upper->owner = btrfs_header_owner(eb); - } - btrfs_backref_link_edge(edge, lower, upper, LINK_LOWER); - - if (rb_node) { - btrfs_put_root(root); - break; - } - lower = upper; - upper = NULL; - } -out: - btrfs_release_path(path); - return ret; -} - -static int handle_one_tree_block(struct btrfs_backref_cache *cache, - struct btrfs_path *path, - struct btrfs_backref_iter *iter, - struct btrfs_key *node_key, - struct btrfs_backref_node *cur) -{ - struct btrfs_fs_info *fs_info = cache->fs_info; - struct btrfs_backref_edge *edge; - struct btrfs_backref_node *exist; - int ret; - - ret = btrfs_backref_iter_start(iter, cur->bytenr); - if (ret < 0) - return ret; - /* - * We skip the first btrfs_tree_block_info, as we don't use the key - * stored in it, but fetch it from the tree block - */ - if (btrfs_backref_has_tree_block_info(iter)) { - ret = btrfs_backref_iter_next(iter); - if (ret < 0) - goto out; - /* No extra backref? This means the tree block is corrupted */ - if (ret > 0) { - ret = -EUCLEAN; - goto out; - } - } - WARN_ON(cur->checked); - if (!list_empty(&cur->upper)) { - /* - * the backref was added previously when processing - * backref of type BTRFS_TREE_BLOCK_REF_KEY - */ - ASSERT(list_is_singular(&cur->upper)); - edge = list_entry(cur->upper.next, struct btrfs_backref_edge, - list[LOWER]); - ASSERT(list_empty(&edge->list[UPPER])); - exist = edge->node[UPPER]; - /* - * add the upper level block to pending list if we need - * check its backrefs - */ - if (!exist->checked) - list_add_tail(&edge->list[UPPER], &cache->pending_edge); - } else { - exist = NULL; - } - - for (; ret == 0; ret = btrfs_backref_iter_next(iter)) { - struct extent_buffer *eb; - struct btrfs_key key; - int type; - - cond_resched(); - eb = btrfs_backref_get_eb(iter); - - key.objectid = iter->bytenr; - if (btrfs_backref_iter_is_inline_ref(iter)) { - struct btrfs_extent_inline_ref *iref; - - /* update key for inline back ref */ - iref = (struct btrfs_extent_inline_ref *) - ((unsigned long)iter->cur_ptr); - type = btrfs_get_extent_inline_ref_type(eb, iref, - BTRFS_REF_TYPE_BLOCK); - if (type == BTRFS_REF_TYPE_INVALID) { - ret = -EUCLEAN; - goto out; - } - key.type = type; - key.offset = btrfs_extent_inline_ref_offset(eb, iref); - } else { - key.type = iter->cur_key.type; - key.offset = iter->cur_key.offset; - } - - /* - * Parent node found and matches current inline ref, no need to - * rebuild this node for this inline ref. - */ - if (exist && - ((key.type == BTRFS_TREE_BLOCK_REF_KEY && - exist->owner == key.offset) || - (key.type == BTRFS_SHARED_BLOCK_REF_KEY && - exist->bytenr == key.offset))) { - exist = NULL; - continue; - } - - /* SHARED_BLOCK_REF means key.offset is the parent bytenr */ - if (key.type == BTRFS_SHARED_BLOCK_REF_KEY) { - ret = handle_direct_tree_backref(cache, &key, cur); - if (ret < 0) - goto out; - continue; - } else if (unlikely(key.type == BTRFS_EXTENT_REF_V0_KEY)) { - ret = -EINVAL; - btrfs_print_v0_err(fs_info); - btrfs_handle_fs_error(fs_info, ret, NULL); - goto out; - } else if (key.type != BTRFS_TREE_BLOCK_REF_KEY) { - continue; - } - - /* - * key.type == BTRFS_TREE_BLOCK_REF_KEY, inline ref offset - * means the root objectid. We need to search the tree to get - * its parent bytenr. - */ - ret = handle_indirect_tree_backref(cache, path, &key, node_key, - cur); - if (ret < 0) - goto out; - } - ret = 0; - cur->checked = 1; - WARN_ON(exist); -out: - btrfs_backref_iter_release(iter); - return ret; -} - /* * In handle_one_tree_backref(), we have only linked the lower node to the edge, * but the upper node hasn't been linked to the edge. @@ -969,7 +615,8 @@ static noinline_for_stack struct btrfs_backref_node *build_backref_tree( /* Breadth-first search to build backref cache */ do { - ret = handle_one_tree_block(cache, path, iter, node_key, cur); + ret = btrfs_backref_add_tree_node(cache, path, iter, node_key, + cur); if (ret < 0) { err = ret; goto out; From fc997ed05a9f9d2185b8804fb2d0273e6d9e921a Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 23 Mar 2020 16:14:08 +0800 Subject: [PATCH 0715/1043] btrfs: backref: rename and move finish_upper_links() This the the 2nd major part of generic backref cache. Move it to backref.c so we can reuse it. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 106 ++++++++++++++++++++++++++++++++++++++ fs/btrfs/backref.h | 3 ++ fs/btrfs/relocation.c | 116 +----------------------------------------- 3 files changed, 110 insertions(+), 115 deletions(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 832071bbb8c9..3b355be95171 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -2830,6 +2830,7 @@ out: * * NOTE: Even if the function returned 0, @cur is not yet cached as its upper * links aren't yet bi-directional. Needs to finish such links. + * Use btrfs_backref_finish_upper_links() to finish such linkage. * * @path: Released path for indirect tree backref lookup * @iter: Released backref iter for extent tree search @@ -2957,3 +2958,108 @@ out: btrfs_backref_iter_release(iter); return ret; } + +/* + * Finish the upwards linkage created by btrfs_backref_add_tree_node() + */ +int btrfs_backref_finish_upper_links(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *start) +{ + struct list_head *useless_node = &cache->useless_node; + struct btrfs_backref_edge *edge; + struct rb_node *rb_node; + LIST_HEAD(pending_edge); + + ASSERT(start->checked); + + /* Insert this node to cache if it's not COW-only */ + if (!start->cowonly) { + rb_node = rb_simple_insert(&cache->rb_root, start->bytenr, + &start->rb_node); + if (rb_node) + btrfs_backref_panic(cache->fs_info, start->bytenr, + -EEXIST); + list_add_tail(&start->lower, &cache->leaves); + } + + /* + * Use breadth first search to iterate all related edges. + * + * The starting points are all the edges of this node + */ + list_for_each_entry(edge, &start->upper, list[LOWER]) + list_add_tail(&edge->list[UPPER], &pending_edge); + + while (!list_empty(&pending_edge)) { + struct btrfs_backref_node *upper; + struct btrfs_backref_node *lower; + struct rb_node *rb_node; + + edge = list_first_entry(&pending_edge, + struct btrfs_backref_edge, list[UPPER]); + list_del_init(&edge->list[UPPER]); + upper = edge->node[UPPER]; + lower = edge->node[LOWER]; + + /* Parent is detached, no need to keep any edges */ + if (upper->detached) { + list_del(&edge->list[LOWER]); + btrfs_backref_free_edge(cache, edge); + + /* Lower node is orphan, queue for cleanup */ + if (list_empty(&lower->upper)) + list_add(&lower->list, useless_node); + continue; + } + + /* + * All new nodes added in current build_backref_tree() haven't + * been linked to the cache rb tree. + * So if we have upper->rb_node populated, this means a cache + * hit. We only need to link the edge, as @upper and all its + * parents have already been linked. + */ + if (!RB_EMPTY_NODE(&upper->rb_node)) { + if (upper->lowest) { + list_del_init(&upper->lower); + upper->lowest = 0; + } + + list_add_tail(&edge->list[UPPER], &upper->lower); + continue; + } + + /* Sanity check, we shouldn't have any unchecked nodes */ + if (!upper->checked) { + ASSERT(0); + return -EUCLEAN; + } + + /* Sanity check, COW-only node has non-COW-only parent */ + if (start->cowonly != upper->cowonly) { + ASSERT(0); + return -EUCLEAN; + } + + /* Only cache non-COW-only (subvolume trees) tree blocks */ + if (!upper->cowonly) { + rb_node = rb_simple_insert(&cache->rb_root, upper->bytenr, + &upper->rb_node); + if (rb_node) { + btrfs_backref_panic(cache->fs_info, + upper->bytenr, -EEXIST); + return -EUCLEAN; + } + } + + list_add_tail(&edge->list[UPPER], &upper->lower); + + /* + * Also queue all the parent edges of this uncached node + * to finish the upper linkage + */ + list_for_each_entry(edge, &upper->upper, list[LOWER]) + list_add_tail(&edge->list[UPPER], &pending_edge); + } + return 0; +} diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 5839da215ff6..80a43359d216 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -369,4 +369,7 @@ int btrfs_backref_add_tree_node(struct btrfs_backref_cache *cache, struct btrfs_key *node_key, struct btrfs_backref_node *cur); +int btrfs_backref_finish_upper_links(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *start); + #endif diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 275e4c9e9685..a3e63b937290 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -377,120 +377,6 @@ static struct btrfs_root *read_fs_root(struct btrfs_fs_info *fs_info, return btrfs_get_fs_root(fs_info, &key, false); } -/* - * In handle_one_tree_backref(), we have only linked the lower node to the edge, - * but the upper node hasn't been linked to the edge. - * This means we can only iterate through btrfs_backref_node::upper to reach - * parent edges, but not through btrfs_backref_node::lower to reach children - * edges. - * - * This function will finish the btrfs_backref_node::lower to related edges, - * so that backref cache can be bi-directionally iterated. - * - * Also, this will add the nodes to backref cache for the next run. - */ -static int finish_upper_links(struct btrfs_backref_cache *cache, - struct btrfs_backref_node *start) -{ - struct list_head *useless_node = &cache->useless_node; - struct btrfs_backref_edge *edge; - struct rb_node *rb_node; - LIST_HEAD(pending_edge); - - ASSERT(start->checked); - - /* Insert this node to cache if it's not COW-only */ - if (!start->cowonly) { - rb_node = rb_simple_insert(&cache->rb_root, start->bytenr, - &start->rb_node); - if (rb_node) - btrfs_backref_panic(cache->fs_info, start->bytenr, - -EEXIST); - list_add_tail(&start->lower, &cache->leaves); - } - - /* - * Use breadth first search to iterate all related edges. - * - * The starting points are all the edges of this node - */ - list_for_each_entry(edge, &start->upper, list[LOWER]) - list_add_tail(&edge->list[UPPER], &pending_edge); - - while (!list_empty(&pending_edge)) { - struct btrfs_backref_node *upper; - struct btrfs_backref_node *lower; - struct rb_node *rb_node; - - edge = list_first_entry(&pending_edge, - struct btrfs_backref_edge, list[UPPER]); - list_del_init(&edge->list[UPPER]); - upper = edge->node[UPPER]; - lower = edge->node[LOWER]; - - /* Parent is detached, no need to keep any edges */ - if (upper->detached) { - list_del(&edge->list[LOWER]); - btrfs_backref_free_edge(cache, edge); - - /* Lower node is orphan, queue for cleanup */ - if (list_empty(&lower->upper)) - list_add(&lower->list, useless_node); - continue; - } - - /* - * All new nodes added in current build_backref_tree() haven't - * been linked to the cache rb tree. - * So if we have upper->rb_node populated, this means a cache - * hit. We only need to link the edge, as @upper and all its - * parent have already been linked. - */ - if (!RB_EMPTY_NODE(&upper->rb_node)) { - if (upper->lowest) { - list_del_init(&upper->lower); - upper->lowest = 0; - } - - list_add_tail(&edge->list[UPPER], &upper->lower); - continue; - } - - /* Sanity check, we shouldn't have any unchecked nodes */ - if (!upper->checked) { - ASSERT(0); - return -EUCLEAN; - } - - /* Sanity check, COW-only node has non-COW-only parent */ - if (start->cowonly != upper->cowonly) { - ASSERT(0); - return -EUCLEAN; - } - - /* Only cache non-COW-only (subvolume trees) tree blocks */ - if (!upper->cowonly) { - rb_node = rb_simple_insert(&cache->rb_root, upper->bytenr, - &upper->rb_node); - if (rb_node) { - btrfs_backref_panic(cache->fs_info, - upper->bytenr, -EEXIST); - return -EUCLEAN; - } - } - - list_add_tail(&edge->list[UPPER], &upper->lower); - - /* - * Also queue all the parent edges of this uncached node to - * finish the upper linkage - */ - list_for_each_entry(edge, &upper->upper, list[LOWER]) - list_add_tail(&edge->list[UPPER], &pending_edge); - } - return 0; -} - /* * For useless nodes, do two major clean ups: * @@ -634,7 +520,7 @@ static noinline_for_stack struct btrfs_backref_node *build_backref_tree( } while (edge); /* Finish the upper linkage of newly added edges/nodes */ - ret = finish_upper_links(cache, node); + ret = btrfs_backref_finish_upper_links(cache, node); if (ret < 0) { err = ret; goto out; From 1b23ea180b6b4186ff79db767dcbec612477968f Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 23 Mar 2020 16:57:15 +0800 Subject: [PATCH 0716/1043] btrfs: reloc: move error handling of build_backref_tree() to backref.c The error cleanup will be extracted as a new function, btrfs_backref_error_cleanup(), and moved to backref.c and exported for later usage. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 54 +++++++++++++++++++++++++++++++++++++++++++ fs/btrfs/backref.h | 3 +++ fs/btrfs/relocation.c | 48 +------------------------------------- 3 files changed, 58 insertions(+), 47 deletions(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 3b355be95171..b38a680e6678 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -3063,3 +3063,57 @@ int btrfs_backref_finish_upper_links(struct btrfs_backref_cache *cache, } return 0; } + +void btrfs_backref_error_cleanup(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node) +{ + struct btrfs_backref_node *lower; + struct btrfs_backref_node *upper; + struct btrfs_backref_edge *edge; + + while (!list_empty(&cache->useless_node)) { + lower = list_first_entry(&cache->useless_node, + struct btrfs_backref_node, list); + list_del_init(&lower->list); + } + while (!list_empty(&cache->pending_edge)) { + edge = list_first_entry(&cache->pending_edge, + struct btrfs_backref_edge, list[UPPER]); + list_del(&edge->list[UPPER]); + list_del(&edge->list[LOWER]); + lower = edge->node[LOWER]; + upper = edge->node[UPPER]; + btrfs_backref_free_edge(cache, edge); + + /* + * Lower is no longer linked to any upper backref nodes and + * isn't in the cache, we can free it ourselves. + */ + if (list_empty(&lower->upper) && + RB_EMPTY_NODE(&lower->rb_node)) + list_add(&lower->list, &cache->useless_node); + + if (!RB_EMPTY_NODE(&upper->rb_node)) + continue; + + /* Add this guy's upper edges to the list to process */ + list_for_each_entry(edge, &upper->upper, list[LOWER]) + list_add_tail(&edge->list[UPPER], + &cache->pending_edge); + if (list_empty(&upper->upper)) + list_add(&upper->list, &cache->useless_node); + } + + while (!list_empty(&cache->useless_node)) { + lower = list_first_entry(&cache->useless_node, + struct btrfs_backref_node, list); + list_del_init(&lower->list); + if (lower == node) + node = NULL; + btrfs_backref_free_node(cache, lower); + } + + btrfs_backref_cleanup_node(cache, node); + ASSERT(list_empty(&cache->useless_node) && + list_empty(&cache->pending_edge)); +} diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 80a43359d216..18393cc05ca2 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -372,4 +372,7 @@ int btrfs_backref_add_tree_node(struct btrfs_backref_cache *cache, int btrfs_backref_finish_upper_links(struct btrfs_backref_cache *cache, struct btrfs_backref_node *start); +void btrfs_backref_error_cleanup(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node); + #endif diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index a3e63b937290..80b58358f688 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -474,8 +474,6 @@ static noinline_for_stack struct btrfs_backref_node *build_backref_tree( /* For searching parent of TREE_BLOCK_REF */ struct btrfs_path *path; struct btrfs_backref_node *cur; - struct btrfs_backref_node *upper; - struct btrfs_backref_node *lower; struct btrfs_backref_node *node = NULL; struct btrfs_backref_edge *edge; int ret; @@ -532,51 +530,7 @@ out: btrfs_backref_iter_free(iter); btrfs_free_path(path); if (err) { - while (!list_empty(&cache->useless_node)) { - lower = list_first_entry(&cache->useless_node, - struct btrfs_backref_node, list); - list_del_init(&lower->list); - } - while (!list_empty(&cache->pending_edge)) { - edge = list_first_entry(&cache->pending_edge, - struct btrfs_backref_edge, list[UPPER]); - list_del(&edge->list[UPPER]); - list_del(&edge->list[LOWER]); - lower = edge->node[LOWER]; - upper = edge->node[UPPER]; - btrfs_backref_free_edge(cache, edge); - - /* - * Lower is no longer linked to any upper backref nodes - * and isn't in the cache, we can free it ourselves. - */ - if (list_empty(&lower->upper) && - RB_EMPTY_NODE(&lower->rb_node)) - list_add(&lower->list, &cache->useless_node); - - if (!RB_EMPTY_NODE(&upper->rb_node)) - continue; - - /* Add this guy's upper edges to the list to process */ - list_for_each_entry(edge, &upper->upper, list[LOWER]) - list_add_tail(&edge->list[UPPER], - &cache->pending_edge); - if (list_empty(&upper->upper)) - list_add(&upper->list, &cache->useless_node); - } - - while (!list_empty(&cache->useless_node)) { - lower = list_first_entry(&cache->useless_node, - struct btrfs_backref_node, list); - list_del_init(&lower->list); - if (lower == node) - node = NULL; - btrfs_backref_free_node(cache, lower); - } - - btrfs_backref_cleanup_node(cache, node); - ASSERT(list_empty(&cache->useless_node) && - list_empty(&cache->pending_edge)); + btrfs_backref_error_cleanup(cache, node); return ERR_PTR(err); } ASSERT(!node || !node->detached); From 876de781b0da240fcf8d29514c34607e147e5a94 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 16 Mar 2020 14:10:01 +0800 Subject: [PATCH 0717/1043] btrfs: backref: distinguish reloc and non-reloc use of indirect resolution For relocation tree detection, relocation backref cache uses btrfs_should_ignore_reloc_root() which uses relocation-specific checks like checking the DEAD_RELOC_ROOT bit. However for general purpose backref cache, we can rely on that check, as it's possible that relocation is also running. For generic purposed backref cache, we detect reloc root by SHARED_BLOCK_REF item. Only reloc root node has its parent bytenr pointing back to itself. And in that case, backref cache will mark the reloc root node useless, dropping any child orphan nodes. So only call btrfs_should_ignore_reloc_root() if the backref cache is for relocation. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index b38a680e6678..ac3c34f47b56 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -2708,7 +2708,17 @@ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, if (btrfs_root_level(&root->root_item) == cur->level) { /* Tree root */ ASSERT(btrfs_root_bytenr(&root->root_item) == cur->bytenr); - if (btrfs_should_ignore_reloc_root(root)) { + /* + * For reloc backref cache, we may ignore reloc root. But for + * general purpose backref cache, we can't rely on + * btrfs_should_ignore_reloc_root() as it may conflict with + * current running relocation and lead to missing root. + * + * For general purpose backref cache, reloc root detection is + * completely relying on direct backref (key->offset is parent + * bytenr), thus only do such check for reloc cache. + */ + if (btrfs_should_ignore_reloc_root(root) && cache->is_reloc) { btrfs_put_root(root); list_add(&cur->list, &cache->useless_node); } else { @@ -2749,7 +2759,9 @@ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, if (!path->nodes[level]) { ASSERT(btrfs_root_bytenr(&root->root_item) == lower->bytenr); - if (btrfs_should_ignore_reloc_root(root)) { + /* Same as previous should_ignore_reloc_root() call */ + if (btrfs_should_ignore_reloc_root(root) && + cache->is_reloc) { btrfs_put_root(root); list_add(&lower->list, &cache->useless_node); } else { From 7f9fe614407692f670601a634621138233ac00d7 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 13 Mar 2020 15:58:05 -0400 Subject: [PATCH 0718/1043] btrfs: improve global reserve stealing logic For unlink transactions and block group removal btrfs_start_transaction_fallback_global_rsv will first try to start an ordinary transaction and if it fails it will fall back to reserving the required amount by stealing from the global reserve. This is problematic because of all the same reasons we had with previous iterations of the ENOSPC handling, thundering herd. We get a bunch of failures all at once, everybody tries to allocate from the global reserve, some win and some lose, we get an ENSOPC. Fix this behavior by introducing BTRFS_RESERVE_FLUSH_ALL_STEAL. It's used to mark unlink reservation. To fix this we need to integrate this logic into the normal ENOSPC infrastructure. We still go through all of the normal flushing work, and at the moment we begin to fail all the tickets we try to satisfy any tickets that are allowed to steal by stealing from the global reserve. If this works we start the flushing system over again just like we would with a normal ticket satisfaction. This serializes our global reserve stealing, so we don't have the thundering herd problem. Reviewed-by: Nikolay Borisov Tested-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 2 +- fs/btrfs/ctree.h | 1 + fs/btrfs/inode.c | 2 +- fs/btrfs/space-info.c | 37 ++++++++++++++++++++++++++++++++++++- fs/btrfs/space-info.h | 1 + fs/btrfs/transaction.c | 42 +++++------------------------------------- fs/btrfs/transaction.h | 3 +-- 7 files changed, 46 insertions(+), 42 deletions(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 696f47103cfc..233c5663f233 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1175,7 +1175,7 @@ struct btrfs_trans_handle *btrfs_start_trans_remove_block_group( free_extent_map(em); return btrfs_start_transaction_fallback_global_rsv(fs_info->extent_root, - num_items, 1); + num_items); } /* diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 01b03e8a671f..c322568231a4 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -2512,6 +2512,7 @@ enum btrfs_reserve_flush_enum { BTRFS_RESERVE_FLUSH_LIMIT, BTRFS_RESERVE_FLUSH_EVICT, BTRFS_RESERVE_FLUSH_ALL, + BTRFS_RESERVE_FLUSH_ALL_STEAL, }; enum btrfs_flush_state { diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 320d1062068d..259239b33370 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -3618,7 +3618,7 @@ static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir) * 1 for the inode ref * 1 for the inode */ - return btrfs_start_transaction_fallback_global_rsv(root, 5, 5); + return btrfs_start_transaction_fallback_global_rsv(root, 5); } static int btrfs_unlink(struct inode *dir, struct dentry *dentry) diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index ff17a4420358..f5ec50e6c48e 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -856,6 +856,34 @@ static inline int need_do_async_reclaim(struct btrfs_fs_info *fs_info, !test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state)); } +static bool steal_from_global_rsv(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info, + struct reserve_ticket *ticket) +{ + struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; + u64 min_bytes; + + if (global_rsv->space_info != space_info) + return false; + + spin_lock(&global_rsv->lock); + min_bytes = div_factor(global_rsv->size, 5); + if (global_rsv->reserved < min_bytes + ticket->bytes) { + spin_unlock(&global_rsv->lock); + return false; + } + global_rsv->reserved -= ticket->bytes; + ticket->bytes = 0; + list_del_init(&ticket->list); + wake_up(&ticket->wait); + space_info->tickets_id++; + if (global_rsv->reserved < global_rsv->size) + global_rsv->full = 0; + spin_unlock(&global_rsv->lock); + + return true; +} + /* * maybe_fail_all_tickets - we've exhausted our flushing, start failing tickets * @fs_info - fs_info for this fs @@ -888,6 +916,10 @@ static bool maybe_fail_all_tickets(struct btrfs_fs_info *fs_info, ticket = list_first_entry(&space_info->tickets, struct reserve_ticket, list); + if (ticket->steal && + steal_from_global_rsv(fs_info, space_info, ticket)) + return true; + /* * may_commit_transaction will avoid committing the transaction * if it doesn't feel like the space reclaimed by the commit @@ -1104,6 +1136,7 @@ static int handle_reserve_ticket(struct btrfs_fs_info *fs_info, switch (flush) { case BTRFS_RESERVE_FLUSH_ALL: + case BTRFS_RESERVE_FLUSH_ALL_STEAL: wait_reserve_ticket(fs_info, space_info, ticket); break; case BTRFS_RESERVE_FLUSH_LIMIT: @@ -1203,7 +1236,9 @@ static int __reserve_metadata_bytes(struct btrfs_fs_info *fs_info, ticket.error = 0; space_info->reclaim_size += ticket.bytes; init_waitqueue_head(&ticket.wait); - if (flush == BTRFS_RESERVE_FLUSH_ALL) { + ticket.steal = (flush == BTRFS_RESERVE_FLUSH_ALL_STEAL); + if (flush == BTRFS_RESERVE_FLUSH_ALL || + flush == BTRFS_RESERVE_FLUSH_ALL_STEAL) { list_add_tail(&ticket.list, &space_info->tickets); if (!space_info->flush) { space_info->flush = 1; diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h index 0a5001ef1481..c3c64019950a 100644 --- a/fs/btrfs/space-info.h +++ b/fs/btrfs/space-info.h @@ -78,6 +78,7 @@ struct btrfs_space_info { struct reserve_ticket { u64 bytes; int error; + bool steal; struct list_head list; wait_queue_head_t wait; }; diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 2d5498136e5e..b5da5d8342dc 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -563,7 +563,8 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, * refill that amount for whatever is missing in the reserve. */ num_bytes = btrfs_calc_insert_metadata_size(fs_info, num_items); - if (delayed_refs_rsv->full == 0) { + if (flush == BTRFS_RESERVE_FLUSH_ALL && + delayed_refs_rsv->full == 0) { delayed_refs_bytes = num_bytes; num_bytes <<= 1; } @@ -699,43 +700,10 @@ struct btrfs_trans_handle *btrfs_start_transaction(struct btrfs_root *root, struct btrfs_trans_handle *btrfs_start_transaction_fallback_global_rsv( struct btrfs_root *root, - unsigned int num_items, - int min_factor) + unsigned int num_items) { - struct btrfs_fs_info *fs_info = root->fs_info; - struct btrfs_trans_handle *trans; - u64 num_bytes; - int ret; - - /* - * We have two callers: unlink and block group removal. The - * former should succeed even if we will temporarily exceed - * quota and the latter operates on the extent root so - * qgroup enforcement is ignored anyway. - */ - trans = start_transaction(root, num_items, TRANS_START, - BTRFS_RESERVE_FLUSH_ALL, false); - if (!IS_ERR(trans) || PTR_ERR(trans) != -ENOSPC) - return trans; - - trans = btrfs_start_transaction(root, 0); - if (IS_ERR(trans)) - return trans; - - num_bytes = btrfs_calc_insert_metadata_size(fs_info, num_items); - ret = btrfs_cond_migrate_bytes(fs_info, &fs_info->trans_block_rsv, - num_bytes, min_factor); - if (ret) { - btrfs_end_transaction(trans); - return ERR_PTR(ret); - } - - trans->block_rsv = &fs_info->trans_block_rsv; - trans->bytes_reserved = num_bytes; - trace_btrfs_space_reservation(fs_info, "transaction", - trans->transid, num_bytes, 1); - - return trans; + return start_transaction(root, num_items, TRANS_START, + BTRFS_RESERVE_FLUSH_ALL_STEAL, false); } struct btrfs_trans_handle *btrfs_join_transaction(struct btrfs_root *root) diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h index 31ae8d273065..bf102e64bfb2 100644 --- a/fs/btrfs/transaction.h +++ b/fs/btrfs/transaction.h @@ -193,8 +193,7 @@ struct btrfs_trans_handle *btrfs_start_transaction(struct btrfs_root *root, unsigned int num_items); struct btrfs_trans_handle *btrfs_start_transaction_fallback_global_rsv( struct btrfs_root *root, - unsigned int num_items, - int min_factor); + unsigned int num_items); struct btrfs_trans_handle *btrfs_join_transaction(struct btrfs_root *root); struct btrfs_trans_handle *btrfs_join_transaction_spacecache(struct btrfs_root *root); struct btrfs_trans_handle *btrfs_join_transaction_nostart(struct btrfs_root *root); From e6549c2aabb9875cf7827bb2283905aaab5b0523 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 13 Mar 2020 15:58:06 -0400 Subject: [PATCH 0719/1043] btrfs: allow to use up to 90% of the global block rsv for unlink We previously had a limit of stealing 50% of the global reserve for unlink. This was from a time when the global reserve was used for the delayed refs as well. However now those reservations are kept separate, so the global reserve can be depleted much more to allow us to make progress for space restoring operations like unlink. Change the minimum amount of space required to be left in the global reserve to 10%. Reviewed-by: Nikolay Borisov Tested-by: Nikolay Borisov Signed-off-by: Josef Bacik Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index f5ec50e6c48e..3a49112ca0ea 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -867,7 +867,7 @@ static bool steal_from_global_rsv(struct btrfs_fs_info *fs_info, return false; spin_lock(&global_rsv->lock); - min_bytes = div_factor(global_rsv->size, 5); + min_bytes = div_factor(global_rsv->size, 1); if (global_rsv->reserved < min_bytes + ticket->bytes) { spin_unlock(&global_rsv->lock); return false; From bb4f58a747f0421b10645fbf75a6acc88da0de50 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 13 Mar 2020 15:58:07 -0400 Subject: [PATCH 0720/1043] btrfs: account for trans_block_rsv in may_commit_transaction On ppc64le with 64k page size (respectively 64k block size) generic/320 was failing and debug output showed we were getting a premature ENOSPC with a bunch of space in btrfs_fs_info::trans_block_rsv. This meant there were still open transaction handles holding space, yet the flusher didn't commit the transaction because it deemed the freed space won't be enough to satisfy the current reserve ticket. Fix this by accounting for space in trans_block_rsv when deciding whether the current transaction should be committed or not. Reviewed-by: Nikolay Borisov Tested-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 3a49112ca0ea..b76ca6b74ebb 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -626,6 +626,7 @@ static int may_commit_transaction(struct btrfs_fs_info *fs_info, struct reserve_ticket *ticket = NULL; struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_block_rsv; struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv; + struct btrfs_block_rsv *trans_rsv = &fs_info->trans_block_rsv; struct btrfs_trans_handle *trans; u64 bytes_needed; u64 reclaim_bytes = 0; @@ -688,6 +689,11 @@ static int may_commit_transaction(struct btrfs_fs_info *fs_info, spin_lock(&delayed_refs_rsv->lock); reclaim_bytes += delayed_refs_rsv->reserved; spin_unlock(&delayed_refs_rsv->lock); + + spin_lock(&trans_rsv->lock); + reclaim_bytes += trans_rsv->reserved; + spin_unlock(&trans_rsv->lock); + if (reclaim_bytes >= bytes_needed) goto commit; bytes_needed -= reclaim_bytes; From 666daa9f977d46010e2597a3b4f6524ba4d28129 Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 13 Mar 2020 15:58:08 -0400 Subject: [PATCH 0721/1043] btrfs: only check priority tickets for priority flushing In debugging a generic/320 failure on ppc64, Nikolay noticed that sometimes we'd ENOSPC out with plenty of space to reclaim if we had committed the transaction. He further discovered that this was because there was a priority ticket that was small enough to fit in the free space currently in the space_info. Consider the following scenario. There is no more space to reclaim in the fs without committing the transaction. Assume there's 1MiB of space free in the space info, but there are pending normal tickets with 2MiB reservations. Now a priority ticket comes in with a .5MiB reservation. Because we have normal tickets pending we add ourselves to the priority list, despite the fact that we could satisfy this reservation. The flushing machinery now gets to the point where it wants to commit the transaction, but because there's a .5MiB ticket on the priority list and we have 1MiB of free space we assume the ticket will be granted soon, so we bail without committing the transaction. Meanwhile the priority flushing does not commit the transaction, and eventually fails with an ENOSPC. Then all other tickets are failed with ENOSPC because we were never able to actually commit the transaction. The fix for this is we should have simply granted the priority flusher his reservation, because there was space to make the reservation. Priority flushers by definition take priority, so they are allowed to make their reservations before any previous normal tickets. By not adding this priority ticket to the list the normal flushing mechanisms will then commit the transaction and everything will continue normally. We still need to serialize ourselves with other priority tickets, so if there are any tickets on the priority list then we need to add ourselves to that list in order to maintain the serialization between priority tickets. Reviewed-by: Nikolay Borisov Tested-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index b76ca6b74ebb..6826d72abc4c 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -1184,6 +1184,16 @@ static int handle_reserve_ticket(struct btrfs_fs_info *fs_info, return ret; } +/* + * This returns true if this flush state will go through the ordinary flushing + * code. + */ +static inline bool is_normal_flushing(enum btrfs_reserve_flush_enum flush) +{ + return (flush == BTRFS_RESERVE_FLUSH_ALL) || + (flush == BTRFS_RESERVE_FLUSH_ALL_STEAL); +} + /** * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space * @root - the root we're allocating for @@ -1214,8 +1224,17 @@ static int __reserve_metadata_bytes(struct btrfs_fs_info *fs_info, spin_lock(&space_info->lock); ret = -ENOSPC; used = btrfs_space_info_used(space_info, true); - pending_tickets = !list_empty(&space_info->tickets) || - !list_empty(&space_info->priority_tickets); + + /* + * We don't want NO_FLUSH allocations to jump everybody, they can + * generally handle ENOSPC in a different way, so treat them the same as + * normal flushers when it comes to skipping pending tickets. + */ + if (is_normal_flushing(flush) || (flush == BTRFS_RESERVE_NO_FLUSH)) + pending_tickets = !list_empty(&space_info->tickets) || + !list_empty(&space_info->priority_tickets); + else + pending_tickets = !list_empty(&space_info->priority_tickets); /* * Carry on if we have enough space (short-circuit) OR call From 42a72cb75374634ef714514b4742a8844391fd9e Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 13 Mar 2020 15:58:09 -0400 Subject: [PATCH 0722/1043] btrfs: run btrfs_try_granting_tickets if a priority ticket fails With normal tickets we could have a large reservation at the front of the list that is unable to be satisfied, but a smaller ticket later on that can be satisfied. The way we handle this is to run btrfs_try_granting_tickets() in maybe_fail_all_tickets(). However no such protection exists for priority tickets. Fix this by handling it in handle_reserve_ticket(). If we've returned after attempting to flush space in a priority related way, we'll still be on the priority list and need to be removed. We rely on the flushing to free up space and wake the ticket, but if there is not enough space to reclaim _but_ there's enough space in the space_info to handle subsequent reservations then we would have gotten an ENOSPC erroneously. Address this by catching where we are still on the list, meaning we were a priority ticket, and removing ourselves and then running btrfs_try_granting_tickets(). This will handle this particular corner case. Reviewed-by: Nikolay Borisov Tested-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 6826d72abc4c..3cb8423c2065 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -1164,11 +1164,17 @@ static int handle_reserve_ticket(struct btrfs_fs_info *fs_info, ret = ticket->error; if (ticket->bytes || ticket->error) { /* - * Need to delete here for priority tickets. For regular tickets - * either the async reclaim job deletes the ticket from the list - * or we delete it ourselves at wait_reserve_ticket(). + * We were a priority ticket, so we need to delete ourselves + * from the list. Because we could have other priority tickets + * behind us that require less space, run + * btrfs_try_granting_tickets() to see if their reservations can + * now be made. */ - remove_ticket(space_info, ticket); + if (!list_empty(&ticket->list)) { + remove_ticket(space_info, ticket); + btrfs_try_granting_tickets(fs_info, space_info); + } + if (!ret) ret = -ENOSPC; } From 9c343784c4328781129bcf9e671645f69fe4b38a Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 13 Mar 2020 15:28:48 -0400 Subject: [PATCH 0723/1043] btrfs: force chunk allocation if our global rsv is larger than metadata Nikolay noticed a bunch of test failures with my global rsv steal patches. At first he thought they were introduced by them, but they've been failing for a while with 64k nodes. The problem is with 64k nodes we have a global reserve that calculates out to 13MiB on a freshly made file system, which only has 8MiB of metadata space. Because of changes I previously made we no longer account for the global reserve in the overcommit logic, which means we correctly allow overcommit to happen even though we are already overcommitted. However in some corner cases, for example btrfs/170, we will allocate the entire file system up with data chunks before we have enough space pressure to allocate a metadata chunk. Then once the fs is full we ENOSPC out because we cannot overcommit and the global reserve is taking up all of the available space. The most ideal way to deal with this is to change our space reservation stuff to take into account the height of the tree's that we're modifying, so that our global reserve calculation does not end up so obscenely large. However that is a huge undertaking. Instead fix this by forcing a chunk allocation if the global reserve is larger than the total metadata space. This gives us essentially the same behavior that happened before, we get a chunk allocated and these tests can pass. This is meant to be a stop-gap measure until we can tackle the "tree height only" project. Fixes: 0096420adb03 ("btrfs: do not account global reserve in can_overcommit") CC: stable@vger.kernel.org # 5.4+ Reviewed-by: Nikolay Borisov Tested-by: Nikolay Borisov Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/block-rsv.c | 3 +++ fs/btrfs/transaction.c | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/fs/btrfs/block-rsv.c b/fs/btrfs/block-rsv.c index 27efec8f7c5b..dbba53e712e6 100644 --- a/fs/btrfs/block-rsv.c +++ b/fs/btrfs/block-rsv.c @@ -5,6 +5,7 @@ #include "block-rsv.h" #include "space-info.h" #include "transaction.h" +#include "block-group.h" /* * HOW DO BLOCK RESERVES WORK @@ -405,6 +406,8 @@ void btrfs_update_global_block_rsv(struct btrfs_fs_info *fs_info) else block_rsv->full = 0; + if (block_rsv->size >= sinfo->total_bytes) + sinfo->force_alloc = CHUNK_ALLOC_FORCE; spin_unlock(&block_rsv->lock); spin_unlock(&sinfo->lock); } diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index b5da5d8342dc..96eb313a5080 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -21,6 +21,7 @@ #include "dev-replace.h" #include "qgroup.h" #include "block-group.h" +#include "space-info.h" #define BTRFS_ROOT_TRANS_TAG 0 @@ -523,6 +524,7 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, u64 num_bytes = 0; u64 qgroup_reserved = 0; bool reloc_reserved = false; + bool do_chunk_alloc = false; int ret; /* Send isn't supposed to start transactions. */ @@ -585,6 +587,9 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, delayed_refs_bytes); num_bytes -= delayed_refs_bytes; } + + if (rsv->space_info->force_alloc) + do_chunk_alloc = true; } else if (num_items == 0 && flush == BTRFS_RESERVE_FLUSH_ALL && !delayed_refs_rsv->full) { /* @@ -666,6 +671,19 @@ got_it: if (!current->journal_info) current->journal_info = h; + /* + * If the space_info is marked ALLOC_FORCE then we'll get upgraded to + * ALLOC_FORCE the first run through, and then we won't allocate for + * anybody else who races in later. We don't care about the return + * value here. + */ + if (do_chunk_alloc && num_bytes) { + u64 flags = h->block_rsv->space_info->flags; + + btrfs_chunk_alloc(h, btrfs_get_alloc_profile(fs_info, flags), + CHUNK_ALLOC_NO_FORCE); + } + /* * btrfs_record_root_in_trans() needs to alloc new extents, and may * call btrfs_join_transaction() while we're also starting a From 72f4f078de3fa81b089522fb8090b08e3ffdd148 Mon Sep 17 00:00:00 2001 From: Zheng Wei Date: Mon, 16 Mar 2020 11:45:57 +0800 Subject: [PATCH 0724/1043] btrfs: tree-checker: remove duplicate definition of 'inode_item_err' Remove the duplicate definition of 'inode_item_err' in the file tree-checker.c that got there by accident in c23c77b097dc ("btrfs: tree-checker: Refactor inode key check into seperate function"). Reviewed-by: Nikolay Borisov Signed-off-by: Zheng Wei Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/tree-checker.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index a92f8a6dd192..517b44300a05 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -957,10 +957,6 @@ static int check_dev_item(struct extent_buffer *leaf, return 0; } -/* Inode item error output has the same format as dir_item_err() */ -#define inode_item_err(eb, slot, fmt, ...) \ - dir_item_err(eb, slot, fmt, __VA_ARGS__) - static int check_inode_item(struct extent_buffer *leaf, struct btrfs_key *key, int slot) { From 2d9faa5a8aaaf4a0559be46cccdbed7dbb10bad0 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Tue, 7 Apr 2020 11:38:58 +0100 Subject: [PATCH 0725/1043] btrfs: remove pointless assertion on reclaim_size counter The reclaim_size counter of a space_info object is unsigned. So its value can never be negative, it's pointless to have an assertion that checks its value is >= 0, therefore remove it. Reviewed-by: Nikolay Borisov Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/space-info.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 3cb8423c2065..41ee88633769 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -1262,7 +1262,6 @@ static int __reserve_metadata_bytes(struct btrfs_fs_info *fs_info, * the list and we will do our own flushing further down. */ if (ret && flush != BTRFS_RESERVE_NO_FLUSH) { - ASSERT(space_info->reclaim_size >= 0); ticket.bytes = orig_bytes; ticket.error = 0; space_info->reclaim_size += ticket.bytes; From 7c09c03091ac562ddca2b393e5d65c1d37da79f1 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 25 Feb 2020 15:05:53 +0100 Subject: [PATCH 0726/1043] btrfs: don't force read-only after error in drop snapshot Deleting a subvolume on a full filesystem leads to ENOSPC followed by a forced read-only. This is not a transaction abort and the filesystem is otherwise ok, so the error should be just propagated to the callers. This is caused by unnecessary call to btrfs_handle_fs_error for all errors, except EAGAIN. This does not make sense as the standard transaction abort mechanism is in btrfs_drop_snapshot so all relevant failures are handled. Originally in commit cb1b69f4508a ("Btrfs: forced readonly when btrfs_drop_snapshot() fails") there was no return value at all, so the btrfs_std_error made some sense but once the error handling and propagation has been implemented we don't need it anymore. Signed-off-by: David Sterba --- fs/btrfs/extent-tree.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 752b229cbb13..faa585d54eb7 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -5502,8 +5502,6 @@ out: */ if (!for_reloc && !root_dropped) btrfs_add_dead_root(root); - if (err && err != -EAGAIN) - btrfs_handle_fs_error(fs_info, err, NULL); return err; } From a7571232b2aa8a43a137f827d77941634c309144 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Fri, 21 Feb 2020 15:11:24 +0200 Subject: [PATCH 0727/1043] btrfs: use list_for_each_entry_safe in free_reloc_roots The function always works on a local copy of the reloc root list, which cannot be modified outside of it so using list_for_each_entry is fine. Additionally the macro handles empty lists so drop list_empty checks of callers. No semantic changes. Reviewed-by: Qu Wenruo Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 80b58358f688..5f8820c4eb3c 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -1884,13 +1884,10 @@ again: static noinline_for_stack void free_reloc_roots(struct list_head *list) { - struct btrfs_root *reloc_root; + struct btrfs_root *reloc_root, *tmp; - while (!list_empty(list)) { - reloc_root = list_entry(list->next, struct btrfs_root, - root_list); + list_for_each_entry_safe(reloc_root, tmp, list, root_list) __del_reloc_root(reloc_root); - } } static noinline_for_stack @@ -1949,15 +1946,13 @@ again: out: if (ret) { btrfs_handle_fs_error(fs_info, ret, NULL); - if (!list_empty(&reloc_roots)) - free_reloc_roots(&reloc_roots); + free_reloc_roots(&reloc_roots); /* new reloc root may be added */ mutex_lock(&fs_info->reloc_mutex); list_splice_init(&rc->reloc_roots, &reloc_roots); mutex_unlock(&fs_info->reloc_mutex); - if (!list_empty(&reloc_roots)) - free_reloc_roots(&reloc_roots); + free_reloc_roots(&reloc_roots); } /* @@ -3869,8 +3864,7 @@ out_unset: unset_reloc_control(rc); free_reloc_control(rc); out: - if (!list_empty(&reloc_roots)) - free_reloc_roots(&reloc_roots); + free_reloc_roots(&reloc_roots); btrfs_free_path(path); From b335eab890ed4c5b8e45e4b3d12f303686298850 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Wed, 15 Apr 2020 15:53:46 +0300 Subject: [PATCH 0728/1043] btrfs: make btrfs_read_disk_super return struct btrfs_disk_super Instead of returning both the page and the super block structure, make btrfs_read_disk_super just return a pointer to struct btrfs_disk_super. As a result the function signature is simplified. Also, read_cache_page_gfp can never return NULL so check its return value only for IS_ERR. Reviewed-by: Johannes Thumshirn Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/volumes.c | 47 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index c1909e5f4506..aa1e0ae32943 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -1251,49 +1251,48 @@ void btrfs_release_disk_super(struct btrfs_super_block *super) put_page(page); } -static int btrfs_read_disk_super(struct block_device *bdev, u64 bytenr, - struct page **page, - struct btrfs_super_block **disk_super) +static struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev, + u64 bytenr) { + struct btrfs_super_block *disk_super; + struct page *page; void *p; pgoff_t index; /* make sure our super fits in the device */ if (bytenr + PAGE_SIZE >= i_size_read(bdev->bd_inode)) - return 1; + return ERR_PTR(-EINVAL); /* make sure our super fits in the page */ - if (sizeof(**disk_super) > PAGE_SIZE) - return 1; + if (sizeof(*disk_super) > PAGE_SIZE) + return ERR_PTR(-EINVAL); /* make sure our super doesn't straddle pages on disk */ index = bytenr >> PAGE_SHIFT; - if ((bytenr + sizeof(**disk_super) - 1) >> PAGE_SHIFT != index) - return 1; + if ((bytenr + sizeof(*disk_super) - 1) >> PAGE_SHIFT != index) + return ERR_PTR(-EINVAL); /* pull in the page with our super */ - *page = read_cache_page_gfp(bdev->bd_inode->i_mapping, - index, GFP_KERNEL); + page = read_cache_page_gfp(bdev->bd_inode->i_mapping, index, GFP_KERNEL); - if (IS_ERR(*page)) - return 1; + if (IS_ERR(page)) + return ERR_CAST(page); - p = page_address(*page); + p = page_address(page); /* align our pointer to the offset of the super block */ - *disk_super = p + offset_in_page(bytenr); + disk_super = p + offset_in_page(bytenr); - if (btrfs_super_bytenr(*disk_super) != bytenr || - btrfs_super_magic(*disk_super) != BTRFS_MAGIC) { + if (btrfs_super_bytenr(disk_super) != bytenr || + btrfs_super_magic(disk_super) != BTRFS_MAGIC) { btrfs_release_disk_super(p); - return 1; + return ERR_PTR(-EINVAL); } - if ((*disk_super)->label[0] && - (*disk_super)->label[BTRFS_LABEL_SIZE - 1]) - (*disk_super)->label[BTRFS_LABEL_SIZE - 1] = '\0'; + if (disk_super->label[0] && disk_super->label[BTRFS_LABEL_SIZE - 1]) + disk_super->label[BTRFS_LABEL_SIZE - 1] = 0; - return 0; + return disk_super; } int btrfs_forget_devices(const char *path) @@ -1319,7 +1318,6 @@ struct btrfs_device *btrfs_scan_one_device(const char *path, fmode_t flags, bool new_device_added = false; struct btrfs_device *device = NULL; struct block_device *bdev; - struct page *page; u64 bytenr; lockdep_assert_held(&uuid_mutex); @@ -1337,8 +1335,9 @@ struct btrfs_device *btrfs_scan_one_device(const char *path, fmode_t flags, if (IS_ERR(bdev)) return ERR_CAST(bdev); - if (btrfs_read_disk_super(bdev, bytenr, &page, &disk_super)) { - device = ERR_PTR(-EINVAL); + disk_super = btrfs_read_disk_super(bdev, bytenr); + if (IS_ERR(disk_super)) { + device = ERR_CAST(disk_super); goto error_bdev_put; } From e3b8336117e515a260da32fa10bb3354ba12c429 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Fri, 17 Apr 2020 15:08:21 +0800 Subject: [PATCH 0729/1043] btrfs: remove the redundant parameter level in btrfs_bin_search() All callers pass the eb::level so we can get read it directly inside the btrfs_bin_search and key_search. This is inspired by the work of Marek in U-boot. CC: Marek Behun Reviewed-by: Johannes Thumshirn Reviewed-by: Nikolay Borisov Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 14 +++++++------- fs/btrfs/ctree.h | 2 +- fs/btrfs/relocation.c | 8 +++----- fs/btrfs/tree-log.c | 3 +-- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index bfedbbe2311f..6c28efe5b14a 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -1733,9 +1733,9 @@ static noinline int generic_bin_search(struct extent_buffer *eb, * leaves vs nodes */ int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key, - int level, int *slot) + int *slot) { - if (level == 0) + if (btrfs_header_level(eb) == 0) return generic_bin_search(eb, offsetof(struct btrfs_leaf, items), sizeof(struct btrfs_item), @@ -2502,10 +2502,10 @@ done: } static int key_search(struct extent_buffer *b, const struct btrfs_key *key, - int level, int *prev_cmp, int *slot) + int *prev_cmp, int *slot) { if (*prev_cmp != 0) { - *prev_cmp = btrfs_bin_search(b, key, level, slot); + *prev_cmp = btrfs_bin_search(b, key, slot); return *prev_cmp; } @@ -2783,7 +2783,7 @@ cow_done: } } - ret = key_search(b, key, level, &prev_cmp, &slot); + ret = key_search(b, key, &prev_cmp, &slot); if (ret < 0) goto done; @@ -2947,7 +2947,7 @@ again: * time. */ prev_cmp = -1; - ret = key_search(b, key, level, &prev_cmp, &slot); + ret = key_search(b, key, &prev_cmp, &slot); if (ret < 0) goto done; @@ -5103,7 +5103,7 @@ again: while (1) { nritems = btrfs_header_nritems(cur); level = btrfs_header_level(cur); - sret = btrfs_bin_search(cur, min_key, level, &slot); + sret = btrfs_bin_search(cur, min_key, &slot); if (sret < 0) { ret = sret; goto out; diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index c322568231a4..e4b8f5f2273d 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -2552,7 +2552,7 @@ void btrfs_wait_for_snapshot_creation(struct btrfs_root *root); /* ctree.c */ int btrfs_bin_search(struct extent_buffer *eb, const struct btrfs_key *key, - int level, int *slot); + int *slot); int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2); int btrfs_previous_item(struct btrfs_root *root, struct btrfs_path *path, u64 min_objectid, diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 5f8820c4eb3c..f25deca18a5d 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -1236,7 +1236,7 @@ again: level = btrfs_header_level(parent); BUG_ON(level < lowest_level); - ret = btrfs_bin_search(parent, &key, level, &slot); + ret = btrfs_bin_search(parent, &key, &slot); if (ret < 0) break; if (ret && slot > 0) @@ -2220,8 +2220,7 @@ static int do_relocation(struct btrfs_trans_handle *trans, if (upper->eb && !upper->locked) { if (!lowest) { - ret = btrfs_bin_search(upper->eb, key, - upper->level, &slot); + ret = btrfs_bin_search(upper->eb, key, &slot); if (ret < 0) { err = ret; goto next; @@ -2259,8 +2258,7 @@ static int do_relocation(struct btrfs_trans_handle *trans, slot = path->slots[upper->level]; btrfs_release_path(path); } else { - ret = btrfs_bin_search(upper->eb, key, upper->level, - &slot); + ret = btrfs_bin_search(upper->eb, key, &slot); if (ret < 0) { err = ret; goto next; diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 02ebdd9edc19..0254e8d10e4a 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -3816,8 +3816,7 @@ static int drop_objectid_items(struct btrfs_trans_handle *trans, found_key.offset = 0; found_key.type = 0; - ret = btrfs_bin_search(path->nodes[0], &found_key, 0, - &start_slot); + ret = btrfs_bin_search(path->nodes[0], &found_key, &start_slot); if (ret < 0) break; From 534cf531ccf6331b6e63a054f2ceb8e1ccff1ec9 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Fri, 17 Apr 2020 16:36:50 +0100 Subject: [PATCH 0730/1043] btrfs: simplify error handling of clean_pinned_extents() At clean_pinned_extents(), whether we end up returning success or failure, we pretty much have to do the same things: 1) unlock unused_bg_unpin_mutex 2) decrement reference count on the previous transaction We also call btrfs_dec_block_group_ro() in case of failure, but that is better done in its caller, btrfs_delete_unused_bgs(), since its the caller that calls inc_block_group_ro(), so it should be responsible for the decrement operation, as it is in case any of the other functions it calls fail. So move the call to btrfs_dec_block_group_ro() from clean_pinned_extents() into btrfs_delete_unused_bgs() and unify the error and success return paths for clean_pinned_extents(), reducing duplicated code and making it simpler. Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 233c5663f233..5627f53ca115 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1284,25 +1284,17 @@ static bool clean_pinned_extents(struct btrfs_trans_handle *trans, ret = clear_extent_bits(&prev_trans->pinned_extents, start, end, EXTENT_DIRTY); if (ret) - goto err; + goto out; } ret = clear_extent_bits(&trans->transaction->pinned_extents, start, end, EXTENT_DIRTY); - if (ret) - goto err; +out: mutex_unlock(&fs_info->unused_bg_unpin_mutex); if (prev_trans) btrfs_put_transaction(prev_trans); - return true; - -err: - mutex_unlock(&fs_info->unused_bg_unpin_mutex); - if (prev_trans) - btrfs_put_transaction(prev_trans); - btrfs_dec_block_group_ro(bg); - return false; + return ret == 0; } /* @@ -1400,8 +1392,10 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) * We could have pending pinned extents for this block group, * just delete them, we don't care about them anymore. */ - if (!clean_pinned_extents(trans, block_group)) + if (!clean_pinned_extents(trans, block_group)) { + btrfs_dec_block_group_ro(block_group); goto end_trans; + } /* * At this point, the block_group is read only and should fail From 1072c12d7d58b5512b6c05c2268f57d32f1ab76c Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:11 -0700 Subject: [PATCH 0731/1043] block: add bio_for_each_bvec_all() An upcoming Btrfs fix needs to know the original size of a non-cloned bios. Rather than accessing the bvec table directly, let's add a bio_for_each_bvec_all() accessor. Reviewed-by: Johannes Thumshirn Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- .clang-format | 1 + Documentation/block/biovecs.rst | 2 ++ include/linux/bio.h | 8 ++++++++ 3 files changed, 11 insertions(+) diff --git a/.clang-format b/.clang-format index e92e6dd1780d..a0a96088c74f 100644 --- a/.clang-format +++ b/.clang-format @@ -80,6 +80,7 @@ ForEachMacros: - 'ax25_uid_for_each' - '__bio_for_each_bvec' - 'bio_for_each_bvec' + - 'bio_for_each_bvec_all' - 'bio_for_each_integrity_vec' - '__bio_for_each_segment' - 'bio_for_each_segment' diff --git a/Documentation/block/biovecs.rst b/Documentation/block/biovecs.rst index ad303a2569d3..36771a131b56 100644 --- a/Documentation/block/biovecs.rst +++ b/Documentation/block/biovecs.rst @@ -129,6 +129,7 @@ Usage of helpers: :: bio_for_each_segment_all() + bio_for_each_bvec_all() bio_first_bvec_all() bio_first_page_all() bio_last_bvec_all() @@ -143,4 +144,5 @@ Usage of helpers: bio_vec' will contain a multi-page IO vector during the iteration:: bio_for_each_bvec() + bio_for_each_bvec_all() rq_for_each_bvec() diff --git a/include/linux/bio.h b/include/linux/bio.h index a0ee494a6329..8e23f51ccfa4 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -169,6 +169,14 @@ static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter, #define bio_for_each_bvec(bvl, bio, iter) \ __bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter) +/* + * Iterate over all multi-page bvecs. Drivers shouldn't use this version for the + * same reasons as bio_for_each_segment_all(). + */ +#define bio_for_each_bvec_all(bvl, bio, i) \ + for (i = 0, bvl = bio_first_bvec_all(bio); \ + i < (bio)->bi_vcnt; i++, bvl++) \ + #define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len) static inline unsigned bio_segments(struct bio *bio) From 6d3113a193e3385c72240096fe397618ecab6e43 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:12 -0700 Subject: [PATCH 0732/1043] btrfs: fix error handling when submitting direct I/O bio In btrfs_submit_direct_hook(), if a direct I/O write doesn't span a RAID stripe or chunk, we submit orig_bio without cloning it. In this case, we don't increment pending_bios. Then, if btrfs_submit_dio_bio() fails, we decrement pending_bios to -1, and we never complete orig_bio. Fix it by initializing pending_bios to 1 instead of incrementing later. Fixing this exposes another bug: we put orig_bio prematurely and then put it again from end_io. Fix it by not putting orig_bio. After this change, pending_bios is really more of a reference count, but I'll leave that cleanup separate to keep the fix small. Fixes: e65e15355429 ("btrfs: fix panic caused by direct IO") CC: stable@vger.kernel.org # 4.4+ Reviewed-by: Nikolay Borisov Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- fs/btrfs/inode.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 259239b33370..b628c319a5b6 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7939,7 +7939,6 @@ static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip) /* bio split */ ASSERT(geom.len <= INT_MAX); - atomic_inc(&dip->pending_bios); do { clone_len = min_t(int, submit_len, geom.len); @@ -7989,7 +7988,8 @@ submit: if (!status) return 0; - bio_put(bio); + if (bio != orig_bio) + bio_put(bio); out_err: dip->errors = 1; /* @@ -8030,7 +8030,7 @@ static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, bio->bi_private = dip; dip->orig_bio = bio; dip->dio_bio = dio_bio; - atomic_set(&dip->pending_bios, 0); + atomic_set(&dip->pending_bios, 1); io_bio = btrfs_io_bio(bio); io_bio->logical = file_offset; From c36cac28cb94e58f7e21ff43bdc6064346dab32c Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:13 -0700 Subject: [PATCH 0733/1043] btrfs: fix double __endio_write_update_ordered in direct I/O In btrfs_submit_direct(), if we fail to allocate the btrfs_dio_private, we complete the ordered extent range. However, we don't mark that the range doesn't need to be cleaned up from btrfs_direct_IO() until later. Therefore, if we fail to allocate the btrfs_dio_private, we complete the ordered extent range twice. We could fix this by updating unsubmitted_oe_range earlier, but it's cleaner to reorganize the code so that creating the btrfs_dio_private and submitting the bios are separate, and once the btrfs_dio_private is created, cleanup always happens through the btrfs_dio_private. The logic around unsubmitted_oe_range_end and unsubmitted_oe_range_start is really subtle. We have the following: 1. btrfs_direct_IO sets those two to the same value. 2. When we call __blockdev_direct_IO unless btrfs_get_blocks_direct->btrfs_get_blocks_direct_write is called to modify unsubmitted_oe_range_start so that start < end. Cleanup won't happen. 3. We come into btrfs_submit_direct - if it dip allocation fails we'd return with oe_range_end now modified so cleanup will happen. 4. If we manage to allocate the dip we reset the unsubmitted range members to be equal so that cleanup happens from btrfs_endio_direct_write. This 4-step logic is not really obvious, especially given it's scattered across 3 functions. Fixes: f28a49287817 ("Btrfs: fix leaking of ordered extents after direct IO write error") Reviewed-by: Johannes Thumshirn Reviewed-by: Nikolay Borisov Signed-off-by: Omar Sandoval [ add range start/end logic explanation from Nikolay ] Signed-off-by: David Sterba --- fs/btrfs/inode.c | 180 +++++++++++++++++++---------------------------- 1 file changed, 71 insertions(+), 109 deletions(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index b628c319a5b6..81020eb898c3 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7903,14 +7903,64 @@ err: return ret; } -static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip) +/* + * If this succeeds, the btrfs_dio_private is responsible for cleaning up locked + * or ordered extents whether or not we submit any bios. + */ +static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio, + struct inode *inode, + loff_t file_offset) { - struct inode *inode = dip->inode; - struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + const bool write = (bio_op(dio_bio) == REQ_OP_WRITE); + struct btrfs_dio_private *dip; struct bio *bio; - struct bio *orig_bio = dip->orig_bio; - u64 start_sector = orig_bio->bi_iter.bi_sector; - u64 file_offset = dip->logical_offset; + + dip = kzalloc(sizeof(*dip), GFP_NOFS); + if (!dip) + return NULL; + + bio = btrfs_bio_clone(dio_bio); + bio->bi_private = dip; + btrfs_io_bio(bio)->logical = file_offset; + + dip->private = dio_bio->bi_private; + dip->inode = inode; + dip->logical_offset = file_offset; + dip->bytes = dio_bio->bi_iter.bi_size; + dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9; + dip->orig_bio = bio; + dip->dio_bio = dio_bio; + atomic_set(&dip->pending_bios, 1); + + if (write) { + struct btrfs_dio_data *dio_data = current->journal_info; + + /* + * Setting range start and end to the same value means that + * no cleanup will happen in btrfs_direct_IO + */ + dio_data->unsubmitted_oe_range_end = dip->logical_offset + + dip->bytes; + dio_data->unsubmitted_oe_range_start = + dio_data->unsubmitted_oe_range_end; + + bio->bi_end_io = btrfs_endio_direct_write; + } else { + bio->bi_end_io = btrfs_endio_direct_read; + dip->subio_endio = btrfs_subio_endio_read; + } + return dip; +} + +static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, + loff_t file_offset) +{ + const bool write = (bio_op(dio_bio) == REQ_OP_WRITE); + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + struct btrfs_dio_private *dip; + struct bio *bio; + struct bio *orig_bio; + u64 start_sector; int async_submit = 0; u64 submit_len; int clone_offset = 0; @@ -7919,11 +7969,24 @@ static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip) blk_status_t status; struct btrfs_io_geometry geom; + dip = btrfs_create_dio_private(dio_bio, inode, file_offset); + if (!dip) { + if (!write) { + unlock_extent(&BTRFS_I(inode)->io_tree, file_offset, + file_offset + dio_bio->bi_iter.bi_size - 1); + } + dio_bio->bi_status = BLK_STS_RESOURCE; + dio_end_io(dio_bio); + return; + } + + orig_bio = dip->orig_bio; + start_sector = orig_bio->bi_iter.bi_sector; submit_len = orig_bio->bi_iter.bi_size; ret = btrfs_get_io_geometry(fs_info, btrfs_op(orig_bio), start_sector << 9, submit_len, &geom); if (ret) - return -EIO; + goto out_err; if (geom.len >= submit_len) { bio = orig_bio; @@ -7986,7 +8049,7 @@ static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip) submit: status = btrfs_submit_dio_bio(bio, inode, file_offset, async_submit); if (!status) - return 0; + return; if (bio != orig_bio) bio_put(bio); @@ -8000,107 +8063,6 @@ out_err: */ if (atomic_dec_and_test(&dip->pending_bios)) bio_io_error(dip->orig_bio); - - /* bio_end_io() will handle error, so we needn't return it */ - return 0; -} - -static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, - loff_t file_offset) -{ - struct btrfs_dio_private *dip = NULL; - struct bio *bio = NULL; - struct btrfs_io_bio *io_bio; - bool write = (bio_op(dio_bio) == REQ_OP_WRITE); - int ret = 0; - - bio = btrfs_bio_clone(dio_bio); - - dip = kzalloc(sizeof(*dip), GFP_NOFS); - if (!dip) { - ret = -ENOMEM; - goto free_ordered; - } - - dip->private = dio_bio->bi_private; - dip->inode = inode; - dip->logical_offset = file_offset; - dip->bytes = dio_bio->bi_iter.bi_size; - dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9; - bio->bi_private = dip; - dip->orig_bio = bio; - dip->dio_bio = dio_bio; - atomic_set(&dip->pending_bios, 1); - io_bio = btrfs_io_bio(bio); - io_bio->logical = file_offset; - - if (write) { - bio->bi_end_io = btrfs_endio_direct_write; - } else { - bio->bi_end_io = btrfs_endio_direct_read; - dip->subio_endio = btrfs_subio_endio_read; - } - - /* - * Reset the range for unsubmitted ordered extents (to a 0 length range) - * even if we fail to submit a bio, because in such case we do the - * corresponding error handling below and it must not be done a second - * time by btrfs_direct_IO(). - */ - if (write) { - struct btrfs_dio_data *dio_data = current->journal_info; - - dio_data->unsubmitted_oe_range_end = dip->logical_offset + - dip->bytes; - dio_data->unsubmitted_oe_range_start = - dio_data->unsubmitted_oe_range_end; - } - - ret = btrfs_submit_direct_hook(dip); - if (!ret) - return; - - btrfs_io_bio_free_csum(io_bio); - -free_ordered: - /* - * If we arrived here it means either we failed to submit the dip - * or we either failed to clone the dio_bio or failed to allocate the - * dip. If we cloned the dio_bio and allocated the dip, we can just - * call bio_endio against our io_bio so that we get proper resource - * cleanup if we fail to submit the dip, otherwise, we must do the - * same as btrfs_endio_direct_[write|read] because we can't call these - * callbacks - they require an allocated dip and a clone of dio_bio. - */ - if (bio && dip) { - bio_io_error(bio); - /* - * The end io callbacks free our dip, do the final put on bio - * and all the cleanup and final put for dio_bio (through - * dio_end_io()). - */ - dip = NULL; - bio = NULL; - } else { - if (write) - __endio_write_update_ordered(inode, - file_offset, - dio_bio->bi_iter.bi_size, - false); - else - unlock_extent(&BTRFS_I(inode)->io_tree, file_offset, - file_offset + dio_bio->bi_iter.bi_size - 1); - - dio_bio->bi_status = BLK_STS_IOERR; - /* - * Releases and cleans up our dio_bio, no need to bio_put() - * nor bio_endio()/bio_io_error() against dio_bio. - */ - dio_end_io(dio_bio); - } - if (bio) - bio_put(bio); - kfree(dip); } static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info, From c7333972b9b571a03bf9aecd1aeecfab81243e8d Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:14 -0700 Subject: [PATCH 0734/1043] btrfs: look at full bi_io_vec for repair decision Read repair does two things: it finds a good copy of data to return to the reader, and it corrects the bad copy on disk. If a read of multiple sectors has an I/O error, repair does an extra "validation" step that issues a separate read for each sector. This allows us to find the exact failing sectors and only rewrite those. This heuristic is implemented in bio_readpage_error()/btrfs_check_repairable() as: failed_bio_pages = failed_bio->bi_iter.bi_size >> PAGE_SHIFT; if (failed_bio_pages > 1) do validation However, at this point, bi_iter may have already been advanced. This means that we'll skip the validation step and rewrite the entire failed read. Fix it by getting the actual size from the biovec (which we can do because this is only called for non-cloned bios, although that will change in a later commit). Fixes: 8a2ee44a371c ("btrfs: look at bi_size for repair decisions") Reviewed-by: Johannes Thumshirn Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 33 +++++++++++++++++++++++++++------ fs/btrfs/extent_io.h | 5 +++-- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 39e45b8a5031..fcf2ff9ae4a8 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2537,8 +2537,9 @@ int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end, return 0; } -bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages, - struct io_failure_record *failrec, int failed_mirror) +bool btrfs_check_repairable(struct inode *inode, bool needs_validation, + struct io_failure_record *failrec, + int failed_mirror) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); int num_copies; @@ -2561,7 +2562,7 @@ bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages, * a) deliver good data to the caller * b) correct the bad sectors on disk */ - if (failed_bio_pages > 1) { + if (needs_validation) { /* * to fulfill b), we need to know the exact failing sectors, as * we don't want to rewrite any more than the failed ones. thus, @@ -2633,6 +2634,24 @@ struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio, return bio; } +static bool btrfs_io_needs_validation(struct inode *inode, struct bio *bio) +{ + struct bio_vec *bvec; + u64 len = 0; + int i; + + /* + * We need to validate each sector individually if the failed I/O was + * for multiple sectors. + */ + bio_for_each_bvec_all(bvec, bio, i) { + len += bvec->bv_len; + if (len > inode->i_sb->s_blocksize) + return true; + } + return false; +} + /* * This is a generic handler for readpage errors. If other copies exist, read * those and write back good data to the failed position. Does not investigate @@ -2647,11 +2666,11 @@ static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset, struct inode *inode = page->mapping->host; struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; + bool need_validation; struct bio *bio; int read_mode = 0; blk_status_t status; int ret; - unsigned failed_bio_pages = failed_bio->bi_iter.bi_size >> PAGE_SHIFT; BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE); @@ -2659,13 +2678,15 @@ static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset, if (ret) return ret; - if (!btrfs_check_repairable(inode, failed_bio_pages, failrec, + need_validation = btrfs_io_needs_validation(inode, failed_bio); + + if (!btrfs_check_repairable(inode, need_validation, failrec, failed_mirror)) { free_io_failure(failure_tree, tree, failrec); return -EIO; } - if (failed_bio_pages > 1) + if (need_validation) read_mode |= REQ_FAILFAST_DEV; phy_offset >>= inode->i_sb->s_blocksize_bits; diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index 2ed65bd0760e..e042c771893d 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -312,8 +312,9 @@ struct io_failure_record { }; -bool btrfs_check_repairable(struct inode *inode, unsigned failed_bio_pages, - struct io_failure_record *failrec, int fail_mirror); +bool btrfs_check_repairable(struct inode *inode, bool needs_validation, + struct io_failure_record *failrec, + int failed_mirror); struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio, struct io_failure_record *failrec, struct page *page, int pg_offset, int icsum, From f337bd7478178f4c5e2b825fdb8dfaa266c344a2 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:15 -0700 Subject: [PATCH 0735/1043] btrfs: don't do repair validation for checksum errors The purpose of the validation step is to distinguish between good and bad sectors in a failed multi-sector read. If a multi-sector read succeeded but some of those sectors had checksum errors, we don't need to validate anything; we know the sectors with bad checksums need to be repaired. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index fcf2ff9ae4a8..cf24b2855462 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2640,6 +2640,14 @@ static bool btrfs_io_needs_validation(struct inode *inode, struct bio *bio) u64 len = 0; int i; + /* + * If bi_status is BLK_STS_OK, then this was a checksum error, not an + * I/O error. In this case, we already know exactly which sector was + * bad, so we don't need to validate. + */ + if (bio->bi_status == BLK_STS_OK) + return false; + /* * We need to validate each sector individually if the failed I/O was * for multiple sectors. From fb30f4707d421add0f4bf9a08bb228daac273b9f Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:16 -0700 Subject: [PATCH 0736/1043] btrfs: clarify btrfs_lookup_bio_sums documentation Fix a couple of issues in the btrfs_lookup_bio_sums documentation: * The bio doesn't need to be a btrfs_io_bio if dst was provided. Move the declaration in the code to make that clear, too. * dst must be large enough to hold nblocks * csum_size, not just csum_size. Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Reviewed-by: Nikolay Borisov Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- fs/btrfs/file-item.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index b618ad5339ba..22cbb4da6d42 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -242,11 +242,13 @@ int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans, /** * btrfs_lookup_bio_sums - Look up checksums for a bio. * @inode: inode that the bio is for. - * @bio: bio embedded in btrfs_io_bio. + * @bio: bio to look up. * @offset: Unless (u64)-1, look up checksums for this offset in the file. * If (u64)-1, use the page offsets from the bio instead. - * @dst: Buffer of size btrfs_super_csum_size() used to return checksum. If - * NULL, the checksum is returned in btrfs_io_bio(bio)->csum instead. + * @dst: Buffer of size nblocks * btrfs_super_csum_size() used to return + * checksum (nblocks = bio->bi_iter.bi_size / fs_info->sectorsize). If + * NULL, the checksum buffer is allocated and returned in + * btrfs_io_bio(bio)->csum instead. * * Return: BLK_STS_RESOURCE if allocating memory fails, BLK_STS_OK otherwise. */ @@ -256,7 +258,6 @@ blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct bio_vec bvec; struct bvec_iter iter; - struct btrfs_io_bio *btrfs_bio = btrfs_io_bio(bio); struct btrfs_csum_item *item = NULL; struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; struct btrfs_path *path; @@ -277,6 +278,8 @@ blk_status_t btrfs_lookup_bio_sums(struct inode *inode, struct bio *bio, nblocks = bio->bi_iter.bi_size >> inode->i_sb->s_blocksize_bits; if (!dst) { + struct btrfs_io_bio *btrfs_bio = btrfs_io_bio(bio); + if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) { btrfs_bio->csum = kmalloc_array(nblocks, csum_size, GFP_NOFS); From 47df7765a803a3333dda7f1452581e1641f984b0 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:17 -0700 Subject: [PATCH 0737/1043] btrfs: rename __readpage_endio_check to check_data_csum __readpage_endio_check() is also used from the direct I/O read code, so give it a more descriptive name. Reviewed-by: Nikolay Borisov Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- fs/btrfs/inode.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 81020eb898c3..b32970d04699 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2726,10 +2726,9 @@ void btrfs_writepage_endio_finish_ordered(struct page *page, u64 start, btrfs_queue_work(wq, &ordered_extent->work); } -static int __readpage_endio_check(struct inode *inode, - struct btrfs_io_bio *io_bio, - int icsum, struct page *page, - int pgoff, u64 start, size_t len) +static int check_data_csum(struct inode *inode, struct btrfs_io_bio *io_bio, + int icsum, struct page *page, int pgoff, u64 start, + size_t len) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); @@ -2790,8 +2789,8 @@ static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio, } phy_offset >>= inode->i_sb->s_blocksize_bits; - return __readpage_endio_check(inode, io_bio, phy_offset, page, offset, - start, (size_t)(end - start + 1)); + return check_data_csum(inode, io_bio, phy_offset, page, offset, start, + (size_t)(end - start + 1)); } /* @@ -7584,9 +7583,9 @@ static void btrfs_retry_endio(struct bio *bio) ASSERT(!bio_flagged(bio, BIO_CLONED)); bio_for_each_segment_all(bvec, bio, iter_all) { - ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page, - bvec->bv_offset, done->start, - bvec->bv_len); + ret = check_data_csum(inode, io_bio, i, bvec->bv_page, + bvec->bv_offset, done->start, + bvec->bv_len); if (!ret) clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree, io_tree, done->start, @@ -7636,8 +7635,9 @@ static blk_status_t __btrfs_subio_endio_read(struct inode *inode, next_block: if (uptodate) { csum_pos = BTRFS_BYTES_TO_BLKS(fs_info, offset); - ret = __readpage_endio_check(inode, io_bio, csum_pos, - bvec.bv_page, pgoff, start, sectorsize); + ret = check_data_csum(inode, io_bio, csum_pos, + bvec.bv_page, pgoff, start, + sectorsize); if (likely(!ret)) goto next; } From ce06d3ec2b5aed70b95ee080a7a3d55ef08ce7f3 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:18 -0700 Subject: [PATCH 0738/1043] btrfs: make btrfs_check_repairable() static Since its introduction in commit 2fe6303e7cd0 ("Btrfs: split bio_readpage_error into several functions"), btrfs_check_repairable() has only been used from extent_io.c where it is defined. Reviewed-by: Nikolay Borisov Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 6 +++--- fs/btrfs/extent_io.h | 3 --- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index cf24b2855462..6ddf6d1c3e70 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2537,9 +2537,9 @@ int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end, return 0; } -bool btrfs_check_repairable(struct inode *inode, bool needs_validation, - struct io_failure_record *failrec, - int failed_mirror) +static bool btrfs_check_repairable(struct inode *inode, bool needs_validation, + struct io_failure_record *failrec, + int failed_mirror) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); int num_copies; diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index e042c771893d..f4dfac756455 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -312,9 +312,6 @@ struct io_failure_record { }; -bool btrfs_check_repairable(struct inode *inode, bool needs_validation, - struct io_failure_record *failrec, - int failed_mirror); struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio, struct io_failure_record *failrec, struct page *page, int pg_offset, int icsum, From 2390a6daf92d241b23bc02687bfb9fabdf93f117 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:19 -0700 Subject: [PATCH 0739/1043] btrfs: remove unused btrfs_dio_private::private We haven't used this since commit 9be3395bcd4a ("Btrfs: use a btrfs bioset instead of abusing bio internals"). Reviewed-by: Nikolay Borisov Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- fs/btrfs/btrfs_inode.h | 1 - fs/btrfs/inode.c | 1 - 2 files changed, 2 deletions(-) diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index 27a1fefce508..ad36685ce046 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -301,7 +301,6 @@ struct btrfs_dio_private { u64 logical_offset; u64 disk_bytenr; u64 bytes; - void *private; /* number of bios pending for this dio */ atomic_t pending_bios; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index b32970d04699..6a240d846017 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7923,7 +7923,6 @@ static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio, bio->bi_private = dip; btrfs_io_bio(bio)->logical = file_offset; - dip->private = dio_bio->bi_private; dip->inode = inode; dip->logical_offset = file_offset; dip->bytes = dio_bio->bi_iter.bi_size; From e3b318d14df7d6d04f37ce10f9a2f33ca3d550be Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:20 -0700 Subject: [PATCH 0740/1043] btrfs: convert btrfs_dio_private->pending_bios to refcount_t This is really a reference count now, so convert it to refcount_t and rename it to refs. Reviewed-by: Nikolay Borisov Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- fs/btrfs/btrfs_inode.h | 8 ++++++-- fs/btrfs/inode.c | 10 +++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index ad36685ce046..b965fa5429ec 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -7,6 +7,7 @@ #define BTRFS_INODE_H #include +#include #include "extent_map.h" #include "extent_io.h" #include "ordered-data.h" @@ -302,8 +303,11 @@ struct btrfs_dio_private { u64 disk_bytenr; u64 bytes; - /* number of bios pending for this dio */ - atomic_t pending_bios; + /* + * References to this structure. There is one reference per in-flight + * bio plus one while we're still setting up. + */ + refcount_t refs; /* IO errors */ int errors; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 6a240d846017..a487454b049d 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7811,7 +7811,7 @@ static void btrfs_end_dio_bio(struct bio *bio) } /* if there are more bios still pending for this dio, just exit */ - if (!atomic_dec_and_test(&dip->pending_bios)) + if (!refcount_dec_and_test(&dip->refs)) goto out; if (dip->errors) { @@ -7929,7 +7929,7 @@ static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio, dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9; dip->orig_bio = bio; dip->dio_bio = dio_bio; - atomic_set(&dip->pending_bios, 1); + refcount_set(&dip->refs, 1); if (write) { struct btrfs_dio_data *dio_data = current->journal_info; @@ -8025,13 +8025,13 @@ static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, * count. Otherwise, the dip might get freed before we're * done setting it up. */ - atomic_inc(&dip->pending_bios); + refcount_inc(&dip->refs); status = btrfs_submit_dio_bio(bio, inode, file_offset, async_submit); if (status) { bio_put(bio); - atomic_dec(&dip->pending_bios); + refcount_dec(&dip->refs); goto out_err; } @@ -8060,7 +8060,7 @@ out_err: * atomic operations with a return value are fully ordered as per * atomic_t.txt */ - if (atomic_dec_and_test(&dip->pending_bios)) + if (refcount_dec_and_test(&dip->refs)) bio_io_error(dip->orig_bio); } From 85879573fc8b06e8f70698686e3bc17c9e6225ba Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:21 -0700 Subject: [PATCH 0741/1043] btrfs: put direct I/O checksums in btrfs_dio_private instead of bio The next commit will get rid of btrfs_dio_private->orig_bio. The only thing we really need it for is containing all of the checksums, but we can easily put the checksum array in btrfs_dio_private and have the submitted bios reference the array. We can also look the checksums up while we're setting up instead of the current awkward logic that looks them up for orig_bio when the first split bio is submitted. (Interestingly, btrfs_dio_private did contain the checksums before commit 23ea8e5a0767 ("Btrfs: load checksum data once when submitting a direct read io"), but it didn't look them up up front.) Reviewed-by: Nikolay Borisov Reviewed-by: Josef Bacik Reviewed-by: Johannes Thumshirn Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- fs/btrfs/btrfs_inode.h | 3 ++ fs/btrfs/inode.c | 70 +++++++++++++++++++----------------------- 2 files changed, 34 insertions(+), 39 deletions(-) diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index b965fa5429ec..7b04389e219c 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -324,6 +324,9 @@ struct btrfs_dio_private { */ blk_status_t (*subio_endio)(struct inode *, struct btrfs_io_bio *, blk_status_t); + + /* Array of checksums */ + u8 csums[]; }; /* diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index a487454b049d..d96c67f3c5a4 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7712,7 +7712,6 @@ static void btrfs_endio_direct_read(struct bio *bio) dio_bio->bi_status = err; dio_end_io(dio_bio); - btrfs_io_bio_free_csum(io_bio); bio_put(bio); } @@ -7824,39 +7823,6 @@ out: bio_put(bio); } -static inline blk_status_t btrfs_lookup_and_bind_dio_csum(struct inode *inode, - struct btrfs_dio_private *dip, - struct bio *bio, - u64 file_offset) -{ - struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); - struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio); - u16 csum_size; - blk_status_t ret; - - /* - * We load all the csum data we need when we submit - * the first bio to reduce the csum tree search and - * contention. - */ - if (dip->logical_offset == file_offset) { - ret = btrfs_lookup_bio_sums(inode, dip->orig_bio, file_offset, - NULL); - if (ret) - return ret; - } - - if (bio == dip->orig_bio) - return 0; - - file_offset -= dip->logical_offset; - file_offset >>= inode->i_sb->s_blocksize_bits; - csum_size = btrfs_super_csum_size(btrfs_sb(inode->i_sb)->super_copy); - io_bio->csum = orig_io_bio->csum + csum_size * file_offset; - - return 0; -} - static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio, struct inode *inode, u64 file_offset, int async_submit) { @@ -7892,10 +7858,12 @@ static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio, if (ret) goto err; } else { - ret = btrfs_lookup_and_bind_dio_csum(inode, dip, bio, - file_offset); - if (ret) - goto err; + u64 csum_offset; + + csum_offset = file_offset - dip->logical_offset; + csum_offset >>= inode->i_sb->s_blocksize_bits; + csum_offset *= btrfs_super_csum_size(fs_info->super_copy); + btrfs_io_bio(bio)->csum = dip->csums + csum_offset; } map: ret = btrfs_map_bio(fs_info, bio, 0); @@ -7912,10 +7880,22 @@ static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio, loff_t file_offset) { const bool write = (bio_op(dio_bio) == REQ_OP_WRITE); + const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM); + size_t dip_size; struct btrfs_dio_private *dip; struct bio *bio; - dip = kzalloc(sizeof(*dip), GFP_NOFS); + dip_size = sizeof(*dip); + if (!write && csum) { + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + size_t nblocks; + + nblocks = dio_bio->bi_iter.bi_size >> inode->i_sb->s_blocksize_bits; + dip_size += csum_size * nblocks; + } + + dip = kzalloc(dip_size, GFP_NOFS); if (!dip) return NULL; @@ -7955,6 +7935,7 @@ static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, loff_t file_offset) { const bool write = (bio_op(dio_bio) == REQ_OP_WRITE); + const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM); struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct btrfs_dio_private *dip; struct bio *bio; @@ -7979,6 +7960,17 @@ static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, return; } + if (!write && csum) { + /* + * Load the csums up front to reduce csum tree searches and + * contention when submitting bios. + */ + status = btrfs_lookup_bio_sums(inode, dio_bio, file_offset, + dip->csums); + if (status != BLK_STS_OK) + goto out_err; + } + orig_bio = dip->orig_bio; start_sector = orig_bio->bi_iter.bi_sector; submit_len = orig_bio->bi_iter.bi_size; From 769b4f2497bdb31cd2c7b53d171bc7c13ef4e793 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:22 -0700 Subject: [PATCH 0742/1043] btrfs: get rid of one layer of bios in direct I/O In the worst case, there are _4_ layers of bios in the Btrfs direct I/O path: 1. The bio created by the generic direct I/O code (dio_bio). 2. A clone of dio_bio we create in btrfs_submit_direct() to represent the entire direct I/O range (orig_bio). 3. A partial clone of orig_bio limited to the size of a RAID stripe that we create in btrfs_submit_direct_hook(). 4. Clones of each of those split bios for each RAID stripe that we create in btrfs_map_bio(). As of the previous commit, the second layer (orig_bio) is no longer needed for anything: we can split dio_bio instead, and complete dio_bio directly when all of the cloned bios complete. This lets us clean up a bunch of cruft, including dip->subio_endio and dip->errors (we can use dio_bio->bi_status instead). It also enables the next big cleanup of direct I/O read repair. Reviewed-by: Josef Bacik Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- fs/btrfs/btrfs_inode.h | 16 ---- fs/btrfs/inode.c | 185 +++++++++++++++-------------------------- 2 files changed, 65 insertions(+), 136 deletions(-) diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index 7b04389e219c..e7d709505cb1 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -294,11 +294,8 @@ static inline int btrfs_inode_in_log(struct btrfs_inode *inode, u64 generation) return ret; } -#define BTRFS_DIO_ORIG_BIO_SUBMITTED 0x1 - struct btrfs_dio_private { struct inode *inode; - unsigned long flags; u64 logical_offset; u64 disk_bytenr; u64 bytes; @@ -309,22 +306,9 @@ struct btrfs_dio_private { */ refcount_t refs; - /* IO errors */ - int errors; - - /* orig_bio is our btrfs_io_bio */ - struct bio *orig_bio; - /* dio_bio came from fs/direct-io.c */ struct bio *dio_bio; - /* - * The original bio may be split to several sub-bios, this is - * done during endio of sub-bios - */ - blk_status_t (*subio_endio)(struct inode *, struct btrfs_io_bio *, - blk_status_t); - /* Array of checksums */ u8 csums[]; }; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index d96c67f3c5a4..63e2f816d9cd 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7356,6 +7356,29 @@ err: return ret; } +static void btrfs_dio_private_put(struct btrfs_dio_private *dip) +{ + /* + * This implies a barrier so that stores to dio_bio->bi_status before + * this and loads of dio_bio->bi_status after this are fully ordered. + */ + if (!refcount_dec_and_test(&dip->refs)) + return; + + if (bio_op(dip->dio_bio) == REQ_OP_WRITE) { + __endio_write_update_ordered(dip->inode, dip->logical_offset, + dip->bytes, + !dip->dio_bio->bi_status); + } else { + unlock_extent(&BTRFS_I(dip->inode)->io_tree, + dip->logical_offset, + dip->logical_offset + dip->bytes - 1); + } + + dio_end_io(dip->dio_bio); + kfree(dip); +} + static inline blk_status_t submit_dio_repair_bio(struct inode *inode, struct bio *bio, int mirror_num) @@ -7678,8 +7701,9 @@ next: return err; } -static blk_status_t btrfs_subio_endio_read(struct inode *inode, - struct btrfs_io_bio *io_bio, blk_status_t err) +static blk_status_t btrfs_check_read_dio_bio(struct inode *inode, + struct btrfs_io_bio *io_bio, + blk_status_t err) { bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; @@ -7693,28 +7717,6 @@ static blk_status_t btrfs_subio_endio_read(struct inode *inode, } } -static void btrfs_endio_direct_read(struct bio *bio) -{ - struct btrfs_dio_private *dip = bio->bi_private; - struct inode *inode = dip->inode; - struct bio *dio_bio; - struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); - blk_status_t err = bio->bi_status; - - if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED) - err = btrfs_subio_endio_read(inode, io_bio, err); - - unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset, - dip->logical_offset + dip->bytes - 1); - dio_bio = dip->dio_bio; - - kfree(dip); - - dio_bio->bi_status = err; - dio_end_io(dio_bio); - bio_put(bio); -} - static void __endio_write_update_ordered(struct inode *inode, const u64 offset, const u64 bytes, const bool uptodate) @@ -7758,21 +7760,6 @@ static void __endio_write_update_ordered(struct inode *inode, } } -static void btrfs_endio_direct_write(struct bio *bio) -{ - struct btrfs_dio_private *dip = bio->bi_private; - struct bio *dio_bio = dip->dio_bio; - - __endio_write_update_ordered(dip->inode, dip->logical_offset, - dip->bytes, !bio->bi_status); - - kfree(dip); - - dio_bio->bi_status = bio->bi_status; - dio_end_io(dio_bio); - bio_put(bio); -} - static blk_status_t btrfs_submit_bio_start_direct_io(void *private_data, struct bio *bio, u64 offset) { @@ -7796,31 +7783,16 @@ static void btrfs_end_dio_bio(struct bio *bio) (unsigned long long)bio->bi_iter.bi_sector, bio->bi_iter.bi_size, err); - if (dip->subio_endio) - err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err); - - if (err) { - /* - * We want to perceive the errors flag being set before - * decrementing the reference count. We don't need a barrier - * since atomic operations with a return value are fully - * ordered as per atomic_t.txt - */ - dip->errors = 1; + if (bio_op(bio) == REQ_OP_READ) { + err = btrfs_check_read_dio_bio(dip->inode, btrfs_io_bio(bio), + err); } - /* if there are more bios still pending for this dio, just exit */ - if (!refcount_dec_and_test(&dip->refs)) - goto out; + if (err) + dip->dio_bio->bi_status = err; - if (dip->errors) { - bio_io_error(dip->orig_bio); - } else { - dip->dio_bio->bi_status = BLK_STS_OK; - bio_endio(dip->orig_bio); - } -out: bio_put(bio); + btrfs_dio_private_put(dip); } static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio, @@ -7883,7 +7855,6 @@ static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio, const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM); size_t dip_size; struct btrfs_dio_private *dip; - struct bio *bio; dip_size = sizeof(*dip); if (!write && csum) { @@ -7899,15 +7870,10 @@ static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio, if (!dip) return NULL; - bio = btrfs_bio_clone(dio_bio); - bio->bi_private = dip; - btrfs_io_bio(bio)->logical = file_offset; - dip->inode = inode; dip->logical_offset = file_offset; dip->bytes = dio_bio->bi_iter.bi_size; dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9; - dip->orig_bio = bio; dip->dio_bio = dio_bio; refcount_set(&dip->refs, 1); @@ -7922,11 +7888,6 @@ static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio, dip->bytes; dio_data->unsubmitted_oe_range_start = dio_data->unsubmitted_oe_range_end; - - bio->bi_end_io = btrfs_endio_direct_write; - } else { - bio->bi_end_io = btrfs_endio_direct_read; - dip->subio_endio = btrfs_subio_endio_read; } return dip; } @@ -7937,9 +7898,10 @@ static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, const bool write = (bio_op(dio_bio) == REQ_OP_WRITE); const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM); struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + const bool raid56 = (btrfs_data_alloc_profile(fs_info) & + BTRFS_BLOCK_GROUP_RAID56_MASK); struct btrfs_dio_private *dip; struct bio *bio; - struct bio *orig_bio; u64 start_sector; int async_submit = 0; u64 submit_len; @@ -7971,89 +7933,72 @@ static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, goto out_err; } - orig_bio = dip->orig_bio; - start_sector = orig_bio->bi_iter.bi_sector; - submit_len = orig_bio->bi_iter.bi_size; - ret = btrfs_get_io_geometry(fs_info, btrfs_op(orig_bio), - start_sector << 9, submit_len, &geom); - if (ret) - goto out_err; + start_sector = dio_bio->bi_iter.bi_sector; + submit_len = dio_bio->bi_iter.bi_size; - if (geom.len >= submit_len) { - bio = orig_bio; - dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED; - goto submit; - } - - /* async crcs make it difficult to collect full stripe writes. */ - if (btrfs_data_alloc_profile(fs_info) & BTRFS_BLOCK_GROUP_RAID56_MASK) - async_submit = 0; - else - async_submit = 1; - - /* bio split */ - ASSERT(geom.len <= INT_MAX); do { + ret = btrfs_get_io_geometry(fs_info, btrfs_op(dio_bio), + start_sector << 9, submit_len, + &geom); + if (ret) { + status = errno_to_blk_status(ret); + goto out_err; + } + ASSERT(geom.len <= INT_MAX); + clone_len = min_t(int, submit_len, geom.len); /* * This will never fail as it's passing GPF_NOFS and * the allocation is backed by btrfs_bioset. */ - bio = btrfs_bio_clone_partial(orig_bio, clone_offset, - clone_len); + bio = btrfs_bio_clone_partial(dio_bio, clone_offset, clone_len); bio->bi_private = dip; bio->bi_end_io = btrfs_end_dio_bio; btrfs_io_bio(bio)->logical = file_offset; ASSERT(submit_len >= clone_len); submit_len -= clone_len; - if (submit_len == 0) - break; /* * Increase the count before we submit the bio so we know * the end IO handler won't happen before we increase the * count. Otherwise, the dip might get freed before we're * done setting it up. + * + * We transfer the initial reference to the last bio, so we + * don't need to increment the reference count for the last one. */ - refcount_inc(&dip->refs); + if (submit_len > 0) { + refcount_inc(&dip->refs); + /* + * If we are submitting more than one bio, submit them + * all asynchronously. The exception is RAID 5 or 6, as + * asynchronous checksums make it difficult to collect + * full stripe writes. + */ + if (!raid56) + async_submit = 1; + } status = btrfs_submit_dio_bio(bio, inode, file_offset, async_submit); if (status) { bio_put(bio); - refcount_dec(&dip->refs); + if (submit_len > 0) + refcount_dec(&dip->refs); goto out_err; } clone_offset += clone_len; start_sector += clone_len >> 9; file_offset += clone_len; - - ret = btrfs_get_io_geometry(fs_info, btrfs_op(orig_bio), - start_sector << 9, submit_len, &geom); - if (ret) - goto out_err; } while (submit_len > 0); + return; -submit: - status = btrfs_submit_dio_bio(bio, inode, file_offset, async_submit); - if (!status) - return; - - if (bio != orig_bio) - bio_put(bio); out_err: - dip->errors = 1; - /* - * Before atomic variable goto zero, we must make sure dip->errors is - * perceived to be set. This ordering is ensured by the fact that an - * atomic operations with a return value are fully ordered as per - * atomic_t.txt - */ - if (refcount_dec_and_test(&dip->refs)) - bio_io_error(dip->orig_bio); + dip->dio_bio->bi_status = status; + btrfs_dio_private_put(dip); } static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info, From fd9d6670edba68cedd87b3db04cbdcdfc492cc0a Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:23 -0700 Subject: [PATCH 0743/1043] btrfs: simplify direct I/O read repair Direct I/O read repair was originally implemented in commit 8b110e393c5a ("Btrfs: implement repair function when direct read fails"). This implementation is unnecessarily complicated. There is major code duplication between __btrfs_subio_endio_read() (checks checksums and handles I/O errors for files with checksums), __btrfs_correct_data_nocsum() (handles I/O errors for files without checksums), btrfs_retry_endio() (checks checksums and handles I/O errors for retries of files with checksums), and btrfs_retry_endio_nocsum() (handles I/O errors for retries of files without checksum). If it sounds like these should be one function, that's because they should. Additionally, these functions are very hard to follow due to their excessive use of goto. This commit replaces the original implementation. After the previous commit getting rid of orig_bio, we can reuse the same endio callback for repair I/O and the original I/O, we just need to track the file offset and original iterator in the repair bio. We can also unify the handling of files with and without checksums and simplify the control flow. We also no longer have to wait for each repair I/O to complete one by one. Reviewed-by: Josef Bacik Reviewed-by: Nikolay Borisov Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 2 + fs/btrfs/inode.c | 284 ++++++++----------------------------------- 2 files changed, 52 insertions(+), 234 deletions(-) diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 6ddf6d1c3e70..2b1f0becd680 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2630,6 +2630,8 @@ struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio, } bio_add_page(bio, page, failrec->len, pg_offset); + btrfs_io_bio(bio)->logical = failrec->start; + btrfs_io_bio(bio)->iter = bio->bi_iter; return bio; } diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 63e2f816d9cd..1462f64c2c4b 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7435,19 +7435,17 @@ static int btrfs_check_dio_repairable(struct inode *inode, static blk_status_t dio_read_error(struct inode *inode, struct bio *failed_bio, struct page *page, unsigned int pgoff, - u64 start, u64 end, int failed_mirror, - bio_end_io_t *repair_endio, void *repair_arg) + u64 start, u64 end, int failed_mirror) { + struct btrfs_dio_private *dip = failed_bio->bi_private; struct io_failure_record *failrec; struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; struct bio *bio; int isector; unsigned int read_mode = 0; - int segs; int ret; blk_status_t status; - struct bio_vec bvec; BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE); @@ -7462,259 +7460,77 @@ static blk_status_t dio_read_error(struct inode *inode, struct bio *failed_bio, return BLK_STS_IOERR; } - segs = bio_segments(failed_bio); - bio_get_first_bvec(failed_bio, &bvec); - if (segs > 1 || - (bvec.bv_len > btrfs_inode_sectorsize(inode))) + if (btrfs_io_bio(failed_bio)->iter.bi_size > inode->i_sb->s_blocksize) read_mode |= REQ_FAILFAST_DEV; isector = start - btrfs_io_bio(failed_bio)->logical; isector >>= inode->i_sb->s_blocksize_bits; - bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page, - pgoff, isector, repair_endio, repair_arg); + bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page, pgoff, + isector, failed_bio->bi_end_io, dip); bio->bi_opf = REQ_OP_READ | read_mode; btrfs_debug(BTRFS_I(inode)->root->fs_info, "repair DIO read error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d", read_mode, failrec->this_mirror, failrec->in_validation); + refcount_inc(&dip->refs); status = submit_dio_repair_bio(inode, bio, failrec->this_mirror); if (status) { free_io_failure(failure_tree, io_tree, failrec); bio_put(bio); + refcount_dec(&dip->refs); } return status; } -struct btrfs_retry_complete { - struct completion done; - struct inode *inode; - u64 start; - int uptodate; -}; - -static void btrfs_retry_endio_nocsum(struct bio *bio) -{ - struct btrfs_retry_complete *done = bio->bi_private; - struct inode *inode = done->inode; - struct bio_vec *bvec; - struct extent_io_tree *io_tree, *failure_tree; - struct bvec_iter_all iter_all; - - if (bio->bi_status) - goto end; - - ASSERT(bio->bi_vcnt == 1); - io_tree = &BTRFS_I(inode)->io_tree; - failure_tree = &BTRFS_I(inode)->io_failure_tree; - ASSERT(bio_first_bvec_all(bio)->bv_len == btrfs_inode_sectorsize(inode)); - - done->uptodate = 1; - ASSERT(!bio_flagged(bio, BIO_CLONED)); - bio_for_each_segment_all(bvec, bio, iter_all) - clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree, - io_tree, done->start, bvec->bv_page, - btrfs_ino(BTRFS_I(inode)), 0); -end: - complete(&done->done); - bio_put(bio); -} - -static blk_status_t __btrfs_correct_data_nocsum(struct inode *inode, - struct btrfs_io_bio *io_bio) -{ - struct btrfs_fs_info *fs_info; - struct bio_vec bvec; - struct bvec_iter iter; - struct btrfs_retry_complete done; - u64 start; - unsigned int pgoff; - u32 sectorsize; - int nr_sectors; - blk_status_t ret; - blk_status_t err = BLK_STS_OK; - - fs_info = BTRFS_I(inode)->root->fs_info; - sectorsize = fs_info->sectorsize; - - start = io_bio->logical; - done.inode = inode; - io_bio->bio.bi_iter = io_bio->iter; - - bio_for_each_segment(bvec, &io_bio->bio, iter) { - nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len); - pgoff = bvec.bv_offset; - -next_block_or_try_again: - done.uptodate = 0; - done.start = start; - init_completion(&done.done); - - ret = dio_read_error(inode, &io_bio->bio, bvec.bv_page, - pgoff, start, start + sectorsize - 1, - io_bio->mirror_num, - btrfs_retry_endio_nocsum, &done); - if (ret) { - err = ret; - goto next; - } - - wait_for_completion_io(&done.done); - - if (!done.uptodate) { - /* We might have another mirror, so try again */ - goto next_block_or_try_again; - } - -next: - start += sectorsize; - - nr_sectors--; - if (nr_sectors) { - pgoff += sectorsize; - ASSERT(pgoff < PAGE_SIZE); - goto next_block_or_try_again; - } - } - - return err; -} - -static void btrfs_retry_endio(struct bio *bio) -{ - struct btrfs_retry_complete *done = bio->bi_private; - struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); - struct extent_io_tree *io_tree, *failure_tree; - struct inode *inode = done->inode; - struct bio_vec *bvec; - int uptodate; - int ret; - int i = 0; - struct bvec_iter_all iter_all; - - if (bio->bi_status) - goto end; - - uptodate = 1; - - ASSERT(bio->bi_vcnt == 1); - ASSERT(bio_first_bvec_all(bio)->bv_len == btrfs_inode_sectorsize(done->inode)); - - io_tree = &BTRFS_I(inode)->io_tree; - failure_tree = &BTRFS_I(inode)->io_failure_tree; - - ASSERT(!bio_flagged(bio, BIO_CLONED)); - bio_for_each_segment_all(bvec, bio, iter_all) { - ret = check_data_csum(inode, io_bio, i, bvec->bv_page, - bvec->bv_offset, done->start, - bvec->bv_len); - if (!ret) - clean_io_failure(BTRFS_I(inode)->root->fs_info, - failure_tree, io_tree, done->start, - bvec->bv_page, - btrfs_ino(BTRFS_I(inode)), - bvec->bv_offset); - else - uptodate = 0; - i++; - } - - done->uptodate = uptodate; -end: - complete(&done->done); - bio_put(bio); -} - -static blk_status_t __btrfs_subio_endio_read(struct inode *inode, - struct btrfs_io_bio *io_bio, blk_status_t err) -{ - struct btrfs_fs_info *fs_info; - struct bio_vec bvec; - struct bvec_iter iter; - struct btrfs_retry_complete done; - u64 start; - u64 offset = 0; - u32 sectorsize; - int nr_sectors; - unsigned int pgoff; - int csum_pos; - bool uptodate = (err == 0); - int ret; - blk_status_t status; - - fs_info = BTRFS_I(inode)->root->fs_info; - sectorsize = fs_info->sectorsize; - - err = BLK_STS_OK; - start = io_bio->logical; - done.inode = inode; - io_bio->bio.bi_iter = io_bio->iter; - - bio_for_each_segment(bvec, &io_bio->bio, iter) { - nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len); - - pgoff = bvec.bv_offset; -next_block: - if (uptodate) { - csum_pos = BTRFS_BYTES_TO_BLKS(fs_info, offset); - ret = check_data_csum(inode, io_bio, csum_pos, - bvec.bv_page, pgoff, start, - sectorsize); - if (likely(!ret)) - goto next; - } -try_again: - done.uptodate = 0; - done.start = start; - init_completion(&done.done); - - status = dio_read_error(inode, &io_bio->bio, bvec.bv_page, - pgoff, start, start + sectorsize - 1, - io_bio->mirror_num, btrfs_retry_endio, - &done); - if (status) { - err = status; - goto next; - } - - wait_for_completion_io(&done.done); - - if (!done.uptodate) { - /* We might have another mirror, so try again */ - goto try_again; - } -next: - offset += sectorsize; - start += sectorsize; - - ASSERT(nr_sectors); - - nr_sectors--; - if (nr_sectors) { - pgoff += sectorsize; - ASSERT(pgoff < PAGE_SIZE); - goto next_block; - } - } - - return err; -} - static blk_status_t btrfs_check_read_dio_bio(struct inode *inode, struct btrfs_io_bio *io_bio, - blk_status_t err) + const bool uptodate) { - bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM; + struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; + const u32 sectorsize = fs_info->sectorsize; + struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; + struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; + const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM); + struct bio_vec bvec; + struct bvec_iter iter; + u64 start = io_bio->logical; + int icsum = 0; + blk_status_t err = BLK_STS_OK; - if (skip_csum) { - if (unlikely(err)) - return __btrfs_correct_data_nocsum(inode, io_bio); - else - return BLK_STS_OK; - } else { - return __btrfs_subio_endio_read(inode, io_bio, err); + __bio_for_each_segment(bvec, &io_bio->bio, iter, io_bio->iter) { + unsigned int i, nr_sectors, pgoff; + + nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len); + pgoff = bvec.bv_offset; + for (i = 0; i < nr_sectors; i++) { + ASSERT(pgoff < PAGE_SIZE); + if (uptodate && + (!csum || !check_data_csum(inode, io_bio, icsum, + bvec.bv_page, pgoff, + start, sectorsize))) { + clean_io_failure(fs_info, failure_tree, io_tree, + start, bvec.bv_page, + btrfs_ino(BTRFS_I(inode)), + pgoff); + } else { + blk_status_t status; + + status = dio_read_error(inode, &io_bio->bio, + bvec.bv_page, pgoff, + start, + start + sectorsize - 1, + io_bio->mirror_num); + if (status) + err = status; + } + start += sectorsize; + icsum++; + pgoff += sectorsize; + } } + return err; } static void __endio_write_update_ordered(struct inode *inode, @@ -7785,7 +7601,7 @@ static void btrfs_end_dio_bio(struct bio *bio) if (bio_op(bio) == REQ_OP_READ) { err = btrfs_check_read_dio_bio(dip->inode, btrfs_io_bio(bio), - err); + !err); } if (err) From 5c047a699aa9433ad92136343a9306d985134c24 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:24 -0700 Subject: [PATCH 0744/1043] btrfs: get rid of endio_repair_workers This was originally added in commit 8b110e393c5a ("Btrfs: implement repair function when direct read fails") to avoid a deadlock. In that commit, the direct I/O read endio executes on the endio_workers workqueue, submits a repair bio, and waits for it to complete. The repair bio endio must execute on a different workqueue, otherwise it could block on the endio_workers workqueue becoming available, which won't happen because the original endio is blocked on the repair bio. As of the previous commit, the original endio doesn't wait for the repair bio, so this separate workqueue is unnecessary. Reviewed-by: Josef Bacik Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 1 - fs/btrfs/disk-io.c | 8 +------- fs/btrfs/disk-io.h | 1 - fs/btrfs/inode.c | 2 +- 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index e4b8f5f2273d..03ea7370aea7 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -758,7 +758,6 @@ struct btrfs_fs_info { struct btrfs_workqueue *endio_workers; struct btrfs_workqueue *endio_meta_workers; struct btrfs_workqueue *endio_raid56_workers; - struct btrfs_workqueue *endio_repair_workers; struct btrfs_workqueue *rmw_workers; struct btrfs_workqueue *endio_meta_write_workers; struct btrfs_workqueue *endio_write_workers; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index d10c7be10f3b..8ad451695d49 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -709,9 +709,7 @@ static void end_workqueue_bio(struct bio *bio) else wq = fs_info->endio_write_workers; } else { - if (unlikely(end_io_wq->metadata == BTRFS_WQ_ENDIO_DIO_REPAIR)) - wq = fs_info->endio_repair_workers; - else if (end_io_wq->metadata == BTRFS_WQ_ENDIO_RAID56) + if (end_io_wq->metadata == BTRFS_WQ_ENDIO_RAID56) wq = fs_info->endio_raid56_workers; else if (end_io_wq->metadata) wq = fs_info->endio_meta_workers; @@ -1942,7 +1940,6 @@ static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info) btrfs_destroy_workqueue(fs_info->workers); btrfs_destroy_workqueue(fs_info->endio_workers); btrfs_destroy_workqueue(fs_info->endio_raid56_workers); - btrfs_destroy_workqueue(fs_info->endio_repair_workers); btrfs_destroy_workqueue(fs_info->rmw_workers); btrfs_destroy_workqueue(fs_info->endio_write_workers); btrfs_destroy_workqueue(fs_info->endio_freespace_worker); @@ -2145,8 +2142,6 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info, fs_info->endio_raid56_workers = btrfs_alloc_workqueue(fs_info, "endio-raid56", flags, max_active, 4); - fs_info->endio_repair_workers = - btrfs_alloc_workqueue(fs_info, "endio-repair", flags, 1, 0); fs_info->rmw_workers = btrfs_alloc_workqueue(fs_info, "rmw", flags, max_active, 2); fs_info->endio_write_workers = @@ -2170,7 +2165,6 @@ static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info, fs_info->flush_workers && fs_info->endio_workers && fs_info->endio_meta_workers && fs_info->endio_meta_write_workers && - fs_info->endio_repair_workers && fs_info->endio_write_workers && fs_info->endio_raid56_workers && fs_info->endio_freespace_worker && fs_info->rmw_workers && fs_info->caching_workers && fs_info->readahead_workers && diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index cd629113f61c..734bc5270b6a 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -25,7 +25,6 @@ enum btrfs_wq_endio_type { BTRFS_WQ_ENDIO_METADATA, BTRFS_WQ_ENDIO_FREE_SPACE, BTRFS_WQ_ENDIO_RAID56, - BTRFS_WQ_ENDIO_DIO_REPAIR, }; static inline u64 btrfs_sb_offset(int mirror) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 1462f64c2c4b..bd7453f02a9d 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7388,7 +7388,7 @@ static inline blk_status_t submit_dio_repair_bio(struct inode *inode, BUG_ON(bio_op(bio) == REQ_OP_WRITE); - ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DIO_REPAIR); + ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA); if (ret) return ret; From 77d5d6893106ea7b19709bed2491f93ff10a86d7 Mon Sep 17 00:00:00 2001 From: Omar Sandoval Date: Thu, 16 Apr 2020 14:46:25 -0700 Subject: [PATCH 0745/1043] btrfs: unify buffered and direct I/O read repair Currently, direct I/O has its own versions of bio_readpage_error() and btrfs_check_repairable() (dio_read_error() and btrfs_check_dio_repairable(), respectively). The main difference is that the direct I/O version doesn't do read validation. The rework of direct I/O repair makes it possible to do validation, so we can get rid of btrfs_check_dio_repairable() and combine bio_readpage_error() and dio_read_error() into a new helper, btrfs_submit_read_repair(). Reviewed-by: Josef Bacik Signed-off-by: Omar Sandoval Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 136 ++++++++++++++++++++----------------------- fs/btrfs/extent_io.h | 17 ++++-- fs/btrfs/inode.c | 105 ++++----------------------------- 3 files changed, 87 insertions(+), 171 deletions(-) diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 2b1f0becd680..22db0b234ffe 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2601,46 +2601,10 @@ static bool btrfs_check_repairable(struct inode *inode, bool needs_validation, return true; } - -struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio, - struct io_failure_record *failrec, - struct page *page, int pg_offset, int icsum, - bio_end_io_t *endio_func, void *data) -{ - struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - struct bio *bio; - struct btrfs_io_bio *btrfs_failed_bio; - struct btrfs_io_bio *btrfs_bio; - - bio = btrfs_io_bio_alloc(1); - bio->bi_end_io = endio_func; - bio->bi_iter.bi_sector = failrec->logical >> 9; - bio->bi_iter.bi_size = 0; - bio->bi_private = data; - - btrfs_failed_bio = btrfs_io_bio(failed_bio); - if (btrfs_failed_bio->csum) { - u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); - - btrfs_bio = btrfs_io_bio(bio); - btrfs_bio->csum = btrfs_bio->csum_inline; - icsum *= csum_size; - memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum, - csum_size); - } - - bio_add_page(bio, page, failrec->len, pg_offset); - btrfs_io_bio(bio)->logical = failrec->start; - btrfs_io_bio(bio)->iter = bio->bi_iter; - - return bio; -} - static bool btrfs_io_needs_validation(struct inode *inode, struct bio *bio) { - struct bio_vec *bvec; u64 len = 0; - int i; + const u32 blocksize = inode->i_sb->s_blocksize; /* * If bi_status is BLK_STS_OK, then this was a checksum error, not an @@ -2653,72 +2617,99 @@ static bool btrfs_io_needs_validation(struct inode *inode, struct bio *bio) /* * We need to validate each sector individually if the failed I/O was * for multiple sectors. + * + * There are a few possible bios that can end up here: + * 1. A buffered read bio, which is not cloned. + * 2. A direct I/O read bio, which is cloned. + * 3. A (buffered or direct) repair bio, which is not cloned. + * + * For cloned bios (case 2), we can get the size from + * btrfs_io_bio->iter; for non-cloned bios (cases 1 and 3), we can get + * it from the bvecs. */ - bio_for_each_bvec_all(bvec, bio, i) { - len += bvec->bv_len; - if (len > inode->i_sb->s_blocksize) + if (bio_flagged(bio, BIO_CLONED)) { + if (btrfs_io_bio(bio)->iter.bi_size > blocksize) return true; + } else { + struct bio_vec *bvec; + int i; + + bio_for_each_bvec_all(bvec, bio, i) { + len += bvec->bv_len; + if (len > blocksize) + return true; + } } return false; } -/* - * This is a generic handler for readpage errors. If other copies exist, read - * those and write back good data to the failed position. Does not investigate - * in remapping the failed extent elsewhere, hoping the device will be smart - * enough to do this as needed - */ -static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset, - struct page *page, u64 start, u64 end, - int failed_mirror) +blk_status_t btrfs_submit_read_repair(struct inode *inode, + struct bio *failed_bio, u64 phy_offset, + struct page *page, unsigned int pgoff, + u64 start, u64 end, int failed_mirror, + submit_bio_hook_t *submit_bio_hook) { struct io_failure_record *failrec; - struct inode *inode = page->mapping->host; + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; + struct btrfs_io_bio *failed_io_bio = btrfs_io_bio(failed_bio); + const int icsum = phy_offset >> inode->i_sb->s_blocksize_bits; bool need_validation; - struct bio *bio; - int read_mode = 0; + struct bio *repair_bio; + struct btrfs_io_bio *repair_io_bio; blk_status_t status; int ret; + btrfs_debug(fs_info, + "repair read error: read error at %llu", start); + BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE); ret = btrfs_get_io_failure_record(inode, start, end, &failrec); if (ret) - return ret; + return errno_to_blk_status(ret); need_validation = btrfs_io_needs_validation(inode, failed_bio); if (!btrfs_check_repairable(inode, need_validation, failrec, failed_mirror)) { free_io_failure(failure_tree, tree, failrec); - return -EIO; + return BLK_STS_IOERR; } + repair_bio = btrfs_io_bio_alloc(1); + repair_io_bio = btrfs_io_bio(repair_bio); + repair_bio->bi_opf = REQ_OP_READ; if (need_validation) - read_mode |= REQ_FAILFAST_DEV; + repair_bio->bi_opf |= REQ_FAILFAST_DEV; + repair_bio->bi_end_io = failed_bio->bi_end_io; + repair_bio->bi_iter.bi_sector = failrec->logical >> 9; + repair_bio->bi_private = failed_bio->bi_private; - phy_offset >>= inode->i_sb->s_blocksize_bits; - bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page, - start - page_offset(page), - (int)phy_offset, failed_bio->bi_end_io, - NULL); - bio->bi_opf = REQ_OP_READ | read_mode; + if (failed_io_bio->csum) { + const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy); + + repair_io_bio->csum = repair_io_bio->csum_inline; + memcpy(repair_io_bio->csum, + failed_io_bio->csum + csum_size * icsum, csum_size); + } + + bio_add_page(repair_bio, page, failrec->len, pgoff); + repair_io_bio->logical = failrec->start; + repair_io_bio->iter = repair_bio->bi_iter; btrfs_debug(btrfs_sb(inode->i_sb), - "Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d", - read_mode, failrec->this_mirror, failrec->in_validation); +"repair read error: submitting new read to mirror %d, in_validation=%d", + failrec->this_mirror, failrec->in_validation); - status = tree->ops->submit_bio_hook(tree->private_data, bio, failrec->this_mirror, - failrec->bio_flags); + status = submit_bio_hook(inode, repair_bio, failrec->this_mirror, + failrec->bio_flags); if (status) { free_io_failure(failure_tree, tree, failrec); - bio_put(bio); - ret = blk_status_to_errno(status); + bio_put(repair_bio); } - - return ret; + return status; } /* lots and lots of room for performance fixes in the end_bio funcs */ @@ -2890,9 +2881,10 @@ static void end_bio_extent_readpage(struct bio *bio) * If it can't handle the error it will return -EIO and * we remain responsible for that page. */ - ret = bio_readpage_error(bio, offset, page, start, end, - mirror); - if (ret == 0) { + if (!btrfs_submit_read_repair(inode, bio, offset, page, + start - page_offset(page), + start, end, mirror, + tree->ops->submit_bio_hook)) { uptodate = !bio->bi_status; offset += len; continue; diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index f4dfac756455..a2842b2d9a98 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -66,6 +66,10 @@ struct btrfs_io_bio; struct io_failure_record; struct extent_io_tree; +typedef blk_status_t (submit_bio_hook_t)(struct inode *inode, struct bio *bio, + int mirror_num, + unsigned long bio_flags); + typedef blk_status_t (extent_submit_bio_start_t)(void *private_data, struct bio *bio, u64 bio_offset); @@ -74,8 +78,7 @@ struct extent_io_ops { * The following callbacks must be always defined, the function * pointer will be called unconditionally. */ - blk_status_t (*submit_bio_hook)(struct inode *inode, struct bio *bio, - int mirror_num, unsigned long bio_flags); + submit_bio_hook_t *submit_bio_hook; int (*readpage_end_io_hook)(struct btrfs_io_bio *io_bio, u64 phy_offset, struct page *page, u64 start, u64 end, int mirror); @@ -312,10 +315,12 @@ struct io_failure_record { }; -struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio, - struct io_failure_record *failrec, - struct page *page, int pg_offset, int icsum, - bio_end_io_t *endio_func, void *data); +blk_status_t btrfs_submit_read_repair(struct inode *inode, + struct bio *failed_bio, u64 phy_offset, + struct page *page, unsigned int pgoff, + u64 start, u64 end, int failed_mirror, + submit_bio_hook_t *submit_bio_hook); + #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS bool find_lock_delalloc_range(struct inode *inode, struct page *locked_page, u64 *start, diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index bd7453f02a9d..5d567082f95a 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7379,10 +7379,11 @@ static void btrfs_dio_private_put(struct btrfs_dio_private *dip) kfree(dip); } -static inline blk_status_t submit_dio_repair_bio(struct inode *inode, - struct bio *bio, - int mirror_num) +static blk_status_t submit_dio_repair_bio(struct inode *inode, struct bio *bio, + int mirror_num, + unsigned long bio_flags) { + struct btrfs_dio_private *dip = bio->bi_private; struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); blk_status_t ret; @@ -7392,96 +7393,11 @@ static inline blk_status_t submit_dio_repair_bio(struct inode *inode, if (ret) return ret; - ret = btrfs_map_bio(fs_info, bio, mirror_num); - - return ret; -} - -static int btrfs_check_dio_repairable(struct inode *inode, - struct bio *failed_bio, - struct io_failure_record *failrec, - int failed_mirror) -{ - struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - int num_copies; - - num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len); - if (num_copies == 1) { - /* - * we only have a single copy of the data, so don't bother with - * all the retry and error correction code that follows. no - * matter what the error is, it is very likely to persist. - */ - btrfs_debug(fs_info, - "Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d", - num_copies, failrec->this_mirror, failed_mirror); - return 0; - } - - failrec->failed_mirror = failed_mirror; - failrec->this_mirror++; - if (failrec->this_mirror == failed_mirror) - failrec->this_mirror++; - - if (failrec->this_mirror > num_copies) { - btrfs_debug(fs_info, - "Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d", - num_copies, failrec->this_mirror, failed_mirror); - return 0; - } - - return 1; -} - -static blk_status_t dio_read_error(struct inode *inode, struct bio *failed_bio, - struct page *page, unsigned int pgoff, - u64 start, u64 end, int failed_mirror) -{ - struct btrfs_dio_private *dip = failed_bio->bi_private; - struct io_failure_record *failrec; - struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; - struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; - struct bio *bio; - int isector; - unsigned int read_mode = 0; - int ret; - blk_status_t status; - - BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE); - - ret = btrfs_get_io_failure_record(inode, start, end, &failrec); - if (ret) - return errno_to_blk_status(ret); - - ret = btrfs_check_dio_repairable(inode, failed_bio, failrec, - failed_mirror); - if (!ret) { - free_io_failure(failure_tree, io_tree, failrec); - return BLK_STS_IOERR; - } - - if (btrfs_io_bio(failed_bio)->iter.bi_size > inode->i_sb->s_blocksize) - read_mode |= REQ_FAILFAST_DEV; - - isector = start - btrfs_io_bio(failed_bio)->logical; - isector >>= inode->i_sb->s_blocksize_bits; - bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page, pgoff, - isector, failed_bio->bi_end_io, dip); - bio->bi_opf = REQ_OP_READ | read_mode; - - btrfs_debug(BTRFS_I(inode)->root->fs_info, - "repair DIO read error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d", - read_mode, failrec->this_mirror, failrec->in_validation); - refcount_inc(&dip->refs); - status = submit_dio_repair_bio(inode, bio, failrec->this_mirror); - if (status) { - free_io_failure(failure_tree, io_tree, failrec); - bio_put(bio); + ret = btrfs_map_bio(fs_info, bio, mirror_num); + if (ret) refcount_dec(&dip->refs); - } - - return status; + return ret; } static blk_status_t btrfs_check_read_dio_bio(struct inode *inode, @@ -7517,11 +7433,14 @@ static blk_status_t btrfs_check_read_dio_bio(struct inode *inode, } else { blk_status_t status; - status = dio_read_error(inode, &io_bio->bio, + status = btrfs_submit_read_repair(inode, + &io_bio->bio, + start - io_bio->logical, bvec.bv_page, pgoff, start, start + sectorsize - 1, - io_bio->mirror_num); + io_bio->mirror_num, + submit_dio_repair_bio); if (status) err = status; } From 0bc2d3c08e2b970117cc2b7c529291123978cae6 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Tue, 21 Apr 2020 11:25:31 +0100 Subject: [PATCH 0746/1043] btrfs: remove useless check for copy_items() return value At btrfs_log_prealloc_extents() we are checking if copy_items() returns a value greater than 0. That used to happen in the past to signal the caller that the path given to it was released and reused for other searches, but as of commit 0e56315ca147b3 ("Btrfs: fix missing hole after hole punching and fsync when using NO_HOLES"), the copy_items() function does not have that behaviour anymore and always returns 0 or a negative value. So just remove that check at btrfs_log_prealloc_extents(), which the previously mentioned commit forgot to remove. Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 0254e8d10e4a..39ec25518ec2 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -4335,12 +4335,9 @@ static int btrfs_log_prealloc_extents(struct btrfs_trans_handle *trans, } } } - if (ins_nr > 0) { + if (ins_nr > 0) ret = copy_items(trans, inode, dst_path, path, start_slot, ins_nr, 1, 0); - if (ret > 0) - ret = 0; - } out: btrfs_release_path(path); btrfs_free_path(dst_path); From 1ed802c972c6c676412e34d168e313a17c3d931e Mon Sep 17 00:00:00 2001 From: Anand Jain Date: Tue, 28 Apr 2020 23:22:25 +0800 Subject: [PATCH 0747/1043] btrfs: drop useless goto in open_fs_devices There is no need of goto out in open_fs_devices() as there is nothing special done there. Reviewed-by: Nikolay Borisov Reviewed-by: Josef Bacik Signed-off-by: Anand Jain Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/volumes.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index aa1e0ae32943..bf953c4895f3 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -1185,7 +1185,6 @@ static int open_fs_devices(struct btrfs_fs_devices *fs_devices, { struct btrfs_device *device; struct btrfs_device *latest_dev = NULL; - int ret = 0; flags |= FMODE_EXCL; @@ -1198,16 +1197,15 @@ static int open_fs_devices(struct btrfs_fs_devices *fs_devices, device->generation > latest_dev->generation) latest_dev = device; } - if (fs_devices->open_devices == 0) { - ret = -EINVAL; - goto out; - } + if (fs_devices->open_devices == 0) + return -EINVAL; + fs_devices->opened = 1; fs_devices->latest_bdev = latest_dev->bdev; fs_devices->total_rw_bytes = 0; fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_REGULAR; -out: - return ret; + + return 0; } static int devid_cmp(void *priv, struct list_head *a, struct list_head *b) From fd08001f17c7f0a5b19bf4026bbaace19d8fbdb6 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 30 Apr 2020 23:51:59 -0700 Subject: [PATCH 0748/1043] btrfs: use crypto_shash_digest() instead of open coding Use crypto_shash_digest() instead of crypto_shash_init() + crypto_shash_update() + crypto_shash_final(). This is more efficient. Signed-off-by: Eric Biggers Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/compression.c | 4 +--- fs/btrfs/disk-io.c | 13 +++++-------- fs/btrfs/file-item.c | 7 +++---- fs/btrfs/inode.c | 4 +--- 4 files changed, 10 insertions(+), 18 deletions(-) diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 9ab610cc9114..1b624f9ef97d 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -194,11 +194,9 @@ static int check_compressed_csum(struct btrfs_inode *inode, for (i = 0; i < cb->nr_pages; i++) { page = cb->compressed_pages[i]; - crypto_shash_init(shash); kaddr = kmap_atomic(page); - crypto_shash_update(shash, kaddr, PAGE_SIZE); + crypto_shash_digest(shash, kaddr, PAGE_SIZE, csum); kunmap_atomic(kaddr); - crypto_shash_final(shash, (u8 *)&csum); if (memcmp(&csum, cb_sum, csum_size)) { btrfs_print_data_csum_error(inode, disk_start, diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 8ad451695d49..714b57553ed6 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -358,16 +358,14 @@ static int btrfs_check_super_csum(struct btrfs_fs_info *fs_info, SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); shash->tfm = fs_info->csum_shash; - crypto_shash_init(shash); /* * The super_block structure does not span the whole * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space is * filled with zeros and is included in the checksum. */ - crypto_shash_update(shash, raw_disk_sb + BTRFS_CSUM_SIZE, - BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE); - crypto_shash_final(shash, result); + crypto_shash_digest(shash, raw_disk_sb + BTRFS_CSUM_SIZE, + BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, result); if (memcmp(disk_sb->csum, result, btrfs_super_csum_size(disk_sb))) return 1; @@ -3504,10 +3502,9 @@ static int write_dev_supers(struct btrfs_device *device, btrfs_set_super_bytenr(sb, bytenr); - crypto_shash_init(shash); - crypto_shash_update(shash, (const char *)sb + BTRFS_CSUM_SIZE, - BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE); - crypto_shash_final(shash, sb->csum); + crypto_shash_digest(shash, (const char *)sb + BTRFS_CSUM_SIZE, + BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, + sb->csum); page = find_or_create_page(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS); diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index 22cbb4da6d42..8cdd06ea0e67 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -601,13 +601,12 @@ blk_status_t btrfs_csum_one_bio(struct inode *inode, struct bio *bio, index = 0; } - crypto_shash_init(shash); data = kmap_atomic(bvec.bv_page); - crypto_shash_update(shash, data + bvec.bv_offset + crypto_shash_digest(shash, data + bvec.bv_offset + (i * fs_info->sectorsize), - fs_info->sectorsize); + fs_info->sectorsize, + sums->sums + index); kunmap_atomic(data); - crypto_shash_final(shash, (char *)(sums->sums + index)); index += csum_size; offset += fs_info->sectorsize; this_sum_bytes += fs_info->sectorsize; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 5d567082f95a..cc94291fdd18 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2742,9 +2742,7 @@ static int check_data_csum(struct inode *inode, struct btrfs_io_bio *io_bio, kaddr = kmap_atomic(page); shash->tfm = fs_info->csum_shash; - crypto_shash_init(shash); - crypto_shash_update(shash, kaddr + pgoff, len); - crypto_shash_final(shash, csum); + crypto_shash_digest(shash, kaddr + pgoff, len, csum); if (memcmp(csum, csum_expected, csum_size)) goto zeroit; From 998a0671961f66e9fad4990ed75f80ba3088c2f1 Mon Sep 17 00:00:00 2001 From: Anand Jain Date: Tue, 5 May 2020 02:58:25 +0800 Subject: [PATCH 0749/1043] btrfs: include non-missing as a qualifier for the latest_bdev btrfs_free_extra_devids() updates fs_devices::latest_bdev to point to the bdev with greatest device::generation number. For a typical-missing device the generation number is zero so fs_devices::latest_bdev will never point to it. But if the missing device is due to alienation [1], then device::generation is not zero and if it is greater or equal to the rest of device generations in the list, then fs_devices::latest_bdev ends up pointing to the missing device and reports the error like [2]. [1] We maintain devices of a fsid (as in fs_device::fsid) in the fs_devices::devices list, a device is considered as an alien device if its fsid does not match with the fs_device::fsid Consider a working filesystem with raid1: $ mkfs.btrfs -f -d raid1 -m raid1 /dev/sda /dev/sdb $ mount /dev/sda /mnt-raid1 $ umount /mnt-raid1 While mnt-raid1 was unmounted the user force-adds one of its devices to another btrfs filesystem: $ mkfs.btrfs -f /dev/sdc $ mount /dev/sdc /mnt-single $ btrfs dev add -f /dev/sda /mnt-single Now the original mnt-raid1 fails to mount in degraded mode, because fs_devices::latest_bdev is pointing to the alien device. $ mount -o degraded /dev/sdb /mnt-raid1 [2] mount: wrong fs type, bad option, bad superblock on /dev/sdb, missing codepage or helper program, or other error In some cases useful info is found in syslog - try dmesg | tail or so. kernel: BTRFS warning (device sdb): devid 1 uuid 072a0192-675b-4d5a-8640-a5cf2b2c704d is missing kernel: BTRFS error (device sdb): failed to read devices kernel: BTRFS error (device sdb): open_ctree failed Fix the root cause by checking if the device is not missing before it can be considered for the fs_devices::latest_bdev. CC: stable@vger.kernel.org # 4.19+ Reviewed-by: Josef Bacik Signed-off-by: Anand Jain Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/volumes.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index bf953c4895f3..c661155f29d9 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -1042,6 +1042,8 @@ again: &device->dev_state)) { if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state) && + !test_bit(BTRFS_DEV_STATE_MISSING, + &device->dev_state) && (!latest_dev || device->generation > latest_dev->generation)) { latest_dev = device; From 7f551d969037cc128eca60688d9c5a300d84e665 Mon Sep 17 00:00:00 2001 From: Anand Jain Date: Tue, 5 May 2020 02:58:26 +0800 Subject: [PATCH 0750/1043] btrfs: free alien device after device add When an old device has new fsid through 'btrfs device add -f ' our fs_devices list has an alien device in one of the fs_devices lists. By having an alien device in fs_devices, we have two issues so far 1. missing device does not not show as missing in the userland 2. degraded mount will fail Both issues are caused by the fact that there's an alien device in the fs_devices list. (Alien means that it does not belong to the filesystem, identified by fsid, or does not contain btrfs filesystem at all, eg. due to overwrite). A device can be scanned/added through the control device ioctls SCAN_DEV, DEVICES_READY or by ADD_DEV. And device coming through the control device is checked against the all other devices in the lists, but this was not the case for ADD_DEV. This patch fixes both issues above by removing the alien device. CC: stable@vger.kernel.org # 5.4+ Signed-off-by: Anand Jain Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/volumes.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index c661155f29d9..be1e047a489e 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -2662,8 +2662,18 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path ret = btrfs_commit_transaction(trans); } - /* Update ctime/mtime for libblkid */ + /* + * Now that we have written a new super block to this device, check all + * other fs_devices list if device_path alienates any other scanned + * device. + * We can ignore the return value as it typically returns -EINVAL and + * only succeeds if the device was an alien. + */ + btrfs_forget_devices(device_path); + + /* Update ctime/mtime for blkid or udev */ update_dev_time(device_path); + return ret; error_sysfs: From d54f814434c5f87356d35bc377f43ef8d0556f15 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 28 Apr 2020 17:10:27 +0200 Subject: [PATCH 0751/1043] btrfs: sort error decoder entries Add the raw errnos and sort them accordingly. Reviewed-by: Anand Jain Signed-off-by: David Sterba --- fs/btrfs/super.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 7932d8d07cff..9e5d723a0d99 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -72,23 +72,23 @@ const char * __attribute_const__ btrfs_decode_error(int errno) char *errstr = "unknown"; switch (errno) { - case -EIO: + case -ENOENT: /* -2 */ + errstr = "No such entry"; + break; + case -EIO: /* -5 */ errstr = "IO failure"; break; - case -ENOMEM: + case -ENOMEM: /* -12*/ errstr = "Out of memory"; break; - case -EROFS: - errstr = "Readonly filesystem"; - break; - case -EEXIST: + case -EEXIST: /* -17 */ errstr = "Object already exists"; break; - case -ENOSPC: + case -ENOSPC: /* -28 */ errstr = "No space left"; break; - case -ENOENT: - errstr = "No such entry"; + case -EROFS: /* -30 */ + errstr = "Readonly filesystem"; break; } From fb8521caa8f21f3153c5dc9feb055b6cd3e45c89 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 28 Apr 2020 17:10:29 +0200 Subject: [PATCH 0752/1043] btrfs: add more codes to decoder table I've grepped logs for 'errno=.*unknown' and found -95, -117 and -122, now added to the table. The wording is adjusted so it makes sense in context of filesystem. Reviewed-by: Anand Jain Signed-off-by: David Sterba --- fs/btrfs/super.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 9e5d723a0d99..438ecba26557 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -90,6 +90,15 @@ const char * __attribute_const__ btrfs_decode_error(int errno) case -EROFS: /* -30 */ errstr = "Readonly filesystem"; break; + case -EOPNOTSUPP: /* -95 */ + errstr = "Operation not supported"; + break; + case -EUCLEAN: /* -117 */ + errstr = "Filesystem corrupted"; + break; + case -EDQUOT: /* -122 */ + errstr = "Quota exceeded"; + break; } return errstr; From eec5b6e097cbcaf8d42209b6ef059cee8aa00790 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Wed, 6 May 2020 21:22:39 +0800 Subject: [PATCH 0753/1043] btrfs: remove unused function heads_to_leaves There's no callers in-tree anymore since commit 64403612b73a ("btrfs: rework btrfs_check_space_for_delayed_refs") Signed-off-by: YueHaibing Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/extent-tree.c | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index faa585d54eb7..3593f8cce9e5 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -2114,22 +2114,6 @@ static u64 find_middle(struct rb_root *root) } #endif -static inline u64 heads_to_leaves(struct btrfs_fs_info *fs_info, u64 heads) -{ - u64 num_bytes; - - num_bytes = heads * (sizeof(struct btrfs_extent_item) + - sizeof(struct btrfs_extent_inline_ref)); - if (!btrfs_fs_incompat(fs_info, SKINNY_METADATA)) - num_bytes += heads * sizeof(struct btrfs_tree_block_info); - - /* - * We don't ever fill up leaves all the way so multiply by 2 just to be - * closer to what we're really going to want to use. - */ - return div_u64(num_bytes, BTRFS_LEAF_DATA_SIZE(fs_info)); -} - /* * Takes the number of bytes to be csumm'ed and figures out how many leaves it * would require to store the csums for that many bytes. From a619b3c7abdd585cdd003effa1773e627aa8a44e Mon Sep 17 00:00:00 2001 From: Robbie Ko Date: Thu, 7 May 2020 10:54:40 +0800 Subject: [PATCH 0754/1043] btrfs: speedup dead root detection during orphan cleanup When mounting, we handle deleted subvolume and orphan items. First, find add orphan roots, then add them to fs_root radix tree. Second, in tree-root, process each orphan item, skip if it is dead root. The original algorithm is based on the list of dead_roots, one by one to visit and check whether the objectid is consistent, the time complexity is O (n ^ 2). When processing 50000 deleted subvols, it takes about 120s. Because btrfs_find_orphan_roots has already ran before us, and added deleted subvol to fs_roots radix tree. The fs root will only be removed from the fs_roots radix tree after the cleaner process is started, and the cleaner will only start execution after the mount is complete. btrfs_orphan_cleanup can be called during the whole filesystem mount lifetime, but only "tree root" will be used in this section of code, and only mount time will be brought into tree root. So we can quickly check whether the orphan item is dead root through the fs_roots radix tree. Reviewed-by: Filipe Manana Signed-off-by: Robbie Ko Signed-off-by: David Sterba --- fs/btrfs/inode.c | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index cc94291fdd18..7b854f711ce1 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2997,18 +2997,16 @@ int btrfs_orphan_cleanup(struct btrfs_root *root) * orphan must not get deleted. * find_dead_roots already ran before us, so if this * is a snapshot deletion, we should find the root - * in the dead_roots list + * in the fs_roots radix tree. */ - spin_lock(&fs_info->trans_lock); - list_for_each_entry(dead_root, &fs_info->dead_roots, - root_list) { - if (dead_root->root_key.objectid == - found_key.objectid) { - is_dead_root = 1; - break; - } - } - spin_unlock(&fs_info->trans_lock); + + spin_lock(&fs_info->fs_roots_radix_lock); + dead_root = radix_tree_lookup(&fs_info->fs_roots_radix, + (unsigned long)found_key.objectid); + if (dead_root && btrfs_root_refs(&dead_root->root_item) == 0) + is_dead_root = 1; + spin_unlock(&fs_info->fs_roots_radix_lock); + if (is_dead_root) { /* prevent this orphan from being found again */ key.offset = found_key.objectid - 1; From cbab8ade585a18c4334b085564d9d046e01a3f70 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Thu, 2 Apr 2020 14:37:35 +0800 Subject: [PATCH 0755/1043] btrfs: qgroup: mark qgroup inconsistent if we're inherting snapshot to a new qgroup [BUG] For the following operation, qgroup is guaranteed to be screwed up due to snapshot adding to a new qgroup: # mkfs.btrfs -f $dev # mount $dev $mnt # btrfs qgroup en $mnt # btrfs subv create $mnt/src # xfs_io -f -c "pwrite 0 1m" $mnt/src/file # sync # btrfs qgroup create 1/0 $mnt/src # btrfs subv snapshot -i 1/0 $mnt/src $mnt/snapshot # btrfs qgroup show -prce $mnt/src qgroupid rfer excl max_rfer max_excl parent child -------- ---- ---- -------- -------- ------ ----- 0/5 16.00KiB 16.00KiB none none --- --- 0/257 1.02MiB 16.00KiB none none --- --- 0/258 1.02MiB 16.00KiB none none 1/0 --- 1/0 0.00B 0.00B none none --- 0/258 ^^^^^^^^^^^^^^^^^^^^ [CAUSE] The problem is in btrfs_qgroup_inherit(), we don't have good enough check to determine if the new relation would break the existing accounting. Unlike btrfs_add_qgroup_relation(), which has proper check to determine if we can do quick update without a rescan, in btrfs_qgroup_inherit() we can even assign a snapshot to multiple qgroups. [FIX] Fix it by manually marking qgroup inconsistent for snapshot inheritance. For subvolume creation, since all its extents are exclusively owned, we don't need to rescan. In theory, we should call relation check like quick_update_accounting() when doing qgroup inheritance and inform user about qgroup accounting inconsistency. But we don't have good mechanism to relay that back to the user in the snapshot creation context, thus we can only silently mark the qgroup inconsistent. Anyway, user shouldn't use qgroup inheritance during snapshot creation, and should add qgroup relationship after snapshot creation by 'btrfs qgroup assign', which has a much better UI to inform user about qgroup inconsistent and kick in rescan automatically. Reviewed-by: Josef Bacik Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/qgroup.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index c3888fb367e7..5bd4089ad0e1 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -2622,6 +2622,7 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid, struct btrfs_root *quota_root; struct btrfs_qgroup *srcgroup; struct btrfs_qgroup *dstgroup; + bool need_rescan = false; u32 level_size = 0; u64 nums; @@ -2765,6 +2766,13 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid, goto unlock; } ++i_qgroups; + + /* + * If we're doing a snapshot, and adding the snapshot to a new + * qgroup, the numbers are guaranteed to be incorrect. + */ + if (srcid) + need_rescan = true; } for (i = 0; i < inherit->num_ref_copies; ++i, i_qgroups += 2) { @@ -2784,6 +2792,9 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid, dst->rfer = src->rfer - level_size; dst->rfer_cmpr = src->rfer_cmpr - level_size; + + /* Manually tweaking numbers certainly needs a rescan */ + need_rescan = true; } for (i = 0; i < inherit->num_excl_copies; ++i, i_qgroups += 2) { struct btrfs_qgroup *src; @@ -2802,6 +2813,7 @@ int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid, dst->excl = src->excl + level_size; dst->excl_cmpr = src->excl_cmpr + level_size; + need_rescan = true; } unlock: @@ -2809,6 +2821,8 @@ unlock: out: if (!committing) mutex_unlock(&fs_info->qgroup_ioctl_lock); + if (need_rescan) + fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; return ret; } From 943aeb0dae9903ec70157129daed246086e8e111 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Sat, 9 May 2020 19:22:43 +0800 Subject: [PATCH 0756/1043] btrfs: remove unused function btrfs_dev_extent_chunk_tree_uuid There's no callers in-tree anymore since commit d24ee97b96db ("btrfs: use new helpers to set uuids in eb") Signed-off-by: YueHaibing Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 03ea7370aea7..0b78ab0213bb 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -1643,13 +1643,6 @@ BTRFS_SETGET_FUNCS(dev_extent_chunk_objectid, struct btrfs_dev_extent, BTRFS_SETGET_FUNCS(dev_extent_chunk_offset, struct btrfs_dev_extent, chunk_offset, 64); BTRFS_SETGET_FUNCS(dev_extent_length, struct btrfs_dev_extent, length, 64); - -static inline unsigned long btrfs_dev_extent_chunk_tree_uuid(struct btrfs_dev_extent *dev) -{ - unsigned long ptr = offsetof(struct btrfs_dev_extent, chunk_tree_uuid); - return (unsigned long)dev + ptr; -} - BTRFS_SETGET_FUNCS(extent_refs, struct btrfs_extent_item, refs, 64); BTRFS_SETGET_FUNCS(extent_generation, struct btrfs_extent_item, generation, 64); From 31344b2fcead3239c5b801016d9bae82506b92c2 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Mon, 11 May 2020 14:49:10 +0200 Subject: [PATCH 0757/1043] btrfs: remove more obsolete v0 extent ref declarations The extent references v0 have been superseded long time go, there are some unused declarations of access helpers. We can safely remove them now. The struct btrfs_extent_ref_v0 is not used anywhere, but struct btrfs_extent_item_v0 is still part of a backward compatibility check in relocation.c and thus not removed. Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 9 --------- include/uapi/linux/btrfs_tree.h | 9 --------- 2 files changed, 18 deletions(-) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 0b78ab0213bb..86ec25250ac5 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -1648,9 +1648,6 @@ BTRFS_SETGET_FUNCS(extent_generation, struct btrfs_extent_item, generation, 64); BTRFS_SETGET_FUNCS(extent_flags, struct btrfs_extent_item, flags, 64); -BTRFS_SETGET_FUNCS(extent_refs_v0, struct btrfs_extent_item_v0, refs, 32); - - BTRFS_SETGET_FUNCS(tree_block_level, struct btrfs_tree_block_info, level, 8); static inline void btrfs_tree_block_key(struct extent_buffer *eb, @@ -1698,12 +1695,6 @@ static inline u32 btrfs_extent_inline_ref_size(int type) return 0; } -BTRFS_SETGET_FUNCS(ref_root_v0, struct btrfs_extent_ref_v0, root, 64); -BTRFS_SETGET_FUNCS(ref_generation_v0, struct btrfs_extent_ref_v0, - generation, 64); -BTRFS_SETGET_FUNCS(ref_objectid_v0, struct btrfs_extent_ref_v0, objectid, 64); -BTRFS_SETGET_FUNCS(ref_count_v0, struct btrfs_extent_ref_v0, count, 32); - /* struct btrfs_node */ BTRFS_SETGET_FUNCS(key_blockptr, struct btrfs_key_ptr, blockptr, 64); BTRFS_SETGET_FUNCS(key_generation, struct btrfs_key_ptr, generation, 64); diff --git a/include/uapi/linux/btrfs_tree.h b/include/uapi/linux/btrfs_tree.h index 8e322e2c7e78..a3f3975df0de 100644 --- a/include/uapi/linux/btrfs_tree.h +++ b/include/uapi/linux/btrfs_tree.h @@ -519,15 +519,6 @@ struct btrfs_extent_inline_ref { __le64 offset; } __attribute__ ((__packed__)); -/* old style backrefs item */ -struct btrfs_extent_ref_v0 { - __le64 root; - __le64 generation; - __le64 objectid; - __le32 count; -} __attribute__ ((__packed__)); - - /* dev extents record free space on individual devices. The owner * field points back to the chunk allocation mapping tree that allocated * the extent. The chunk tree uuid field is a way to double check the owner From 2473d24f2b77da0ffabcbb916793e58e7f57440b Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Fri, 8 May 2020 11:01:10 +0100 Subject: [PATCH 0758/1043] btrfs: fix a race between scrub and block group removal/allocation When scrub is verifying the extents of a block group for a device, it is possible that the corresponding block group gets removed and its logical address and device extents get used for a new block group allocation. When this happens scrub incorrectly reports that errors were detected and, if the the new block group has a different profile then the old one, deleted block group, we can crash due to a null pointer dereference. Possibly other unexpected and weird consequences can happen as well. Consider the following sequence of actions that leads to the null pointer dereference crash when scrub is running in parallel with balance: 1) Balance sets block group X to read-only mode and starts relocating it. Block group X is a metadata block group, has a raid1 profile (two device extents, each one in a different device) and a logical address of 19424870400; 2) Scrub is running and finds device extent E, which belongs to block group X. It enters scrub_stripe() to find all extents allocated to block group X, the search is done using the extent tree; 3) Balance finishes relocating block group X and removes block group X; 4) Balance starts relocating another block group and when trying to commit the current transaction as part of the preparation step (prepare_to_relocate()), it blocks because scrub is running; 5) The scrub task finds the metadata extent at the logical address 19425001472 and marks the pages of the extent to be read by a bio (struct scrub_bio). The extent item's flags, which have the bit BTRFS_EXTENT_FLAG_TREE_BLOCK set, are added to each page (struct scrub_page). It is these flags in the scrub pages that tells the bio's end io function (scrub_bio_end_io_worker) which type of extent it is dealing with. At this point we end up with 4 pages in a bio which is ready for submission (the metadata extent has a size of 16Kb, so that gives 4 pages on x86); 6) At the next iteration of scrub_stripe(), scrub checks that there is a pause request from the relocation task trying to commit a transaction, therefore it submits the pending bio and pauses, waiting for the transaction commit to complete before resuming; 7) The relocation task commits the transaction. The device extent E, that was used by our block group X, is now available for allocation, since the commit root for the device tree was swapped by the transaction commit; 8) Another task doing a direct IO write allocates a new data block group Y which ends using device extent E. This new block group Y also ends up getting the same logical address that block group X had: 19424870400. This happens because block group X was the block group with the highest logical address and, when allocating Y, find_next_chunk() returns the end offset of the current last block group to be used as the logical address for the new block group, which is 18351128576 + 1073741824 = 19424870400 So our new block group Y has the same logical address and device extent that block group X had. However Y is a data block group, while X was a metadata one, and Y has a raid0 profile, while X had a raid1 profile; 9) After allocating block group Y, the direct IO submits a bio to write to device extent E; 10) The read bio submitted by scrub reads the 4 pages (16Kb) from device extent E, which now correspond to the data written by the task that did a direct IO write. Then at the end io function associated with the bio, scrub_bio_end_io_worker(), we call scrub_block_complete() which calls scrub_checksum(). This later function checks the flags of the first page, and sees that the bit BTRFS_EXTENT_FLAG_TREE_BLOCK is set in the flags, so it assumes it has a metadata extent and then calls scrub_checksum_tree_block(). That functions returns an error, since interpreting data as a metadata extent causes the checksum verification to fail. So this makes scrub_checksum() call scrub_handle_errored_block(), which determines 'failed_mirror_index' to be 1, since the device extent E was allocated as the second mirror of block group X. It allocates BTRFS_MAX_MIRRORS scrub_block structures as an array at 'sblocks_for_recheck', and all the memory is initialized to zeroes by kcalloc(). After that it calls scrub_setup_recheck_block(), which is responsible for filling each of those structures. However, when that function calls btrfs_map_sblock() against the logical address of the metadata extent, 19425001472, it gets a struct btrfs_bio ('bbio') that matches the current block group Y. However block group Y has a raid0 profile and not a raid1 profile like X had, so the following call returns 1: scrub_nr_raid_mirrors(bbio) And as a result scrub_setup_recheck_block() only initializes the first (index 0) scrub_block structure in 'sblocks_for_recheck'. Then scrub_recheck_block() is called by scrub_handle_errored_block() with the second (index 1) scrub_block structure as the argument, because 'failed_mirror_index' was previously set to 1. This scrub_block was not initialized by scrub_setup_recheck_block(), so it has zero pages, its 'page_count' member is 0 and its 'pagev' page array has all members pointing to NULL. Finally when scrub_recheck_block() calls scrub_recheck_block_checksum() we have a NULL pointer dereference when accessing the flags of the first page, as pavev[0] is NULL: static void scrub_recheck_block_checksum(struct scrub_block *sblock) { (...) if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA) scrub_checksum_data(sblock); (...) } Producing a stack trace like the following: [542998.008985] BUG: kernel NULL pointer dereference, address: 0000000000000028 [542998.010238] #PF: supervisor read access in kernel mode [542998.010878] #PF: error_code(0x0000) - not-present page [542998.011516] PGD 0 P4D 0 [542998.011929] Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC PTI [542998.012786] CPU: 3 PID: 4846 Comm: kworker/u8:1 Tainted: G B W 5.6.0-rc7-btrfs-next-58 #1 [542998.014524] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.12.0-59-gc9ba5276e321-prebuilt.qemu.org 04/01/2014 [542998.016065] Workqueue: btrfs-scrub btrfs_work_helper [btrfs] [542998.017255] RIP: 0010:scrub_recheck_block_checksum+0xf/0x20 [btrfs] [542998.018474] Code: 4c 89 e6 ... [542998.021419] RSP: 0018:ffffa7af0375fbd8 EFLAGS: 00010202 [542998.022120] RAX: 0000000000000000 RBX: ffff9792e674d120 RCX: 0000000000000000 [542998.023178] RDX: 0000000000000001 RSI: ffff9792e674d120 RDI: ffff9792e674d120 [542998.024465] RBP: 0000000000000000 R08: 0000000000000067 R09: 0000000000000001 [542998.025462] R10: ffffa7af0375fa50 R11: 0000000000000000 R12: ffff9791f61fe800 [542998.026357] R13: ffff9792e674d120 R14: 0000000000000001 R15: ffffffffc0e3dfc0 [542998.027237] FS: 0000000000000000(0000) GS:ffff9792fb200000(0000) knlGS:0000000000000000 [542998.028327] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [542998.029261] CR2: 0000000000000028 CR3: 00000000b3b18003 CR4: 00000000003606e0 [542998.030301] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [542998.031316] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [542998.032380] Call Trace: [542998.032752] scrub_recheck_block+0x162/0x400 [btrfs] [542998.033500] ? __alloc_pages_nodemask+0x31e/0x460 [542998.034228] scrub_handle_errored_block+0x6f8/0x1920 [btrfs] [542998.035170] scrub_bio_end_io_worker+0x100/0x520 [btrfs] [542998.035991] btrfs_work_helper+0xaa/0x720 [btrfs] [542998.036735] process_one_work+0x26d/0x6a0 [542998.037275] worker_thread+0x4f/0x3e0 [542998.037740] ? process_one_work+0x6a0/0x6a0 [542998.038378] kthread+0x103/0x140 [542998.038789] ? kthread_create_worker_on_cpu+0x70/0x70 [542998.039419] ret_from_fork+0x3a/0x50 [542998.039875] Modules linked in: dm_snapshot dm_thin_pool ... [542998.047288] CR2: 0000000000000028 [542998.047724] ---[ end trace bde186e176c7f96a ]--- This issue has been around for a long time, possibly since scrub exists. The last time I ran into it was over 2 years ago. After recently fixing fstests to pass the "--full-balance" command line option to btrfs-progs when doing balance, several tests started to more heavily exercise balance with fsstress, scrub and other operations in parallel, and therefore started to hit this issue again (with btrfs/061 for example). Fix this by having scrub increment the 'trimming' counter of the block group, which pins the block group in such a way that it guarantees neither its logical address nor device extents can be reused by future block group allocations until we decrement the 'trimming' counter. Also make sure that on each iteration of scrub_stripe() we stop scrubbing the block group if it was removed already. A later patch in the series will rename the block group's 'trimming' counter and its helpers to a more generic name, since now it is not used exclusively for pinning while trimming anymore. CC: stable@vger.kernel.org # 4.4+ Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/scrub.c | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index adaf8ab694d5..7c50ac5b6876 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -3046,7 +3046,8 @@ out: static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx, struct map_lookup *map, struct btrfs_device *scrub_dev, - int num, u64 base, u64 length) + int num, u64 base, u64 length, + struct btrfs_block_group *cache) { struct btrfs_path *path, *ppath; struct btrfs_fs_info *fs_info = sctx->fs_info; @@ -3284,6 +3285,20 @@ static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx, break; } + /* + * If our block group was removed in the meanwhile, just + * stop scrubbing since there is no point in continuing. + * Continuing would prevent reusing its device extents + * for new block groups for a long time. + */ + spin_lock(&cache->lock); + if (cache->removed) { + spin_unlock(&cache->lock); + ret = 0; + goto out; + } + spin_unlock(&cache->lock); + extent = btrfs_item_ptr(l, slot, struct btrfs_extent_item); flags = btrfs_extent_flags(l, extent); @@ -3457,7 +3472,7 @@ static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx, if (map->stripes[i].dev->bdev == scrub_dev->bdev && map->stripes[i].physical == dev_offset) { ret = scrub_stripe(sctx, map, scrub_dev, i, - chunk_offset, length); + chunk_offset, length, cache); if (ret) goto out; } @@ -3554,6 +3569,23 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, if (!cache) goto skip; + /* + * Make sure that while we are scrubbing the corresponding block + * group doesn't get its logical address and its device extents + * reused for another block group, which can possibly be of a + * different type and different profile. We do this to prevent + * false error detections and crashes due to bogus attempts to + * repair extents. + */ + spin_lock(&cache->lock); + if (cache->removed) { + spin_unlock(&cache->lock); + btrfs_put_block_group(cache); + goto skip; + } + btrfs_get_block_group_trimming(cache); + spin_unlock(&cache->lock); + /* * we need call btrfs_inc_block_group_ro() with scrubs_paused, * to avoid deadlock caused by: @@ -3609,6 +3641,7 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, } else { btrfs_warn(fs_info, "failed setting block group ro: %d", ret); + btrfs_put_block_group_trimming(cache); btrfs_put_block_group(cache); scrub_pause_off(fs_info); break; @@ -3695,6 +3728,7 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, spin_unlock(&cache->lock); } + btrfs_put_block_group_trimming(cache); btrfs_put_block_group(cache); if (ret) break; From 6b7304af62d02d77d740defd4cfddf2ef3188067 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Fri, 8 May 2020 11:01:47 +0100 Subject: [PATCH 0759/1043] btrfs: rename member 'trimming' of block group to a more generic name Back in 2014, commit 04216820fe83d5 ("Btrfs: fix race between fs trimming and block group remove/allocation"), I added the 'trimming' member to the block group structure. Its purpose was to prevent races between trimming and block group deletion/allocation by pinning the block group in a way that prevents its logical address and device extents from being reused while trimming is in progress for a block group, so that if another task deletes the block group and then another task allocates a new block group that gets the same logical address and device extents while the trimming task is still in progress. After the previous fix for scrub (patch "btrfs: fix a race between scrub and block group removal/allocation"), scrub now also has the same needs that trimming has, so the member name 'trimming' no longer makes sense. Since there is already a 'pinned' member in the block group that refers to space reservations (pinned bytes), rename the member to 'frozen', add a comment on top of it to describe its general purpose and rename the helpers to increment and decrement the counter as well, to match the new member name. The next patch in the series will move the helpers into a more suitable file (from free-space-cache.c to block-group.c). Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 29 ++++++++++++++++------------- fs/btrfs/block-group.h | 11 ++++++++++- fs/btrfs/ctree.h | 4 ++-- fs/btrfs/extent-tree.c | 2 +- fs/btrfs/free-space-cache.c | 25 +++++++++++++------------ fs/btrfs/scrub.c | 6 +++--- fs/btrfs/transaction.c | 2 +- 7 files changed, 46 insertions(+), 33 deletions(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 5627f53ca115..3ceaacdd4ba5 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1073,18 +1073,21 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, spin_lock(&block_group->lock); block_group->removed = 1; /* - * At this point trimming can't start on this block group, because we - * removed the block group from the tree fs_info->block_group_cache_tree - * so no one can't find it anymore and even if someone already got this - * block group before we removed it from the rbtree, they have already - * incremented block_group->trimming - if they didn't, they won't find - * any free space entries because we already removed them all when we - * called btrfs_remove_free_space_cache(). + * At this point trimming or scrub can't start on this block group, + * because we removed the block group from the rbtree + * fs_info->block_group_cache_tree so no one can't find it anymore and + * even if someone already got this block group before we removed it + * from the rbtree, they have already incremented block_group->frozen - + * if they didn't, for the trimming case they won't find any free space + * entries because we already removed them all when we called + * btrfs_remove_free_space_cache(). * * And we must not remove the extent map from the fs_info->mapping_tree * to prevent the same logical address range and physical device space - * ranges from being reused for a new block group. This is because our - * fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is + * ranges from being reused for a new block group. This is needed to + * avoid races with trimming and scrub. + * + * An fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is * completely transactionless, so while it is trimming a range the * currently running transaction might finish and a new one start, * allowing for new block groups to be created that can reuse the same @@ -1095,7 +1098,7 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, * in place until the extents have been discarded completely when * the transaction commit has completed. */ - remove_em = (atomic_read(&block_group->trimming) == 0); + remove_em = (atomic_read(&block_group->frozen) == 0); spin_unlock(&block_group->lock); mutex_unlock(&fs_info->chunk_mutex); @@ -1444,7 +1447,7 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) /* Implicit trim during transaction commit. */ if (trimming) - btrfs_get_block_group_trimming(block_group); + btrfs_freeze_block_group(block_group); /* * Btrfs_remove_chunk will abort the transaction if things go @@ -1454,7 +1457,7 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) if (ret) { if (trimming) - btrfs_put_block_group_trimming(block_group); + btrfs_unfreeze_block_group(block_group); goto end_trans; } @@ -1803,7 +1806,7 @@ static struct btrfs_block_group *btrfs_create_block_group_cache( INIT_LIST_HEAD(&cache->dirty_list); INIT_LIST_HEAD(&cache->io_list); btrfs_init_free_space_ctl(cache); - atomic_set(&cache->trimming, 0); + atomic_set(&cache->frozen, 0); mutex_init(&cache->free_space_lock); btrfs_init_full_stripe_locks_tree(&cache->full_stripe_locks_root); diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 107bb557ca8d..04967ea7ba2c 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -129,8 +129,17 @@ struct btrfs_block_group { /* For read-only block groups */ struct list_head ro_list; + /* + * When non-zero it means the block group's logical address and its + * device extents can not be reused for future block group allocations + * until the counter goes down to 0. This is to prevent them from being + * reused while some task is still using the block group after it was + * deleted - we want to make sure they can only be reused for new block + * groups after that task is done with the deleted block group. + */ + atomic_t frozen; + /* For discard operations */ - atomic_t trimming; struct list_head discard_list; int discard_index; u64 discard_eligible_time; diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 86ec25250ac5..242f974a4451 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -2481,8 +2481,8 @@ int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, struct btrfs_ref *generic_ref); int btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr); -void btrfs_get_block_group_trimming(struct btrfs_block_group *cache); -void btrfs_put_block_group_trimming(struct btrfs_block_group *cache); +void btrfs_freeze_block_group(struct btrfs_block_group *cache); +void btrfs_unfreeze_block_group(struct btrfs_block_group *cache); void btrfs_clear_space_info_full(struct btrfs_fs_info *info); enum btrfs_reserve_flush_enum { diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 3593f8cce9e5..140011edcad7 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -2916,7 +2916,7 @@ int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans) &trimmed); list_del_init(&block_group->bg_list); - btrfs_put_block_group_trimming(block_group); + btrfs_unfreeze_block_group(block_group); btrfs_put_block_group(block_group); if (ret) { diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index 3613da065a73..e9cfe9da6bbe 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -3762,12 +3762,12 @@ out: return ret; } -void btrfs_get_block_group_trimming(struct btrfs_block_group *cache) +void btrfs_freeze_block_group(struct btrfs_block_group *cache) { - atomic_inc(&cache->trimming); + atomic_inc(&cache->frozen); } -void btrfs_put_block_group_trimming(struct btrfs_block_group *block_group) +void btrfs_unfreeze_block_group(struct btrfs_block_group *block_group) { struct btrfs_fs_info *fs_info = block_group->fs_info; struct extent_map_tree *em_tree; @@ -3775,7 +3775,7 @@ void btrfs_put_block_group_trimming(struct btrfs_block_group *block_group) bool cleanup; spin_lock(&block_group->lock); - cleanup = (atomic_dec_and_test(&block_group->trimming) && + cleanup = (atomic_dec_and_test(&block_group->frozen) && block_group->removed); spin_unlock(&block_group->lock); @@ -3795,8 +3795,9 @@ void btrfs_put_block_group_trimming(struct btrfs_block_group *block_group) free_extent_map(em); /* - * We've left one free space entry and other tasks trimming - * this block group have left 1 entry each one. Free them. + * We may have left one free space entry and other possible + * tasks trimming this block group have left 1 entry each one. + * Free them if any. */ __btrfs_remove_free_space_cache(block_group->free_space_ctl); } @@ -3816,7 +3817,7 @@ int btrfs_trim_block_group(struct btrfs_block_group *block_group, spin_unlock(&block_group->lock); return 0; } - btrfs_get_block_group_trimming(block_group); + btrfs_freeze_block_group(block_group); spin_unlock(&block_group->lock); ret = trim_no_bitmap(block_group, trimmed, start, end, minlen, false); @@ -3829,7 +3830,7 @@ int btrfs_trim_block_group(struct btrfs_block_group *block_group, if (rem) reset_trimming_bitmap(ctl, offset_to_bitmap(ctl, end)); out: - btrfs_put_block_group_trimming(block_group); + btrfs_unfreeze_block_group(block_group); return ret; } @@ -3846,11 +3847,11 @@ int btrfs_trim_block_group_extents(struct btrfs_block_group *block_group, spin_unlock(&block_group->lock); return 0; } - btrfs_get_block_group_trimming(block_group); + btrfs_freeze_block_group(block_group); spin_unlock(&block_group->lock); ret = trim_no_bitmap(block_group, trimmed, start, end, minlen, async); - btrfs_put_block_group_trimming(block_group); + btrfs_unfreeze_block_group(block_group); return ret; } @@ -3868,13 +3869,13 @@ int btrfs_trim_block_group_bitmaps(struct btrfs_block_group *block_group, spin_unlock(&block_group->lock); return 0; } - btrfs_get_block_group_trimming(block_group); + btrfs_freeze_block_group(block_group); spin_unlock(&block_group->lock); ret = trim_bitmaps(block_group, trimmed, start, end, minlen, maxlen, async); - btrfs_put_block_group_trimming(block_group); + btrfs_unfreeze_block_group(block_group); return ret; } diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 7c50ac5b6876..2486f58d8205 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -3583,7 +3583,7 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, btrfs_put_block_group(cache); goto skip; } - btrfs_get_block_group_trimming(cache); + btrfs_freeze_block_group(cache); spin_unlock(&cache->lock); /* @@ -3641,7 +3641,7 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, } else { btrfs_warn(fs_info, "failed setting block group ro: %d", ret); - btrfs_put_block_group_trimming(cache); + btrfs_unfreeze_block_group(cache); btrfs_put_block_group(cache); scrub_pause_off(fs_info); break; @@ -3728,7 +3728,7 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, spin_unlock(&cache->lock); } - btrfs_put_block_group_trimming(cache); + btrfs_unfreeze_block_group(cache); btrfs_put_block_group(cache); if (ret) break; diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index 96eb313a5080..f58d0fdc5078 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -142,7 +142,7 @@ void btrfs_put_transaction(struct btrfs_transaction *transaction) struct btrfs_block_group, bg_list); list_del_init(&cache->bg_list); - btrfs_put_block_group_trimming(cache); + btrfs_unfreeze_block_group(cache); btrfs_put_block_group(cache); } WARN_ON(!list_empty(&transaction->dev_update_list)); From 684b752b0933ac287fdd1f4cdc53c4a937e90e46 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Fri, 8 May 2020 11:01:59 +0100 Subject: [PATCH 0760/1043] btrfs: move the block group freeze/unfreeze helpers into block-group.c The helpers btrfs_freeze_block_group() and btrfs_unfreeze_block_group() used to be named btrfs_get_block_group_trimming() and btrfs_put_block_group_trimming() respectively. At the time they were added to free-space-cache.c, by commit e33e17ee1098 ("btrfs: add missing discards when unpinning extents with -o discard") because all the trimming related functions were in free-space-cache.c. Now that the helpers were renamed and are used in scrub context as well, move them to block-group.c, a much more logical location for them. Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 41 +++++++++++++++++++++++++++++++++++++ fs/btrfs/block-group.h | 3 +++ fs/btrfs/ctree.h | 2 -- fs/btrfs/free-space-cache.c | 41 ------------------------------------- 4 files changed, 44 insertions(+), 43 deletions(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 3ceaacdd4ba5..c68d8e06f794 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -3381,3 +3381,44 @@ int btrfs_free_block_groups(struct btrfs_fs_info *info) } return 0; } + +void btrfs_freeze_block_group(struct btrfs_block_group *cache) +{ + atomic_inc(&cache->frozen); +} + +void btrfs_unfreeze_block_group(struct btrfs_block_group *block_group) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct extent_map_tree *em_tree; + struct extent_map *em; + bool cleanup; + + spin_lock(&block_group->lock); + cleanup = (atomic_dec_and_test(&block_group->frozen) && + block_group->removed); + spin_unlock(&block_group->lock); + + if (cleanup) { + mutex_lock(&fs_info->chunk_mutex); + em_tree = &fs_info->mapping_tree; + write_lock(&em_tree->lock); + em = lookup_extent_mapping(em_tree, block_group->start, + 1); + BUG_ON(!em); /* logic error, can't happen */ + remove_extent_mapping(em_tree, em); + write_unlock(&em_tree->lock); + mutex_unlock(&fs_info->chunk_mutex); + + /* once for us and once for the tree */ + free_extent_map(em); + free_extent_map(em); + + /* + * We may have left one free space entry and other possible + * tasks trimming this block group have left 1 entry each one. + * Free them if any. + */ + __btrfs_remove_free_space_cache(block_group->free_space_ctl); + } +} diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h index 04967ea7ba2c..b6ee70a039c7 100644 --- a/fs/btrfs/block-group.h +++ b/fs/btrfs/block-group.h @@ -292,6 +292,9 @@ static inline int btrfs_block_group_done(struct btrfs_block_group *cache) cache->cached == BTRFS_CACHE_ERROR; } +void btrfs_freeze_block_group(struct btrfs_block_group *cache); +void btrfs_unfreeze_block_group(struct btrfs_block_group *cache); + #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, u64 physical, u64 **logical, int *naddrs, int *stripe_len); diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 242f974a4451..6e226c371972 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -2481,8 +2481,6 @@ int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, struct btrfs_ref *generic_ref); int btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr); -void btrfs_freeze_block_group(struct btrfs_block_group *cache); -void btrfs_unfreeze_block_group(struct btrfs_block_group *cache); void btrfs_clear_space_info_full(struct btrfs_fs_info *info); enum btrfs_reserve_flush_enum { diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index e9cfe9da6bbe..3c353a337b91 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -3762,47 +3762,6 @@ out: return ret; } -void btrfs_freeze_block_group(struct btrfs_block_group *cache) -{ - atomic_inc(&cache->frozen); -} - -void btrfs_unfreeze_block_group(struct btrfs_block_group *block_group) -{ - struct btrfs_fs_info *fs_info = block_group->fs_info; - struct extent_map_tree *em_tree; - struct extent_map *em; - bool cleanup; - - spin_lock(&block_group->lock); - cleanup = (atomic_dec_and_test(&block_group->frozen) && - block_group->removed); - spin_unlock(&block_group->lock); - - if (cleanup) { - mutex_lock(&fs_info->chunk_mutex); - em_tree = &fs_info->mapping_tree; - write_lock(&em_tree->lock); - em = lookup_extent_mapping(em_tree, block_group->start, - 1); - BUG_ON(!em); /* logic error, can't happen */ - remove_extent_mapping(em_tree, em); - write_unlock(&em_tree->lock); - mutex_unlock(&fs_info->chunk_mutex); - - /* once for us and once for the tree */ - free_extent_map(em); - free_extent_map(em); - - /* - * We may have left one free space entry and other possible - * tasks trimming this block group have left 1 entry each one. - * Free them if any. - */ - __btrfs_remove_free_space_cache(block_group->free_space_ctl); - } -} - int btrfs_trim_block_group(struct btrfs_block_group *block_group, u64 *trimmed, u64 start, u64 end, u64 minlen) { From 89490303a4294238b84c83541c140e540faa17ba Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Fri, 8 May 2020 11:02:07 +0100 Subject: [PATCH 0761/1043] btrfs: scrub, only lookup for csums if we are dealing with a data extent When scrubbing a stripe, whenever we find an extent we lookup for its checksums in the checksum tree. However we do it even for metadata extents which don't have checksum items stored in the checksum tree, that is only for data extents. So make the lookup for checksums only if we are processing with a data extent. Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/scrub.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 2486f58d8205..0f7740970553 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -3343,13 +3343,14 @@ again: &extent_dev, &extent_mirror_num); - ret = btrfs_lookup_csums_range(csum_root, - extent_logical, - extent_logical + - extent_len - 1, - &sctx->csum_list, 1); - if (ret) - goto out; + if (flags & BTRFS_EXTENT_FLAG_DATA) { + ret = btrfs_lookup_csums_range(csum_root, + extent_logical, + extent_logical + extent_len - 1, + &sctx->csum_list, 1); + if (ret) + goto out; + } ret = scrub_extent(sctx, map, extent_logical, extent_len, extent_physical, extent_dev, flags, From 89efda52e6b6930f80f5adda9c3c9edfb1397191 Mon Sep 17 00:00:00 2001 From: Marcos Paulo de Souza Date: Sun, 10 May 2020 23:15:07 -0300 Subject: [PATCH 0762/1043] btrfs: send: emit file capabilities after chown Whenever a chown is executed, all capabilities of the file being touched are lost. When doing incremental send with a file with capabilities, there is a situation where the capability can be lost on the receiving side. The sequence of actions bellow shows the problem: $ mount /dev/sda fs1 $ mount /dev/sdb fs2 $ touch fs1/foo.bar $ setcap cap_sys_nice+ep fs1/foo.bar $ btrfs subvolume snapshot -r fs1 fs1/snap_init $ btrfs send fs1/snap_init | btrfs receive fs2 $ chgrp adm fs1/foo.bar $ setcap cap_sys_nice+ep fs1/foo.bar $ btrfs subvolume snapshot -r fs1 fs1/snap_complete $ btrfs subvolume snapshot -r fs1 fs1/snap_incremental $ btrfs send fs1/snap_complete | btrfs receive fs2 $ btrfs send -p fs1/snap_init fs1/snap_incremental | btrfs receive fs2 At this point, only a chown was emitted by "btrfs send" since only the group was changed. This makes the cap_sys_nice capability to be dropped from fs2/snap_incremental/foo.bar To fix that, only emit capabilities after chown is emitted. The current code first checks for xattrs that are new/changed, emits them, and later emit the chown. Now, __process_new_xattr skips capabilities, letting only finish_inode_if_needed to emit them, if they exist, for the inode being processed. This behavior was being worked around in "btrfs receive" side by caching the capability and only applying it after chown. Now, xattrs are only emmited _after_ chown, making that workaround not needed anymore. Link: https://github.com/kdave/btrfs-progs/issues/202 CC: stable@vger.kernel.org # 4.4+ Suggested-by: Filipe Manana Reviewed-by: Filipe Manana Signed-off-by: Marcos Paulo de Souza Signed-off-by: David Sterba --- fs/btrfs/send.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index c5f41bd86765..4f3b8d2bb56b 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -23,6 +23,7 @@ #include "btrfs_inode.h" #include "transaction.h" #include "compression.h" +#include "xattr.h" /* * Maximum number of references an extent can have in order for us to attempt to @@ -4545,6 +4546,10 @@ static int __process_new_xattr(int num, struct btrfs_key *di_key, struct fs_path *p; struct posix_acl_xattr_header dummy_acl; + /* Capabilities are emitted by finish_inode_if_needed */ + if (!strncmp(name, XATTR_NAME_CAPS, name_len)) + return 0; + p = fs_path_alloc(); if (!p) return -ENOMEM; @@ -5107,6 +5112,64 @@ static int send_extent_data(struct send_ctx *sctx, return 0; } +/* + * Search for a capability xattr related to sctx->cur_ino. If the capability is + * found, call send_set_xattr function to emit it. + * + * Return 0 if there isn't a capability, or when the capability was emitted + * successfully, or < 0 if an error occurred. + */ +static int send_capabilities(struct send_ctx *sctx) +{ + struct fs_path *fspath = NULL; + struct btrfs_path *path; + struct btrfs_dir_item *di; + struct extent_buffer *leaf; + unsigned long data_ptr; + char *buf = NULL; + int buf_len; + int ret = 0; + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + di = btrfs_lookup_xattr(NULL, sctx->send_root, path, sctx->cur_ino, + XATTR_NAME_CAPS, strlen(XATTR_NAME_CAPS), 0); + if (!di) { + /* There is no xattr for this inode */ + goto out; + } else if (IS_ERR(di)) { + ret = PTR_ERR(di); + goto out; + } + + leaf = path->nodes[0]; + buf_len = btrfs_dir_data_len(leaf, di); + + fspath = fs_path_alloc(); + buf = kmalloc(buf_len, GFP_KERNEL); + if (!fspath || !buf) { + ret = -ENOMEM; + goto out; + } + + ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, fspath); + if (ret < 0) + goto out; + + data_ptr = (unsigned long)(di + 1) + btrfs_dir_name_len(leaf, di); + read_extent_buffer(leaf, buf, data_ptr, buf_len); + + ret = send_set_xattr(sctx, fspath, XATTR_NAME_CAPS, + strlen(XATTR_NAME_CAPS), buf, buf_len); +out: + kfree(buf); + fs_path_free(fspath); + btrfs_free_path(path); + return ret; +} + static int clone_range(struct send_ctx *sctx, struct clone_root *clone_root, const u64 disk_byte, @@ -5972,6 +6035,10 @@ static int finish_inode_if_needed(struct send_ctx *sctx, int at_end) goto out; } + ret = send_capabilities(sctx); + if (ret < 0) + goto out; + /* * If other directory inodes depended on our current directory * inode's move/rename, now do their move/rename operations. From 83fe9e12b0558eae519351cff00da1e06bc054d2 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 5 May 2020 07:58:19 +0800 Subject: [PATCH 0763/1043] btrfs: block-group: don't set the wrong READA flag for btrfs_read_block_groups() Regular block group items in extent tree are scattered inside the huge tree, thus forward readahead makes no sense. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index c68d8e06f794..1137597fbca5 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1988,7 +1988,6 @@ int btrfs_read_block_groups(struct btrfs_fs_info *info) path = btrfs_alloc_path(); if (!path) return -ENOMEM; - path->reada = READA_FORWARD; cache_gen = btrfs_super_cache_generation(info->super_copy); if (btrfs_test_opt(info, SPACE_CACHE) && From 9afc66498a0b5f76928f9823b58c55c5dfb06ae5 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 5 May 2020 07:58:20 +0800 Subject: [PATCH 0764/1043] btrfs: block-group: refactor how we read one block group item Structure btrfs_block_group has the following members which are currently read from on-disk block group item and key: - length - from item key - used - flags - from block group item However for incoming skinny block group tree, we are going to read those members from different sources. This patch will refactor such read by: - Don't initialize btrfs_block_group::length at allocation Caller should initialize them manually. Also to avoid possible (well, only two callers) missing initialization, add extra ASSERT() in btrfs_add_block_group_cache(). - Refactor length/used/flags initialization into one function The new function, fill_one_block_group() will handle the initialization of such members. - Use btrfs_block_group::length to replace key::offset Since skinny block group item would have a different meaning for its key offset. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 47 ++++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 1137597fbca5..e9232ba3d7ca 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -161,6 +161,8 @@ static int btrfs_add_block_group_cache(struct btrfs_fs_info *info, struct rb_node *parent = NULL; struct btrfs_block_group *cache; + ASSERT(block_group->length != 0); + spin_lock(&info->block_group_cache_lock); p = &info->block_group_cache_tree.rb_node; @@ -1771,7 +1773,7 @@ static void link_block_group(struct btrfs_block_group *cache) } static struct btrfs_block_group *btrfs_create_block_group_cache( - struct btrfs_fs_info *fs_info, u64 start, u64 size) + struct btrfs_fs_info *fs_info, u64 start) { struct btrfs_block_group *cache; @@ -1787,7 +1789,6 @@ static struct btrfs_block_group *btrfs_create_block_group_cache( } cache->start = start; - cache->length = size; cache->fs_info = fs_info; cache->full_stripe_len = btrfs_full_stripe_len(fs_info, start); @@ -1867,25 +1868,44 @@ static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info) return ret; } +static int read_block_group_item(struct btrfs_block_group *cache, + struct btrfs_path *path, + const struct btrfs_key *key) +{ + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_block_group_item bgi; + int slot = path->slots[0]; + + cache->length = key->offset; + + read_extent_buffer(leaf, &bgi, btrfs_item_ptr_offset(leaf, slot), + sizeof(bgi)); + cache->used = btrfs_stack_block_group_used(&bgi); + cache->flags = btrfs_stack_block_group_flags(&bgi); + + return 0; +} + static int read_one_block_group(struct btrfs_fs_info *info, struct btrfs_path *path, const struct btrfs_key *key, int need_clear) { - struct extent_buffer *leaf = path->nodes[0]; struct btrfs_block_group *cache; struct btrfs_space_info *space_info; - struct btrfs_block_group_item bgi; const bool mixed = btrfs_fs_incompat(info, MIXED_GROUPS); - int slot = path->slots[0]; int ret; ASSERT(key->type == BTRFS_BLOCK_GROUP_ITEM_KEY); - cache = btrfs_create_block_group_cache(info, key->objectid, key->offset); + cache = btrfs_create_block_group_cache(info, key->objectid); if (!cache) return -ENOMEM; + ret = read_block_group_item(cache, path, key); + if (ret < 0) + goto error; + if (need_clear) { /* * When we mount with old space cache, we need to @@ -1900,10 +1920,6 @@ static int read_one_block_group(struct btrfs_fs_info *info, if (btrfs_test_opt(info, SPACE_CACHE)) cache->disk_cache_state = BTRFS_DC_CLEAR; } - read_extent_buffer(leaf, &bgi, btrfs_item_ptr_offset(leaf, slot), - sizeof(bgi)); - cache->used = btrfs_stack_block_group_used(&bgi); - cache->flags = btrfs_stack_block_group_flags(&bgi); if (!mixed && ((cache->flags & BTRFS_BLOCK_GROUP_METADATA) && (cache->flags & BTRFS_BLOCK_GROUP_DATA))) { btrfs_err(info, @@ -1931,15 +1947,15 @@ static int read_one_block_group(struct btrfs_fs_info *info, * are empty, and we can just add all the space in and be done with it. * This saves us _a_lot_ of time, particularly in the full case. */ - if (key->offset == cache->used) { + if (cache->length == cache->used) { cache->last_byte_to_unpin = (u64)-1; cache->cached = BTRFS_CACHE_FINISHED; btrfs_free_excluded_extents(cache); } else if (cache->used == 0) { cache->last_byte_to_unpin = (u64)-1; cache->cached = BTRFS_CACHE_FINISHED; - add_new_free_space(cache, key->objectid, - key->objectid + key->offset); + add_new_free_space(cache, cache->start, + cache->start + cache->length); btrfs_free_excluded_extents(cache); } @@ -1949,7 +1965,7 @@ static int read_one_block_group(struct btrfs_fs_info *info, goto error; } trace_btrfs_add_block_group(info, cache, 0); - btrfs_update_space_info(info, cache->flags, key->offset, + btrfs_update_space_info(info, cache->flags, cache->length, cache->used, cache->bytes_super, &space_info); cache->space_info = space_info; @@ -2096,10 +2112,11 @@ int btrfs_make_block_group(struct btrfs_trans_handle *trans, u64 bytes_used, btrfs_set_log_full_commit(trans); - cache = btrfs_create_block_group_cache(fs_info, chunk_offset, size); + cache = btrfs_create_block_group_cache(fs_info, chunk_offset); if (!cache) return -ENOMEM; + cache->length = size; cache->used = bytes_used; cache->flags = type; cache->last_byte_to_unpin = (u64)-1; From 7357623a7f4beb4ac76005f8fac9fc0230f9a67e Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 5 May 2020 07:58:21 +0800 Subject: [PATCH 0765/1043] btrfs: block-group: refactor how we delete one block group item When deleting a block group item, it's pretty straight forward, just delete the item pointed by the key. However it will not be that straight-forward for incoming skinny block group item. So refactor the block group item deletion into a new function, remove_block_group_item(), also to make the already lengthy btrfs_remove_block_group() a little shorter. Reviewed-by: Johannes Thumshirn Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index e9232ba3d7ca..af990f82448f 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -865,11 +865,34 @@ static void clear_incompat_bg_bits(struct btrfs_fs_info *fs_info, u64 flags) } } +static int remove_block_group_item(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_block_group *block_group) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *root; + struct btrfs_key key; + int ret; + + root = fs_info->extent_root; + key.objectid = block_group->start; + key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; + key.offset = block_group->length; + + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret > 0) + ret = -ENOENT; + if (ret < 0) + return ret; + + ret = btrfs_del_item(trans, root, path); + return ret; +} + int btrfs_remove_block_group(struct btrfs_trans_handle *trans, u64 group_start, struct extent_map *em) { struct btrfs_fs_info *fs_info = trans->fs_info; - struct btrfs_root *root = fs_info->extent_root; struct btrfs_path *path; struct btrfs_block_group *block_group; struct btrfs_free_cluster *cluster; @@ -1067,10 +1090,6 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, spin_unlock(&block_group->space_info->lock); - key.objectid = block_group->start; - key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; - key.offset = block_group->length; - mutex_lock(&fs_info->chunk_mutex); spin_lock(&block_group->lock); block_group->removed = 1; @@ -1112,16 +1131,10 @@ int btrfs_remove_block_group(struct btrfs_trans_handle *trans, /* Once for the block groups rbtree */ btrfs_put_block_group(block_group); - ret = btrfs_search_slot(trans, root, &key, path, -1, 1); - if (ret > 0) - ret = -EIO; + ret = remove_block_group_item(trans, path, block_group); if (ret < 0) goto out; - ret = btrfs_del_item(trans, root, path); - if (ret) - goto out; - if (remove_em) { struct extent_map_tree *em_tree; From 97f4728af88822ddd13941c3ac84a4ce3a9ff823 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 5 May 2020 07:58:22 +0800 Subject: [PATCH 0766/1043] btrfs: block-group: refactor how we insert a block group item Currently the block group item insert is pretty straight forward, fill the block group item structure and insert it into extent tree. However the incoming skinny block group feature is going to change this, so this patch will refactor insertion into a new function, insert_block_group_item(), to make the incoming feature easier to add. Reviewed-by: Johannes Thumshirn Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index af990f82448f..0b5849f46835 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -2071,13 +2071,32 @@ error: return ret; } +static int insert_block_group_item(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_block_group_item bgi; + struct btrfs_root *root; + struct btrfs_key key; + + spin_lock(&block_group->lock); + btrfs_set_stack_block_group_used(&bgi, block_group->used); + btrfs_set_stack_block_group_chunk_objectid(&bgi, + BTRFS_FIRST_CHUNK_TREE_OBJECTID); + btrfs_set_stack_block_group_flags(&bgi, block_group->flags); + key.objectid = block_group->start; + key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; + key.offset = block_group->length; + spin_unlock(&block_group->lock); + + root = fs_info->extent_root; + return btrfs_insert_item(trans, root, &key, &bgi, sizeof(bgi)); +} + void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans) { struct btrfs_fs_info *fs_info = trans->fs_info; struct btrfs_block_group *block_group; - struct btrfs_root *extent_root = fs_info->extent_root; - struct btrfs_block_group_item item; - struct btrfs_key key; int ret = 0; if (!trans->can_flush_pending_bgs) @@ -2090,21 +2109,11 @@ void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans) if (ret) goto next; - spin_lock(&block_group->lock); - btrfs_set_stack_block_group_used(&item, block_group->used); - btrfs_set_stack_block_group_chunk_objectid(&item, - BTRFS_FIRST_CHUNK_TREE_OBJECTID); - btrfs_set_stack_block_group_flags(&item, block_group->flags); - key.objectid = block_group->start; - key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; - key.offset = block_group->length; - spin_unlock(&block_group->lock); - - ret = btrfs_insert_item(trans, extent_root, &key, &item, - sizeof(item)); + ret = insert_block_group_item(trans, block_group); if (ret) btrfs_abort_transaction(trans, ret); - ret = btrfs_finish_chunk_alloc(trans, key.objectid, key.offset); + ret = btrfs_finish_chunk_alloc(trans, block_group->start, + block_group->length); if (ret) btrfs_abort_transaction(trans, ret); add_block_group_free_space(trans, block_group); From 3be4d8efe3cf218350cf2c3f39136a636698c572 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 5 May 2020 07:58:23 +0800 Subject: [PATCH 0767/1043] btrfs: block-group: rename write_one_cache_group() The name of this function contains the word "cache", which is left from the times where btrfs_block_group was called btrfs_block_group_cache. Now this "cache" doesn't match anything, and we have better namings for functions like read/insert/remove_block_group_item(). Rename it to update_block_group_item(). Reviewed-by: Johannes Thumshirn Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 0b5849f46835..048b3384f588 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -2349,13 +2349,13 @@ void btrfs_dec_block_group_ro(struct btrfs_block_group *cache) spin_unlock(&sinfo->lock); } -static int write_one_cache_group(struct btrfs_trans_handle *trans, - struct btrfs_path *path, - struct btrfs_block_group *cache) +static int update_block_group_item(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_block_group *cache) { struct btrfs_fs_info *fs_info = trans->fs_info; int ret; - struct btrfs_root *extent_root = fs_info->extent_root; + struct btrfs_root *root = fs_info->extent_root; unsigned long bi; struct extent_buffer *leaf; struct btrfs_block_group_item bgi; @@ -2365,7 +2365,7 @@ static int write_one_cache_group(struct btrfs_trans_handle *trans, key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; key.offset = cache->length; - ret = btrfs_search_slot(trans, extent_root, &key, path, 0, 1); + ret = btrfs_search_slot(trans, root, &key, path, 0, 1); if (ret) { if (ret > 0) ret = -ENOENT; @@ -2677,7 +2677,7 @@ again: } } if (!ret) { - ret = write_one_cache_group(trans, path, cache); + ret = update_block_group_item(trans, path, cache); /* * Our block group might still be attached to the list * of new block groups in the transaction handle of some @@ -2826,7 +2826,7 @@ int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans) } } if (!ret) { - ret = write_one_cache_group(trans, path, cache); + ret = update_block_group_item(trans, path, cache); /* * One of the free space endio workers might have * created a new block group while updating a free space @@ -2843,7 +2843,7 @@ int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans) if (ret == -ENOENT) { wait_event(cur_trans->writer_wait, atomic_read(&cur_trans->num_writers) == 1); - ret = write_one_cache_group(trans, path, cache); + ret = update_block_group_item(trans, path, cache); } if (ret) btrfs_abort_transaction(trans, ret); From f2998ebd32e25e432e3d0058552e4777c5aba75a Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Tue, 12 May 2020 11:55:03 +0800 Subject: [PATCH 0768/1043] btrfs: remove duplicated include in block-group.c disk-io.h is included more than once in block-group.c, remove it. Reviewed-by: Johannes Thumshirn Signed-off-by: Tiezhu Yang Signed-off-by: David Sterba --- fs/btrfs/block-group.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 048b3384f588..176e8a292fd1 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -7,7 +7,6 @@ #include "disk-io.h" #include "free-space-cache.h" #include "free-space-tree.h" -#include "disk-io.h" #include "volumes.h" #include "transaction.h" #include "ref-verify.h" From 4dae666a6286ce5d9f4890f8b88af646473686f0 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 02:13:57 +0200 Subject: [PATCH 0769/1043] btrfs: use the token::eb for all set/get helpers The token stores a copy of the extent buffer pointer but does not make any use of it besides sanity checks. We can use it and drop the eb parameter from several functions, this patch only switches the use inside the set/get helpers. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 73f7987143df..7928d310f698 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -62,12 +62,12 @@ u##bits btrfs_get_token_##bits(const struct extent_buffer *eb, \ res = get_unaligned_le##bits(p + off); \ return res; \ } \ - err = map_private_extent_buffer(eb, offset, size, \ + err = map_private_extent_buffer(token->eb, offset, size, \ &kaddr, &map_start, &map_len); \ if (err) { \ __le##bits leres; \ \ - read_extent_buffer(eb, &leres, offset, size); \ + read_extent_buffer(token->eb, &leres, offset, size); \ return le##bits##_to_cpu(leres); \ } \ p = kaddr + part_offset - map_start; \ @@ -125,13 +125,13 @@ void btrfs_set_token_##bits(struct extent_buffer *eb, \ put_unaligned_le##bits(val, p + off); \ return; \ } \ - err = map_private_extent_buffer(eb, offset, size, \ + err = map_private_extent_buffer(token->eb, offset, size, \ &kaddr, &map_start, &map_len); \ if (err) { \ __le##bits val2; \ \ val2 = cpu_to_le##bits(val); \ - write_extent_buffer(eb, &val2, offset, size); \ + write_extent_buffer(token->eb, &val2, offset, size); \ return; \ } \ p = kaddr + part_offset - map_start; \ From cc4c13d55cba8a0b81bc18243eabc57be1aa44d2 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 02:15:56 +0200 Subject: [PATCH 0770/1043] btrfs: drop eb parameter from set/get token helpers Now that all set/get helpers use the eb from the token, we don't need to pass it to many btrfs_token_*/btrfs_set_token_* helpers, saving some stack space. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 53 +++++++++----------- fs/btrfs/ctree.h | 27 +++++----- fs/btrfs/inode.c | 61 +++++++++++------------ fs/btrfs/struct-funcs.c | 12 ++--- fs/btrfs/tree-log.c | 107 ++++++++++++++++++---------------------- 5 files changed, 116 insertions(+), 144 deletions(-) diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 6c28efe5b14a..576111cdea1d 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -3517,9 +3517,9 @@ static int leaf_space_used(struct extent_buffer *l, int start, int nr) btrfs_init_map_token(&token, l); start_item = btrfs_item_nr(start); end_item = btrfs_item_nr(end); - data_len = btrfs_token_item_offset(l, start_item, &token) + - btrfs_token_item_size(l, start_item, &token); - data_len = data_len - btrfs_token_item_offset(l, end_item, &token); + data_len = btrfs_token_item_offset(&token, start_item) + + btrfs_token_item_size(&token, start_item); + data_len = data_len - btrfs_token_item_offset(&token, end_item); data_len += sizeof(struct btrfs_item) * nr; WARN_ON(data_len < 0); return data_len; @@ -3650,8 +3650,8 @@ static noinline int __push_leaf_right(struct btrfs_path *path, push_space = BTRFS_LEAF_DATA_SIZE(fs_info); for (i = 0; i < right_nritems; i++) { item = btrfs_item_nr(i); - push_space -= btrfs_token_item_size(right, item, &token); - btrfs_set_token_item_offset(right, item, push_space, &token); + push_space -= btrfs_token_item_size(&token, item); + btrfs_set_token_item_offset(&token, item, push_space); } left_nritems -= push_items; @@ -3859,10 +3859,9 @@ static noinline int __push_leaf_left(struct btrfs_path *path, int data_size, item = btrfs_item_nr(i); - ioff = btrfs_token_item_offset(left, item, &token); - btrfs_set_token_item_offset(left, item, - ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size), - &token); + ioff = btrfs_token_item_offset(&token, item); + btrfs_set_token_item_offset(&token, item, + ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size)); } btrfs_set_header_nritems(left, old_left_nritems + push_items); @@ -3892,9 +3891,8 @@ static noinline int __push_leaf_left(struct btrfs_path *path, int data_size, for (i = 0; i < right_nritems; i++) { item = btrfs_item_nr(i); - push_space = push_space - btrfs_token_item_size(right, - item, &token); - btrfs_set_token_item_offset(right, item, push_space, &token); + push_space = push_space - btrfs_token_item_size(&token, item); + btrfs_set_token_item_offset(&token, item, push_space); } btrfs_mark_buffer_dirty(left); @@ -4036,9 +4034,8 @@ static noinline void copy_for_split(struct btrfs_trans_handle *trans, struct btrfs_item *item = btrfs_item_nr(i); u32 ioff; - ioff = btrfs_token_item_offset(right, item, &token); - btrfs_set_token_item_offset(right, item, - ioff + rt_data_off, &token); + ioff = btrfs_token_item_offset(&token, item); + btrfs_set_token_item_offset(&token, item, ioff + rt_data_off); } btrfs_set_header_nritems(l, mid); @@ -4541,9 +4538,8 @@ void btrfs_truncate_item(struct btrfs_path *path, u32 new_size, int from_end) u32 ioff; item = btrfs_item_nr(i); - ioff = btrfs_token_item_offset(leaf, item, &token); - btrfs_set_token_item_offset(leaf, item, - ioff + size_diff, &token); + ioff = btrfs_token_item_offset(&token, item); + btrfs_set_token_item_offset(&token, item, ioff + size_diff); } /* shift the data */ @@ -4640,9 +4636,8 @@ void btrfs_extend_item(struct btrfs_path *path, u32 data_size) u32 ioff; item = btrfs_item_nr(i); - ioff = btrfs_token_item_offset(leaf, item, &token); - btrfs_set_token_item_offset(leaf, item, - ioff - data_size, &token); + ioff = btrfs_token_item_offset(&token, item); + btrfs_set_token_item_offset(&token, item, ioff - data_size); } /* shift the data */ @@ -4718,9 +4713,9 @@ void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, u32 ioff; item = btrfs_item_nr(i); - ioff = btrfs_token_item_offset(leaf, item, &token); - btrfs_set_token_item_offset(leaf, item, - ioff - total_data, &token); + ioff = btrfs_token_item_offset(&token, item); + btrfs_set_token_item_offset(&token, item, + ioff - total_data); } /* shift the items */ memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot + nr), @@ -4739,10 +4734,9 @@ void setup_items_for_insert(struct btrfs_root *root, struct btrfs_path *path, btrfs_cpu_key_to_disk(&disk_key, cpu_key + i); btrfs_set_item_key(leaf, &disk_key, slot + i); item = btrfs_item_nr(slot + i); - btrfs_set_token_item_offset(leaf, item, - data_end - data_size[i], &token); + btrfs_set_token_item_offset(&token, item, data_end - data_size[i]); data_end -= data_size[i]; - btrfs_set_token_item_size(leaf, item, data_size[i], &token); + btrfs_set_token_item_size(&token, item, data_size[i]); } btrfs_set_header_nritems(leaf, nritems + nr); @@ -4930,9 +4924,8 @@ int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, u32 ioff; item = btrfs_item_nr(i); - ioff = btrfs_token_item_offset(leaf, item, &token); - btrfs_set_token_item_offset(leaf, item, - ioff + dsize, &token); + ioff = btrfs_token_item_offset(&token, item); + btrfs_set_token_item_offset(&token, item, ioff + dsize); } memmove_extent_buffer(leaf, btrfs_item_nr_offset(slot), diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 6e226c371972..dcb82d690e76 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -1340,7 +1340,7 @@ do { \ BTRFS_INODE_ROOT_ITEM_INIT) struct btrfs_map_token { - const struct extent_buffer *eb; + struct extent_buffer *eb; char *kaddr; unsigned long offset; }; @@ -1376,12 +1376,11 @@ static inline void btrfs_init_map_token(struct btrfs_map_token *token, sizeof(((type *)0)->member))) #define DECLARE_BTRFS_SETGET_BITS(bits) \ -u##bits btrfs_get_token_##bits(const struct extent_buffer *eb, \ - const void *ptr, unsigned long off, \ - struct btrfs_map_token *token); \ -void btrfs_set_token_##bits(struct extent_buffer *eb, const void *ptr, \ - unsigned long off, u##bits val, \ - struct btrfs_map_token *token); \ +u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ + const void *ptr, unsigned long off); \ +void btrfs_set_token_##bits(struct btrfs_map_token *token, \ + const void *ptr, unsigned long off, \ + u##bits val); \ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ const void *ptr, unsigned long off); \ void btrfs_set_##bits(struct extent_buffer *eb, void *ptr, \ @@ -1405,19 +1404,17 @@ static inline void btrfs_set_##name(struct extent_buffer *eb, type *s, \ BUILD_BUG_ON(sizeof(u##bits) != sizeof(((type *)0))->member); \ btrfs_set_##bits(eb, s, offsetof(type, member), val); \ } \ -static inline u##bits btrfs_token_##name(const struct extent_buffer *eb,\ - const type *s, \ - struct btrfs_map_token *token) \ +static inline u##bits btrfs_token_##name(struct btrfs_map_token *token, \ + const type *s) \ { \ BUILD_BUG_ON(sizeof(u##bits) != sizeof(((type *)0))->member); \ - return btrfs_get_token_##bits(eb, s, offsetof(type, member), token); \ + return btrfs_get_token_##bits(token, s, offsetof(type, member));\ } \ -static inline void btrfs_set_token_##name(struct extent_buffer *eb, \ - type *s, u##bits val, \ - struct btrfs_map_token *token) \ +static inline void btrfs_set_token_##name(struct btrfs_map_token *token,\ + type *s, u##bits val) \ { \ BUILD_BUG_ON(sizeof(u##bits) != sizeof(((type *)0))->member); \ - btrfs_set_token_##bits(eb, s, offsetof(type, member), val, token); \ + btrfs_set_token_##bits(token, s, offsetof(type, member), val); \ } #define BTRFS_SETGET_HEADER_FUNCS(name, type, member, bits) \ diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 7b854f711ce1..3ea694ee1c90 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -3352,43 +3352,40 @@ static void fill_inode_item(struct btrfs_trans_handle *trans, btrfs_init_map_token(&token, leaf); - btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token); - btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token); - btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size, - &token); - btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token); - btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token); + btrfs_set_token_inode_uid(&token, item, i_uid_read(inode)); + btrfs_set_token_inode_gid(&token, item, i_gid_read(inode)); + btrfs_set_token_inode_size(&token, item, BTRFS_I(inode)->disk_i_size); + btrfs_set_token_inode_mode(&token, item, inode->i_mode); + btrfs_set_token_inode_nlink(&token, item, inode->i_nlink); - btrfs_set_token_timespec_sec(leaf, &item->atime, - inode->i_atime.tv_sec, &token); - btrfs_set_token_timespec_nsec(leaf, &item->atime, - inode->i_atime.tv_nsec, &token); + btrfs_set_token_timespec_sec(&token, &item->atime, + inode->i_atime.tv_sec); + btrfs_set_token_timespec_nsec(&token, &item->atime, + inode->i_atime.tv_nsec); - btrfs_set_token_timespec_sec(leaf, &item->mtime, - inode->i_mtime.tv_sec, &token); - btrfs_set_token_timespec_nsec(leaf, &item->mtime, - inode->i_mtime.tv_nsec, &token); + btrfs_set_token_timespec_sec(&token, &item->mtime, + inode->i_mtime.tv_sec); + btrfs_set_token_timespec_nsec(&token, &item->mtime, + inode->i_mtime.tv_nsec); - btrfs_set_token_timespec_sec(leaf, &item->ctime, - inode->i_ctime.tv_sec, &token); - btrfs_set_token_timespec_nsec(leaf, &item->ctime, - inode->i_ctime.tv_nsec, &token); + btrfs_set_token_timespec_sec(&token, &item->ctime, + inode->i_ctime.tv_sec); + btrfs_set_token_timespec_nsec(&token, &item->ctime, + inode->i_ctime.tv_nsec); - btrfs_set_token_timespec_sec(leaf, &item->otime, - BTRFS_I(inode)->i_otime.tv_sec, &token); - btrfs_set_token_timespec_nsec(leaf, &item->otime, - BTRFS_I(inode)->i_otime.tv_nsec, &token); + btrfs_set_token_timespec_sec(&token, &item->otime, + BTRFS_I(inode)->i_otime.tv_sec); + btrfs_set_token_timespec_nsec(&token, &item->otime, + BTRFS_I(inode)->i_otime.tv_nsec); - btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode), - &token); - btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation, - &token); - btrfs_set_token_inode_sequence(leaf, item, inode_peek_iversion(inode), - &token); - btrfs_set_token_inode_transid(leaf, item, trans->transid, &token); - btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token); - btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token); - btrfs_set_token_inode_block_group(leaf, item, 0, &token); + btrfs_set_token_inode_nbytes(&token, item, inode_get_bytes(inode)); + btrfs_set_token_inode_generation(&token, item, + BTRFS_I(inode)->generation); + btrfs_set_token_inode_sequence(&token, item, inode_peek_iversion(inode)); + btrfs_set_token_inode_transid(&token, item, trans->transid); + btrfs_set_token_inode_rdev(&token, item, inode->i_rdev); + btrfs_set_token_inode_flags(&token, item, BTRFS_I(inode)->flags); + btrfs_set_token_inode_block_group(&token, item, 0); } /* diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 7928d310f698..cebd0b5e4f37 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -38,9 +38,8 @@ static inline void put_unaligned_le8(u8 val, void *p) */ #define DEFINE_BTRFS_SETGET_BITS(bits) \ -u##bits btrfs_get_token_##bits(const struct extent_buffer *eb, \ - const void *ptr, unsigned long off, \ - struct btrfs_map_token *token) \ +u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ + const void *ptr, unsigned long off) \ { \ unsigned long part_offset = (unsigned long)ptr; \ unsigned long offset = part_offset + off; \ @@ -53,7 +52,6 @@ u##bits btrfs_get_token_##bits(const struct extent_buffer *eb, \ u##bits res; \ \ ASSERT(token); \ - ASSERT(token->eb == eb); \ \ if (token->kaddr && token->offset <= offset && \ (token->offset + PAGE_SIZE >= offset + size)) { \ @@ -101,10 +99,9 @@ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ res = get_unaligned_le##bits(p + off); \ return res; \ } \ -void btrfs_set_token_##bits(struct extent_buffer *eb, \ +void btrfs_set_token_##bits(struct btrfs_map_token *token, \ const void *ptr, unsigned long off, \ - u##bits val, \ - struct btrfs_map_token *token) \ + u##bits val) \ { \ unsigned long part_offset = (unsigned long)ptr; \ unsigned long offset = part_offset + off; \ @@ -116,7 +113,6 @@ void btrfs_set_token_##bits(struct extent_buffer *eb, \ int size = sizeof(u##bits); \ \ ASSERT(token); \ - ASSERT(token->eb == eb); \ \ if (token->kaddr && token->offset <= offset && \ (token->offset + PAGE_SIZE >= offset + size)) { \ diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 39ec25518ec2..ee1c627bd618 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -509,8 +509,8 @@ insert: struct btrfs_map_token token; btrfs_init_map_token(&token, dst_eb); - btrfs_set_token_inode_size(dst_eb, dst_item, - ino_size, &token); + btrfs_set_token_inode_size(&token, dst_item, + ino_size); } goto no_copy; } @@ -3852,44 +3852,41 @@ static void fill_inode_item(struct btrfs_trans_handle *trans, * just to say 'this inode exists' and a logging * to say 'update this inode with these values' */ - btrfs_set_token_inode_generation(leaf, item, 0, &token); - btrfs_set_token_inode_size(leaf, item, logged_isize, &token); + btrfs_set_token_inode_generation(&token, item, 0); + btrfs_set_token_inode_size(&token, item, logged_isize); } else { - btrfs_set_token_inode_generation(leaf, item, - BTRFS_I(inode)->generation, - &token); - btrfs_set_token_inode_size(leaf, item, inode->i_size, &token); + btrfs_set_token_inode_generation(&token, item, + BTRFS_I(inode)->generation); + btrfs_set_token_inode_size(&token, item, inode->i_size); } - btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token); - btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token); - btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token); - btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token); + btrfs_set_token_inode_uid(&token, item, i_uid_read(inode)); + btrfs_set_token_inode_gid(&token, item, i_gid_read(inode)); + btrfs_set_token_inode_mode(&token, item, inode->i_mode); + btrfs_set_token_inode_nlink(&token, item, inode->i_nlink); - btrfs_set_token_timespec_sec(leaf, &item->atime, - inode->i_atime.tv_sec, &token); - btrfs_set_token_timespec_nsec(leaf, &item->atime, - inode->i_atime.tv_nsec, &token); + btrfs_set_token_timespec_sec(&token, &item->atime, + inode->i_atime.tv_sec); + btrfs_set_token_timespec_nsec(&token, &item->atime, + inode->i_atime.tv_nsec); - btrfs_set_token_timespec_sec(leaf, &item->mtime, - inode->i_mtime.tv_sec, &token); - btrfs_set_token_timespec_nsec(leaf, &item->mtime, - inode->i_mtime.tv_nsec, &token); + btrfs_set_token_timespec_sec(&token, &item->mtime, + inode->i_mtime.tv_sec); + btrfs_set_token_timespec_nsec(&token, &item->mtime, + inode->i_mtime.tv_nsec); - btrfs_set_token_timespec_sec(leaf, &item->ctime, - inode->i_ctime.tv_sec, &token); - btrfs_set_token_timespec_nsec(leaf, &item->ctime, - inode->i_ctime.tv_nsec, &token); + btrfs_set_token_timespec_sec(&token, &item->ctime, + inode->i_ctime.tv_sec); + btrfs_set_token_timespec_nsec(&token, &item->ctime, + inode->i_ctime.tv_nsec); - btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode), - &token); + btrfs_set_token_inode_nbytes(&token, item, inode_get_bytes(inode)); - btrfs_set_token_inode_sequence(leaf, item, - inode_peek_iversion(inode), &token); - btrfs_set_token_inode_transid(leaf, item, trans->transid, &token); - btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token); - btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token); - btrfs_set_token_inode_block_group(leaf, item, 0, &token); + btrfs_set_token_inode_sequence(&token, item, inode_peek_iversion(inode)); + btrfs_set_token_inode_transid(&token, item, trans->transid); + btrfs_set_token_inode_rdev(&token, item, inode->i_rdev); + btrfs_set_token_inode_flags(&token, item, BTRFS_I(inode)->flags); + btrfs_set_token_inode_block_group(&token, item, 0); } static int log_inode_item(struct btrfs_trans_handle *trans, @@ -4163,43 +4160,35 @@ static int log_one_extent(struct btrfs_trans_handle *trans, fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); - btrfs_set_token_file_extent_generation(leaf, fi, trans->transid, - &token); + btrfs_set_token_file_extent_generation(&token, fi, trans->transid); if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) - btrfs_set_token_file_extent_type(leaf, fi, - BTRFS_FILE_EXTENT_PREALLOC, - &token); + btrfs_set_token_file_extent_type(&token, fi, + BTRFS_FILE_EXTENT_PREALLOC); else - btrfs_set_token_file_extent_type(leaf, fi, - BTRFS_FILE_EXTENT_REG, - &token); + btrfs_set_token_file_extent_type(&token, fi, + BTRFS_FILE_EXTENT_REG); block_len = max(em->block_len, em->orig_block_len); if (em->compress_type != BTRFS_COMPRESS_NONE) { - btrfs_set_token_file_extent_disk_bytenr(leaf, fi, - em->block_start, - &token); - btrfs_set_token_file_extent_disk_num_bytes(leaf, fi, block_len, - &token); + btrfs_set_token_file_extent_disk_bytenr(&token, fi, + em->block_start); + btrfs_set_token_file_extent_disk_num_bytes(&token, fi, block_len); } else if (em->block_start < EXTENT_MAP_LAST_BYTE) { - btrfs_set_token_file_extent_disk_bytenr(leaf, fi, + btrfs_set_token_file_extent_disk_bytenr(&token, fi, em->block_start - - extent_offset, &token); - btrfs_set_token_file_extent_disk_num_bytes(leaf, fi, block_len, - &token); + extent_offset); + btrfs_set_token_file_extent_disk_num_bytes(&token, fi, block_len); } else { - btrfs_set_token_file_extent_disk_bytenr(leaf, fi, 0, &token); - btrfs_set_token_file_extent_disk_num_bytes(leaf, fi, 0, - &token); + btrfs_set_token_file_extent_disk_bytenr(&token, fi, 0); + btrfs_set_token_file_extent_disk_num_bytes(&token, fi, 0); } - btrfs_set_token_file_extent_offset(leaf, fi, extent_offset, &token); - btrfs_set_token_file_extent_num_bytes(leaf, fi, em->len, &token); - btrfs_set_token_file_extent_ram_bytes(leaf, fi, em->ram_bytes, &token); - btrfs_set_token_file_extent_compression(leaf, fi, em->compress_type, - &token); - btrfs_set_token_file_extent_encryption(leaf, fi, 0, &token); - btrfs_set_token_file_extent_other_encoding(leaf, fi, 0, &token); + btrfs_set_token_file_extent_offset(&token, fi, extent_offset); + btrfs_set_token_file_extent_num_bytes(&token, fi, em->len); + btrfs_set_token_file_extent_ram_bytes(&token, fi, em->ram_bytes); + btrfs_set_token_file_extent_compression(&token, fi, em->compress_type); + btrfs_set_token_file_extent_encryption(&token, fi, 0); + btrfs_set_token_file_extent_other_encoding(&token, fi, 0); btrfs_mark_buffer_dirty(leaf); btrfs_release_path(path); From 60d48e2e4575dd848a43ceeda6cbc38a87427330 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 15:29:53 +0200 Subject: [PATCH 0771/1043] btrfs: don't use set/get token for single assignment in overwrite_item The set/get token is supposed to cache the last page that was accessed so it speeds up subsequential access to the eb. It does not make sense to use that for just one change, which is the case of inode size in overwrite_item. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/tree-log.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index ee1c627bd618..60febf2082ee 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -505,13 +505,8 @@ insert: */ if (S_ISREG(btrfs_inode_mode(eb, src_item)) && S_ISREG(btrfs_inode_mode(dst_eb, dst_item)) && - ino_size != 0) { - struct btrfs_map_token token; - - btrfs_init_map_token(&token, dst_eb); - btrfs_set_token_inode_size(&token, dst_item, - ino_size); - } + ino_size != 0) + btrfs_set_inode_size(dst_eb, dst_item, ino_size); goto no_copy; } From a31356b9e263b723d4991383efc87b71c6e87991 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 22:56:01 +0200 Subject: [PATCH 0772/1043] btrfs: don't use set/get token in leaf_space_used The token is supposed to cache the last page used by the set/get helpers. In leaf_space_used the first and last items are accessed, it's not likely they'd be on the same page so there's some overhead caused updating the token address but not using it. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 576111cdea1d..6dbeb23c59ec 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -3507,19 +3507,17 @@ static int leaf_space_used(struct extent_buffer *l, int start, int nr) { struct btrfs_item *start_item; struct btrfs_item *end_item; - struct btrfs_map_token token; int data_len; int nritems = btrfs_header_nritems(l); int end = min(nritems, start + nr) - 1; if (!nr) return 0; - btrfs_init_map_token(&token, l); start_item = btrfs_item_nr(start); end_item = btrfs_item_nr(end); - data_len = btrfs_token_item_offset(&token, start_item) + - btrfs_token_item_size(&token, start_item); - data_len = data_len - btrfs_token_item_offset(&token, end_item); + data_len = btrfs_item_offset(l, start_item) + + btrfs_item_size(l, start_item); + data_len = data_len - btrfs_item_offset(l, end_item); data_len += sizeof(struct btrfs_item) * nr; WARN_ON(data_len < 0); return data_len; From 870b388db02cac33eebe40a1cbeb056cf13e9f40 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 19:29:04 +0200 Subject: [PATCH 0773/1043] btrfs: preset set/get token with first page and drop condition All the set/get helpers first check if the token contains a cached address. After first use the address is always valid, but the extra check is done for each call. The token initialization can optimistically set it to the first extent buffer page, that we know always exists. Then the condition in all btrfs_token_*/btrfs_set_token_* can be simplified by removing the address check from the condition, but for development the assertion still makes sure it's valid. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 3 ++- fs/btrfs/struct-funcs.c | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index dcb82d690e76..557de0dc904a 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -1352,7 +1352,8 @@ static inline void btrfs_init_map_token(struct btrfs_map_token *token, struct extent_buffer *eb) { token->eb = eb; - token->kaddr = NULL; + token->kaddr = page_address(eb->pages[0]); + token->offset = 0; } /* some macros to generate set/get functions for the struct fields. This diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index cebd0b5e4f37..cef628a5a9e0 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -52,8 +52,8 @@ u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ u##bits res; \ \ ASSERT(token); \ - \ - if (token->kaddr && token->offset <= offset && \ + ASSERT(token->kaddr); \ + if (token->offset <= offset && \ (token->offset + PAGE_SIZE >= offset + size)) { \ kaddr = token->kaddr; \ p = kaddr + part_offset - token->offset; \ @@ -113,8 +113,8 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ int size = sizeof(u##bits); \ \ ASSERT(token); \ - \ - if (token->kaddr && token->offset <= offset && \ + ASSERT(token->kaddr); \ + if (token->offset <= offset && \ (token->offset + PAGE_SIZE >= offset + size)) { \ kaddr = token->kaddr; \ p = kaddr + part_offset - token->offset; \ From 5e3946890c184a17b4d74baec89c96dce518f002 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 30 Apr 2020 23:38:11 +0200 Subject: [PATCH 0774/1043] btrfs: add separate bounds checker for set/get helpers The bounds checking is now done in map_private_extent_buffer but that will be removed in following patches and some sanity checks should still be done. There are two separate checks to see the kind of out of bounds access: partial (start offset is in the buffer) or complete (both start and end are out). Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index cef628a5a9e0..68c02997e60d 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -17,6 +17,27 @@ static inline void put_unaligned_le8(u8 val, void *p) *(u8 *)p = val; } +static bool check_setget_bounds(const struct extent_buffer *eb, + const void *ptr, unsigned off, int size) +{ + const unsigned long member_offset = (unsigned long)ptr + off; + + if (member_offset > eb->len) { + btrfs_warn(eb->fs_info, + "bad eb member start: ptr 0x%lx start %llu member offset %lu size %d", + (unsigned long)ptr, eb->start, member_offset, size); + return false; + } + if (member_offset + size > eb->len) { + btrfs_warn(eb->fs_info, + "bad eb member end: ptr 0x%lx start %llu member offset %lu size %d", + (unsigned long)ptr, eb->start, member_offset, size); + return false; + } + + return true; +} + /* * this is some deeply nasty code. * @@ -53,6 +74,7 @@ u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ \ ASSERT(token); \ ASSERT(token->kaddr); \ + ASSERT(check_setget_bounds(token->eb, ptr, off, size)); \ if (token->offset <= offset && \ (token->offset + PAGE_SIZE >= offset + size)) { \ kaddr = token->kaddr; \ @@ -87,6 +109,7 @@ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ int size = sizeof(u##bits); \ u##bits res; \ \ + ASSERT(check_setget_bounds(eb, ptr, off, size)); \ err = map_private_extent_buffer(eb, offset, size, \ &kaddr, &map_start, &map_len); \ if (err) { \ @@ -114,6 +137,7 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ \ ASSERT(token); \ ASSERT(token->kaddr); \ + ASSERT(check_setget_bounds(token->eb, ptr, off, size)); \ if (token->offset <= offset && \ (token->offset + PAGE_SIZE >= offset + size)) { \ kaddr = token->kaddr; \ @@ -147,6 +171,7 @@ void btrfs_set_##bits(struct extent_buffer *eb, void *ptr, \ unsigned long map_len; \ int size = sizeof(u##bits); \ \ + ASSERT(check_setget_bounds(eb, ptr, off, size)); \ err = map_private_extent_buffer(eb, offset, size, \ &kaddr, &map_start, &map_len); \ if (err) { \ From 1441ed9b7a0f53f47afd0b5ccdcb447e50559165 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 16:04:44 +0200 Subject: [PATCH 0775/1043] btrfs: speed up btrfs_get_##bits helpers The helpers unconditionally call map_private_extent_buffer to get the address of page containing the requested offset plus the mapping start and length. Depending on the return value, the fast path uses unaligned read to get data within a page, or fall back to read_extent_buffer that can handle reads spanning more pages. This is all wasteful. We know the number of bytes to read, 1/2/4/8 and can find out the page. Then simply check if it's contained or the fallback is needed. This saves one function call to map_private_extent_buffer and several unnecessary temporary variables. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 68c02997e60d..e6d2bd019444 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -99,28 +99,19 @@ u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ const void *ptr, unsigned long off) \ { \ - unsigned long part_offset = (unsigned long)ptr; \ - unsigned long offset = part_offset + off; \ - void *p; \ - int err; \ - char *kaddr; \ - unsigned long map_start; \ - unsigned long map_len; \ - int size = sizeof(u##bits); \ - u##bits res; \ + const unsigned long member_offset = (unsigned long)ptr + off; \ + const unsigned long oip = offset_in_page(member_offset); \ + const int size = sizeof(u##bits); \ + __le##bits leres; \ \ ASSERT(check_setget_bounds(eb, ptr, off, size)); \ - err = map_private_extent_buffer(eb, offset, size, \ - &kaddr, &map_start, &map_len); \ - if (err) { \ - __le##bits leres; \ - \ - read_extent_buffer(eb, &leres, offset, size); \ - return le##bits##_to_cpu(leres); \ + if (oip + size <= PAGE_SIZE) { \ + const unsigned long idx = member_offset >> PAGE_SHIFT; \ + const char *kaddr = page_address(eb->pages[idx]); \ + return get_unaligned_le##bits(kaddr + oip); \ } \ - p = kaddr + part_offset - map_start; \ - res = get_unaligned_le##bits(p + off); \ - return res; \ + read_extent_buffer(eb, &leres, member_offset, size); \ + return le##bits##_to_cpu(leres); \ } \ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ const void *ptr, unsigned long off, \ From 8f9da810ee1fac6b587e3b871ee73696906bcba3 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 17:45:33 +0200 Subject: [PATCH 0776/1043] btrfs: speed up btrfs_get_token_##bits helpers The set/get token helpers either use the cached address in the token or unconditionally call map_private_extent_buffer to get the address of page containing the requested offset plus the mapping start and length. Depending on the return value, the fast path uses unaligned read to get data within a page, or fall back to read_extent_buffer that can handle reads spanning more pages. This is all wasteful. We know the number of bytes to read, 1/2/4/8 and can find out the page. Then simply check if it's contained or the fallback is needed. The token address is updated to the page, or the on the next index, expecting that the next read will use that. This saves one function call to map_private_extent_buffer and several unnecessary temporary variables. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 43 +++++++++++++++-------------------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index e6d2bd019444..e357e0bab397 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -62,39 +62,28 @@ static bool check_setget_bounds(const struct extent_buffer *eb, u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ const void *ptr, unsigned long off) \ { \ - unsigned long part_offset = (unsigned long)ptr; \ - unsigned long offset = part_offset + off; \ - void *p; \ - int err; \ - char *kaddr; \ - unsigned long map_start; \ - unsigned long map_len; \ - int size = sizeof(u##bits); \ - u##bits res; \ + const unsigned long member_offset = (unsigned long)ptr + off; \ + const unsigned long idx = member_offset >> PAGE_SHIFT; \ + const unsigned long oip = offset_in_page(member_offset); \ + const int size = sizeof(u##bits); \ + __le##bits leres; \ \ ASSERT(token); \ ASSERT(token->kaddr); \ ASSERT(check_setget_bounds(token->eb, ptr, off, size)); \ - if (token->offset <= offset && \ - (token->offset + PAGE_SIZE >= offset + size)) { \ - kaddr = token->kaddr; \ - p = kaddr + part_offset - token->offset; \ - res = get_unaligned_le##bits(p + off); \ - return res; \ + if (token->offset <= member_offset && \ + member_offset + size <= token->offset + PAGE_SIZE) { \ + return get_unaligned_le##bits(token->kaddr + oip); \ } \ - err = map_private_extent_buffer(token->eb, offset, size, \ - &kaddr, &map_start, &map_len); \ - if (err) { \ - __le##bits leres; \ - \ - read_extent_buffer(token->eb, &leres, offset, size); \ - return le##bits##_to_cpu(leres); \ + if (oip + size <= PAGE_SIZE) { \ + token->kaddr = page_address(token->eb->pages[idx]); \ + token->offset = idx << PAGE_SHIFT; \ + return get_unaligned_le##bits(token->kaddr + oip); \ } \ - p = kaddr + part_offset - map_start; \ - res = get_unaligned_le##bits(p + off); \ - token->kaddr = kaddr; \ - token->offset = map_start; \ - return res; \ + token->kaddr = page_address(token->eb->pages[idx + 1]); \ + token->offset = (idx + 1) << PAGE_SHIFT; \ + read_extent_buffer(token->eb, &leres, member_offset, size); \ + return le##bits##_to_cpu(leres); \ } \ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ const void *ptr, unsigned long off) \ From 029e4a42a2b22f3c82e5b325820219963aa3f13b Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 18:07:04 +0200 Subject: [PATCH 0777/1043] btrfs: speed up btrfs_set_##bits helpers The helpers unconditionally call map_private_extent_buffer to get the address of page containing the requested offset plus the mapping start and length. Depending on the return value, the fast path uses unaligned put to write data within a page, or fall back to write_extent_buffer that can handle writes spanning more pages. This is all wasteful. We know the number of bytes to write, 1/2/4/8 and can find out the page. Then simply check if it's contained or the fallback is needed. This saves one function call to map_private_extent_buffer and several unnecessary temporary variables. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index e357e0bab397..f8a0357d10fd 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -142,27 +142,20 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ void btrfs_set_##bits(struct extent_buffer *eb, void *ptr, \ unsigned long off, u##bits val) \ { \ - unsigned long part_offset = (unsigned long)ptr; \ - unsigned long offset = part_offset + off; \ - void *p; \ - int err; \ - char *kaddr; \ - unsigned long map_start; \ - unsigned long map_len; \ - int size = sizeof(u##bits); \ + const unsigned long member_offset = (unsigned long)ptr + off; \ + const unsigned long oip = offset_in_page(member_offset); \ + const int size = sizeof(u##bits); \ + __le##bits leres; \ \ ASSERT(check_setget_bounds(eb, ptr, off, size)); \ - err = map_private_extent_buffer(eb, offset, size, \ - &kaddr, &map_start, &map_len); \ - if (err) { \ - __le##bits val2; \ - \ - val2 = cpu_to_le##bits(val); \ - write_extent_buffer(eb, &val2, offset, size); \ + if (oip + size <= PAGE_SIZE) { \ + const unsigned long idx = member_offset >> PAGE_SHIFT; \ + char *kaddr = page_address(eb->pages[idx]); \ + put_unaligned_le##bits(val, kaddr + oip); \ return; \ } \ - p = kaddr + part_offset - map_start; \ - put_unaligned_le##bits(val, p + off); \ + leres = cpu_to_le##bits(val); \ + write_extent_buffer(eb, &leres, member_offset, size); \ } DEFINE_BTRFS_SETGET_BITS(8) From ce7afe8782a5293edbf4bc2165ac4a866ed430db Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 18:23:37 +0200 Subject: [PATCH 0778/1043] btrfs: speed up btrfs_set_token_##bits helpers The set/get token helpers either use the cached address in the token or unconditionally call map_private_extent_buffer to get the address of page containing the requested offset plus the mapping start and length. Depending on the return value, the fast path uses unaligned put to write data within a page, or fall back to write_extent_buffer that can handle writes spanning more pages. This is all wasteful. We know the number of bytes to write, 1/2/4/8 and can find out the page. Then simply check if it's contained or the fallback is needed. The token address is updated to the page, or the on the next index, expecting that the next write will use that. This saves one function call to map_private_extent_buffer and several unnecessary temporary variables. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index f8a0357d10fd..67dfd1287c3e 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -106,38 +106,30 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ const void *ptr, unsigned long off, \ u##bits val) \ { \ - unsigned long part_offset = (unsigned long)ptr; \ - unsigned long offset = part_offset + off; \ - void *p; \ - int err; \ - char *kaddr; \ - unsigned long map_start; \ - unsigned long map_len; \ - int size = sizeof(u##bits); \ + const unsigned long member_offset = (unsigned long)ptr + off; \ + const unsigned long idx = member_offset >> PAGE_SHIFT; \ + const unsigned long oip = offset_in_page(member_offset); \ + const int size = sizeof(u##bits); \ + __le##bits leres; \ \ ASSERT(token); \ ASSERT(token->kaddr); \ ASSERT(check_setget_bounds(token->eb, ptr, off, size)); \ - if (token->offset <= offset && \ - (token->offset + PAGE_SIZE >= offset + size)) { \ - kaddr = token->kaddr; \ - p = kaddr + part_offset - token->offset; \ - put_unaligned_le##bits(val, p + off); \ + if (token->offset <= member_offset && \ + member_offset + size <= token->offset + PAGE_SIZE) { \ + put_unaligned_le##bits(val, token->kaddr + oip); \ return; \ } \ - err = map_private_extent_buffer(token->eb, offset, size, \ - &kaddr, &map_start, &map_len); \ - if (err) { \ - __le##bits val2; \ - \ - val2 = cpu_to_le##bits(val); \ - write_extent_buffer(token->eb, &val2, offset, size); \ + if (oip + size <= PAGE_SIZE) { \ + token->kaddr = page_address(token->eb->pages[idx]); \ + token->offset = idx << PAGE_SHIFT; \ + put_unaligned_le##bits(val, token->kaddr + oip); \ return; \ } \ - p = kaddr + part_offset - map_start; \ - put_unaligned_le##bits(val, p + off); \ - token->kaddr = kaddr; \ - token->offset = map_start; \ + token->kaddr = page_address(token->eb->pages[idx + 1]); \ + token->offset = (idx + 1) << PAGE_SHIFT; \ + leres = cpu_to_le##bits(val); \ + write_extent_buffer(token->eb, &leres, member_offset, size); \ } \ void btrfs_set_##bits(struct extent_buffer *eb, void *ptr, \ unsigned long off, u##bits val) \ From 5cd17f343bd1c47dc673260fa2973abc14ecc549 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 23:23:37 +0200 Subject: [PATCH 0779/1043] btrfs: speed up and simplify generic_bin_search The bin search jumps over the extent buffer item keys, comparing directly the bytes if the key is in one page, or storing it in a temporary buffer in case it spans two pages. The mapping start and length are obtained from map_private_extent_buffer, which is heavy weight compared to what we need. We know the key size and can find out the eb page in a simple way. For keys spanning two pages the fallback read_extent_buffer is used. The temporary variables are reduced and moved to the scope of use. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 43 +++++++++++++++---------------------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 6dbeb23c59ec..746dec22f250 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -1668,15 +1668,8 @@ static noinline int generic_bin_search(struct extent_buffer *eb, { int low = 0; int high = max; - int mid; int ret; - struct btrfs_disk_key *tmp = NULL; - struct btrfs_disk_key unaligned; - unsigned long offset; - char *kaddr = NULL; - unsigned long map_start = 0; - unsigned long map_len = 0; - int err; + const int key_size = sizeof(struct btrfs_disk_key); if (low > high) { btrfs_err(eb->fs_info, @@ -1687,32 +1680,26 @@ static noinline int generic_bin_search(struct extent_buffer *eb, } while (low < high) { + unsigned long oip; + unsigned long offset; + struct btrfs_disk_key *tmp; + struct btrfs_disk_key unaligned; + int mid; + mid = (low + high) / 2; offset = p + mid * item_size; + oip = offset_in_page(offset); - if (!kaddr || offset < map_start || - (offset + sizeof(struct btrfs_disk_key)) > - map_start + map_len) { - - err = map_private_extent_buffer(eb, offset, - sizeof(struct btrfs_disk_key), - &kaddr, &map_start, &map_len); - - if (!err) { - tmp = (struct btrfs_disk_key *)(kaddr + offset - - map_start); - } else if (err == 1) { - read_extent_buffer(eb, &unaligned, - offset, sizeof(unaligned)); - tmp = &unaligned; - } else { - return err; - } + if (oip + key_size <= PAGE_SIZE) { + const unsigned long idx = offset >> PAGE_SHIFT; + char *kaddr = page_address(eb->pages[idx]); + tmp = (struct btrfs_disk_key *)(kaddr + oip); } else { - tmp = (struct btrfs_disk_key *)(kaddr + offset - - map_start); + read_extent_buffer(eb, &unaligned, offset, key_size); + tmp = &unaligned; } + ret = comp_keys(tmp, key); if (ret < 0) From db3756c879773c4c7986fce3dac8355f210df807 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 23:36:03 +0200 Subject: [PATCH 0780/1043] btrfs: remove unused map_private_extent_buffer All uses of map_private_extent_buffer have been replaced by more effective way. The set/get helpers have their own bounds checker. The function name was confusing since the non-private helper was removed in a65917156e34 ("Btrfs: stop using highmem for extent_buffers") many years ago. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 42 ------------------------------------------ fs/btrfs/extent_io.h | 4 ---- 2 files changed, 46 deletions(-) diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 22db0b234ffe..67ee46a0a9c8 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -5685,48 +5685,6 @@ int read_extent_buffer_to_user(const struct extent_buffer *eb, return ret; } -/* - * return 0 if the item is found within a page. - * return 1 if the item spans two pages. - * return -EINVAL otherwise. - */ -int map_private_extent_buffer(const struct extent_buffer *eb, - unsigned long start, unsigned long min_len, - char **map, unsigned long *map_start, - unsigned long *map_len) -{ - size_t offset; - char *kaddr; - struct page *p; - size_t start_offset = offset_in_page(eb->start); - unsigned long i = (start_offset + start) >> PAGE_SHIFT; - unsigned long end_i = (start_offset + start + min_len - 1) >> - PAGE_SHIFT; - - if (start + min_len > eb->len) { - WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, wanted %lu %lu\n", - eb->start, eb->len, start, min_len); - return -EINVAL; - } - - if (i != end_i) - return 1; - - if (i == 0) { - offset = start_offset; - *map_start = 0; - } else { - offset = 0; - *map_start = ((u64)i << PAGE_SHIFT) - start_offset; - } - - p = eb->pages[i]; - kaddr = page_address(p); - *map = kaddr + offset; - *map_len = PAGE_SIZE - offset; - return 0; -} - int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv, unsigned long start, unsigned long len) { diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index a2842b2d9a98..9ed89c01e2da 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -271,10 +271,6 @@ bool set_extent_buffer_dirty(struct extent_buffer *eb); void set_extent_buffer_uptodate(struct extent_buffer *eb); void clear_extent_buffer_uptodate(struct extent_buffer *eb); int extent_buffer_under_io(struct extent_buffer *eb); -int map_private_extent_buffer(const struct extent_buffer *eb, - unsigned long offset, unsigned long min_len, - char **map, unsigned long *map_start, - unsigned long *map_len); void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end); void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end); void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end, From 2b48966a4da4bcb35f0883bc23dcaf63fcb8557f Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 03:04:10 +0200 Subject: [PATCH 0781/1043] btrfs: constify extent_buffer in the API functions There are many helpers around extent buffers, found in extent_io.h and ctree.h. Most of them can be converted to take constified eb as there are no changes to the extent buffer structure itself but rather the pages. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 28 +++++++++++++------------- fs/btrfs/extent_io.c | 44 ++++++++++++++++++++++------------------- fs/btrfs/extent_io.h | 44 ++++++++++++++++++++++------------------- fs/btrfs/struct-funcs.c | 2 +- 4 files changed, 63 insertions(+), 55 deletions(-) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 557de0dc904a..0a1fa1526c43 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -1384,7 +1384,7 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ u##bits val); \ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ const void *ptr, unsigned long off); \ -void btrfs_set_##bits(struct extent_buffer *eb, void *ptr, \ +void btrfs_set_##bits(const struct extent_buffer *eb, void *ptr, \ unsigned long off, u##bits val); DECLARE_BTRFS_SETGET_BITS(8) @@ -1399,7 +1399,7 @@ static inline u##bits btrfs_##name(const struct extent_buffer *eb, \ BUILD_BUG_ON(sizeof(u##bits) != sizeof(((type *)0))->member); \ return btrfs_get_##bits(eb, s, offsetof(type, member)); \ } \ -static inline void btrfs_set_##name(struct extent_buffer *eb, type *s, \ +static inline void btrfs_set_##name(const struct extent_buffer *eb, type *s, \ u##bits val) \ { \ BUILD_BUG_ON(sizeof(u##bits) != sizeof(((type *)0))->member); \ @@ -1425,7 +1425,7 @@ static inline u##bits btrfs_##name(const struct extent_buffer *eb) \ u##bits res = le##bits##_to_cpu(p->member); \ return res; \ } \ -static inline void btrfs_set_##name(struct extent_buffer *eb, \ +static inline void btrfs_set_##name(const struct extent_buffer *eb, \ u##bits val) \ { \ type *p = page_address(eb->pages[0]); \ @@ -1443,7 +1443,7 @@ static inline void btrfs_set_##name(type *s, u##bits val) \ } -static inline u64 btrfs_device_total_bytes(struct extent_buffer *eb, +static inline u64 btrfs_device_total_bytes(const struct extent_buffer *eb, struct btrfs_dev_item *s) { BUILD_BUG_ON(sizeof(u64) != @@ -1451,7 +1451,7 @@ static inline u64 btrfs_device_total_bytes(struct extent_buffer *eb, return btrfs_get_64(eb, s, offsetof(struct btrfs_dev_item, total_bytes)); } -static inline void btrfs_set_device_total_bytes(struct extent_buffer *eb, +static inline void btrfs_set_device_total_bytes(const struct extent_buffer *eb, struct btrfs_dev_item *s, u64 val) { @@ -1555,13 +1555,13 @@ static inline char *btrfs_stripe_dev_uuid_nr(struct btrfs_chunk *c, int nr) return btrfs_stripe_dev_uuid(btrfs_stripe_nr(c, nr)); } -static inline u64 btrfs_stripe_offset_nr(struct extent_buffer *eb, +static inline u64 btrfs_stripe_offset_nr(const struct extent_buffer *eb, struct btrfs_chunk *c, int nr) { return btrfs_stripe_offset(eb, btrfs_stripe_nr(c, nr)); } -static inline u64 btrfs_stripe_devid_nr(struct extent_buffer *eb, +static inline u64 btrfs_stripe_devid_nr(const struct extent_buffer *eb, struct btrfs_chunk *c, int nr) { return btrfs_stripe_devid(eb, btrfs_stripe_nr(c, nr)); @@ -1648,14 +1648,14 @@ BTRFS_SETGET_FUNCS(extent_flags, struct btrfs_extent_item, flags, 64); BTRFS_SETGET_FUNCS(tree_block_level, struct btrfs_tree_block_info, level, 8); -static inline void btrfs_tree_block_key(struct extent_buffer *eb, +static inline void btrfs_tree_block_key(const struct extent_buffer *eb, struct btrfs_tree_block_info *item, struct btrfs_disk_key *key) { read_eb_member(eb, item, struct btrfs_tree_block_info, key, key); } -static inline void btrfs_set_tree_block_key(struct extent_buffer *eb, +static inline void btrfs_set_tree_block_key(const struct extent_buffer *eb, struct btrfs_tree_block_info *item, struct btrfs_disk_key *key) { @@ -1701,7 +1701,7 @@ BTRFS_SETGET_STACK_FUNCS(stack_key_blockptr, struct btrfs_key_ptr, BTRFS_SETGET_STACK_FUNCS(stack_key_generation, struct btrfs_key_ptr, generation, 64); -static inline u64 btrfs_node_blockptr(struct extent_buffer *eb, int nr) +static inline u64 btrfs_node_blockptr(const struct extent_buffer *eb, int nr) { unsigned long ptr; ptr = offsetof(struct btrfs_node, ptrs) + @@ -1709,7 +1709,7 @@ static inline u64 btrfs_node_blockptr(struct extent_buffer *eb, int nr) return btrfs_key_blockptr(eb, (struct btrfs_key_ptr *)ptr); } -static inline void btrfs_set_node_blockptr(struct extent_buffer *eb, +static inline void btrfs_set_node_blockptr(const struct extent_buffer *eb, int nr, u64 val) { unsigned long ptr; @@ -1718,7 +1718,7 @@ static inline void btrfs_set_node_blockptr(struct extent_buffer *eb, btrfs_set_key_blockptr(eb, (struct btrfs_key_ptr *)ptr, val); } -static inline u64 btrfs_node_ptr_generation(struct extent_buffer *eb, int nr) +static inline u64 btrfs_node_ptr_generation(const struct extent_buffer *eb, int nr) { unsigned long ptr; ptr = offsetof(struct btrfs_node, ptrs) + @@ -1726,7 +1726,7 @@ static inline u64 btrfs_node_ptr_generation(struct extent_buffer *eb, int nr) return btrfs_key_generation(eb, (struct btrfs_key_ptr *)ptr); } -static inline void btrfs_set_node_ptr_generation(struct extent_buffer *eb, +static inline void btrfs_set_node_ptr_generation(const struct extent_buffer *eb, int nr, u64 val) { unsigned long ptr; @@ -1744,7 +1744,7 @@ static inline unsigned long btrfs_node_key_ptr_offset(int nr) void btrfs_node_key(const struct extent_buffer *eb, struct btrfs_disk_key *disk_key, int nr); -static inline void btrfs_set_node_key(struct extent_buffer *eb, +static inline void btrfs_set_node_key(const struct extent_buffer *eb, struct btrfs_disk_key *disk_key, int nr) { unsigned long ptr; diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 67ee46a0a9c8..da6f0c1ed80c 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -2333,7 +2333,7 @@ int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, return 0; } -int btrfs_repair_eb_io_failure(struct extent_buffer *eb, int mirror_num) +int btrfs_repair_eb_io_failure(const struct extent_buffer *eb, int mirror_num) { struct btrfs_fs_info *fs_info = eb->fs_info; u64 start = eb->start; @@ -4910,7 +4910,7 @@ static void __free_extent_buffer(struct extent_buffer *eb) kmem_cache_free(extent_buffer_cache, eb); } -int extent_buffer_under_io(struct extent_buffer *eb) +int extent_buffer_under_io(const struct extent_buffer *eb) { return (atomic_read(&eb->io_pages) || test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) || @@ -5018,7 +5018,7 @@ __alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start, return eb; } -struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src) +struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src) { int i; struct page *p; @@ -5424,7 +5424,7 @@ void free_extent_buffer_stale(struct extent_buffer *eb) release_extent_buffer(eb); } -void clear_extent_buffer_dirty(struct extent_buffer *eb) +void clear_extent_buffer_dirty(const struct extent_buffer *eb) { int i; int num_pages; @@ -5720,7 +5720,7 @@ int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv, return ret; } -void write_extent_buffer_chunk_tree_uuid(struct extent_buffer *eb, +void write_extent_buffer_chunk_tree_uuid(const struct extent_buffer *eb, const void *srcv) { char *kaddr; @@ -5731,7 +5731,7 @@ void write_extent_buffer_chunk_tree_uuid(struct extent_buffer *eb, BTRFS_FSID_SIZE); } -void write_extent_buffer_fsid(struct extent_buffer *eb, const void *srcv) +void write_extent_buffer_fsid(const struct extent_buffer *eb, const void *srcv) { char *kaddr; @@ -5741,7 +5741,7 @@ void write_extent_buffer_fsid(struct extent_buffer *eb, const void *srcv) BTRFS_FSID_SIZE); } -void write_extent_buffer(struct extent_buffer *eb, const void *srcv, +void write_extent_buffer(const struct extent_buffer *eb, const void *srcv, unsigned long start, unsigned long len) { size_t cur; @@ -5772,7 +5772,7 @@ void write_extent_buffer(struct extent_buffer *eb, const void *srcv, } } -void memzero_extent_buffer(struct extent_buffer *eb, unsigned long start, +void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start, unsigned long len) { size_t cur; @@ -5801,8 +5801,8 @@ void memzero_extent_buffer(struct extent_buffer *eb, unsigned long start, } } -void copy_extent_buffer_full(struct extent_buffer *dst, - struct extent_buffer *src) +void copy_extent_buffer_full(const struct extent_buffer *dst, + const struct extent_buffer *src) { int i; int num_pages; @@ -5815,7 +5815,8 @@ void copy_extent_buffer_full(struct extent_buffer *dst, page_address(src->pages[i])); } -void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src, +void copy_extent_buffer(const struct extent_buffer *dst, + const struct extent_buffer *src, unsigned long dst_offset, unsigned long src_offset, unsigned long len) { @@ -5860,7 +5861,7 @@ void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src, * This helper hides the ugliness of finding the byte in an extent buffer which * contains a given bit. */ -static inline void eb_bitmap_offset(struct extent_buffer *eb, +static inline void eb_bitmap_offset(const struct extent_buffer *eb, unsigned long start, unsigned long nr, unsigned long *page_index, size_t *page_offset) @@ -5886,7 +5887,7 @@ static inline void eb_bitmap_offset(struct extent_buffer *eb, * @start: offset of the bitmap item in the extent buffer * @nr: bit number to test */ -int extent_buffer_test_bit(struct extent_buffer *eb, unsigned long start, +int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start, unsigned long nr) { u8 *kaddr; @@ -5908,7 +5909,7 @@ int extent_buffer_test_bit(struct extent_buffer *eb, unsigned long start, * @pos: bit number of the first bit * @len: number of bits to set */ -void extent_buffer_bitmap_set(struct extent_buffer *eb, unsigned long start, +void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start, unsigned long pos, unsigned long len) { u8 *kaddr; @@ -5950,8 +5951,9 @@ void extent_buffer_bitmap_set(struct extent_buffer *eb, unsigned long start, * @pos: bit number of the first bit * @len: number of bits to clear */ -void extent_buffer_bitmap_clear(struct extent_buffer *eb, unsigned long start, - unsigned long pos, unsigned long len) +void extent_buffer_bitmap_clear(const struct extent_buffer *eb, + unsigned long start, unsigned long pos, + unsigned long len) { u8 *kaddr; struct page *page; @@ -6012,8 +6014,9 @@ static void copy_pages(struct page *dst_page, struct page *src_page, memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len); } -void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, - unsigned long src_offset, unsigned long len) +void memcpy_extent_buffer(const struct extent_buffer *dst, + unsigned long dst_offset, unsigned long src_offset, + unsigned long len) { struct btrfs_fs_info *fs_info = dst->fs_info; size_t cur; @@ -6057,8 +6060,9 @@ void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, } } -void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, - unsigned long src_offset, unsigned long len) +void memmove_extent_buffer(const struct extent_buffer *dst, + unsigned long dst_offset, unsigned long src_offset, + unsigned long len) { struct btrfs_fs_info *fs_info = dst->fs_info; size_t cur; diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index 9ed89c01e2da..9a10681b12bf 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -213,7 +213,7 @@ struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info, u64 start, unsigned long len); struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info, u64 start); -struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src); +struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src); struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info, u64 start); void free_extent_buffer(struct extent_buffer *eb); @@ -231,7 +231,7 @@ static inline int num_extent_pages(const struct extent_buffer *eb) (eb->start >> PAGE_SHIFT); } -static inline int extent_buffer_uptodate(struct extent_buffer *eb) +static inline int extent_buffer_uptodate(const struct extent_buffer *eb) { return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); } @@ -244,33 +244,37 @@ void read_extent_buffer(const struct extent_buffer *eb, void *dst, int read_extent_buffer_to_user(const struct extent_buffer *eb, void __user *dst, unsigned long start, unsigned long len); -void write_extent_buffer_fsid(struct extent_buffer *eb, const void *src); -void write_extent_buffer_chunk_tree_uuid(struct extent_buffer *eb, +void write_extent_buffer_fsid(const struct extent_buffer *eb, const void *src); +void write_extent_buffer_chunk_tree_uuid(const struct extent_buffer *eb, const void *src); -void write_extent_buffer(struct extent_buffer *eb, const void *src, +void write_extent_buffer(const struct extent_buffer *eb, const void *src, unsigned long start, unsigned long len); -void copy_extent_buffer_full(struct extent_buffer *dst, - struct extent_buffer *src); -void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src, +void copy_extent_buffer_full(const struct extent_buffer *dst, + const struct extent_buffer *src); +void copy_extent_buffer(const struct extent_buffer *dst, + const struct extent_buffer *src, unsigned long dst_offset, unsigned long src_offset, unsigned long len); -void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, - unsigned long src_offset, unsigned long len); -void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset, - unsigned long src_offset, unsigned long len); -void memzero_extent_buffer(struct extent_buffer *eb, unsigned long start, +void memcpy_extent_buffer(const struct extent_buffer *dst, + unsigned long dst_offset, unsigned long src_offset, + unsigned long len); +void memmove_extent_buffer(const struct extent_buffer *dst, + unsigned long dst_offset, unsigned long src_offset, unsigned long len); -int extent_buffer_test_bit(struct extent_buffer *eb, unsigned long start, +void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start, + unsigned long len); +int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start, unsigned long pos); -void extent_buffer_bitmap_set(struct extent_buffer *eb, unsigned long start, +void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start, unsigned long pos, unsigned long len); -void extent_buffer_bitmap_clear(struct extent_buffer *eb, unsigned long start, - unsigned long pos, unsigned long len); -void clear_extent_buffer_dirty(struct extent_buffer *eb); +void extent_buffer_bitmap_clear(const struct extent_buffer *eb, + unsigned long start, unsigned long pos, + unsigned long len); +void clear_extent_buffer_dirty(const struct extent_buffer *eb); bool set_extent_buffer_dirty(struct extent_buffer *eb); void set_extent_buffer_uptodate(struct extent_buffer *eb); void clear_extent_buffer_uptodate(struct extent_buffer *eb); -int extent_buffer_under_io(struct extent_buffer *eb); +int extent_buffer_under_io(const struct extent_buffer *eb); void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end); void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end); void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end, @@ -289,7 +293,7 @@ int repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, u64 length, u64 logical, struct page *page, unsigned int pg_offset, int mirror_num); void end_extent_writepage(struct page *page, int err, u64 start, u64 end); -int btrfs_repair_eb_io_failure(struct extent_buffer *eb, int mirror_num); +int btrfs_repair_eb_io_failure(const struct extent_buffer *eb, int mirror_num); /* * When IO fails, either with EIO or csum verification fails, we diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 67dfd1287c3e..0b23aa0a32d5 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -131,7 +131,7 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ leres = cpu_to_le##bits(val); \ write_extent_buffer(token->eb, &leres, member_offset, size); \ } \ -void btrfs_set_##bits(struct extent_buffer *eb, void *ptr, \ +void btrfs_set_##bits(const struct extent_buffer *eb, void *ptr, \ unsigned long off, u##bits val) \ { \ const unsigned long member_offset = (unsigned long)ptr + off; \ From c60ac0ffd682fc04f564dd6f4cc1a77c0f94608b Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 29 Apr 2020 23:41:20 +0200 Subject: [PATCH 0782/1043] btrfs: drop unnecessary offset_in_page in extent buffer helpers Helpers that iterate over extent buffer pages set up several variables, one of them is finding out offset of the extent buffer start within a page. Right now we have extent buffers aligned to page sizes so this is effectively storing zero. This makes the code harder the follow and can be simplified. The same change is done in all the helpers: * remove: size_t start_offset = offset_in_page(eb->start); * simplify code using start_offset Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/extent_io.c | 51 ++++++++++++++++++-------------------------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index da6f0c1ed80c..c59e07360083 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -5622,8 +5622,7 @@ void read_extent_buffer(const struct extent_buffer *eb, void *dstv, struct page *page; char *kaddr; char *dst = (char *)dstv; - size_t start_offset = offset_in_page(eb->start); - unsigned long i = (start_offset + start) >> PAGE_SHIFT; + unsigned long i = start >> PAGE_SHIFT; if (start + len > eb->len) { WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, wanted %lu %lu\n", @@ -5632,7 +5631,7 @@ void read_extent_buffer(const struct extent_buffer *eb, void *dstv, return; } - offset = offset_in_page(start_offset + start); + offset = offset_in_page(start); while (len > 0) { page = eb->pages[i]; @@ -5657,14 +5656,13 @@ int read_extent_buffer_to_user(const struct extent_buffer *eb, struct page *page; char *kaddr; char __user *dst = (char __user *)dstv; - size_t start_offset = offset_in_page(eb->start); - unsigned long i = (start_offset + start) >> PAGE_SHIFT; + unsigned long i = start >> PAGE_SHIFT; int ret = 0; WARN_ON(start > eb->len); WARN_ON(start + len > eb->start + eb->len); - offset = offset_in_page(start_offset + start); + offset = offset_in_page(start); while (len > 0) { page = eb->pages[i]; @@ -5693,14 +5691,13 @@ int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv, struct page *page; char *kaddr; char *ptr = (char *)ptrv; - size_t start_offset = offset_in_page(eb->start); - unsigned long i = (start_offset + start) >> PAGE_SHIFT; + unsigned long i = start >> PAGE_SHIFT; int ret = 0; WARN_ON(start > eb->len); WARN_ON(start + len > eb->start + eb->len); - offset = offset_in_page(start_offset + start); + offset = offset_in_page(start); while (len > 0) { page = eb->pages[i]; @@ -5749,13 +5746,12 @@ void write_extent_buffer(const struct extent_buffer *eb, const void *srcv, struct page *page; char *kaddr; char *src = (char *)srcv; - size_t start_offset = offset_in_page(eb->start); - unsigned long i = (start_offset + start) >> PAGE_SHIFT; + unsigned long i = start >> PAGE_SHIFT; WARN_ON(start > eb->len); WARN_ON(start + len > eb->start + eb->len); - offset = offset_in_page(start_offset + start); + offset = offset_in_page(start); while (len > 0) { page = eb->pages[i]; @@ -5779,13 +5775,12 @@ void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start, size_t offset; struct page *page; char *kaddr; - size_t start_offset = offset_in_page(eb->start); - unsigned long i = (start_offset + start) >> PAGE_SHIFT; + unsigned long i = start >> PAGE_SHIFT; WARN_ON(start > eb->len); WARN_ON(start + len > eb->start + eb->len); - offset = offset_in_page(start_offset + start); + offset = offset_in_page(start); while (len > 0) { page = eb->pages[i]; @@ -5825,12 +5820,11 @@ void copy_extent_buffer(const struct extent_buffer *dst, size_t offset; struct page *page; char *kaddr; - size_t start_offset = offset_in_page(dst->start); - unsigned long i = (start_offset + dst_offset) >> PAGE_SHIFT; + unsigned long i = dst_offset >> PAGE_SHIFT; WARN_ON(src->len != dst_len); - offset = offset_in_page(start_offset + dst_offset); + offset = offset_in_page(dst_offset); while (len > 0) { page = dst->pages[i]; @@ -5866,7 +5860,6 @@ static inline void eb_bitmap_offset(const struct extent_buffer *eb, unsigned long *page_index, size_t *page_offset) { - size_t start_offset = offset_in_page(eb->start); size_t byte_offset = BIT_BYTE(nr); size_t offset; @@ -5875,7 +5868,7 @@ static inline void eb_bitmap_offset(const struct extent_buffer *eb, * the bitmap item in the extent buffer + the offset of the byte in the * bitmap item. */ - offset = start_offset + start + byte_offset; + offset = start + byte_offset; *page_index = offset >> PAGE_SHIFT; *page_offset = offset_in_page(offset); @@ -6022,7 +6015,6 @@ void memcpy_extent_buffer(const struct extent_buffer *dst, size_t cur; size_t dst_off_in_page; size_t src_off_in_page; - size_t start_offset = offset_in_page(dst->start); unsigned long dst_i; unsigned long src_i; @@ -6040,11 +6032,11 @@ void memcpy_extent_buffer(const struct extent_buffer *dst, } while (len > 0) { - dst_off_in_page = offset_in_page(start_offset + dst_offset); - src_off_in_page = offset_in_page(start_offset + src_offset); + dst_off_in_page = offset_in_page(dst_offset); + src_off_in_page = offset_in_page(src_offset); - dst_i = (start_offset + dst_offset) >> PAGE_SHIFT; - src_i = (start_offset + src_offset) >> PAGE_SHIFT; + dst_i = dst_offset >> PAGE_SHIFT; + src_i = src_offset >> PAGE_SHIFT; cur = min(len, (unsigned long)(PAGE_SIZE - src_off_in_page)); @@ -6070,7 +6062,6 @@ void memmove_extent_buffer(const struct extent_buffer *dst, size_t src_off_in_page; unsigned long dst_end = dst_offset + len - 1; unsigned long src_end = src_offset + len - 1; - size_t start_offset = offset_in_page(dst->start); unsigned long dst_i; unsigned long src_i; @@ -6091,11 +6082,11 @@ void memmove_extent_buffer(const struct extent_buffer *dst, return; } while (len > 0) { - dst_i = (start_offset + dst_end) >> PAGE_SHIFT; - src_i = (start_offset + src_end) >> PAGE_SHIFT; + dst_i = dst_end >> PAGE_SHIFT; + src_i = src_end >> PAGE_SHIFT; - dst_off_in_page = offset_in_page(start_offset + dst_end); - src_off_in_page = offset_in_page(start_offset + src_end); + dst_off_in_page = offset_in_page(dst_end); + src_off_in_page = offset_in_page(src_end); cur = min_t(unsigned long, len, src_off_in_page + 1); cur = min(cur, dst_off_in_page + 1); From 84da071f3df5a9c0df0352d1c52b0c36ddb1c53f Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 30 Apr 2020 17:57:55 +0200 Subject: [PATCH 0783/1043] btrfs: optimize split page read in btrfs_get_##bits The helper read_extent_buffer is called to do read of the data spanning two extent buffer pages. As the size is known, we can do the read directly in two steps. This removes one function call and compiler can optimize memcpy as the sizes are known at compile time. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 0b23aa0a32d5..46a7269bee07 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -90,17 +90,20 @@ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ { \ const unsigned long member_offset = (unsigned long)ptr + off; \ const unsigned long oip = offset_in_page(member_offset); \ + const unsigned long idx = member_offset >> PAGE_SHIFT; \ + char *kaddr = page_address(eb->pages[idx]); \ const int size = sizeof(u##bits); \ - __le##bits leres; \ + const int part = PAGE_SIZE - oip; \ + u8 lebytes[sizeof(u##bits)]; \ \ ASSERT(check_setget_bounds(eb, ptr, off, size)); \ - if (oip + size <= PAGE_SIZE) { \ - const unsigned long idx = member_offset >> PAGE_SHIFT; \ - const char *kaddr = page_address(eb->pages[idx]); \ + if (oip + size <= PAGE_SIZE) \ return get_unaligned_le##bits(kaddr + oip); \ - } \ - read_extent_buffer(eb, &leres, member_offset, size); \ - return le##bits##_to_cpu(leres); \ + \ + memcpy(lebytes, kaddr + oip, part); \ + kaddr = page_address(eb->pages[idx + 1]); \ + memcpy(lebytes + part, kaddr, size - part); \ + return get_unaligned_le##bits(lebytes); \ } \ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ const void *ptr, unsigned long off, \ From ba8a9a0537770df69d9dc38c11312c9b0f840cf2 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 30 Apr 2020 17:57:55 +0200 Subject: [PATCH 0784/1043] btrfs: optimize split page read in btrfs_get_token_##bits The fallback path calls helper read_extent_buffer to do read of the data spanning two extent buffer pages. As the size is known, we can do the read directly in two steps. This removes one function call and compiler can optimize memcpy as the sizes are known at compile time. The cached token address is set to the second page. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 46a7269bee07..63cab91507f8 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -66,7 +66,8 @@ u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ const unsigned long idx = member_offset >> PAGE_SHIFT; \ const unsigned long oip = offset_in_page(member_offset); \ const int size = sizeof(u##bits); \ - __le##bits leres; \ + u8 lebytes[sizeof(u##bits)]; \ + const int part = PAGE_SIZE - oip; \ \ ASSERT(token); \ ASSERT(token->kaddr); \ @@ -75,15 +76,16 @@ u##bits btrfs_get_token_##bits(struct btrfs_map_token *token, \ member_offset + size <= token->offset + PAGE_SIZE) { \ return get_unaligned_le##bits(token->kaddr + oip); \ } \ - if (oip + size <= PAGE_SIZE) { \ - token->kaddr = page_address(token->eb->pages[idx]); \ - token->offset = idx << PAGE_SHIFT; \ + token->kaddr = page_address(token->eb->pages[idx]); \ + token->offset = idx << PAGE_SHIFT; \ + if (oip + size <= PAGE_SIZE) \ return get_unaligned_le##bits(token->kaddr + oip); \ - } \ + \ + memcpy(lebytes, token->kaddr + oip, part); \ token->kaddr = page_address(token->eb->pages[idx + 1]); \ token->offset = (idx + 1) << PAGE_SHIFT; \ - read_extent_buffer(token->eb, &leres, member_offset, size); \ - return le##bits##_to_cpu(leres); \ + memcpy(lebytes + part, token->kaddr, size - part); \ + return get_unaligned_le##bits(lebytes); \ } \ u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ const void *ptr, unsigned long off) \ From f4ca8c51d12631f1297d093f767c3a5ce5b95aff Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 30 Apr 2020 17:57:55 +0200 Subject: [PATCH 0785/1043] btrfs: optimize split page write in btrfs_set_##bits The helper write_extent_buffer is called to do write of the data spanning two extent buffer pages. As the size is known, we can do the write directly in two steps. This removes one function call and compiler can optimize memcpy as the sizes are known at compile time. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 63cab91507f8..7987d3910660 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -141,18 +141,22 @@ void btrfs_set_##bits(const struct extent_buffer *eb, void *ptr, \ { \ const unsigned long member_offset = (unsigned long)ptr + off; \ const unsigned long oip = offset_in_page(member_offset); \ + const unsigned long idx = member_offset >> PAGE_SHIFT; \ + char *kaddr = page_address(eb->pages[idx]); \ const int size = sizeof(u##bits); \ - __le##bits leres; \ + const int part = PAGE_SIZE - oip; \ + u8 lebytes[sizeof(u##bits)]; \ \ ASSERT(check_setget_bounds(eb, ptr, off, size)); \ if (oip + size <= PAGE_SIZE) { \ - const unsigned long idx = member_offset >> PAGE_SHIFT; \ - char *kaddr = page_address(eb->pages[idx]); \ put_unaligned_le##bits(val, kaddr + oip); \ return; \ } \ - leres = cpu_to_le##bits(val); \ - write_extent_buffer(eb, &leres, member_offset, size); \ + \ + put_unaligned_le##bits(val, lebytes); \ + memcpy(kaddr + oip, lebytes, part); \ + kaddr = page_address(eb->pages[idx + 1]); \ + memcpy(kaddr, lebytes + part, size - part); \ } DEFINE_BTRFS_SETGET_BITS(8) From f472d3c28385397b1a2891c4ec264b90cd6d2187 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 30 Apr 2020 17:57:55 +0200 Subject: [PATCH 0786/1043] btrfs: optimize split page write in btrfs_set_token_##bits The fallback path calls helper write_extent_buffer to do write of the data spanning two extent buffer pages. As the size is known, we can do the write directly in two steps. This removes one function call and compiler can optimize memcpy as the sizes are known at compile time. The cached token address is set to the second page. Reviewed-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 7987d3910660..225ef6d7e949 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -115,7 +115,8 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ const unsigned long idx = member_offset >> PAGE_SHIFT; \ const unsigned long oip = offset_in_page(member_offset); \ const int size = sizeof(u##bits); \ - __le##bits leres; \ + u8 lebytes[sizeof(u##bits)]; \ + const int part = PAGE_SIZE - oip; \ \ ASSERT(token); \ ASSERT(token->kaddr); \ @@ -125,16 +126,17 @@ void btrfs_set_token_##bits(struct btrfs_map_token *token, \ put_unaligned_le##bits(val, token->kaddr + oip); \ return; \ } \ + token->kaddr = page_address(token->eb->pages[idx]); \ + token->offset = idx << PAGE_SHIFT; \ if (oip + size <= PAGE_SIZE) { \ - token->kaddr = page_address(token->eb->pages[idx]); \ - token->offset = idx << PAGE_SHIFT; \ put_unaligned_le##bits(val, token->kaddr + oip); \ return; \ } \ + put_unaligned_le##bits(val, lebytes); \ + memcpy(token->kaddr + oip, lebytes, part); \ token->kaddr = page_address(token->eb->pages[idx + 1]); \ token->offset = (idx + 1) << PAGE_SHIFT; \ - leres = cpu_to_le##bits(val); \ - write_extent_buffer(token->eb, &leres, member_offset, size); \ + memcpy(token->kaddr, lebytes + part, size - part); \ } \ void btrfs_set_##bits(const struct extent_buffer *eb, void *ptr, \ unsigned long off, u##bits val) \ From 583e4a2384db843fa8ae9a67aff8ea0eed140e61 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Wed, 6 May 2020 20:54:13 +0200 Subject: [PATCH 0787/1043] btrfs: update documentation of set/get helpers Signed-off-by: David Sterba --- fs/btrfs/struct-funcs.c | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/fs/btrfs/struct-funcs.c b/fs/btrfs/struct-funcs.c index 225ef6d7e949..079b059818e9 100644 --- a/fs/btrfs/struct-funcs.c +++ b/fs/btrfs/struct-funcs.c @@ -39,23 +39,27 @@ static bool check_setget_bounds(const struct extent_buffer *eb, } /* - * this is some deeply nasty code. + * Macro templates that define helpers to read/write extent buffer data of a + * given size, that are also used via ctree.h for access to item members by + * specialized helpers. * - * The end result is that anyone who #includes ctree.h gets a - * declaration for the btrfs_set_foo functions and btrfs_foo functions, - * which are wrappers of btrfs_set_token_#bits functions and - * btrfs_get_token_#bits functions, which are defined in this file. + * Generic helpers: + * - btrfs_set_8 (for 8/16/32/64) + * - btrfs_get_8 (for 8/16/32/64) * - * These setget functions do all the extent_buffer related mapping - * required to efficiently read and write specific fields in the extent - * buffers. Every pointer to metadata items in btrfs is really just - * an unsigned long offset into the extent buffer which has been - * cast to a specific type. This gives us all the gcc type checking. + * Generic helpers with a token (cached address of the most recently accessed + * page): + * - btrfs_set_token_8 (for 8/16/32/64) + * - btrfs_get_token_8 (for 8/16/32/64) * - * The extent buffer api is used to do the page spanning work required to - * have a metadata blocksize different from the page size. + * The set/get functions handle data spanning two pages transparently, in case + * metadata block size is larger than page. Every pointer to metadata items is + * an offset into the extent buffer page array, cast to a specific type. This + * gives us all the type checking. * - * There are 2 variants defined, one with a token pointer and one without. + * The extent buffer pages stored in the array pages do not form a contiguous + * phyusical range, but the API functions assume the linear offset to the range + * from 0 to metadata node size. */ #define DEFINE_BTRFS_SETGET_BITS(bits) \ From ae3e715f85fff878b317369b898cf4b89785c80a Mon Sep 17 00:00:00 2001 From: Anand Jain Date: Thu, 14 May 2020 01:42:45 +0800 Subject: [PATCH 0788/1043] btrfs: drop stale reference to volume_mutex Commit dccdb07bc996 ("btrfs: kill btrfs_fs_info::volume_mutex") removed the last use of the volume_mutex, forgetting to update the comment. Signed-off-by: Anand Jain Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/volumes.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index be1e047a489e..0d6e785bcb98 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -280,10 +280,9 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, * ============ * * uuid_mutex - * volume_mutex - * device_list_mutex - * chunk_mutex - * balance_mutex + * device_list_mutex + * chunk_mutex + * balance_mutex * * * Exclusive operations, BTRFS_FS_EXCL_OP From 92a7cc4252231d1641b36c38cf845cfc50308ab0 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Fri, 15 May 2020 14:01:40 +0800 Subject: [PATCH 0789/1043] btrfs: rename BTRFS_ROOT_REF_COWS to BTRFS_ROOT_SHAREABLE The name BTRFS_ROOT_REF_COWS is not very clear about the meaning. In fact, that bit can only be set to those trees: - Subvolume roots - Data reloc root - Reloc roots for above roots All other trees won't get this bit set. So just by the result, it is obvious that, roots with this bit set can have tree blocks shared with other trees. Either shared by snapshots, or by reloc roots (an special snapshot created by relocation). This patch will rename BTRFS_ROOT_REF_COWS to BTRFS_ROOT_SHAREABLE to make it easier to understand, and update all comment mentioning "reference counted" to follow the rename. Signed-off-by: Qu Wenruo Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/backref.c | 4 ++-- fs/btrfs/backref.h | 2 +- fs/btrfs/block-rsv.c | 2 +- fs/btrfs/ctree.c | 26 +++++++++++++------------- fs/btrfs/ctree.h | 25 +++++++++++++++++++++++-- fs/btrfs/disk-io.c | 13 +++++++------ fs/btrfs/extent-tree.c | 2 +- fs/btrfs/file.c | 2 +- fs/btrfs/inode.c | 17 +++++++++-------- fs/btrfs/ioctl.c | 2 +- fs/btrfs/relocation.c | 25 ++++++++++++++----------- fs/btrfs/transaction.c | 14 +++++++------- fs/btrfs/tree-defrag.c | 2 +- 13 files changed, 81 insertions(+), 55 deletions(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index ac3c34f47b56..41ef3fa91864 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -2702,7 +2702,7 @@ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, root = btrfs_get_fs_root(fs_info, &root_key, false); if (IS_ERR(root)) return PTR_ERR(root); - if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) cur->cowonly = 1; if (btrfs_root_level(&root->root_item) == cur->level) { @@ -2789,7 +2789,7 @@ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, goto out; } upper->owner = btrfs_header_owner(eb); - if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) upper->cowonly = 1; /* diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h index 18393cc05ca2..ff705cc564a9 100644 --- a/fs/btrfs/backref.h +++ b/fs/btrfs/backref.h @@ -184,7 +184,7 @@ struct btrfs_backref_node { struct extent_buffer *eb; /* Level of the tree block */ unsigned int level:8; - /* Is the block in non-reference counted tree */ + /* Is the block in a non-shareable tree */ unsigned int cowonly:1; /* 1 if no child node is in the cache */ unsigned int lowest:1; diff --git a/fs/btrfs/block-rsv.c b/fs/btrfs/block-rsv.c index dbba53e712e6..7e1549a84fcc 100644 --- a/fs/btrfs/block-rsv.c +++ b/fs/btrfs/block-rsv.c @@ -458,7 +458,7 @@ static struct btrfs_block_rsv *get_block_rsv( struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_block_rsv *block_rsv = NULL; - if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || + if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) || (root == fs_info->csum_root && trans->adding_csums) || (root == fs_info->uuid_root)) block_rsv = trans->block_rsv; diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 746dec22f250..92775554d1cc 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -144,9 +144,10 @@ struct extent_buffer *btrfs_root_node(struct btrfs_root *root) return eb; } -/* cowonly root (everything not a reference counted cow subvolume), just get - * put onto a simple dirty list. transaction.c walks this to make sure they - * get properly updated on disk. +/* + * Cowonly root (not-shareable trees, everything not subvolume or reloc roots), + * just get put onto a simple dirty list. Transaction walks this list to make + * sure they get properly updated on disk. */ static void add_root_to_dirty_list(struct btrfs_root *root) { @@ -185,9 +186,9 @@ int btrfs_copy_root(struct btrfs_trans_handle *trans, int level; struct btrfs_disk_key disk_key; - WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && + WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && trans->transid != fs_info->running_transaction->transid); - WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && + WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && trans->transid != root->last_trans); level = btrfs_header_level(buf); @@ -826,12 +827,11 @@ int btrfs_block_can_be_shared(struct btrfs_root *root, struct extent_buffer *buf) { /* - * Tree blocks not in reference counted trees and tree roots - * are never shared. If a block was allocated after the last - * snapshot and the block was not allocated by tree relocation, - * we know the block is not shared. + * Tree blocks not in shareable trees and tree roots are never shared. + * If a block was allocated after the last snapshot and the block was + * not allocated by tree relocation, we know the block is not shared. */ - if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) && + if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && buf != root->node && buf != root->commit_root && (btrfs_header_generation(buf) <= btrfs_root_last_snapshot(&root->root_item) || @@ -1024,9 +1024,9 @@ static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, btrfs_assert_tree_locked(buf); - WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && + WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && trans->transid != fs_info->running_transaction->transid); - WARN_ON(test_bit(BTRFS_ROOT_REF_COWS, &root->state) && + WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && trans->transid != root->last_trans); level = btrfs_header_level(buf); @@ -1065,7 +1065,7 @@ static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, return ret; } - if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) { + if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) { ret = btrfs_reloc_cow_block(trans, root, buf, cow); if (ret) { btrfs_abort_transaction(trans, ret); diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 0a1fa1526c43..bf46093be76e 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -969,7 +969,28 @@ enum { * is used to tell us when more checks are required */ BTRFS_ROOT_IN_TRANS_SETUP, - BTRFS_ROOT_REF_COWS, + + /* + * Set if tree blocks of this root can be shared by other roots. + * Only subvolume trees and their reloc trees have this bit set. + * Conflicts with TRACK_DIRTY bit. + * + * This affects two things: + * + * - How balance works + * For shareable roots, we need to use reloc tree and do path + * replacement for balance, and need various pre/post hooks for + * snapshot creation to handle them. + * + * While for non-shareable trees, we just simply do a tree search + * with COW. + * + * - How dirty roots are tracked + * For shareable roots, btrfs_record_root_in_trans() is needed to + * track them, while non-subvolume roots have TRACK_DIRTY bit, they + * don't need to set this manually. + */ + BTRFS_ROOT_SHAREABLE, BTRFS_ROOT_TRACK_DIRTY, BTRFS_ROOT_IN_RADIX, BTRFS_ROOT_ORPHAN_ITEM_INSERTED, @@ -1055,7 +1076,7 @@ struct btrfs_root { struct btrfs_key defrag_progress; struct btrfs_key defrag_max; - /* the dirty list is only used by non-reference counted roots */ + /* The dirty list is only used by non-shareable roots */ struct list_head dirty_list; struct list_head root_list; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 714b57553ed6..248086cca124 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -1273,12 +1273,13 @@ static struct btrfs_root *alloc_log_tree(struct btrfs_trans_handle *trans, root->root_key.offset = BTRFS_TREE_LOG_OBJECTID; /* - * DON'T set REF_COWS for log trees + * DON'T set SHAREABLE bit for log trees. * - * log trees do not get reference counted because they go away - * before a real commit is actually done. They do store pointers - * to file data extents, and those reference counts still get - * updated (along with back refs to the log tree). + * Log trees are not exposed to user space thus can't be snapshotted, + * and they go away before a real commit is actually done. + * + * They do store pointers to file data extents, and those reference + * counts still get updated (along with back refs to the log tree). */ leaf = btrfs_alloc_tree_block(trans, root, 0, BTRFS_TREE_LOG_OBJECTID, @@ -1417,7 +1418,7 @@ static int btrfs_init_fs_root(struct btrfs_root *root) goto fail; if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { - set_bit(BTRFS_ROOT_REF_COWS, &root->state); + set_bit(BTRFS_ROOT_SHAREABLE, &root->state); btrfs_check_and_init_root_item(&root->root_item); } diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 140011edcad7..c0bc35f932bf 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -2426,7 +2426,7 @@ static int __btrfs_mod_ref(struct btrfs_trans_handle *trans, nritems = btrfs_header_nritems(buf); level = btrfs_header_level(buf); - if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state) && level == 0) + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && level == 0) return 0; if (full_backref) diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 719e68ab552c..606c2f3c1a38 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -775,7 +775,7 @@ int __btrfs_drop_extents(struct btrfs_trans_handle *trans, if (start >= BTRFS_I(inode)->disk_i_size && !replace_extent) modify_tree = 0; - update_refs = (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || + update_refs = (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) || root == fs_info->tree_root); while (1) { recow = 0; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 3ea694ee1c90..08c02bd148a6 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -4100,11 +4100,12 @@ int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY); /* - * for non-free space inodes and ref cows, we want to back off from - * time to time + * For non-free space inodes and non-shareable roots, we want to back + * off from time to time. This means all inodes in subvolume roots, + * reloc roots, and data reloc roots. */ if (!btrfs_is_free_space_inode(BTRFS_I(inode)) && - test_bit(BTRFS_ROOT_REF_COWS, &root->state)) + test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) be_nice = true; path = btrfs_alloc_path(); @@ -4121,7 +4122,7 @@ int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, * not block aligned since we will be keeping the last block of the * extent just the way it is. */ - if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || + if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) || root == fs_info->tree_root) btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size, fs_info->sectorsize), @@ -4233,7 +4234,7 @@ search_again: extent_num_bytes); num_dec = (orig_num_bytes - extent_num_bytes); - if (test_bit(BTRFS_ROOT_REF_COWS, + if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && extent_start != 0) inode_sub_bytes(inode, num_dec); @@ -4249,7 +4250,7 @@ search_again: num_dec = btrfs_file_extent_num_bytes(leaf, fi); if (extent_start != 0) { found_extent = 1; - if (test_bit(BTRFS_ROOT_REF_COWS, + if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) inode_sub_bytes(inode, num_dec); } @@ -4285,7 +4286,7 @@ search_again: clear_len = fs_info->sectorsize; } - if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) + if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) inode_sub_bytes(inode, item_end + 1 - new_size); } delete: @@ -4326,7 +4327,7 @@ delete: should_throttle = false; if (found_extent && - (test_bit(BTRFS_ROOT_REF_COWS, &root->state) || + (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) || root == fs_info->tree_root)) { struct btrfs_ref ref = { 0 }; diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 40b729dce91c..709d9446896a 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -750,7 +750,7 @@ static int create_snapshot(struct btrfs_root *root, struct inode *dir, int ret; bool snapshot_force_cow = false; - if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) return -EINVAL; if (atomic_read(&root->nr_swapfiles)) { diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index f25deca18a5d..1dd7b5310ffd 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -321,7 +321,7 @@ int btrfs_should_ignore_reloc_root(struct btrfs_root *root) { struct btrfs_root *reloc_root; - if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) return 0; /* This root has been merged with its reloc tree, we can ignore it */ @@ -808,7 +808,7 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, reloc_root = btrfs_read_tree_root(fs_info->tree_root, &root_key); BUG_ON(IS_ERR(reloc_root)); - set_bit(BTRFS_ROOT_REF_COWS, &reloc_root->state); + set_bit(BTRFS_ROOT_SHAREABLE, &reloc_root->state); reloc_root->last_trans = trans->transid; return reloc_root; } @@ -2018,7 +2018,7 @@ struct btrfs_root *select_reloc_root(struct btrfs_trans_handle *trans, next = walk_up_backref(next, edges, &index); root = next->root; BUG_ON(!root); - BUG_ON(!test_bit(BTRFS_ROOT_REF_COWS, &root->state)); + BUG_ON(!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)); if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID) { record_reloc_root_in_trans(trans, root); @@ -2062,10 +2062,13 @@ struct btrfs_root *select_reloc_root(struct btrfs_trans_handle *trans, } /* - * select a tree root for relocation. return NULL if the block - * is reference counted. we should use do_relocation() in this - * case. return a tree root pointer if the block isn't reference - * counted. return -ENOENT if the block is root of reloc tree. + * Select a tree root for relocation. + * + * Return NULL if the block is not shareable. We should use do_relocation() in + * this case. + * + * Return a tree root pointer if the block is shareable. + * Return -ENOENT if the block is root of reloc tree. */ static noinline_for_stack struct btrfs_root *select_one_root(struct btrfs_backref_node *node) @@ -2083,8 +2086,8 @@ struct btrfs_root *select_one_root(struct btrfs_backref_node *node) root = next->root; BUG_ON(!root); - /* no other choice for non-references counted tree */ - if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) + /* No other choice for non-shareable tree */ + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) return root; if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) @@ -2480,7 +2483,7 @@ static int relocate_tree_block(struct btrfs_trans_handle *trans, } if (root) { - if (test_bit(BTRFS_ROOT_REF_COWS, &root->state)) { + if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) { BUG_ON(node->new_bytenr); BUG_ON(!list_empty(&node->list)); btrfs_record_root_in_trans(trans, root); @@ -3765,7 +3768,7 @@ int btrfs_recover_relocation(struct btrfs_root *root) goto out; } - set_bit(BTRFS_ROOT_REF_COWS, &reloc_root->state); + set_bit(BTRFS_ROOT_SHAREABLE, &reloc_root->state); list_add(&reloc_root->root_list, &reloc_roots); if (btrfs_root_refs(&reloc_root->root_item) > 0) { diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index f58d0fdc5078..a8cc2ddffc17 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -349,10 +349,10 @@ loop: } /* - * this does all the record keeping required to make sure that a reference - * counted root is properly recorded in a given transaction. This is required - * to make sure the old root from before we joined the transaction is deleted - * when the transaction commits + * This does all the record keeping required to make sure that a shareable root + * is properly recorded in a given transaction. This is required to make sure + * the old root from before we joined the transaction is deleted when the + * transaction commits. */ static int record_root_in_trans(struct btrfs_trans_handle *trans, struct btrfs_root *root, @@ -360,7 +360,7 @@ static int record_root_in_trans(struct btrfs_trans_handle *trans, { struct btrfs_fs_info *fs_info = root->fs_info; - if ((test_bit(BTRFS_ROOT_REF_COWS, &root->state) && + if ((test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && root->last_trans < trans->transid) || force) { WARN_ON(root == fs_info->extent_root); WARN_ON(!force && root->commit_root != root->node); @@ -439,7 +439,7 @@ int btrfs_record_root_in_trans(struct btrfs_trans_handle *trans, { struct btrfs_fs_info *fs_info = root->fs_info; - if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) return 0; /* @@ -504,7 +504,7 @@ static inline bool need_reserve_reloc_root(struct btrfs_root *root) struct btrfs_fs_info *fs_info = root->fs_info; if (!fs_info->reloc_ctl || - !test_bit(BTRFS_ROOT_REF_COWS, &root->state) || + !test_bit(BTRFS_ROOT_SHAREABLE, &root->state) || root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID || root->reloc_root) return false; diff --git a/fs/btrfs/tree-defrag.c b/fs/btrfs/tree-defrag.c index 5f9e2dd413af..16c3a6d2586d 100644 --- a/fs/btrfs/tree-defrag.c +++ b/fs/btrfs/tree-defrag.c @@ -35,7 +35,7 @@ int btrfs_defrag_leaves(struct btrfs_trans_handle *trans, goto out; } - if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state)) + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) goto out; path = btrfs_alloc_path(); From 82028e0a2a9b7a3db6ef1f1885c580a37ab9eebb Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Fri, 15 May 2020 14:01:41 +0800 Subject: [PATCH 0790/1043] btrfs: inode: cleanup the log-tree exceptions in btrfs_truncate_inode_items() There are a lot of root owner checks in btrfs_truncate_inode_items() like: if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) || root == fs_info->tree_root) But considering that, only these trees can have INODE_ITEMs: - tree root (for v1 space cache) - subvolume trees - tree reloc trees - data reloc tree - log trees And since subvolume/tree reloc/data reloc trees all have SHAREABLE bit, and we're checking tree root manually, so above check is just excluding log trees. This patch will replace two of such checks to a simpler one: if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) This would merge btrfs_drop_extent_cache() and lock_extent_bits() call into the same if branch. Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/inode.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 08c02bd148a6..5be7820afe90 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -4113,20 +4113,19 @@ int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, return -ENOMEM; path->reada = READA_BACK; - if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) + if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, (u64)-1, &cached_state); - /* - * We want to drop from the next block forward in case this new size is - * not block aligned since we will be keeping the last block of the - * extent just the way it is. - */ - if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) || - root == fs_info->tree_root) + /* + * We want to drop from the next block forward in case this + * new size is not block aligned since we will be keeping the + * last block of the extent just the way it is. + */ btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size, fs_info->sectorsize), (u64)-1, 0); + } /* * This function is also used to drop the items in the log tree before @@ -4327,8 +4326,7 @@ delete: should_throttle = false; if (found_extent && - (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) || - root == fs_info->tree_root)) { + root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { struct btrfs_ref ref = { 0 }; bytes_deleted += extent_num_bytes; From aeb935a455812e0ec15e15801f7a42d887e6c22f Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Fri, 15 May 2020 14:01:42 +0800 Subject: [PATCH 0791/1043] btrfs: don't set SHAREABLE flag for data reloc tree SHAREABLE flag is set for subvolumes because users can create snapshot for subvolumes, thus sharing tree blocks of them. But data reloc tree is not exposed to user space, as it's only an internal tree for data relocation, thus it doesn't need the full path replacement handling at all. This patch will make data reloc tree a non-shareable tree, and add btrfs_fs_info::data_reloc_root for data reloc tree, so relocation code can grab it from fs_info directly. This would slightly improve tree relocation, as now data reloc tree can go through regular COW routine to get relocated, without bothering the complex tree reloc tree routine. Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 1 + fs/btrfs/disk-io.c | 18 +++++++++++++++++- fs/btrfs/relocation.c | 16 +++++----------- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index bf46093be76e..616fdbc53695 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -582,6 +582,7 @@ struct btrfs_fs_info { struct btrfs_root *quota_root; struct btrfs_root *uuid_root; struct btrfs_root *free_space_root; + struct btrfs_root *data_reloc_root; /* the log root tree is a directory of all the other log roots */ struct btrfs_root *log_root_tree; diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 248086cca124..f1143741f6ba 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -1417,7 +1417,8 @@ static int btrfs_init_fs_root(struct btrfs_root *root) if (ret) goto fail; - if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) { + if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID && + root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID) { set_bit(BTRFS_ROOT_SHAREABLE, &root->state); btrfs_check_and_init_root_item(&root->root_item); } @@ -1523,6 +1524,7 @@ void btrfs_free_fs_info(struct btrfs_fs_info *fs_info) btrfs_put_root(fs_info->uuid_root); btrfs_put_root(fs_info->free_space_root); btrfs_put_root(fs_info->fs_root); + btrfs_put_root(fs_info->data_reloc_root); btrfs_check_leaked_roots(fs_info); btrfs_extent_buffer_leak_debug_check(fs_info); kfree(fs_info->super_copy); @@ -1979,6 +1981,7 @@ static void free_root_pointers(struct btrfs_fs_info *info, bool free_chunk_root) free_root_extent_buffers(info->quota_root); free_root_extent_buffers(info->uuid_root); free_root_extent_buffers(info->fs_root); + free_root_extent_buffers(info->data_reloc_root); if (free_chunk_root) free_root_extent_buffers(info->chunk_root); free_root_extent_buffers(info->free_space_root); @@ -2285,6 +2288,19 @@ static int btrfs_read_roots(struct btrfs_fs_info *fs_info) set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state); fs_info->csum_root = root; + /* + * This tree can share blocks with some other fs tree during relocation + * and we need a proper setup by btrfs_get_fs_root + */ + location.objectid = BTRFS_DATA_RELOC_TREE_OBJECTID; + root = btrfs_get_fs_root(tree_root->fs_info, &location, true); + if (IS_ERR(root)) { + ret = PTR_ERR(root); + goto out; + } + set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state); + fs_info->data_reloc_root = root; + location.objectid = BTRFS_QUOTA_TREE_OBJECTID; root = btrfs_read_tree_root(tree_root, &location); if (!IS_ERR(root)) { diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 1dd7b5310ffd..58f56e01de0d 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -3476,10 +3476,7 @@ struct inode *create_reloc_inode(struct btrfs_fs_info *fs_info, u64 objectid; int err = 0; - root = read_fs_root(fs_info, BTRFS_DATA_RELOC_TREE_OBJECTID); - if (IS_ERR(root)) - return ERR_CAST(root); - + root = btrfs_grab_root(fs_info->data_reloc_root); trans = btrfs_start_transaction(root, 6); if (IS_ERR(trans)) { btrfs_put_root(root); @@ -3871,13 +3868,10 @@ out: if (err == 0) { /* cleanup orphan inode in data relocation tree */ - fs_root = read_fs_root(fs_info, BTRFS_DATA_RELOC_TREE_OBJECTID); - if (IS_ERR(fs_root)) { - err = PTR_ERR(fs_root); - } else { - err = btrfs_orphan_cleanup(fs_root); - btrfs_put_root(fs_root); - } + fs_root = btrfs_grab_root(fs_info->data_reloc_root); + ASSERT(fs_root); + err = btrfs_orphan_cleanup(fs_root); + btrfs_put_root(fs_root); } return err; } From c11fbb6ed0ddc11b992f9c668b79505d31956368 Mon Sep 17 00:00:00 2001 From: Robbie Ko Date: Thu, 14 May 2020 17:19:18 +0800 Subject: [PATCH 0792/1043] btrfs: reduce lock contention when creating snapshot When creating a snapshot, ordered extents need to be flushed and this can take a long time. In create_snapshot there are two locks held when this happens: 1. Destination directory inode lock 2. Global subvolume semaphore This will unnecessarily block other operations like subvolume destroy, create, or setflag until the snapshot is created. We can fix that by moving the flush outside the locked section as this does not depend on the aforementioned locks. The code factors out the snapshot related work from create_snapshot to btrfs_mksnapshot. __btrfs_ioctl_snap_create btrfs_mksubvol create_subvol btrfs_mksnapshot btrfs_mksubvol create_snapshot Reviewed-by: Filipe Manana Signed-off-by: Robbie Ko Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ioctl.c | 70 ++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 709d9446896a..973236b72a97 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -748,7 +748,6 @@ static int create_snapshot(struct btrfs_root *root, struct inode *dir, struct btrfs_pending_snapshot *pending_snapshot; struct btrfs_trans_handle *trans; int ret; - bool snapshot_force_cow = false; if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) return -EINVAL; @@ -771,27 +770,6 @@ static int create_snapshot(struct btrfs_root *root, struct inode *dir, goto free_pending; } - /* - * Force new buffered writes to reserve space even when NOCOW is - * possible. This is to avoid later writeback (running dealloc) to - * fallback to COW mode and unexpectedly fail with ENOSPC. - */ - btrfs_drew_read_lock(&root->snapshot_lock); - - ret = btrfs_start_delalloc_snapshot(root); - if (ret) - goto dec_and_free; - - /* - * All previous writes have started writeback in NOCOW mode, so now - * we force future writes to fallback to COW mode during snapshot - * creation. - */ - atomic_inc(&root->snapshot_force_cow); - snapshot_force_cow = true; - - btrfs_wait_ordered_extents(root, U64_MAX, 0, (u64)-1); - btrfs_init_block_rsv(&pending_snapshot->block_rsv, BTRFS_BLOCK_RSV_TEMP); /* @@ -806,7 +784,7 @@ static int create_snapshot(struct btrfs_root *root, struct inode *dir, &pending_snapshot->block_rsv, 8, false); if (ret) - goto dec_and_free; + goto free_pending; pending_snapshot->dentry = dentry; pending_snapshot->root = root; @@ -848,11 +826,6 @@ static int create_snapshot(struct btrfs_root *root, struct inode *dir, fail: btrfs_put_root(pending_snapshot->snap); btrfs_subvolume_release_metadata(fs_info, &pending_snapshot->block_rsv); -dec_and_free: - if (snapshot_force_cow) - atomic_dec(&root->snapshot_force_cow); - btrfs_drew_read_unlock(&root->snapshot_lock); - free_pending: kfree(pending_snapshot->root_item); btrfs_free_path(pending_snapshot->path); @@ -983,6 +956,45 @@ out_unlock: return error; } +static noinline int btrfs_mksnapshot(const struct path *parent, + const char *name, int namelen, + struct btrfs_root *root, + bool readonly, + struct btrfs_qgroup_inherit *inherit) +{ + int ret; + bool snapshot_force_cow = false; + + /* + * Force new buffered writes to reserve space even when NOCOW is + * possible. This is to avoid later writeback (running dealloc) to + * fallback to COW mode and unexpectedly fail with ENOSPC. + */ + btrfs_drew_read_lock(&root->snapshot_lock); + + ret = btrfs_start_delalloc_snapshot(root); + if (ret) + goto out; + + /* + * All previous writes have started writeback in NOCOW mode, so now + * we force future writes to fallback to COW mode during snapshot + * creation. + */ + atomic_inc(&root->snapshot_force_cow); + snapshot_force_cow = true; + + btrfs_wait_ordered_extents(root, U64_MAX, 0, (u64)-1); + + ret = btrfs_mksubvol(parent, name, namelen, + root, readonly, inherit); +out: + if (snapshot_force_cow) + atomic_dec(&root->snapshot_force_cow); + btrfs_drew_read_unlock(&root->snapshot_lock); + return ret; +} + /* * When we're defragging a range, we don't want to kick it off again * if it is really just waiting for delalloc to send it down. @@ -1762,7 +1774,7 @@ static noinline int __btrfs_ioctl_snap_create(struct file *file, */ ret = -EPERM; } else { - ret = btrfs_mksubvol(&file->f_path, name, namelen, + ret = btrfs_mksnapshot(&file->f_path, name, namelen, BTRFS_I(src_inode)->root, readonly, inherit); } From 51415b6c1b117e223bc083e30af675cb5c5498f3 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 19 May 2020 10:13:20 +0800 Subject: [PATCH 0793/1043] btrfs: reloc: fix reloc root leak and NULL pointer dereference [BUG] When balance is canceled, there is a pretty high chance that unmounting the fs can lead to lead the NULL pointer dereference: BTRFS warning (device dm-3): page private not zero on page 223158272 ... BTRFS warning (device dm-3): page private not zero on page 223162368 BTRFS error (device dm-3): leaked root 18446744073709551608-304 refcount 1 BUG: kernel NULL pointer dereference, address: 0000000000000168 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page PGD 0 P4D 0 Oops: 0000 [#1] PREEMPT SMP NOPTI CPU: 2 PID: 5793 Comm: umount Tainted: G O 5.7.0-rc5-custom+ #53 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 0.0.0 02/06/2015 RIP: 0010:__lock_acquire+0x5dc/0x24c0 Call Trace: lock_acquire+0xab/0x390 _raw_spin_lock+0x39/0x80 btrfs_release_extent_buffer_pages+0xd7/0x200 [btrfs] release_extent_buffer+0xb2/0x170 [btrfs] free_extent_buffer+0x66/0xb0 [btrfs] btrfs_put_root+0x8e/0x130 [btrfs] btrfs_check_leaked_roots.cold+0x5/0x5d [btrfs] btrfs_free_fs_info+0xe5/0x120 [btrfs] btrfs_kill_super+0x1f/0x30 [btrfs] deactivate_locked_super+0x3b/0x80 deactivate_super+0x3e/0x50 cleanup_mnt+0x109/0x160 __cleanup_mnt+0x12/0x20 task_work_run+0x67/0xa0 exit_to_usermode_loop+0xc5/0xd0 syscall_return_slowpath+0x205/0x360 do_syscall_64+0x6e/0xb0 entry_SYSCALL_64_after_hwframe+0x49/0xb3 RIP: 0033:0x7fd028ef740b [CAUSE] When balance is canceled, all reloc roots are marked as orphan, and orphan reloc roots are going to be cleaned up. However for orphan reloc roots and merged reloc roots, their lifespan are quite different: Merged reloc roots | Orphan reloc roots by cancel -------------------------------------------------------------------- create_reloc_root() | create_reloc_root() |- refs == 1 | |- refs == 1 | btrfs_grab_root(reloc_root); | btrfs_grab_root(reloc_root); |- refs == 2 | |- refs == 2 | root->reloc_root = reloc_root; | root->reloc_root = reloc_root; >>> No difference so far <<< | prepare_to_merge() | prepare_to_merge() |- btrfs_set_root_refs(item, 1);| |- if (!err) (err == -EINTR) | merge_reloc_roots() | merge_reloc_roots() |- merge_reloc_root() | |- Doing nothing to put reloc root |- insert_dirty_subvol() | |- refs == 2 |- __del_reloc_root() | |- btrfs_put_root() | |- refs == 1 | >>> Now orphan reloc roots still have refs 2 <<< | clean_dirty_subvols() | clean_dirty_subvols() |- btrfs_drop_snapshot() | |- btrfS_drop_snapshot() |- reloc_root get freed | |- reloc_root still has refs 2 | related ebs get freed, but | reloc_root still recorded in | allocated_roots btrfs_check_leaked_roots() | btrfs_check_leaked_roots() |- No leaked roots | |- Leaked reloc_roots detected | |- btrfs_put_root() | |- free_extent_buffer(root->node); | |- eb already freed, caused NULL | pointer dereference [FIX] The fix is to clear fs_root->reloc_root and put it at merge_reloc_roots() time, so that we won't leak reloc roots. Fixes: d2311e698578 ("btrfs: relocation: Delay reloc tree deletion after merge_reloc_roots") CC: stable@vger.kernel.org # 5.1+ Tested-by: Johannes Thumshirn Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 58f56e01de0d..81b076e46143 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -1917,12 +1917,10 @@ again: reloc_root = list_entry(reloc_roots.next, struct btrfs_root, root_list); + root = read_fs_root(fs_info, reloc_root->root_key.offset); if (btrfs_root_refs(&reloc_root->root_item) > 0) { - root = read_fs_root(fs_info, - reloc_root->root_key.offset); BUG_ON(IS_ERR(root)); BUG_ON(root->reloc_root != reloc_root); - ret = merge_reloc_root(rc, root); btrfs_put_root(root); if (ret) { @@ -1932,6 +1930,14 @@ again: goto out; } } else { + if (!IS_ERR(root)) { + if (root->reloc_root == reloc_root) { + root->reloc_root = NULL; + btrfs_put_root(reloc_root); + } + btrfs_put_root(root); + } + list_del_init(&reloc_root->root_list); /* Don't forget to queue this reloc root for cleanup */ list_add_tail(&reloc_root->reloc_dirty_list, From 1dae7e0e58b484eaa43d530f211098fdeeb0f404 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Wed, 20 May 2020 14:58:51 +0800 Subject: [PATCH 0794/1043] btrfs: reloc: clear DEAD_RELOC_TREE bit for orphan roots to prevent runaway balance [BUG] There are several reported runaway balance, that balance is flooding the log with "found X extents" where the X never changes. [CAUSE] Commit d2311e698578 ("btrfs: relocation: Delay reloc tree deletion after merge_reloc_roots") introduced BTRFS_ROOT_DEAD_RELOC_TREE bit to indicate that one subvolume has finished its tree blocks swap with its reloc tree. However if balance is canceled or hits ENOSPC halfway, we didn't clear the BTRFS_ROOT_DEAD_RELOC_TREE bit, leaving that bit hanging forever until unmount. Any subvolume root with that bit, would cause backref cache to skip this tree block, as it has finished its tree block swap. This would cause all tree blocks of that root be ignored by balance, leading to runaway balance. [FIX] Fix the problem by also clearing the BTRFS_ROOT_DEAD_RELOC_TREE bit for the original subvolume of orphan reloc root. Add an umount check for the stale bit still set. Fixes: d2311e698578 ("btrfs: relocation: Delay reloc tree deletion after merge_reloc_roots") Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 1 + fs/btrfs/relocation.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index f1143741f6ba..7b66a8e4ca54 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -1994,6 +1994,7 @@ void btrfs_put_root(struct btrfs_root *root) if (refcount_dec_and_test(&root->refs)) { WARN_ON(!RB_EMPTY_ROOT(&root->inode_tree)); + WARN_ON(test_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state)); if (root->anon_dev) free_anon_bdev(root->anon_dev); btrfs_drew_lock_destroy(&root->snapshot_lock); diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 81b076e46143..68dd7a771e5e 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -1935,6 +1935,8 @@ again: root->reloc_root = NULL; btrfs_put_root(reloc_root); } + clear_bit(BTRFS_ROOT_DEAD_RELOC_TREE, + &root->state); btrfs_put_root(root); } From 56e9357a1e8167134388d4c70654795353765c7b Mon Sep 17 00:00:00 2001 From: David Sterba Date: Fri, 15 May 2020 19:35:55 +0200 Subject: [PATCH 0795/1043] btrfs: simplify root lookup by id The main function to lookup a root by its id btrfs_get_fs_root takes the whole key, while only using the objectid. The value of offset is preset to (u64)-1 but not actually used until btrfs_find_root that does the actual search. Switch btrfs_get_fs_root to use only objectid and remove all local variables that existed just for the lookup. The actual key for search is set up in btrfs_get_fs_root, reusing another key variable. Signed-off-by: David Sterba --- fs/btrfs/backref.c | 13 ++----------- fs/btrfs/disk-io.c | 39 ++++++++++++++++++--------------------- fs/btrfs/disk-io.h | 3 +-- fs/btrfs/export.c | 6 +----- fs/btrfs/file.c | 6 +----- fs/btrfs/inode.c | 2 +- fs/btrfs/ioctl.c | 28 ++++++---------------------- fs/btrfs/relocation.c | 8 +------- fs/btrfs/root-tree.c | 12 +++++------- fs/btrfs/scrub.c | 6 +----- fs/btrfs/send.c | 15 ++++----------- fs/btrfs/super.c | 5 +---- fs/btrfs/transaction.c | 2 +- fs/btrfs/tree-log.c | 8 ++------ fs/btrfs/uuid-tree.c | 6 +----- 15 files changed, 46 insertions(+), 113 deletions(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 41ef3fa91864..d888e71e66b6 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -538,18 +538,13 @@ static int resolve_indirect_ref(struct btrfs_fs_info *fs_info, const u64 *extent_item_pos, bool ignore_offset) { struct btrfs_root *root; - struct btrfs_key root_key; struct extent_buffer *eb; int ret = 0; int root_level; int level = ref->level; struct btrfs_key search_key = ref->key_for_search; - root_key.objectid = ref->root_id; - root_key.type = BTRFS_ROOT_ITEM_KEY; - root_key.offset = (u64)-1; - - root = btrfs_get_fs_root(fs_info, &root_key, false); + root = btrfs_get_fs_root(fs_info, ref->root_id, false); if (IS_ERR(root)) { ret = PTR_ERR(root); goto out_free; @@ -2690,16 +2685,12 @@ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, struct btrfs_backref_edge *edge; struct extent_buffer *eb; struct btrfs_root *root; - struct btrfs_key root_key; struct rb_node *rb_node; int level; bool need_check = true; int ret; - root_key.objectid = ref_key->offset; - root_key.type = BTRFS_ROOT_ITEM_KEY; - root_key.offset = (u64)-1; - root = btrfs_get_fs_root(fs_info, &root_key, false); + root = btrfs_get_fs_root(fs_info, ref_key->offset, false); if (IS_ERR(root)) return PTR_ERR(root); if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 7b66a8e4ca54..f2f2864f5978 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -1534,35 +1534,34 @@ void btrfs_free_fs_info(struct btrfs_fs_info *fs_info) struct btrfs_root *btrfs_get_fs_root(struct btrfs_fs_info *fs_info, - struct btrfs_key *location, - bool check_ref) + u64 objectid, bool check_ref) { struct btrfs_root *root; struct btrfs_path *path; struct btrfs_key key; int ret; - if (location->objectid == BTRFS_ROOT_TREE_OBJECTID) + if (objectid == BTRFS_ROOT_TREE_OBJECTID) return btrfs_grab_root(fs_info->tree_root); - if (location->objectid == BTRFS_EXTENT_TREE_OBJECTID) + if (objectid == BTRFS_EXTENT_TREE_OBJECTID) return btrfs_grab_root(fs_info->extent_root); - if (location->objectid == BTRFS_CHUNK_TREE_OBJECTID) + if (objectid == BTRFS_CHUNK_TREE_OBJECTID) return btrfs_grab_root(fs_info->chunk_root); - if (location->objectid == BTRFS_DEV_TREE_OBJECTID) + if (objectid == BTRFS_DEV_TREE_OBJECTID) return btrfs_grab_root(fs_info->dev_root); - if (location->objectid == BTRFS_CSUM_TREE_OBJECTID) + if (objectid == BTRFS_CSUM_TREE_OBJECTID) return btrfs_grab_root(fs_info->csum_root); - if (location->objectid == BTRFS_QUOTA_TREE_OBJECTID) + if (objectid == BTRFS_QUOTA_TREE_OBJECTID) return btrfs_grab_root(fs_info->quota_root) ? fs_info->quota_root : ERR_PTR(-ENOENT); - if (location->objectid == BTRFS_UUID_TREE_OBJECTID) + if (objectid == BTRFS_UUID_TREE_OBJECTID) return btrfs_grab_root(fs_info->uuid_root) ? fs_info->uuid_root : ERR_PTR(-ENOENT); - if (location->objectid == BTRFS_FREE_SPACE_TREE_OBJECTID) + if (objectid == BTRFS_FREE_SPACE_TREE_OBJECTID) return btrfs_grab_root(fs_info->free_space_root) ? fs_info->free_space_root : ERR_PTR(-ENOENT); again: - root = btrfs_lookup_fs_root(fs_info, location->objectid); + root = btrfs_lookup_fs_root(fs_info, objectid); if (root) { if (check_ref && btrfs_root_refs(&root->root_item) == 0) { btrfs_put_root(root); @@ -1571,7 +1570,10 @@ again: return root; } - root = btrfs_read_tree_root(fs_info->tree_root, location); + key.objectid = objectid; + key.type = BTRFS_ROOT_ITEM_KEY; + key.offset = (u64)-1; + root = btrfs_read_tree_root(fs_info->tree_root, &key); if (IS_ERR(root)) return root; @@ -1591,7 +1593,7 @@ again: } key.objectid = BTRFS_ORPHAN_OBJECTID; key.type = BTRFS_ORPHAN_ITEM_KEY; - key.offset = location->objectid; + key.offset = objectid; ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); btrfs_free_path(path); @@ -2293,8 +2295,8 @@ static int btrfs_read_roots(struct btrfs_fs_info *fs_info) * This tree can share blocks with some other fs tree during relocation * and we need a proper setup by btrfs_get_fs_root */ - location.objectid = BTRFS_DATA_RELOC_TREE_OBJECTID; - root = btrfs_get_fs_root(tree_root->fs_info, &location, true); + root = btrfs_get_fs_root(tree_root->fs_info, + BTRFS_DATA_RELOC_TREE_OBJECTID, true); if (IS_ERR(root)) { ret = PTR_ERR(root); goto out; @@ -2839,7 +2841,6 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device u64 generation; u64 features; u16 csum_type; - struct btrfs_key location; struct btrfs_super_block *disk_super; struct btrfs_fs_info *fs_info = btrfs_sb(sb); struct btrfs_root *tree_root; @@ -3253,11 +3254,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device } } - location.objectid = BTRFS_FS_TREE_OBJECTID; - location.type = BTRFS_ROOT_ITEM_KEY; - location.offset = 0; - - fs_info->fs_root = btrfs_get_fs_root(fs_info, &location, true); + fs_info->fs_root = btrfs_get_fs_root(fs_info, BTRFS_FS_TREE_OBJECTID, true); if (IS_ERR(fs_info->fs_root)) { err = PTR_ERR(fs_info->fs_root); btrfs_warn(fs_info, "failed to read fs tree: %d", err); diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index 734bc5270b6a..bf43245406c4 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -66,8 +66,7 @@ int btrfs_insert_fs_root(struct btrfs_fs_info *fs_info, void btrfs_free_fs_roots(struct btrfs_fs_info *fs_info); struct btrfs_root *btrfs_get_fs_root(struct btrfs_fs_info *fs_info, - struct btrfs_key *key, - bool check_ref); + u64 objectid, bool check_ref); void btrfs_free_fs_info(struct btrfs_fs_info *fs_info); int btrfs_cleanup_fs_roots(struct btrfs_fs_info *fs_info); diff --git a/fs/btrfs/export.c b/fs/btrfs/export.c index 2bb25d2dc44b..e7cc98b4d7dc 100644 --- a/fs/btrfs/export.c +++ b/fs/btrfs/export.c @@ -69,11 +69,7 @@ struct dentry *btrfs_get_dentry(struct super_block *sb, u64 objectid, if (objectid < BTRFS_FIRST_FREE_OBJECTID) return ERR_PTR(-ESTALE); - key.objectid = root_objectid; - key.type = BTRFS_ROOT_ITEM_KEY; - key.offset = (u64)-1; - - root = btrfs_get_fs_root(fs_info, &key, true); + root = btrfs_get_fs_root(fs_info, root_objectid, true); if (IS_ERR(root)) return ERR_CAST(root); diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 606c2f3c1a38..14e1464870ab 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -281,11 +281,7 @@ static int __btrfs_run_defrag_inode(struct btrfs_fs_info *fs_info, int ret; /* get the inode */ - key.objectid = defrag->root; - key.type = BTRFS_ROOT_ITEM_KEY; - key.offset = (u64)-1; - - inode_root = btrfs_get_fs_root(fs_info, &key, true); + inode_root = btrfs_get_fs_root(fs_info, defrag->root, true); if (IS_ERR(inode_root)) { ret = PTR_ERR(inode_root); goto cleanup; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 5be7820afe90..e0e8f74fa262 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -5145,7 +5145,7 @@ static int fixup_tree_root_location(struct btrfs_fs_info *fs_info, btrfs_release_path(path); - new_root = btrfs_get_fs_root(fs_info, location, true); + new_root = btrfs_get_fs_root(fs_info, location->objectid, true); if (IS_ERR(new_root)) { err = PTR_ERR(new_root); goto out; diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 973236b72a97..05a8688ed4fd 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -660,7 +660,7 @@ static noinline int create_subvol(struct inode *dir, goto fail; key.offset = (u64)-1; - new_root = btrfs_get_fs_root(fs_info, &key, true); + new_root = btrfs_get_fs_root(fs_info, objectid, true); if (IS_ERR(new_root)) { ret = PTR_ERR(new_root); btrfs_abort_transaction(trans, ret); @@ -2139,10 +2139,7 @@ static noinline int search_ioctl(struct inode *inode, /* search the root of the inode that was passed */ root = btrfs_grab_root(BTRFS_I(inode)->root); } else { - key.objectid = sk->tree_id; - key.type = BTRFS_ROOT_ITEM_KEY; - key.offset = (u64)-1; - root = btrfs_get_fs_root(info, &key, true); + root = btrfs_get_fs_root(info, sk->tree_id, true); if (IS_ERR(root)) { btrfs_free_path(path); return PTR_ERR(root); @@ -2275,10 +2272,7 @@ static noinline int btrfs_search_path_in_tree(struct btrfs_fs_info *info, ptr = &name[BTRFS_INO_LOOKUP_PATH_MAX - 1]; - key.objectid = tree_id; - key.type = BTRFS_ROOT_ITEM_KEY; - key.offset = (u64)-1; - root = btrfs_get_fs_root(info, &key, true); + root = btrfs_get_fs_root(info, tree_id, true); if (IS_ERR(root)) { ret = PTR_ERR(root); root = NULL; @@ -2371,10 +2365,7 @@ static int btrfs_search_path_in_tree_user(struct inode *inode, if (dirid != upper_limit.objectid) { ptr = &args->path[BTRFS_INO_LOOKUP_USER_PATH_MAX - 1]; - key.objectid = treeid; - key.type = BTRFS_ROOT_ITEM_KEY; - key.offset = (u64)-1; - root = btrfs_get_fs_root(fs_info, &key, true); + root = btrfs_get_fs_root(fs_info, treeid, true); if (IS_ERR(root)) { ret = PTR_ERR(root); goto out; @@ -2620,9 +2611,7 @@ static int btrfs_ioctl_get_subvol_info(struct file *file, void __user *argp) /* Get root_item of inode's subvolume */ key.objectid = BTRFS_I(inode)->root->root_key.objectid; - key.type = BTRFS_ROOT_ITEM_KEY; - key.offset = (u64)-1; - root = btrfs_get_fs_root(fs_info, &key, true); + root = btrfs_get_fs_root(fs_info, key.objectid, true); if (IS_ERR(root)) { ret = PTR_ERR(root); goto out_free; @@ -3290,7 +3279,6 @@ static long btrfs_ioctl_default_subvol(struct file *file, void __user *argp) struct btrfs_dir_item *di; struct btrfs_trans_handle *trans; struct btrfs_path *path = NULL; - struct btrfs_key location; struct btrfs_disk_key disk_key; u64 objectid = 0; u64 dir_id; @@ -3311,11 +3299,7 @@ static long btrfs_ioctl_default_subvol(struct file *file, void __user *argp) if (!objectid) objectid = BTRFS_FS_TREE_OBJECTID; - location.objectid = objectid; - location.type = BTRFS_ROOT_ITEM_KEY; - location.offset = (u64)-1; - - new_root = btrfs_get_fs_root(fs_info, &location, true); + new_root = btrfs_get_fs_root(fs_info, objectid, true); if (IS_ERR(new_root)) { ret = PTR_ERR(new_root); goto out; diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 68dd7a771e5e..19db203a8f3b 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -368,13 +368,7 @@ struct btrfs_root *find_reloc_root(struct btrfs_fs_info *fs_info, u64 bytenr) static struct btrfs_root *read_fs_root(struct btrfs_fs_info *fs_info, u64 root_objectid) { - struct btrfs_key key; - - key.objectid = root_objectid; - key.type = BTRFS_ROOT_ITEM_KEY; - key.offset = (u64)-1; - - return btrfs_get_fs_root(fs_info, &key, false); + return btrfs_get_fs_root(fs_info, root_objectid, false); } /* diff --git a/fs/btrfs/root-tree.c b/fs/btrfs/root-tree.c index 668f22844017..c89697486366 100644 --- a/fs/btrfs/root-tree.c +++ b/fs/btrfs/root-tree.c @@ -210,7 +210,6 @@ int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info) struct extent_buffer *leaf; struct btrfs_path *path; struct btrfs_key key; - struct btrfs_key root_key; struct btrfs_root *root; int err = 0; int ret; @@ -223,10 +222,9 @@ int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info) key.type = BTRFS_ORPHAN_ITEM_KEY; key.offset = 0; - root_key.type = BTRFS_ROOT_ITEM_KEY; - root_key.offset = (u64)-1; - while (1) { + u64 root_objectid; + ret = btrfs_search_slot(NULL, tree_root, &key, path, 0, 0); if (ret < 0) { err = ret; @@ -250,10 +248,10 @@ int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info) key.type != BTRFS_ORPHAN_ITEM_KEY) break; - root_key.objectid = key.offset; + root_objectid = key.offset; key.offset++; - root = btrfs_get_fs_root(fs_info, &root_key, false); + root = btrfs_get_fs_root(fs_info, root_objectid, false); err = PTR_ERR_OR_ZERO(root); if (err && err != -ENOENT) { break; @@ -270,7 +268,7 @@ int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info) break; } err = btrfs_del_orphan_item(trans, tree_root, - root_key.objectid); + root_objectid); btrfs_end_transaction(trans); if (err) { btrfs_handle_fs_error(fs_info, err, diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 0f7740970553..016a025e36c7 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -647,13 +647,9 @@ static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root, struct btrfs_fs_info *fs_info = swarn->dev->fs_info; struct inode_fs_paths *ipath = NULL; struct btrfs_root *local_root; - struct btrfs_key root_key; struct btrfs_key key; - root_key.objectid = root; - root_key.type = BTRFS_ROOT_ITEM_KEY; - root_key.offset = (u64)-1; - local_root = btrfs_get_fs_root(fs_info, &root_key, true); + local_root = btrfs_get_fs_root(fs_info, root, true); if (IS_ERR(local_root)) { ret = PTR_ERR(local_root); goto err; diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 4f3b8d2bb56b..3ddd3b9778c7 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -7088,7 +7088,6 @@ long btrfs_ioctl_send(struct file *mnt_file, struct btrfs_ioctl_send_args *arg) struct btrfs_root *send_root = BTRFS_I(file_inode(mnt_file))->root; struct btrfs_fs_info *fs_info = send_root->fs_info; struct btrfs_root *clone_root; - struct btrfs_key key; struct send_ctx *sctx = NULL; u32 i; u64 *clone_sources_tmp = NULL; @@ -7217,11 +7216,8 @@ long btrfs_ioctl_send(struct file *mnt_file, struct btrfs_ioctl_send_args *arg) } for (i = 0; i < arg->clone_sources_count; i++) { - key.objectid = clone_sources_tmp[i]; - key.type = BTRFS_ROOT_ITEM_KEY; - key.offset = (u64)-1; - - clone_root = btrfs_get_fs_root(fs_info, &key, true); + clone_root = btrfs_get_fs_root(fs_info, + clone_sources_tmp[i], true); if (IS_ERR(clone_root)) { ret = PTR_ERR(clone_root); goto out; @@ -7252,11 +7248,8 @@ long btrfs_ioctl_send(struct file *mnt_file, struct btrfs_ioctl_send_args *arg) } if (arg->parent_root) { - key.objectid = arg->parent_root; - key.type = BTRFS_ROOT_ITEM_KEY; - key.offset = (u64)-1; - - sctx->parent_root = btrfs_get_fs_root(fs_info, &key, true); + sctx->parent_root = btrfs_get_fs_root(fs_info, arg->parent_root, + true); if (IS_ERR(sctx->parent_root)) { ret = PTR_ERR(sctx->parent_root); goto out; diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 438ecba26557..6bbf84a26501 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -1102,10 +1102,7 @@ char *btrfs_get_subvol_name_from_objectid(struct btrfs_fs_info *fs_info, dirid = btrfs_root_ref_dirid(path->nodes[0], root_ref); btrfs_release_path(path); - key.objectid = subvol_objectid; - key.type = BTRFS_ROOT_ITEM_KEY; - key.offset = (u64)-1; - fs_root = btrfs_get_fs_root(fs_info, &key, true); + fs_root = btrfs_get_fs_root(fs_info, subvol_objectid, true); if (IS_ERR(fs_root)) { ret = PTR_ERR(fs_root); fs_root = NULL; diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index a8cc2ddffc17..b359d4b17658 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -1630,7 +1630,7 @@ static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans, } key.offset = (u64)-1; - pending->snap = btrfs_get_fs_root(fs_info, &key, true); + pending->snap = btrfs_get_fs_root(fs_info, objectid, true); if (IS_ERR(pending->snap)) { ret = PTR_ERR(pending->snap); btrfs_abort_transaction(trans, ret); diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 60febf2082ee..d3662e102b2e 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -6112,7 +6112,6 @@ int btrfs_recover_log_trees(struct btrfs_root *log_root_tree) struct btrfs_trans_handle *trans; struct btrfs_key key; struct btrfs_key found_key; - struct btrfs_key tmp_key; struct btrfs_root *log; struct btrfs_fs_info *fs_info = log_root_tree->fs_info; struct walk_control wc = { @@ -6174,11 +6173,8 @@ again: goto error; } - tmp_key.objectid = found_key.offset; - tmp_key.type = BTRFS_ROOT_ITEM_KEY; - tmp_key.offset = (u64)-1; - - wc.replay_dest = btrfs_get_fs_root(fs_info, &tmp_key, true); + wc.replay_dest = btrfs_get_fs_root(fs_info, found_key.offset, + true); if (IS_ERR(wc.replay_dest)) { ret = PTR_ERR(wc.replay_dest); diff --git a/fs/btrfs/uuid-tree.c b/fs/btrfs/uuid-tree.c index 76671a6bcb61..28525ad7ff8c 100644 --- a/fs/btrfs/uuid-tree.c +++ b/fs/btrfs/uuid-tree.c @@ -257,7 +257,6 @@ out: static int btrfs_check_uuid_tree_entry(struct btrfs_fs_info *fs_info, u8 *uuid, u8 type, u64 subvolid) { - struct btrfs_key key; int ret = 0; struct btrfs_root *subvol_root; @@ -265,10 +264,7 @@ static int btrfs_check_uuid_tree_entry(struct btrfs_fs_info *fs_info, type != BTRFS_UUID_KEY_RECEIVED_SUBVOL) goto out; - key.objectid = subvolid; - key.type = BTRFS_ROOT_ITEM_KEY; - key.offset = (u64)-1; - subvol_root = btrfs_get_fs_root(fs_info, &key, true); + subvol_root = btrfs_get_fs_root(fs_info, subvolid, true); if (IS_ERR(subvol_root)) { ret = PTR_ERR(subvol_root); if (ret == -ENOENT) From a820feb546842a4ab5678279a62d4450320e6696 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Fri, 15 May 2020 19:35:57 +0200 Subject: [PATCH 0796/1043] btrfs: open code read_fs_root After the update to btrfs_get_fs_root, read_fs_root has become trivial wrapper that can be open coded. Signed-off-by: David Sterba --- fs/btrfs/relocation.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 19db203a8f3b..018f830e31b5 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -365,12 +365,6 @@ struct btrfs_root *find_reloc_root(struct btrfs_fs_info *fs_info, u64 bytenr) return btrfs_grab_root(root); } -static struct btrfs_root *read_fs_root(struct btrfs_fs_info *fs_info, - u64 root_objectid) -{ - return btrfs_get_fs_root(fs_info, root_objectid, false); -} - /* * For useless nodes, do two major clean ups: * @@ -1850,7 +1844,8 @@ again: struct btrfs_root, root_list); list_del_init(&reloc_root->root_list); - root = read_fs_root(fs_info, reloc_root->root_key.offset); + root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, + false); BUG_ON(IS_ERR(root)); BUG_ON(root->reloc_root != reloc_root); @@ -1911,7 +1906,8 @@ again: reloc_root = list_entry(reloc_roots.next, struct btrfs_root, root_list); - root = read_fs_root(fs_info, reloc_root->root_key.offset); + root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, + false); if (btrfs_root_refs(&reloc_root->root_item) > 0) { BUG_ON(IS_ERR(root)); BUG_ON(root->reloc_root != reloc_root); @@ -1995,7 +1991,7 @@ static int record_reloc_root_in_trans(struct btrfs_trans_handle *trans, if (reloc_root->last_trans == trans->transid) return 0; - root = read_fs_root(fs_info, reloc_root->root_key.offset); + root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, false); BUG_ON(IS_ERR(root)); BUG_ON(root->reloc_root != reloc_root); ret = btrfs_record_root_in_trans(trans, root); @@ -3771,8 +3767,8 @@ int btrfs_recover_relocation(struct btrfs_root *root) list_add(&reloc_root->root_list, &reloc_roots); if (btrfs_root_refs(&reloc_root->root_item) > 0) { - fs_root = read_fs_root(fs_info, - reloc_root->root_key.offset); + fs_root = btrfs_get_fs_root(fs_info, + reloc_root->root_key.offset, false); if (IS_ERR(fs_root)) { ret = PTR_ERR(fs_root); if (ret != -ENOENT) { @@ -3828,7 +3824,8 @@ int btrfs_recover_relocation(struct btrfs_root *root) continue; } - fs_root = read_fs_root(fs_info, reloc_root->root_key.offset); + fs_root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, + false); if (IS_ERR(fs_root)) { err = PTR_ERR(fs_root); list_add_tail(&reloc_root->root_list, &reloc_roots); From 0202e83fdab05b3bf641804afea57a2bfcbcbd70 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Fri, 15 May 2020 19:35:59 +0200 Subject: [PATCH 0797/1043] btrfs: simplify iget helpers The inode lookup starting at btrfs_iget takes the full location key, while only the objectid is used to match the inode, because the lookup happens inside the given root thus the inode number is unique. The entire location key is properly set up in btrfs_init_locked_inode. Simplify the helpers and pass only inode number, renaming it to 'ino' instead of 'objectid'. This allows to remove temporary variables key, saving some stack space. Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 5 ++--- fs/btrfs/export.c | 11 ++-------- fs/btrfs/file.c | 6 +----- fs/btrfs/free-space-cache.c | 2 +- fs/btrfs/inode.c | 40 +++++++++++++++++++------------------ fs/btrfs/ioctl.c | 2 +- fs/btrfs/props.c | 9 ++------- fs/btrfs/relocation.c | 13 ++---------- fs/btrfs/send.c | 7 +------ fs/btrfs/super.c | 6 +----- fs/btrfs/tree-log.c | 24 ++++++++++------------ 11 files changed, 44 insertions(+), 81 deletions(-) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 616fdbc53695..5afeb17a3f1a 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -2898,10 +2898,9 @@ void btrfs_free_inode(struct inode *inode); int btrfs_drop_inode(struct inode *inode); int __init btrfs_init_cachep(void); void __cold btrfs_destroy_cachep(void); -struct inode *btrfs_iget_path(struct super_block *s, struct btrfs_key *location, +struct inode *btrfs_iget_path(struct super_block *s, u64 ino, struct btrfs_root *root, struct btrfs_path *path); -struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location, - struct btrfs_root *root); +struct inode *btrfs_iget(struct super_block *s, u64 ino, struct btrfs_root *root); struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, struct page *page, size_t pg_offset, u64 start, u64 end); diff --git a/fs/btrfs/export.c b/fs/btrfs/export.c index e7cc98b4d7dc..1a8d419d9e1f 100644 --- a/fs/btrfs/export.c +++ b/fs/btrfs/export.c @@ -64,7 +64,6 @@ struct dentry *btrfs_get_dentry(struct super_block *sb, u64 objectid, struct btrfs_fs_info *fs_info = btrfs_sb(sb); struct btrfs_root *root; struct inode *inode; - struct btrfs_key key; if (objectid < BTRFS_FIRST_FREE_OBJECTID) return ERR_PTR(-ESTALE); @@ -73,11 +72,7 @@ struct dentry *btrfs_get_dentry(struct super_block *sb, u64 objectid, if (IS_ERR(root)) return ERR_CAST(root); - key.objectid = objectid; - key.type = BTRFS_INODE_ITEM_KEY; - key.offset = 0; - - inode = btrfs_iget(sb, &key, root); + inode = btrfs_iget(sb, objectid, root); btrfs_put_root(root); if (IS_ERR(inode)) return ERR_CAST(inode); @@ -196,9 +191,7 @@ struct dentry *btrfs_get_parent(struct dentry *child) found_key.offset, 0, 0); } - key.type = BTRFS_INODE_ITEM_KEY; - key.offset = 0; - return d_obtain_alias(btrfs_iget(fs_info->sb, &key, root)); + return d_obtain_alias(btrfs_iget(fs_info->sb, key.objectid, root)); fail: btrfs_free_path(path); return ERR_PTR(ret); diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 14e1464870ab..2c14312b05e8 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -275,7 +275,6 @@ static int __btrfs_run_defrag_inode(struct btrfs_fs_info *fs_info, { struct btrfs_root *inode_root; struct inode *inode; - struct btrfs_key key; struct btrfs_ioctl_defrag_range_args range; int num_defrag; int ret; @@ -287,10 +286,7 @@ static int __btrfs_run_defrag_inode(struct btrfs_fs_info *fs_info, goto cleanup; } - key.objectid = defrag->ino; - key.type = BTRFS_INODE_ITEM_KEY; - key.offset = 0; - inode = btrfs_iget(fs_info->sb, &key, inode_root); + inode = btrfs_iget(fs_info->sb, defrag->ino, inode_root); btrfs_put_root(inode_root); if (IS_ERR(inode)) { ret = PTR_ERR(inode); diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index 3c353a337b91..525bc5a250da 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -82,7 +82,7 @@ static struct inode *__lookup_free_space_inode(struct btrfs_root *root, * sure NOFS is set to keep us from deadlocking. */ nofs_flag = memalloc_nofs_save(); - inode = btrfs_iget_path(fs_info->sb, &location, root, path); + inode = btrfs_iget_path(fs_info->sb, location.objectid, root, path); btrfs_release_path(path); memalloc_nofs_restore(nofs_flag); if (IS_ERR(inode)) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index e0e8f74fa262..4ec7f34e6cd9 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -51,7 +51,7 @@ #include "block-group.h" struct btrfs_iget_args { - struct btrfs_key *location; + u64 ino; struct btrfs_root *root; }; @@ -2978,7 +2978,7 @@ int btrfs_orphan_cleanup(struct btrfs_root *root) found_key.objectid = found_key.offset; found_key.type = BTRFS_INODE_ITEM_KEY; found_key.offset = 0; - inode = btrfs_iget(fs_info->sb, &found_key, root); + inode = btrfs_iget(fs_info->sb, last_objectid, root); ret = PTR_ERR_OR_ZERO(inode); if (ret && ret != -ENOENT) goto out; @@ -5223,9 +5223,11 @@ static void inode_tree_del(struct inode *inode) static int btrfs_init_locked_inode(struct inode *inode, void *p) { struct btrfs_iget_args *args = p; - inode->i_ino = args->location->objectid; - memcpy(&BTRFS_I(inode)->location, args->location, - sizeof(*args->location)); + + inode->i_ino = args->ino; + BTRFS_I(inode)->location.objectid = args->ino; + BTRFS_I(inode)->location.type = BTRFS_INODE_ITEM_KEY; + BTRFS_I(inode)->location.offset = 0; BTRFS_I(inode)->root = btrfs_grab_root(args->root); BUG_ON(args->root && !BTRFS_I(inode)->root); return 0; @@ -5234,19 +5236,19 @@ static int btrfs_init_locked_inode(struct inode *inode, void *p) static int btrfs_find_actor(struct inode *inode, void *opaque) { struct btrfs_iget_args *args = opaque; - return args->location->objectid == BTRFS_I(inode)->location.objectid && + + return args->ino == BTRFS_I(inode)->location.objectid && args->root == BTRFS_I(inode)->root; } -static struct inode *btrfs_iget_locked(struct super_block *s, - struct btrfs_key *location, +static struct inode *btrfs_iget_locked(struct super_block *s, u64 ino, struct btrfs_root *root) { struct inode *inode; struct btrfs_iget_args args; - unsigned long hashval = btrfs_inode_hash(location->objectid, root); + unsigned long hashval = btrfs_inode_hash(ino, root); - args.location = location; + args.ino = ino; args.root = root; inode = iget5_locked(s, hashval, btrfs_find_actor, @@ -5256,17 +5258,17 @@ static struct inode *btrfs_iget_locked(struct super_block *s, } /* - * Get an inode object given its location and corresponding root. + * Get an inode object given its inode number and corresponding root. * Path can be preallocated to prevent recursing back to iget through * allocator. NULL is also valid but may require an additional allocation * later. */ -struct inode *btrfs_iget_path(struct super_block *s, struct btrfs_key *location, +struct inode *btrfs_iget_path(struct super_block *s, u64 ino, struct btrfs_root *root, struct btrfs_path *path) { struct inode *inode; - inode = btrfs_iget_locked(s, location, root); + inode = btrfs_iget_locked(s, ino, root); if (!inode) return ERR_PTR(-ENOMEM); @@ -5293,10 +5295,9 @@ struct inode *btrfs_iget_path(struct super_block *s, struct btrfs_key *location, return inode; } -struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location, - struct btrfs_root *root) +struct inode *btrfs_iget(struct super_block *s, u64 ino, struct btrfs_root *root) { - return btrfs_iget_path(s, location, root, NULL); + return btrfs_iget_path(s, ino, root, NULL); } static struct inode *new_simple_dir(struct super_block *s, @@ -5365,7 +5366,7 @@ struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry) return ERR_PTR(ret); if (location.type == BTRFS_INODE_ITEM_KEY) { - inode = btrfs_iget(dir->i_sb, &location, root); + inode = btrfs_iget(dir->i_sb, location.objectid, root); if (IS_ERR(inode)) return inode; @@ -5389,7 +5390,7 @@ struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry) else inode = new_simple_dir(dir->i_sb, &location, sub_root); } else { - inode = btrfs_iget(dir->i_sb, &location, sub_root); + inode = btrfs_iget(dir->i_sb, location.objectid, sub_root); } if (root != sub_root) btrfs_put_root(sub_root); @@ -5770,7 +5771,8 @@ int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index) static int btrfs_insert_inode_locked(struct inode *inode) { struct btrfs_iget_args args; - args.location = &BTRFS_I(inode)->location; + + args.ino = BTRFS_I(inode)->location.objectid; args.root = BTRFS_I(inode)->root; return insert_inode_locked4(inode, diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 05a8688ed4fd..168deb8ef68a 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -2424,7 +2424,7 @@ static int btrfs_search_path_in_tree_user(struct inode *inode, goto out_put; } - temp_inode = btrfs_iget(sb, &key2, root); + temp_inode = btrfs_iget(sb, key2.objectid, root); if (IS_ERR(temp_inode)) { ret = PTR_ERR(temp_inode); goto out_put; diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c index ff1ff90e48b1..2dcb1cb21634 100644 --- a/fs/btrfs/props.c +++ b/fs/btrfs/props.c @@ -408,19 +408,14 @@ int btrfs_subvol_inherit_props(struct btrfs_trans_handle *trans, struct btrfs_root *parent_root) { struct super_block *sb = root->fs_info->sb; - struct btrfs_key key; struct inode *parent_inode, *child_inode; int ret; - key.objectid = BTRFS_FIRST_FREE_OBJECTID; - key.type = BTRFS_INODE_ITEM_KEY; - key.offset = 0; - - parent_inode = btrfs_iget(sb, &key, parent_root); + parent_inode = btrfs_iget(sb, BTRFS_FIRST_FREE_OBJECTID, parent_root); if (IS_ERR(parent_inode)) return PTR_ERR(parent_inode); - child_inode = btrfs_iget(sb, &key, root); + child_inode = btrfs_iget(sb, BTRFS_FIRST_FREE_OBJECTID, root); if (IS_ERR(child_inode)) { iput(parent_inode); return PTR_ERR(child_inode); diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index 018f830e31b5..3bbae80c752f 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -2970,7 +2970,6 @@ static int delete_block_group_cache(struct btrfs_fs_info *fs_info, struct inode *inode, u64 ino) { - struct btrfs_key key; struct btrfs_root *root = fs_info->tree_root; struct btrfs_trans_handle *trans; int ret = 0; @@ -2978,11 +2977,7 @@ static int delete_block_group_cache(struct btrfs_fs_info *fs_info, if (inode) goto truncate; - key.objectid = ino; - key.type = BTRFS_INODE_ITEM_KEY; - key.offset = 0; - - inode = btrfs_iget(fs_info->sb, &key, root); + inode = btrfs_iget(fs_info->sb, ino, root); if (IS_ERR(inode)) return -ENOENT; @@ -3470,7 +3465,6 @@ struct inode *create_reloc_inode(struct btrfs_fs_info *fs_info, struct inode *inode = NULL; struct btrfs_trans_handle *trans; struct btrfs_root *root; - struct btrfs_key key; u64 objectid; int err = 0; @@ -3488,10 +3482,7 @@ struct inode *create_reloc_inode(struct btrfs_fs_info *fs_info, err = __insert_orphan_inode(trans, root, objectid); BUG_ON(err); - key.objectid = objectid; - key.type = BTRFS_INODE_ITEM_KEY; - key.offset = 0; - inode = btrfs_iget(fs_info->sb, &key, root); + inode = btrfs_iget(fs_info->sb, objectid, root); BUG_ON(IS_ERR(inode)); BTRFS_I(inode)->index_cnt = group->start; diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 3ddd3b9778c7..0f37660b14b2 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -4806,17 +4806,12 @@ static ssize_t fill_read_buf(struct send_ctx *sctx, u64 offset, u32 len) struct inode *inode; struct page *page; char *addr; - struct btrfs_key key; pgoff_t index = offset >> PAGE_SHIFT; pgoff_t last_index; unsigned pg_offset = offset_in_page(offset); ssize_t ret = 0; - key.objectid = sctx->cur_ino; - key.type = BTRFS_INODE_ITEM_KEY; - key.offset = 0; - - inode = btrfs_iget(fs_info->sb, &key, root); + inode = btrfs_iget(fs_info->sb, sctx->cur_ino, root); if (IS_ERR(inode)) return PTR_ERR(inode); diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 6bbf84a26501..bc73fd670702 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -1217,7 +1217,6 @@ static int btrfs_fill_super(struct super_block *sb, { struct inode *inode; struct btrfs_fs_info *fs_info = btrfs_sb(sb); - struct btrfs_key key; int err; sb->s_maxbytes = MAX_LFS_FILESIZE; @@ -1245,10 +1244,7 @@ static int btrfs_fill_super(struct super_block *sb, return err; } - key.objectid = BTRFS_FIRST_FREE_OBJECTID; - key.type = BTRFS_INODE_ITEM_KEY; - key.offset = 0; - inode = btrfs_iget(sb, &key, fs_info->fs_root); + inode = btrfs_iget(sb, BTRFS_FIRST_FREE_OBJECTID, fs_info->fs_root); if (IS_ERR(inode)) { err = PTR_ERR(inode); goto fail_close; diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index d3662e102b2e..67fa7087f707 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -550,13 +550,9 @@ no_copy: static noinline struct inode *read_one_inode(struct btrfs_root *root, u64 objectid) { - struct btrfs_key key; struct inode *inode; - key.objectid = objectid; - key.type = BTRFS_INODE_ITEM_KEY; - key.offset = 0; - inode = btrfs_iget(root->fs_info->sb, &key, root); + inode = btrfs_iget(root->fs_info->sb, objectid, root); if (IS_ERR(inode)) inode = NULL; return inode; @@ -4815,10 +4811,7 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, btrfs_release_path(path); - key.objectid = ino; - key.type = BTRFS_INODE_ITEM_KEY; - key.offset = 0; - inode = btrfs_iget(fs_info->sb, &key, root); + inode = btrfs_iget(fs_info->sb, ino, root); /* * If the other inode that had a conflicting dir entry was * deleted in the current transaction, we need to log its parent @@ -4827,8 +4820,7 @@ static int log_conflicting_inodes(struct btrfs_trans_handle *trans, if (IS_ERR(inode)) { ret = PTR_ERR(inode); if (ret == -ENOENT) { - key.objectid = parent; - inode = btrfs_iget(fs_info->sb, &key, root); + inode = btrfs_iget(fs_info->sb, parent, root); if (IS_ERR(inode)) { ret = PTR_ERR(inode); } else { @@ -5567,7 +5559,7 @@ process_leaf: continue; btrfs_release_path(path); - di_inode = btrfs_iget(fs_info->sb, &di_key, root); + di_inode = btrfs_iget(fs_info->sb, di_key.objectid, root); if (IS_ERR(di_inode)) { ret = PTR_ERR(di_inode); goto next_dir_inode; @@ -5693,7 +5685,8 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, cur_offset = item_size; } - dir_inode = btrfs_iget(fs_info->sb, &inode_key, root); + dir_inode = btrfs_iget(fs_info->sb, inode_key.objectid, + root); /* * If the parent inode was deleted, return an error to * fallback to a transaction commit. This is to prevent @@ -5760,14 +5753,17 @@ static int log_new_ancestors(struct btrfs_trans_handle *trans, int slot = path->slots[0]; struct btrfs_key search_key; struct inode *inode; + u64 ino; int ret = 0; btrfs_release_path(path); + ino = found_key.offset; + search_key.objectid = found_key.offset; search_key.type = BTRFS_INODE_ITEM_KEY; search_key.offset = 0; - inode = btrfs_iget(fs_info->sb, &search_key, root); + inode = btrfs_iget(fs_info->sb, ino, root); if (IS_ERR(inode)) return PTR_ERR(inode); From adbab6420c973f37079b1ce9b7e8a12f54810250 Mon Sep 17 00:00:00 2001 From: Anand Jain Date: Mon, 11 May 2020 22:37:51 -0700 Subject: [PATCH 0798/1043] btrfs: unexport btrfs_compress_set_level() btrfs_compress_set_level() can be static function in the file compression.c. Reviewed-by: Johannes Thumshirn Signed-off-by: Anand Jain Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/compression.c | 32 ++++++++++++++++---------------- fs/btrfs/compression.h | 2 -- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 1b624f9ef97d..c6e648603f85 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -1139,6 +1139,22 @@ static void put_workspace(int type, struct list_head *ws) } } +/* + * Adjust @level according to the limits of the compression algorithm or + * fallback to default + */ +static unsigned int btrfs_compress_set_level(int type, unsigned level) +{ + const struct btrfs_compress_op *ops = btrfs_compress_op[type]; + + if (level == 0) + level = ops->default_level; + else + level = min(level, ops->max_level); + + return level; +} + /* * Given an address space and start and length, compress the bytes into @pages * that are allocated on demand. @@ -1746,19 +1762,3 @@ unsigned int btrfs_compress_str2level(unsigned int type, const char *str) return level; } - -/* - * Adjust @level according to the limits of the compression algorithm or - * fallback to default - */ -unsigned int btrfs_compress_set_level(int type, unsigned level) -{ - const struct btrfs_compress_op *ops = btrfs_compress_op[type]; - - if (level == 0) - level = ops->default_level; - else - level = min(level, ops->max_level); - - return level; -} diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h index d253f7aa8ed5..284a3ad31350 100644 --- a/fs/btrfs/compression.h +++ b/fs/btrfs/compression.h @@ -140,8 +140,6 @@ extern const struct btrfs_compress_op btrfs_zstd_compress; const char* btrfs_compress_type2str(enum btrfs_compression_type type); bool btrfs_compress_is_valid_type(const char *str, size_t len); -unsigned int btrfs_compress_set_level(int type, unsigned level); - int btrfs_compress_heuristic(struct inode *inode, u64 start, u64 end); #endif From e289f03ea79bbc6574b78ac25682555423a91cbb Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Mon, 18 May 2020 12:14:50 +0100 Subject: [PATCH 0799/1043] btrfs: fix corrupt log due to concurrent fsync of inodes with shared extents When we have extents shared amongst different inodes in the same subvolume, if we fsync them in parallel we can end up with checksum items in the log tree that represent ranges which overlap. For example, consider we have inodes A and B, both sharing an extent that covers the logical range from X to X + 64KiB: 1) Task A starts an fsync on inode A; 2) Task B starts an fsync on inode B; 3) Task A calls btrfs_csum_file_blocks(), and the first search in the log tree, through btrfs_lookup_csum(), returns -EFBIG because it finds an existing checksum item that covers the range from X - 64KiB to X; 4) Task A checks that the checksum item has not reached the maximum possible size (MAX_CSUM_ITEMS) and then releases the search path before it does another path search for insertion (through a direct call to btrfs_search_slot()); 5) As soon as task A releases the path and before it does the search for insertion, task B calls btrfs_csum_file_blocks() and gets -EFBIG too, because there is an existing checksum item that has an end offset that matches the start offset (X) of the checksum range we want to log; 6) Task B releases the path; 7) Task A does the path search for insertion (through btrfs_search_slot()) and then verifies that the checksum item that ends at offset X still exists and extends its size to insert the checksums for the range from X to X + 64KiB; 8) Task A releases the path and returns from btrfs_csum_file_blocks(), having inserted the checksums into an existing checksum item that got its size extended. At this point we have one checksum item in the log tree that covers the logical range from X - 64KiB to X + 64KiB; 9) Task B now does a search for insertion using btrfs_search_slot() too, but it finds that the previous checksum item no longer ends at the offset X, it now ends at an of offset X + 64KiB, so it leaves that item untouched. Then it releases the path and calls btrfs_insert_empty_item() that inserts a checksum item with a key offset corresponding to X and a size for inserting a single checksum (4 bytes in case of crc32c). Subsequent iterations end up extending this new checksum item so that it contains the checksums for the range from X to X + 64KiB. So after task B returns from btrfs_csum_file_blocks() we end up with two checksum items in the log tree that have overlapping ranges, one for the range from X - 64KiB to X + 64KiB, and another for the range from X to X + 64KiB. Having checksum items that represent ranges which overlap, regardless of being in the log tree or in the chekcsums tree, can lead to problems where checksums for a file range end up not being found. This type of problem has happened a few times in the past and the following commits fixed them and explain in detail why having checksum items with overlapping ranges is problematic: 27b9a8122ff71a "Btrfs: fix csum tree corruption, duplicate and outdated checksums" b84b8390d6009c "Btrfs: fix file read corruption after extent cloning and fsync" 40e046acbd2f36 "Btrfs: fix missing data checksums after replaying a log tree" Since this specific instance of the problem can only happen when logging inodes, because it is the only case where concurrent attempts to insert checksums for the same range can happen, fix the issue by using an extent io tree as a range lock to serialize checksum insertion during inode logging. This issue could often be reproduced by the test case generic/457 from fstests. When it happens it produces the following trace: BTRFS critical (device dm-0): corrupt leaf: root=18446744073709551610 block=30625792 slot=42, csum end range (15020032) goes beyond the start range (15015936) of the next csum item BTRFS info (device dm-0): leaf 30625792 gen 7 total ptrs 49 free space 2402 owner 18446744073709551610 BTRFS info (device dm-0): refs 1 lock (w:0 r:0 bw:0 br:0 sw:0 sr:0) lock_owner 0 current 15884 item 0 key (18446744073709551606 128 13979648) itemoff 3991 itemsize 4 item 1 key (18446744073709551606 128 13983744) itemoff 3987 itemsize 4 item 2 key (18446744073709551606 128 13987840) itemoff 3983 itemsize 4 item 3 key (18446744073709551606 128 13991936) itemoff 3979 itemsize 4 item 4 key (18446744073709551606 128 13996032) itemoff 3975 itemsize 4 item 5 key (18446744073709551606 128 14000128) itemoff 3971 itemsize 4 (...) BTRFS error (device dm-0): block=30625792 write time tree block corruption detected ------------[ cut here ]------------ WARNING: CPU: 1 PID: 15884 at fs/btrfs/disk-io.c:539 btree_csum_one_bio+0x268/0x2d0 [btrfs] Modules linked in: btrfs dm_thin_pool ... CPU: 1 PID: 15884 Comm: fsx Tainted: G W 5.6.0-rc7-btrfs-next-58 #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.12.0-59-gc9ba5276e321-prebuilt.qemu.org 04/01/2014 RIP: 0010:btree_csum_one_bio+0x268/0x2d0 [btrfs] Code: c7 c7 ... RSP: 0018:ffffbb0109e6f8e0 EFLAGS: 00010296 RAX: 0000000000000000 RBX: ffffe1c0847b6080 RCX: 0000000000000000 RDX: 0000000000000000 RSI: ffffffffaa963988 RDI: 0000000000000001 RBP: ffff956a4f4d2000 R08: 0000000000000000 R09: 0000000000000001 R10: 0000000000000526 R11: 0000000000000000 R12: ffff956a5cd28bb0 R13: 0000000000000000 R14: ffff956a649c9388 R15: 000000011ed82000 FS: 00007fb419959e80(0000) GS:ffff956a7aa00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000000fe6d54 CR3: 0000000138696005 CR4: 00000000003606e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: btree_submit_bio_hook+0x67/0xc0 [btrfs] submit_one_bio+0x31/0x50 [btrfs] btree_write_cache_pages+0x2db/0x4b0 [btrfs] ? __filemap_fdatawrite_range+0xb1/0x110 do_writepages+0x23/0x80 __filemap_fdatawrite_range+0xd2/0x110 btrfs_write_marked_extents+0x15e/0x180 [btrfs] btrfs_sync_log+0x206/0x10a0 [btrfs] ? kmem_cache_free+0x315/0x3b0 ? btrfs_log_inode+0x1e8/0xf90 [btrfs] ? __mutex_unlock_slowpath+0x45/0x2a0 ? lockref_put_or_lock+0x9/0x30 ? dput+0x2d/0x580 ? dput+0xb5/0x580 ? btrfs_sync_file+0x464/0x4d0 [btrfs] btrfs_sync_file+0x464/0x4d0 [btrfs] do_fsync+0x38/0x60 __x64_sys_fsync+0x10/0x20 do_syscall_64+0x5c/0x280 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x7fb41953a6d0 Code: 48 3d ... RSP: 002b:00007ffcc86bd218 EFLAGS: 00000246 ORIG_RAX: 000000000000004a RAX: ffffffffffffffda RBX: 000000000000000d RCX: 00007fb41953a6d0 RDX: 0000000000000009 RSI: 0000000000040000 RDI: 0000000000000003 RBP: 0000000000040000 R08: 0000000000000001 R09: 0000000000000009 R10: 0000000000000064 R11: 0000000000000246 R12: 0000556cf4b2c060 R13: 0000000000000100 R14: 0000000000000000 R15: 0000556cf322b420 irq event stamp: 0 hardirqs last enabled at (0): [<0000000000000000>] 0x0 hardirqs last disabled at (0): [] copy_process+0x74f/0x2020 softirqs last enabled at (0): [] copy_process+0x74f/0x2020 softirqs last disabled at (0): [<0000000000000000>] 0x0 ---[ end trace d543fc76f5ad7fd8 ]--- In that trace the tree checker detected the overlapping checksum items at the time when we triggered writeback for the log tree when syncing the log. Another trace that can happen is due to BUG_ON() when deleting checksum items while logging an inode: BTRFS critical (device dm-0): slot 81 key (18446744073709551606 128 13635584) new key (18446744073709551606 128 13635584) BTRFS info (device dm-0): leaf 30949376 gen 7 total ptrs 98 free space 8527 owner 18446744073709551610 BTRFS info (device dm-0): refs 4 lock (w:1 r:0 bw:0 br:0 sw:1 sr:0) lock_owner 13473 current 13473 item 0 key (257 1 0) itemoff 16123 itemsize 160 inode generation 7 size 262144 mode 100600 item 1 key (257 12 256) itemoff 16103 itemsize 20 item 2 key (257 108 0) itemoff 16050 itemsize 53 extent data disk bytenr 13631488 nr 4096 extent data offset 0 nr 131072 ram 131072 (...) ------------[ cut here ]------------ kernel BUG at fs/btrfs/ctree.c:3153! invalid opcode: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC PTI CPU: 1 PID: 13473 Comm: fsx Not tainted 5.6.0-rc7-btrfs-next-58 #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.12.0-59-gc9ba5276e321-prebuilt.qemu.org 04/01/2014 RIP: 0010:btrfs_set_item_key_safe+0x1ea/0x270 [btrfs] Code: 0f b6 ... RSP: 0018:ffff95e3889179d0 EFLAGS: 00010282 RAX: 0000000000000000 RBX: 0000000000000051 RCX: 0000000000000000 RDX: 0000000000000000 RSI: ffffffffb7763988 RDI: 0000000000000001 RBP: fffffffffffffff6 R08: 0000000000000000 R09: 0000000000000001 R10: 00000000000009ef R11: 0000000000000000 R12: ffff8912a8ba5a08 R13: ffff95e388917a06 R14: ffff89138dcf68c8 R15: ffff95e388917ace FS: 00007fe587084e80(0000) GS:ffff8913baa00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007fe587091000 CR3: 0000000126dac005 CR4: 00000000003606e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: btrfs_del_csums+0x2f4/0x540 [btrfs] copy_items+0x4b5/0x560 [btrfs] btrfs_log_inode+0x910/0xf90 [btrfs] btrfs_log_inode_parent+0x2a0/0xe40 [btrfs] ? dget_parent+0x5/0x370 btrfs_log_dentry_safe+0x4a/0x70 [btrfs] btrfs_sync_file+0x42b/0x4d0 [btrfs] __x64_sys_msync+0x199/0x200 do_syscall_64+0x5c/0x280 entry_SYSCALL_64_after_hwframe+0x49/0xbe RIP: 0033:0x7fe586c65760 Code: 00 f7 ... RSP: 002b:00007ffe250f98b8 EFLAGS: 00000246 ORIG_RAX: 000000000000001a RAX: ffffffffffffffda RBX: 00000000000040e1 RCX: 00007fe586c65760 RDX: 0000000000000004 RSI: 0000000000006b51 RDI: 00007fe58708b000 RBP: 0000000000006a70 R08: 0000000000000003 R09: 00007fe58700cb61 R10: 0000000000000100 R11: 0000000000000246 R12: 00000000000000e1 R13: 00007fe58708b000 R14: 0000000000006b51 R15: 0000558de021a420 Modules linked in: dm_log_writes ... ---[ end trace c92a7f447a8515f5 ]--- CC: stable@vger.kernel.org # 4.4+ Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 3 +++ fs/btrfs/disk-io.c | 5 ++++- fs/btrfs/extent-io-tree.h | 1 + fs/btrfs/tree-log.c | 22 +++++++++++++++++++--- include/trace/events/btrfs.h | 1 + 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 5afeb17a3f1a..30ce7039bc27 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -1167,6 +1167,9 @@ struct btrfs_root { /* Record pairs of swapped blocks for qgroup */ struct btrfs_qgroup_swapped_blocks swapped_blocks; + /* Used only by log trees, when logging csum items */ + struct extent_io_tree log_csum_range; + #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS u64 alloc_bytenr; #endif diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index f2f2864f5978..f8ec2d8606fd 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -1133,9 +1133,12 @@ static void __setup_root(struct btrfs_root *root, struct btrfs_fs_info *fs_info, root->log_transid = 0; root->log_transid_committed = -1; root->last_log_commit = 0; - if (!dummy) + if (!dummy) { extent_io_tree_init(fs_info, &root->dirty_log_pages, IO_TREE_ROOT_DIRTY_LOG_PAGES, NULL); + extent_io_tree_init(fs_info, &root->log_csum_range, + IO_TREE_LOG_CSUM_RANGE, NULL); + } memset(&root->root_key, 0, sizeof(root->root_key)); memset(&root->root_item, 0, sizeof(root->root_item)); diff --git a/fs/btrfs/extent-io-tree.h b/fs/btrfs/extent-io-tree.h index b4a7bad3e82e..b6561455b3c4 100644 --- a/fs/btrfs/extent-io-tree.h +++ b/fs/btrfs/extent-io-tree.h @@ -44,6 +44,7 @@ enum { IO_TREE_TRANS_DIRTY_PAGES, IO_TREE_ROOT_DIRTY_LOG_PAGES, IO_TREE_INODE_FILE_EXTENT, + IO_TREE_LOG_CSUM_RANGE, IO_TREE_SELFTEST, }; diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index 67fa7087f707..920cee312f4e 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -3290,6 +3290,7 @@ static void free_log_tree(struct btrfs_trans_handle *trans, clear_extent_bits(&log->dirty_log_pages, 0, (u64)-1, EXTENT_DIRTY | EXTENT_NEW | EXTENT_NEED_WAIT); + extent_io_tree_release(&log->log_csum_range); btrfs_put_root(log); } @@ -3903,8 +3904,20 @@ static int log_csums(struct btrfs_trans_handle *trans, struct btrfs_root *log_root, struct btrfs_ordered_sum *sums) { + const u64 lock_end = sums->bytenr + sums->len - 1; + struct extent_state *cached_state = NULL; int ret; + /* + * Serialize logging for checksums. This is to avoid racing with the + * same checksum being logged by another task that is logging another + * file which happens to refer to the same extent as well. Such races + * can leave checksum items in the log with overlapping ranges. + */ + ret = lock_extent_bits(&log_root->log_csum_range, sums->bytenr, + lock_end, &cached_state); + if (ret) + return ret; /* * Due to extent cloning, we might have logged a csum item that covers a * subrange of a cloned extent, and later we can end up logging a csum @@ -3915,10 +3928,13 @@ static int log_csums(struct btrfs_trans_handle *trans, * trim and adjust) any existing csum items in the log for this range. */ ret = btrfs_del_csums(trans, log_root, sums->bytenr, sums->len); - if (ret) - return ret; + if (!ret) + ret = btrfs_csum_file_blocks(trans, log_root, sums); - return btrfs_csum_file_blocks(trans, log_root, sums); + unlock_extent_cached(&log_root->log_csum_range, sums->bytenr, lock_end, + &cached_state); + + return ret; } static noinline int copy_items(struct btrfs_trans_handle *trans, diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index bcbc763b8814..360b0f9d2220 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -89,6 +89,7 @@ TRACE_DEFINE_ENUM(COMMIT_TRANS); { IO_TREE_TRANS_DIRTY_PAGES, "TRANS_DIRTY_PAGES" }, \ { IO_TREE_ROOT_DIRTY_LOG_PAGES, "ROOT_DIRTY_LOG_PAGES" }, \ { IO_TREE_INODE_FILE_EXTENT, "INODE_FILE_EXTENT" }, \ + { IO_TREE_LOG_CSUM_RANGE, "LOG_CSUM_RANGE" }, \ { IO_TREE_SELFTEST, "SELFTEST" }) #define BTRFS_GROUP_FLAGS \ From cc14600c1516f6c679cab1c503a34841d58050a6 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Mon, 18 May 2020 12:15:00 +0100 Subject: [PATCH 0800/1043] btrfs: make checksum item extension more efficient When we want to add checksums into the checksums tree, or a log tree, we try whenever possible to extend existing checksum items, as this helps reduce amount of metadata space used, since adding a new item uses extra metadata space for a btrfs_item structure (25 bytes). However we have two inefficiencies in the current approach: 1) After finding a checksum item that covers a range with an end offset that matches the start offset of the checksum range we want to insert, we release the search path populated by btrfs_lookup_csum() and then do another COW search on tree with the goal of getting additional space for at least one checksum. Doing this path release and then searching again is a waste of time because very often the leaf already has enough free space for at least one more checksum; 2) After the COW search that guarantees we get free space in the leaf for at least one more checksum, we end up not doing the extension of the previous checksum item, and fallback to insertion of a new checksum item, if the leaf doesn't have an amount of free space larger then the space required for 2 checksums plus one btrfs_item structure - this is pointless for two reasons: a) We want to extend an existing item, so we don't need to account for a btrfs_item structure (25 bytes); b) We made the COW search with an insertion size for 1 single checksum, so if the leaf ends up with a free space amount smaller then 2 checksums plus the size of a btrfs_item structure, we give up on the extension of the existing item and jump to the 'insert' label, where we end up releasing the path and then doing yet another search to insert a new checksum item for a single checksum. Fix these inefficiencies by doing the following: - For case 1), before releasing the path just check if the leaf already has enough space for at least 1 more checksum, and if it does, jump directly to the item extension code, with releasing our current path, which was already COWed by btrfs_lookup_csum(); - For case 2), fix the logic so that for item extension we require only that the leaf has enough free space for 1 checksum, and not a minimum of 2 checksums plus space for a btrfs_item structure. Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/file-item.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index 8cdd06ea0e67..687529c61d13 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -907,9 +907,22 @@ again: } /* - * at this point, we know the tree has an item, but it isn't big - * enough yet to put our csum in. Grow it + * At this point, we know the tree has a checksum item that ends at an + * offset matching the start of the checksum range we want to insert. + * We try to extend that item as much as possible and then add as many + * checksums to it as they fit. + * + * First check if the leaf has enough free space for at least one + * checksum. If it has go directly to the item extension code, otherwise + * release the path and do a search for insertion before the extension. */ + if (btrfs_leaf_free_space(leaf) >= csum_size) { + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + csum_offset = (bytenr - found_key.offset) >> + fs_info->sb->s_blocksize_bits; + goto extend_csum; + } + btrfs_release_path(path); ret = btrfs_search_slot(trans, root, &file_key, path, csum_size, 1); @@ -933,19 +946,13 @@ again: goto insert; } +extend_csum: if (csum_offset == btrfs_item_size_nr(leaf, path->slots[0]) / csum_size) { int extend_nr; u64 tmp; u32 diff; - u32 free_space; - if (btrfs_leaf_free_space(leaf) < - sizeof(struct btrfs_item) + csum_size * 2) - goto insert; - - free_space = btrfs_leaf_free_space(leaf) - - sizeof(struct btrfs_item) - csum_size; tmp = sums->len - total_bytes; tmp >>= fs_info->sb->s_blocksize_bits; WARN_ON(tmp < 1); @@ -956,7 +963,7 @@ again: MAX_CSUM_ITEMS(fs_info, csum_size) * csum_size); diff = diff - btrfs_item_size_nr(leaf, path->slots[0]); - diff = min(free_space, diff); + diff = min_t(u32, btrfs_leaf_free_space(leaf), diff); diff /= csum_size; diff *= csum_size; From 7e4a3f7ed5d54926ec671bbb13e171cfe179cc50 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Mon, 18 May 2020 12:15:09 +0100 Subject: [PATCH 0801/1043] btrfs: do not ignore error from btrfs_next_leaf() when inserting checksums We are currently treating any non-zero return value from btrfs_next_leaf() the same way, by going to the code that inserts a new checksum item in the tree. However if btrfs_next_leaf() returns an error (a value < 0), we should just stop and return the error, and not behave as if nothing has happened, since in that case we do not have a way to know if there is a next leaf or we are currently at the last leaf already. So fix that by returning the error from btrfs_next_leaf(). Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/file-item.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index 687529c61d13..c3f3c25dda38 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -889,10 +889,12 @@ again: nritems = btrfs_header_nritems(path->nodes[0]); if (!nritems || (path->slots[0] >= nritems - 1)) { ret = btrfs_next_leaf(root, path); - if (ret == 1) + if (ret < 0) { + goto out; + } else if (ret > 0) { found_next = 1; - if (ret != 0) goto insert; + } slot = path->slots[0]; } btrfs_item_key_to_cpu(path->nodes[0], &found_key, slot); From 918cdf442326e5eaa808258e403c6a6e2dc23113 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Mon, 18 May 2020 12:15:18 +0100 Subject: [PATCH 0802/1043] btrfs: remove useless 'fail_unlock' label from btrfs_csum_file_blocks() The label 'fail_unlock' is pointless, all it does is to jump to the label 'out', so just remove it. Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/file-item.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index c3f3c25dda38..706a3128e192 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -871,7 +871,7 @@ again: } ret = PTR_ERR(item); if (ret != -EFBIG && ret != -ENOENT) - goto fail_unlock; + goto out; if (ret == -EFBIG) { u32 item_size; @@ -929,7 +929,7 @@ again: ret = btrfs_search_slot(trans, root, &file_key, path, csum_size, 1); if (ret < 0) - goto fail_unlock; + goto out; if (ret > 0) { if (path->slots[0] == 0) @@ -996,9 +996,9 @@ insert: ins_size); path->leave_spinning = 0; if (ret < 0) - goto fail_unlock; + goto out; if (WARN_ON(ret != 0)) - goto fail_unlock; + goto out; leaf = path->nodes[0]; csum: item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item); @@ -1028,9 +1028,6 @@ found: out: btrfs_free_path(path); return ret; - -fail_unlock: - goto out; } void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode, From 2e69a7a60d8d93b19f520bf4179614188a6cfdf5 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Mon, 18 May 2020 17:34:11 +0100 Subject: [PATCH 0803/1043] btrfs: include error on messages about failure to write space/inode caches Currently the error messages logged when we fail to write a free space cache or an inode cache are not very useful as they don't mention what was the error. So include the error number in the messages. Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/free-space-cache.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index 525bc5a250da..01e8451c47d5 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -1193,8 +1193,8 @@ out: if (block_group) { #ifdef CONFIG_BTRFS_DEBUG btrfs_err(root->fs_info, - "failed to write free space cache for block group %llu", - block_group->start); + "failed to write free space cache for block group %llu error %d", + block_group->start, ret); #endif } } @@ -1417,8 +1417,8 @@ int btrfs_write_out_cache(struct btrfs_trans_handle *trans, if (ret) { #ifdef CONFIG_BTRFS_DEBUG btrfs_err(fs_info, - "failed to write free space cache for block group %llu", - block_group->start); + "failed to write free space cache for block group %llu error %d", + block_group->start, ret); #endif spin_lock(&block_group->lock); block_group->disk_cache_state = BTRFS_DC_ERROR; @@ -3997,8 +3997,8 @@ int btrfs_write_out_ino_cache(struct btrfs_root *root, inode->i_size, true); #ifdef CONFIG_BTRFS_DEBUG btrfs_err(fs_info, - "failed to write free ino cache for root %llu", - root->root_key.objectid); + "failed to write free ino cache for root %llu error %d", + root->root_key.objectid, ret); #endif } From bbcd1f4d52587d4f6a90692d896cf06a37b6554a Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Mon, 18 May 2020 17:34:23 +0100 Subject: [PATCH 0804/1043] btrfs: turn space cache writeout failure messages into debug messages Since commit 1afb648e945428 ("btrfs: use standard debug config option to enable free-space-cache debug prints"), we started to log error messages that were never logged before since there was no DEBUG macro defined anywhere. This started to make test case btrfs/187 to fail very often, as it greps for any btrfs error messages in dmesg/syslog and fails if any is found: (...) btrfs/186 1s ... 2s btrfs/187 - output mismatch (see .../results//btrfs/187.out.bad) \--- tests/btrfs/187.out 2019-05-17 12:48:32.537340749 +0100 \+++ /home/fdmanana/git/hub/xfstests/results//btrfs/187.out.bad ... \@@ -1,3 +1,8 @@ QA output created by 187 Create a readonly snapshot of 'SCRATCH_MNT' in 'SCRATCH_MNT/snap1' Create a readonly snapshot of 'SCRATCH_MNT' in 'SCRATCH_MNT/snap2' +[268364.139958] BTRFS error (device sdc): failed to write free space cache for block group 30408704 +[268380.156503] BTRFS error (device sdc): failed to write free space cache for block group 30408704 +[268380.161703] BTRFS error (device sdc): failed to write free space cache for block group 30408704 +[268380.253180] BTRFS error (device sdc): failed to write free space cache for block group 30408704 ... (Run 'diff -u /home/fdmanana/git/hub/xfstests/tests/btrfs/187.out ... btrfs/188 4s ... 2s (...) The space cache write failures happen due to ENOSPC when attempting to update the free space cache items in the root tree. This happens because when starting or joining a transaction we don't know how many block groups we will end up changing (due to extent allocation or release) and therefore never reserve space for updating free space cache items. More often than not, the free space cache writeout succeeds since the metadata space info is not yet full nor very close to being full, but when it is, the space cache writeout fails with ENOSPC. Occasional failures to write space caches are not considered critical since they can be rebuilt when mounting the filesystem or the next attempt to write a free space cache in the next transaction commit might succeed, so we used to hide those error messages with a preprocessor check for the existence of the DEBUG macro that was never enabled anywhere. A few other generic test cases also trigger the error messages due to ENOSPC failure when writing free space caches as well, however they don't fail since they don't grep dmesg/syslog for any btrfs specific error messages. So change the messages from 'error' level to 'debug' level, as it doesn't make much sense to have error messages triggered only if the debug macro is enabled plus, more importantly, the error is not serious nor highly unexpected. Signed-off-by: Filipe Manana Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/free-space-cache.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index 01e8451c47d5..55955bd424d7 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -1190,13 +1190,10 @@ out: if (ret) { invalidate_inode_pages2(inode->i_mapping); BTRFS_I(inode)->generation = 0; - if (block_group) { -#ifdef CONFIG_BTRFS_DEBUG - btrfs_err(root->fs_info, + if (block_group) + btrfs_debug(root->fs_info, "failed to write free space cache for block group %llu error %d", block_group->start, ret); -#endif - } } btrfs_update_inode(trans, root, inode); @@ -1415,11 +1412,9 @@ int btrfs_write_out_cache(struct btrfs_trans_handle *trans, ret = __btrfs_write_out_cache(fs_info->tree_root, inode, ctl, block_group, &block_group->io_ctl, trans); if (ret) { -#ifdef CONFIG_BTRFS_DEBUG - btrfs_err(fs_info, + btrfs_debug(fs_info, "failed to write free space cache for block group %llu error %d", block_group->start, ret); -#endif spin_lock(&block_group->lock); block_group->disk_cache_state = BTRFS_DC_ERROR; spin_unlock(&block_group->lock); @@ -3995,11 +3990,9 @@ int btrfs_write_out_ino_cache(struct btrfs_root *root, if (release_metadata) btrfs_delalloc_release_metadata(BTRFS_I(inode), inode->i_size, true); -#ifdef CONFIG_BTRFS_DEBUG - btrfs_err(fs_info, + btrfs_debug(fs_info, "failed to write free ino cache for root %llu error %d", root->root_key.objectid, ret); -#endif } return ret; From cc86432aa8cc5a81f99d79eea2a29099da694df3 Mon Sep 17 00:00:00 2001 From: Valentin Schneider Date: Thu, 21 May 2020 23:35:00 +0100 Subject: [PATCH 0805/1043] irqchip/gic-v2, v3: Drop extra IRQ_NOAUTOEN setting for (E)PPIs (E)PPIs are per-CPU interrupts, so we want each CPU to go and enable them via enable_percpu_irq(); this also means we want IRQ_NOAUTOEN for them as the autoenable would lead to calling irq_enable() instead of the more appropriate irq_percpu_enable(). Calling irq_set_percpu_devid() is enough to get just that since it trickles down to irq_set_percpu_devid_flags(), which gives us IRQ_NOAUTOEN (and a few others). Setting IRQ_NOAUTOEN *again* right after this call is just redundant, so don't do it. Signed-off-by: Valentin Schneider Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200521223500.834-1-valentin.schneider@arm.com --- drivers/irqchip/irq-gic-v3.c | 1 - drivers/irqchip/irq-gic.c | 1 - 2 files changed, 2 deletions(-) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 98c886dab02d..cc46bc2d634b 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -1282,7 +1282,6 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, irq_set_percpu_devid(irq); irq_domain_set_info(d, irq, hw, chip, d->host_data, handle_percpu_devid_irq, NULL, NULL); - irq_set_status_flags(irq, IRQ_NOAUTOEN); break; case SPI_RANGE: diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index 30ab623343d3..00de05abd3c3 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -982,7 +982,6 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, irq_set_percpu_devid(irq); irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data, handle_percpu_devid_irq, NULL, NULL); - irq_set_status_flags(irq, IRQ_NOAUTOEN); } else { irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data, handle_fasteoi_irq, NULL, NULL); From 2458ed31e9b9ab40d78a452ab2650a0857556e85 Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 18 May 2020 14:44:39 +0530 Subject: [PATCH 0806/1043] irqchip/sifive-plic: Set default irq affinity in plic_irqdomain_map() For multiple PLIC instances, each PLIC can only target a subset of CPUs which is represented by "lmask" in the "struct plic_priv". Currently, the default irq affinity for each PLIC interrupt is all online CPUs which is illegal value for default irq affinity when we have multiple PLIC instances. To fix this, we now set "lmask" as the default irq affinity in for each interrupt in plic_irqdomain_map(). Fixes: f1ad1133b18f ("irqchip/sifive-plic: Add support for multiple PLICs") Signed-off-by: Anup Patel Signed-off-by: Marc Zyngier Reviewed-by: Palmer Dabbelt Acked-by: Palmer Dabbelt Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20200518091441.94843-2-anup.patel@wdc.com --- drivers/irqchip/irq-sifive-plic.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c index 822e074c0600..9f7f8ce88c00 100644 --- a/drivers/irqchip/irq-sifive-plic.c +++ b/drivers/irqchip/irq-sifive-plic.c @@ -176,9 +176,12 @@ static struct irq_chip plic_chip = { static int plic_irqdomain_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hwirq) { + struct plic_priv *priv = d->host_data; + irq_domain_set_info(d, irq, hwirq, &plic_chip, d->host_data, handle_fasteoi_irq, NULL, NULL); irq_set_noprobe(irq); + irq_set_affinity(irq, &priv->lmask); return 0; } From 2234ae846ccb9ebdf4c391824cb79e73674dceda Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 18 May 2020 14:44:40 +0530 Subject: [PATCH 0807/1043] irqchip/sifive-plic: Setup cpuhp once after boot CPU handler is present For multiple PLIC instances, the plic_init() is called once for each PLIC instance. Due to this we have two issues: 1. cpuhp_setup_state() is called multiple times 2. plic_starting_cpu() can crash for boot CPU if cpuhp_setup_state() is called before boot CPU PLIC handler is available. Address both issues by only initializing the HP notifiers when the boot CPU setup is complete. Fixes: f1ad1133b18f ("irqchip/sifive-plic: Add support for multiple PLICs") Signed-off-by: Anup Patel Signed-off-by: Marc Zyngier Reviewed-by: Palmer Dabbelt Acked-by: Palmer Dabbelt Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20200518091441.94843-3-anup.patel@wdc.com --- drivers/irqchip/irq-sifive-plic.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c index 9f7f8ce88c00..6c54abf5cc5e 100644 --- a/drivers/irqchip/irq-sifive-plic.c +++ b/drivers/irqchip/irq-sifive-plic.c @@ -76,6 +76,7 @@ struct plic_handler { void __iomem *enable_base; struct plic_priv *priv; }; +static bool plic_cpuhp_setup_done; static DEFINE_PER_CPU(struct plic_handler, plic_handlers); static inline void plic_toggle(struct plic_handler *handler, @@ -285,6 +286,7 @@ static int __init plic_init(struct device_node *node, int error = 0, nr_contexts, nr_handlers = 0, i; u32 nr_irqs; struct plic_priv *priv; + struct plic_handler *handler; priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) @@ -313,7 +315,6 @@ static int __init plic_init(struct device_node *node, for (i = 0; i < nr_contexts; i++) { struct of_phandle_args parent; - struct plic_handler *handler; irq_hw_number_t hwirq; int cpu, hartid; @@ -367,9 +368,18 @@ done: nr_handlers++; } - cpuhp_setup_state(CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING, + /* + * We can have multiple PLIC instances so setup cpuhp state only + * when context handler for current/boot CPU is present. + */ + handler = this_cpu_ptr(&plic_handlers); + if (handler->present && !plic_cpuhp_setup_done) { + cpuhp_setup_state(CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING, "irqchip/sifive/plic:starting", plic_starting_cpu, plic_dying_cpu); + plic_cpuhp_setup_done = true; + } + pr_info("mapped %d interrupts with %d handlers for %d contexts.\n", nr_irqs, nr_handlers, nr_contexts); set_handle_irq(plic_handle_irq); From 0e375f51017bcc86c23979118b10445c424ef5ad Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Mon, 18 May 2020 14:44:41 +0530 Subject: [PATCH 0808/1043] irqchip/sifive-plic: Improve boot prints for multiple PLIC instances We improve PLIC banner to help distinguish multiple PLIC instances in boot time prints. Signed-off-by: Anup Patel Signed-off-by: Marc Zyngier Reviewed-by: Palmer Dabbelt Acked-by: Palmer Dabbelt Link: https://lore.kernel.org/r/20200518091441.94843-4-anup.patel@wdc.com --- drivers/irqchip/irq-sifive-plic.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c index 6c54abf5cc5e..d9c53f85a68e 100644 --- a/drivers/irqchip/irq-sifive-plic.c +++ b/drivers/irqchip/irq-sifive-plic.c @@ -380,8 +380,8 @@ done: plic_cpuhp_setup_done = true; } - pr_info("mapped %d interrupts with %d handlers for %d contexts.\n", - nr_irqs, nr_handlers, nr_contexts); + pr_info("%pOFP: mapped %d interrupts with %d handlers for" + " %d contexts.\n", node, nr_irqs, nr_handlers, nr_contexts); set_handle_irq(plic_handle_irq); return 0; From d85dc2e116fdce776280224ed2bee4c78e5e5af2 Mon Sep 17 00:00:00 2001 From: Goldwyn Rodrigues Date: Fri, 30 Aug 2019 12:09:24 -0500 Subject: [PATCH 0809/1043] fs: export generic_file_buffered_read() Export generic_file_buffered_read() to be used to supplement incomplete direct reads. Reviewed-by: Johannes Thumshirn Reviewed-by: Christoph Hellwig Signed-off-by: Goldwyn Rodrigues Reviewed-by: David Sterba Signed-off-by: David Sterba --- include/linux/fs.h | 2 ++ mm/filemap.c | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/include/linux/fs.h b/include/linux/fs.h index 45cc10cdf6dd..366c533d30cd 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3124,6 +3124,8 @@ extern int generic_file_rw_checks(struct file *file_in, struct file *file_out); extern int generic_copy_file_checks(struct file *file_in, loff_t pos_in, struct file *file_out, loff_t pos_out, size_t *count, unsigned int flags); +extern ssize_t generic_file_buffered_read(struct kiocb *iocb, + struct iov_iter *to, ssize_t already_read); extern ssize_t generic_file_read_iter(struct kiocb *, struct iov_iter *); extern ssize_t __generic_file_write_iter(struct kiocb *, struct iov_iter *); extern ssize_t generic_file_write_iter(struct kiocb *, struct iov_iter *); diff --git a/mm/filemap.c b/mm/filemap.c index 23a051a7ef0f..ad82672a9941 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1991,7 +1991,7 @@ static void shrink_readahead_size_eio(struct file_ra_state *ra) * * total number of bytes copied, including those the were already @written * * negative error code if nothing was copied */ -static ssize_t generic_file_buffered_read(struct kiocb *iocb, +ssize_t generic_file_buffered_read(struct kiocb *iocb, struct iov_iter *iter, ssize_t written) { struct file *filp = iocb->ki_filp; @@ -2243,6 +2243,7 @@ out: file_accessed(filp); return written ? written : error; } +EXPORT_SYMBOL_GPL(generic_file_buffered_read); /** * generic_file_read_iter - generic filesystem read routine From 8cecd0ba854799cda72d03a470e7de9eed3ed6c4 Mon Sep 17 00:00:00 2001 From: Goldwyn Rodrigues Date: Tue, 14 May 2019 18:54:27 -0500 Subject: [PATCH 0810/1043] iomap: add a filesystem hook for direct I/O bio submission This helps filesystems to perform tasks on the bio while submitting for I/O. This could be post-write operations such as data CRC or data replication for fs-handled RAID. Reviewed-by: Johannes Thumshirn Reviewed-by: Nikolay Borisov Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Goldwyn Rodrigues Signed-off-by: David Sterba --- fs/iomap/direct-io.c | 15 ++++++++++----- include/linux/iomap.h | 2 ++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index 20dde5aadcdd..f88ba6e7f6af 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -59,7 +59,7 @@ int iomap_dio_iopoll(struct kiocb *kiocb, bool spin) EXPORT_SYMBOL_GPL(iomap_dio_iopoll); static void iomap_dio_submit_bio(struct iomap_dio *dio, struct iomap *iomap, - struct bio *bio) + struct bio *bio, loff_t pos) { atomic_inc(&dio->ref); @@ -67,7 +67,12 @@ static void iomap_dio_submit_bio(struct iomap_dio *dio, struct iomap *iomap, bio_set_polled(bio, dio->iocb); dio->submit.last_queue = bdev_get_queue(iomap->bdev); - dio->submit.cookie = submit_bio(bio); + if (dio->dops && dio->dops->submit_io) + dio->submit.cookie = dio->dops->submit_io( + file_inode(dio->iocb->ki_filp), + iomap, bio, pos); + else + dio->submit.cookie = submit_bio(bio); } static ssize_t iomap_dio_complete(struct iomap_dio *dio) @@ -191,7 +196,7 @@ iomap_dio_zero(struct iomap_dio *dio, struct iomap *iomap, loff_t pos, get_page(page); __bio_add_page(bio, page, len, 0); bio_set_op_attrs(bio, REQ_OP_WRITE, flags); - iomap_dio_submit_bio(dio, iomap, bio); + iomap_dio_submit_bio(dio, iomap, bio, pos); } static loff_t @@ -299,11 +304,11 @@ iomap_dio_bio_actor(struct inode *inode, loff_t pos, loff_t length, } dio->size += n; - pos += n; copied += n; nr_pages = iov_iter_npages(dio->submit.iter, BIO_MAX_PAGES); - iomap_dio_submit_bio(dio, iomap, bio); + iomap_dio_submit_bio(dio, iomap, bio, pos); + pos += n; } while (nr_pages); /* diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 8b09463dae0d..5b4875344874 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -252,6 +252,8 @@ int iomap_writepages(struct address_space *mapping, struct iomap_dio_ops { int (*end_io)(struct kiocb *iocb, ssize_t size, int error, unsigned flags); + blk_qc_t (*submit_io)(struct inode *inode, struct iomap *iomap, + struct bio *bio, loff_t file_offset); }; ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, From 3ad99bec6e82e32fa9faf2f84e74b134586b46f7 Mon Sep 17 00:00:00 2001 From: Goldwyn Rodrigues Date: Sat, 30 Nov 2019 09:59:25 -0600 Subject: [PATCH 0811/1043] iomap: remove lockdep_assert_held() Filesystems such as btrfs can perform direct I/O without holding the inode->i_rwsem in some of the cases like writing within i_size. So, remove the check for lockdep_assert_held() in iomap_dio_rw(). Reviewed-by: Darrick J. Wong Signed-off-by: Goldwyn Rodrigues Signed-off-by: David Sterba --- fs/iomap/direct-io.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index f88ba6e7f6af..e4addfc58107 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -416,8 +416,6 @@ iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, struct blk_plug plug; struct iomap_dio *dio; - lockdep_assert_held(&inode->i_rwsem); - if (!count) return 0; From 6af2aa7f6695ba73436083ce2e4c1a4ebc6af765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gonz=C3=A1lez=20Cabanelas?= Date: Sun, 24 May 2020 20:59:53 +0200 Subject: [PATCH 0812/1043] MIPS: BCM63XX: fix BCM6358 GPIO count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The BCM6358 SoC has only 38 available GPIOs. Fix it. Signed-off-by: Daniel González Cabanelas Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mach-bcm63xx/bcm63xx_gpio.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/mips/include/asm/mach-bcm63xx/bcm63xx_gpio.h b/arch/mips/include/asm/mach-bcm63xx/bcm63xx_gpio.h index 8fe88c2251e4..9212429d5edd 100644 --- a/arch/mips/include/asm/mach-bcm63xx/bcm63xx_gpio.h +++ b/arch/mips/include/asm/mach-bcm63xx/bcm63xx_gpio.h @@ -13,16 +13,16 @@ static inline unsigned long bcm63xx_gpio_count(void) case BCM6328_CPU_ID: return 32; case BCM3368_CPU_ID: - case BCM6358_CPU_ID: return 40; case BCM6338_CPU_ID: return 8; case BCM6345_CPU_ID: return 16; - case BCM6362_CPU_ID: - return 48; + case BCM6358_CPU_ID: case BCM6368_CPU_ID: return 38; + case BCM6362_CPU_ID: + return 48; case BCM6348_CPU_ID: default: return 37; From 71b3ec5f221b8b3ff545639be83ddfcd5d7c9800 Mon Sep 17 00:00:00 2001 From: David Brazdil Date: Fri, 15 May 2020 16:20:56 +0100 Subject: [PATCH 0813/1043] KVM: arm64: Clean up cpu_init_hyp_mode() Pull bits of code to the only place where it is used. Remove empty function __cpu_init_stage2(). Remove redundant has_vhe() check since this function is nVHE-only. No functional changes intended. Signed-off-by: David Brazdil Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200515152056.83158-1-dbrazdil@google.com --- arch/arm64/include/asm/kvm_asm.h | 2 ++ arch/arm64/include/asm/kvm_host.h | 35 ------------------------------- arch/arm64/kvm/arm.c | 32 +++++++++++++++++++++++----- 3 files changed, 29 insertions(+), 40 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 59e314f38e43..0c9b5fc4ba0a 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -70,6 +70,8 @@ extern int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu); extern int __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu); +extern void __kvm_enable_ssbs(void); + extern u64 __vgic_v3_get_ich_vtr_el2(void); extern u64 __vgic_v3_read_vmcr(void); extern void __vgic_v3_write_vmcr(u32 vmcr); diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index a723f84fab83..69a338a390a6 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -533,39 +533,6 @@ static inline void kvm_init_host_cpu_context(struct kvm_cpu_context *cpu_ctxt) cpu_ctxt->sys_regs[MPIDR_EL1] = read_cpuid_mpidr(); } -void __kvm_enable_ssbs(void); - -static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr, - unsigned long hyp_stack_ptr, - unsigned long vector_ptr) -{ - /* - * Calculate the raw per-cpu offset without a translation from the - * kernel's mapping to the linear mapping, and store it in tpidr_el2 - * so that we can use adr_l to access per-cpu variables in EL2. - */ - u64 tpidr_el2 = ((u64)this_cpu_ptr(&kvm_host_data) - - (u64)kvm_ksym_ref(kvm_host_data)); - - /* - * Call initialization code, and switch to the full blown HYP code. - * If the cpucaps haven't been finalized yet, something has gone very - * wrong, and hyp will crash and burn when it uses any - * cpus_have_const_cap() wrapper. - */ - BUG_ON(!system_capabilities_finalized()); - __kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr, tpidr_el2); - - /* - * Disabling SSBD on a non-VHE system requires us to enable SSBS - * at EL2. - */ - if (!has_vhe() && this_cpu_has_cap(ARM64_SSBS) && - arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) { - kvm_call_hyp(__kvm_enable_ssbs); - } -} - static inline bool kvm_arch_requires_vhe(void) { /* @@ -601,8 +568,6 @@ int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu, int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); -static inline void __cpu_init_stage2(void) {} - /* Guest/host FPSIMD coordination helpers */ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index e01d44df98df..b0b569f2cdd0 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -1273,19 +1273,41 @@ static void cpu_init_hyp_mode(void) { phys_addr_t pgd_ptr; unsigned long hyp_stack_ptr; - unsigned long stack_page; unsigned long vector_ptr; + unsigned long tpidr_el2; /* Switch from the HYP stub to our own HYP init vector */ __hyp_set_vectors(kvm_get_idmap_vector()); + /* + * Calculate the raw per-cpu offset without a translation from the + * kernel's mapping to the linear mapping, and store it in tpidr_el2 + * so that we can use adr_l to access per-cpu variables in EL2. + */ + tpidr_el2 = ((unsigned long)this_cpu_ptr(&kvm_host_data) - + (unsigned long)kvm_ksym_ref(kvm_host_data)); + pgd_ptr = kvm_mmu_get_httbr(); - stack_page = __this_cpu_read(kvm_arm_hyp_stack_page); - hyp_stack_ptr = stack_page + PAGE_SIZE; + hyp_stack_ptr = __this_cpu_read(kvm_arm_hyp_stack_page) + PAGE_SIZE; vector_ptr = (unsigned long)kvm_get_hyp_vector(); - __cpu_init_hyp_mode(pgd_ptr, hyp_stack_ptr, vector_ptr); - __cpu_init_stage2(); + /* + * Call initialization code, and switch to the full blown HYP code. + * If the cpucaps haven't been finalized yet, something has gone very + * wrong, and hyp will crash and burn when it uses any + * cpus_have_const_cap() wrapper. + */ + BUG_ON(!system_capabilities_finalized()); + __kvm_call_hyp((void *)pgd_ptr, hyp_stack_ptr, vector_ptr, tpidr_el2); + + /* + * Disabling SSBD on a non-VHE system requires us to enable SSBS + * at EL2. + */ + if (this_cpu_has_cap(ARM64_SSBS) && + arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) { + kvm_call_hyp(__kvm_enable_ssbs); + } } static void cpu_hyp_reset(void) From 438f711ce1d889632467be80779c8f5762b107d7 Mon Sep 17 00:00:00 2001 From: David Brazdil Date: Fri, 15 May 2020 16:25:50 +0100 Subject: [PATCH 0814/1043] KVM: arm64: Fix incorrect comment on kvm_get_hyp_vector() The comment used to say that kvm_get_hyp_vector is only called on VHE systems. In fact, it is also called from the nVHE init function cpu_init_hyp_mode(). Fix the comment to stop confusing devs. Signed-off-by: David Brazdil Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200515152550.83810-1-dbrazdil@google.com --- arch/arm64/include/asm/kvm_mmu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 30b0e8d6b895..796f6a2e794a 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -473,7 +473,7 @@ static inline int kvm_write_guest_lock(struct kvm *kvm, gpa_t gpa, extern void *__kvm_bp_vect_base; extern int __kvm_harden_el2_vector_slot; -/* This is only called on a VHE system */ +/* This is called on both VHE and !VHE systems */ static inline void *kvm_get_hyp_vector(void) { struct bp_hardening_data *data = arm64_get_bp_hardening_data(); From 0a78791c0d12fcd5d3f486668defb9ab055e3729 Mon Sep 17 00:00:00 2001 From: Andrew Scull Date: Tue, 19 May 2020 11:40:36 +0100 Subject: [PATCH 0815/1043] KVM: arm64: Remove obsolete kvm_virt_to_phys abstraction This abstraction was introduced to hide the difference between arm and arm64 but, with the former no longer supported, this abstraction can be removed and the canonical kernel API used directly instead. Signed-off-by: Andrew Scull Signed-off-by: Marc Zyngier CC: Marc Zyngier CC: James Morse CC: Suzuki K Poulose Link: https://lore.kernel.org/r/20200519104036.259917-1-ascull@google.com --- arch/arm64/include/asm/kvm_mmu.h | 2 -- arch/arm64/kvm/mmu.c | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 796f6a2e794a..53bd4d517a4d 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -363,8 +363,6 @@ static inline void __kvm_flush_dcache_pud(pud_t pud) } } -#define kvm_virt_to_phys(x) __pa_symbol(x) - void kvm_set_way_flush(struct kvm_vcpu *vcpu); void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index ddf85bf21897..a1f6bc70c4e4 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2197,11 +2197,11 @@ int kvm_mmu_init(void) { int err; - hyp_idmap_start = kvm_virt_to_phys(__hyp_idmap_text_start); + hyp_idmap_start = __pa_symbol(__hyp_idmap_text_start); hyp_idmap_start = ALIGN_DOWN(hyp_idmap_start, PAGE_SIZE); - hyp_idmap_end = kvm_virt_to_phys(__hyp_idmap_text_end); + hyp_idmap_end = __pa_symbol(__hyp_idmap_text_end); hyp_idmap_end = ALIGN(hyp_idmap_end, PAGE_SIZE); - hyp_idmap_vector = kvm_virt_to_phys(__kvm_hyp_init); + hyp_idmap_vector = __pa_symbol(__kvm_hyp_init); /* * We rely on the linker script to ensure at build time that the HYP From 3ead2f97bd44a9a106572d306cb04a878c569cb2 Mon Sep 17 00:00:00 2001 From: Chris Packham Date: Tue, 26 May 2020 11:04:13 +1200 Subject: [PATCH 0816/1043] xtensa: Fix spelling/grammar in comment Change 'excpetion' to 'exception', 'handeled' to 'handled' and 'the the' to 'the'. Signed-off-by: Chris Packham Message-Id: <20200525230413.15551-1-chris.packham@alliedtelesis.co.nz> Signed-off-by: Max Filippov --- arch/xtensa/kernel/entry.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/xtensa/kernel/entry.S b/arch/xtensa/kernel/entry.S index 06fbb0a171f1..fae33ddcaebb 100644 --- a/arch/xtensa/kernel/entry.S +++ b/arch/xtensa/kernel/entry.S @@ -959,14 +959,14 @@ ENDPROC(unrecoverable_exception) * of the proper size instead. * * This algorithm simply backs out the register changes started by the user - * excpetion handler, makes it appear that we have started a window underflow + * exception handler, makes it appear that we have started a window underflow * by rotating the window back and then setting the old window base (OWB) in * the 'ps' register with the rolled back window base. The 'movsp' instruction * will be re-executed and this time since the next window frames is in the * active AR registers it won't cause an exception. * * If the WindowUnderflow code gets a TLB miss the page will get mapped - * the the partial windeowUnderflow will be handeled in the double exception + * the partial WindowUnderflow will be handled in the double exception * handler. * * Entry condition: From 4518a3cc273cf82efdd36522fb1f13baad173c70 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 26 May 2020 20:34:02 +0300 Subject: [PATCH 0817/1043] io_uring: fix flush req->refs underflow In io_uring_cancel_files(), after refcount_sub_and_test() leaves 0 req->refs, it calls io_put_req(), which would also put a ref. Call io_free_req() instead. Cc: stable@vger.kernel.org Fixes: 2ca10259b418 ("io_uring: prune request from overflow list on flush") Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 0b51f21e5432..37422fcdaa7f 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7534,7 +7534,7 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, * all we had, then we're done with this request. */ if (refcount_sub_and_test(2, &cancel_req->refs)) { - io_put_req(cancel_req); + io_free_req(cancel_req); finish_wait(&ctx->inflight_wait, &wait); continue; } From 733f5c95e6fdabd05b8dfc15e04512809c9652c2 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 26 May 2020 20:34:03 +0300 Subject: [PATCH 0818/1043] io_uring: simplify io_timeout locking Move spin_lock_irq() earlier to have only 1 call site of it in io_timeout(). It makes the flow easier. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 37422fcdaa7f..4be8f9eb71e4 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4845,6 +4845,7 @@ static int io_timeout(struct io_kiocb *req) u32 seq = req->sequence; data = &req->io->timeout; + spin_lock_irq(&ctx->completion_lock); /* * sqe->off holds how many events that need to occur for this @@ -4853,7 +4854,6 @@ static int io_timeout(struct io_kiocb *req) */ if (!count) { req->flags |= REQ_F_TIMEOUT_NOSEQ; - spin_lock_irq(&ctx->completion_lock); entry = ctx->timeout_list.prev; goto add; } @@ -4864,7 +4864,6 @@ static int io_timeout(struct io_kiocb *req) * Insertion sort, ensuring the first entry in the list is always * the one we need first. */ - spin_lock_irq(&ctx->completion_lock); list_for_each_prev(entry, &ctx->timeout_list) { struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list); unsigned nxt_seq; From 56080b02ed6e71fbc0add2d05a32ed7361dd736a Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 26 May 2020 20:34:04 +0300 Subject: [PATCH 0819/1043] io_uring: don't re-read sqe->off in timeout_prep() SQEs are user writable, don't read sqe->off twice in io_timeout_prep() Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 4be8f9eb71e4..f888b20d0a68 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -4803,18 +4803,19 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, { struct io_timeout_data *data; unsigned flags; + u32 off = READ_ONCE(sqe->off); if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) return -EINVAL; if (sqe->ioprio || sqe->buf_index || sqe->len != 1) return -EINVAL; - if (sqe->off && is_timeout_link) + if (off && is_timeout_link) return -EINVAL; flags = READ_ONCE(sqe->timeout_flags); if (flags & ~IORING_TIMEOUT_ABS) return -EINVAL; - req->timeout.count = READ_ONCE(sqe->off); + req->timeout.count = off; if (!req->io && io_alloc_async_ctx(req)) return -ENOMEM; From 0451894522108d6c72934aff6ef89023743a9ed4 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 26 May 2020 20:34:05 +0300 Subject: [PATCH 0820/1043] io_uring: separate DRAIN flushing into a cold path io_commit_cqring() assembly doesn't look good with extra code handling drained requests. IOSQE_IO_DRAIN is slow and discouraged to be used in a hot path, so try to minimise its impact by putting it into a helper and doing a fast check. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index f888b20d0a68..0d98a529a93e 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -982,19 +982,6 @@ static inline bool req_need_defer(struct io_kiocb *req) return false; } -static struct io_kiocb *io_get_deferred_req(struct io_ring_ctx *ctx) -{ - struct io_kiocb *req; - - req = list_first_entry_or_null(&ctx->defer_list, struct io_kiocb, list); - if (req && !req_need_defer(req)) { - list_del_init(&req->list); - return req; - } - - return NULL; -} - static struct io_kiocb *io_get_timeout_req(struct io_ring_ctx *ctx) { struct io_kiocb *req; @@ -1127,6 +1114,19 @@ static void io_kill_timeouts(struct io_ring_ctx *ctx) spin_unlock_irq(&ctx->completion_lock); } +static void __io_queue_deferred(struct io_ring_ctx *ctx) +{ + do { + struct io_kiocb *req = list_first_entry(&ctx->defer_list, + struct io_kiocb, list); + + if (req_need_defer(req)) + break; + list_del_init(&req->list); + io_queue_async_work(req); + } while (!list_empty(&ctx->defer_list)); +} + static void io_commit_cqring(struct io_ring_ctx *ctx) { struct io_kiocb *req; @@ -1136,8 +1136,8 @@ static void io_commit_cqring(struct io_ring_ctx *ctx) __io_commit_cqring(ctx); - while ((req = io_get_deferred_req(ctx)) != NULL) - io_queue_async_work(req); + if (unlikely(!list_empty(&ctx->defer_list))) + __io_queue_deferred(ctx); } static struct io_uring_cqe *io_get_cqring(struct io_ring_ctx *ctx) From 0bf0eefdab52d9f9f3a1eeda32a4fc7afe4e9219 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Tue, 26 May 2020 20:34:06 +0300 Subject: [PATCH 0821/1043] io_uring: get rid of manual punting in io_close io_close() was punting async manually to skip grabbing files. Use REQ_F_NO_FILE_TABLE instead, and pass it through the generic path with -EAGAIN. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 0d98a529a93e..8c7a6e3e7669 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -3492,25 +3492,15 @@ static int io_close(struct io_kiocb *req, bool force_nonblock) req->close.put_file = NULL; ret = __close_fd_get_file(req->close.fd, &req->close.put_file); - if (ret < 0) { - if (ret == -ENOENT) - ret = -EBADF; - return ret; - } + if (ret < 0) + return (ret == -ENOENT) ? -EBADF : ret; /* if the file has a flush method, be safe and punt to async */ if (req->close.put_file->f_op->flush && force_nonblock) { - /* submission ref will be dropped, take it for async */ - refcount_inc(&req->refs); - + /* avoid grabbing files - we don't need the files */ + req->flags |= REQ_F_NO_FILE_TABLE | REQ_F_MUST_PUNT; req->work.func = io_close_finish; - /* - * Do manual async queue here to avoid grabbing files - we don't - * need the files, and it'll cause io_close_finish() to close - * the file again and cause a double CQE entry for this request - */ - io_queue_async_work(req); - return 0; + return -EAGAIN; } /* From 263c61581a38d0a5ad1f5f4a9143b27d68caeffd Mon Sep 17 00:00:00 2001 From: Jiri Kosina Date: Tue, 26 May 2020 11:49:18 +0200 Subject: [PATCH 0822/1043] block/floppy: fix contended case in floppy_queue_rq() Since the switch of floppy driver to blk-mq, the contended (fdc_busy) case in floppy_queue_rq() is not handled correctly. In case we reach floppy_queue_rq() with fdc_busy set (i.e. with the floppy locked due to another request still being in-flight), we put the request on the list of requests and return BLK_STS_OK to the block core, without actually scheduling delayed work / doing further processing of the request. This means that processing of this request is postponed until another request comes and passess uncontended. Which in some cases might actually never happen and we keep waiting indefinitely. The simple testcase is for i in `seq 1 2000`; do echo -en $i '\r'; blkid --info /dev/fd0 2> /dev/null; done run in quemu. That reliably causes blkid eventually indefinitely hanging in __floppy_read_block_0() waiting for completion, as the BIO callback never happens, and no further IO is ever submitted on the (non-existent) floppy device. This was observed reliably on qemu-emulated device. Fix that by not queuing the request in the contended case, and return BLK_STS_RESOURCE instead, so that blk core handles the request rescheduling and let it pass properly non-contended later. Fixes: a9f38e1dec107a ("floppy: convert to blk-mq") Cc: stable@vger.kernel.org Tested-by: Libor Pechacek Signed-off-by: Jiri Kosina Signed-off-by: Jens Axboe --- drivers/block/floppy.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 064c1acb9f00..3e9db22db2a8 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -2950,17 +2950,17 @@ static blk_status_t floppy_queue_rq(struct blk_mq_hw_ctx *hctx, (unsigned long long) current_req->cmd_flags)) return BLK_STS_IOERR; - spin_lock_irq(&floppy_lock); - list_add_tail(&bd->rq->queuelist, &floppy_reqs); - spin_unlock_irq(&floppy_lock); - if (test_and_set_bit(0, &fdc_busy)) { /* fdc busy, this new request will be treated when the current one is done */ is_alive(__func__, "old request running"); - return BLK_STS_OK; + return BLK_STS_RESOURCE; } + spin_lock_irq(&floppy_lock); + list_add_tail(&bd->rq->queuelist, &floppy_reqs); + spin_unlock_irq(&floppy_lock); + command_status = FD_COMMAND_NONE; __reschedule_timeout(MAXTIMEOUT, "fd_request"); set_fdc(0); From 1d9e1288039a47dc1189c3c1fed5cf3c215e94b7 Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 22 May 2020 21:31:16 -0700 Subject: [PATCH 0823/1043] io_uring: add io_statx structure Separate statx data from open in io_kiocb. No functional changes. Signed-off-by: Bijan Mottahedeh Signed-off-by: Jens Axboe --- fs/io_uring.c | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 8c7a6e3e7669..2d86bd7b2787 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -425,11 +425,7 @@ struct io_sr_msg { struct io_open { struct file *file; int dfd; - union { - unsigned mask; - }; struct filename *filename; - struct statx __user *buffer; struct open_how how; unsigned long nofile; }; @@ -481,6 +477,15 @@ struct io_provide_buf { __u16 bid; }; +struct io_statx { + struct file *file; + int dfd; + unsigned int mask; + unsigned int flags; + struct filename *filename; + struct statx __user *buffer; +}; + struct io_async_connect { struct sockaddr_storage address; }; @@ -622,6 +627,7 @@ struct io_kiocb { struct io_epoll epoll; struct io_splice splice; struct io_provide_buf pbuf; + struct io_statx statx; }; struct io_async_ctx *io; @@ -3381,19 +3387,19 @@ static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (req->flags & REQ_F_NEED_CLEANUP) return 0; - req->open.dfd = READ_ONCE(sqe->fd); - req->open.mask = READ_ONCE(sqe->len); + req->statx.dfd = READ_ONCE(sqe->fd); + req->statx.mask = READ_ONCE(sqe->len); fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); - req->open.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2)); - req->open.how.flags = READ_ONCE(sqe->statx_flags); + req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2)); + req->statx.flags = READ_ONCE(sqe->statx_flags); - if (vfs_stat_set_lookup_flags(&lookup_flags, req->open.how.flags)) + if (vfs_stat_set_lookup_flags(&lookup_flags, req->statx.flags)) return -EINVAL; - req->open.filename = getname_flags(fname, lookup_flags, NULL); - if (IS_ERR(req->open.filename)) { - ret = PTR_ERR(req->open.filename); - req->open.filename = NULL; + req->statx.filename = getname_flags(fname, lookup_flags, NULL); + if (IS_ERR(req->statx.filename)) { + ret = PTR_ERR(req->statx.filename); + req->statx.filename = NULL; return ret; } @@ -3403,7 +3409,7 @@ static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) static int io_statx(struct io_kiocb *req, bool force_nonblock) { - struct io_open *ctx = &req->open; + struct io_statx *ctx = &req->statx; unsigned lookup_flags; struct path path; struct kstat stat; @@ -3416,7 +3422,7 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock) return -EAGAIN; } - if (vfs_stat_set_lookup_flags(&lookup_flags, ctx->how.flags)) + if (vfs_stat_set_lookup_flags(&lookup_flags, ctx->flags)) return -EINVAL; retry: @@ -3428,7 +3434,7 @@ retry: if (ret) goto err; - ret = vfs_getattr(&path, &stat, ctx->mask, ctx->how.flags); + ret = vfs_getattr(&path, &stat, ctx->mask, ctx->flags); path_put(&path); if (retry_estale(ret, lookup_flags)) { lookup_flags |= LOOKUP_REVAL; From 0018784fc84f636d473a0d2a65a34f9d01893c0a Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 22 May 2020 21:31:17 -0700 Subject: [PATCH 0824/1043] statx: allow system call to be invoked from io_uring This is a prepatory patch to allow io_uring to invoke statx directly. Signed-off-by: Bijan Mottahedeh Signed-off-by: Jens Axboe --- fs/internal.h | 2 ++ fs/stat.c | 32 +++++++++++++++++++------------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/fs/internal.h b/fs/internal.h index aa5d45524e87..88a9793f96a1 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -188,3 +188,5 @@ int sb_init_dio_done_wq(struct super_block *sb); */ unsigned vfs_stat_set_lookup_flags(unsigned *lookup_flags, int flags); int cp_statx(const struct kstat *stat, struct statx __user *buffer); +int do_statx(int dfd, const char __user *filename, unsigned flags, + unsigned int mask, struct statx __user *buffer); diff --git a/fs/stat.c b/fs/stat.c index 030008796479..65cf51fda2aa 100644 --- a/fs/stat.c +++ b/fs/stat.c @@ -567,6 +567,24 @@ cp_statx(const struct kstat *stat, struct statx __user *buffer) return copy_to_user(buffer, &tmp, sizeof(tmp)) ? -EFAULT : 0; } +int do_statx(int dfd, const char __user *filename, unsigned flags, + unsigned int mask, struct statx __user *buffer) +{ + struct kstat stat; + int error; + + if (mask & STATX__RESERVED) + return -EINVAL; + if ((flags & AT_STATX_SYNC_TYPE) == AT_STATX_SYNC_TYPE) + return -EINVAL; + + error = vfs_statx(dfd, filename, flags, &stat, mask); + if (error) + return error; + + return cp_statx(&stat, buffer); +} + /** * sys_statx - System call to get enhanced stats * @dfd: Base directory to pathwalk from *or* fd to stat. @@ -583,19 +601,7 @@ SYSCALL_DEFINE5(statx, unsigned int, mask, struct statx __user *, buffer) { - struct kstat stat; - int error; - - if (mask & STATX__RESERVED) - return -EINVAL; - if ((flags & AT_STATX_SYNC_TYPE) == AT_STATX_SYNC_TYPE) - return -EINVAL; - - error = vfs_statx(dfd, filename, flags, &stat, mask); - if (error) - return error; - - return cp_statx(&stat, buffer); + return do_statx(dfd, filename, flags, mask, buffer); } #ifdef CONFIG_COMPAT From e62753e4e2926f249d088cc0517be5ed4efec6d6 Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 22 May 2020 21:31:18 -0700 Subject: [PATCH 0825/1043] io_uring: call statx directly Calling statx directly both simplifies the interface and avoids potential incompatibilities between sync and async invokations. Signed-off-by: Bijan Mottahedeh Signed-off-by: Jens Axboe --- fs/io_uring.c | 50 ++++---------------------------------------------- 1 file changed, 4 insertions(+), 46 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 2d86bd7b2787..de6547e68626 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -482,7 +482,7 @@ struct io_statx { int dfd; unsigned int mask; unsigned int flags; - struct filename *filename; + const char __user *filename; struct statx __user *buffer; }; @@ -3376,43 +3376,23 @@ static int io_fadvise(struct io_kiocb *req, bool force_nonblock) static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - const char __user *fname; - unsigned lookup_flags; - int ret; - if (sqe->ioprio || sqe->buf_index) return -EINVAL; if (req->flags & REQ_F_FIXED_FILE) return -EBADF; - if (req->flags & REQ_F_NEED_CLEANUP) - return 0; req->statx.dfd = READ_ONCE(sqe->fd); req->statx.mask = READ_ONCE(sqe->len); - fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); + req->statx.filename = u64_to_user_ptr(READ_ONCE(sqe->addr)); req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2)); req->statx.flags = READ_ONCE(sqe->statx_flags); - if (vfs_stat_set_lookup_flags(&lookup_flags, req->statx.flags)) - return -EINVAL; - - req->statx.filename = getname_flags(fname, lookup_flags, NULL); - if (IS_ERR(req->statx.filename)) { - ret = PTR_ERR(req->statx.filename); - req->statx.filename = NULL; - return ret; - } - - req->flags |= REQ_F_NEED_CLEANUP; return 0; } static int io_statx(struct io_kiocb *req, bool force_nonblock) { struct io_statx *ctx = &req->statx; - unsigned lookup_flags; - struct path path; - struct kstat stat; int ret; if (force_nonblock) { @@ -3422,29 +3402,9 @@ static int io_statx(struct io_kiocb *req, bool force_nonblock) return -EAGAIN; } - if (vfs_stat_set_lookup_flags(&lookup_flags, ctx->flags)) - return -EINVAL; + ret = do_statx(ctx->dfd, ctx->filename, ctx->flags, ctx->mask, + ctx->buffer); -retry: - /* filename_lookup() drops it, keep a reference */ - ctx->filename->refcnt++; - - ret = filename_lookup(ctx->dfd, ctx->filename, lookup_flags, &path, - NULL); - if (ret) - goto err; - - ret = vfs_getattr(&path, &stat, ctx->mask, ctx->flags); - path_put(&path); - if (retry_estale(ret, lookup_flags)) { - lookup_flags |= LOOKUP_REVAL; - goto retry; - } - if (!ret) - ret = cp_statx(&stat, ctx->buffer); -err: - putname(ctx->filename); - req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < 0) req_set_fail_links(req); io_cqring_add_event(req, ret); @@ -5196,8 +5156,6 @@ static void io_cleanup_req(struct io_kiocb *req) break; case IORING_OP_OPENAT: case IORING_OP_OPENAT2: - case IORING_OP_STATX: - putname(req->open.filename); break; case IORING_OP_SPLICE: case IORING_OP_TEE: From 6f88cc176a3358c54bb6c38c8afee3f3a42faf54 Mon Sep 17 00:00:00 2001 From: Bijan Mottahedeh Date: Fri, 22 May 2020 21:31:19 -0700 Subject: [PATCH 0826/1043] statx: hide interfaces no longer used by io_uring The io_uring interfaces have been replaced by do_statx() and are no longer needed. Signed-off-by: Bijan Mottahedeh Signed-off-by: Jens Axboe --- fs/internal.h | 2 -- fs/stat.c | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/internal.h b/fs/internal.h index 88a9793f96a1..cf2043c8ac3d 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -186,7 +186,5 @@ int sb_init_dio_done_wq(struct super_block *sb); /* * fs/stat.c: */ -unsigned vfs_stat_set_lookup_flags(unsigned *lookup_flags, int flags); -int cp_statx(const struct kstat *stat, struct statx __user *buffer); int do_statx(int dfd, const char __user *filename, unsigned flags, unsigned int mask, struct statx __user *buffer); diff --git a/fs/stat.c b/fs/stat.c index 65cf51fda2aa..1b509d87265b 100644 --- a/fs/stat.c +++ b/fs/stat.c @@ -152,7 +152,8 @@ int vfs_statx_fd(unsigned int fd, struct kstat *stat, } EXPORT_SYMBOL(vfs_statx_fd); -inline unsigned vfs_stat_set_lookup_flags(unsigned *lookup_flags, int flags) +static inline unsigned vfs_stat_set_lookup_flags(unsigned *lookup_flags, + int flags) { if ((flags & ~(AT_SYMLINK_NOFOLLOW | AT_NO_AUTOMOUNT | AT_EMPTY_PATH | KSTAT_QUERY_FLAGS)) != 0) @@ -533,7 +534,7 @@ SYSCALL_DEFINE4(fstatat64, int, dfd, const char __user *, filename, } #endif /* __ARCH_WANT_STAT64 || __ARCH_WANT_COMPAT_STAT64 */ -noinline_for_stack int +static noinline_for_stack int cp_statx(const struct kstat *stat, struct statx __user *buffer) { struct statx tmp; From 09bb8986c99cd3395e96635df9e712f5da45bc07 Mon Sep 17 00:00:00 2001 From: Chen Zhou Date: Fri, 8 May 2020 19:59:06 +0800 Subject: [PATCH 0827/1043] nvmet: replace kstrndup() with kmemdup_nul() It is more efficient to use kmemdup_nul() if the size is known exactly. The doc in kernel: "Note: Use kmemdup_nul() instead if the size is known exactly." Signed-off-by: Chen Zhou Signed-off-by: Christoph Hellwig --- drivers/nvme/target/configfs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index ae8fb4489a10..19bd5e1c681c 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -324,7 +324,7 @@ static ssize_t nvmet_ns_device_path_store(struct config_item *item, kfree(ns->device_path); ret = -ENOMEM; - ns->device_path = kstrndup(page, len, GFP_KERNEL); + ns->device_path = kmemdup_nul(page, len, GFP_KERNEL); if (!ns->device_path) goto out_unlock; @@ -960,7 +960,7 @@ static ssize_t nvmet_subsys_attr_model_store(struct config_item *item, return -EINVAL; } - new_model_number = kstrndup(page, len, GFP_KERNEL); + new_model_number = kmemdup_nul(page, len, GFP_KERNEL); if (!new_model_number) return -ENOMEM; From 7425596945d7f8bf79f42b2a9c8463d16e24fc34 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 12 May 2020 18:19:13 +0200 Subject: [PATCH 0828/1043] nvmet: mark nvmet_ana_state static Signed-off-by: Christoph Hellwig Reviewed-by: Max Gurtovoy Reviewed-by: Chaitanya Kulkarni --- drivers/nvme/target/configfs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 19bd5e1c681c..24eb4cf53b4f 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -1146,7 +1146,7 @@ static const struct config_item_type nvmet_referrals_type = { .ct_group_ops = &nvmet_referral_group_ops, }; -struct nvmet_type_name_map nvmet_ana_state[] = { +static struct nvmet_type_name_map nvmet_ana_state[] = { { NVME_ANA_OPTIMIZED, "optimized" }, { NVME_ANA_NONOPTIMIZED, "non-optimized" }, { NVME_ANA_INACCESSIBLE, "inaccessible" }, From 5bb052d7aad1f4063631f2aa05452c700139f0f3 Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Mon, 4 May 2020 22:20:01 -0700 Subject: [PATCH 0829/1043] nvme-tcp: set MSG_SENDPAGE_NOTLAST with MSG_MORE when we have more to send We can signal the stack that this is not the last page coming and the stack can build a larger tso segment, so go ahead and use it. Signed-off-by: Sagi Grimberg Signed-off-by: Christoph Hellwig --- drivers/nvme/host/tcp.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index c79e248b9f43..7c7c1886642f 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -885,7 +885,7 @@ static int nvme_tcp_try_send_data(struct nvme_tcp_request *req) if (last && !queue->data_digest) flags |= MSG_EOR; else - flags |= MSG_MORE; + flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; /* can't zcopy slab pages */ if (unlikely(PageSlab(page))) { @@ -924,11 +924,16 @@ static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req) struct nvme_tcp_queue *queue = req->queue; struct nvme_tcp_cmd_pdu *pdu = req->pdu; bool inline_data = nvme_tcp_has_inline_data(req); - int flags = MSG_DONTWAIT | (inline_data ? MSG_MORE : MSG_EOR); u8 hdgst = nvme_tcp_hdgst_len(queue); int len = sizeof(*pdu) + hdgst - req->offset; + int flags = MSG_DONTWAIT; int ret; + if (inline_data) + flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; + else + flags |= MSG_EOR; + if (queue->hdr_digest && !req->offset) nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); @@ -967,7 +972,7 @@ static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req) ret = kernel_sendpage(queue->sock, virt_to_page(pdu), offset_in_page(pdu) + req->offset, len, - MSG_DONTWAIT | MSG_MORE); + MSG_DONTWAIT | MSG_MORE | MSG_SENDPAGE_NOTLAST); if (unlikely(ret <= 0)) return ret; From 4eea804364628c30facd99c30499b2b01c6272c6 Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Mon, 4 May 2020 22:20:02 -0700 Subject: [PATCH 0830/1043] nvmet-tcp: set MSG_SENDPAGE_NOTLAST with MSG_MORE when we have more to send We can signal the stack that this is not the last page coming and the stack can build a larger tso segment, so go ahead and use it. Signed-off-by: Sagi Grimberg Signed-off-by: Christoph Hellwig --- drivers/nvme/target/tcp.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index f0da04e960f4..c08aec62115e 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -510,7 +510,7 @@ static int nvmet_try_send_data_pdu(struct nvmet_tcp_cmd *cmd) ret = kernel_sendpage(cmd->queue->sock, virt_to_page(cmd->data_pdu), offset_in_page(cmd->data_pdu) + cmd->offset, - left, MSG_DONTWAIT | MSG_MORE); + left, MSG_DONTWAIT | MSG_MORE | MSG_SENDPAGE_NOTLAST); if (ret <= 0) return ret; @@ -538,7 +538,7 @@ static int nvmet_try_send_data(struct nvmet_tcp_cmd *cmd, bool last_in_batch) if ((!last_in_batch && cmd->queue->send_list_len) || cmd->wbytes_done + left < cmd->req.transfer_len || queue->data_digest || !queue->nvme_sq.sqhd_disabled) - flags |= MSG_MORE; + flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; ret = kernel_sendpage(cmd->queue->sock, page, cmd->offset, left, flags); @@ -585,7 +585,7 @@ static int nvmet_try_send_response(struct nvmet_tcp_cmd *cmd, int ret; if (!last_in_batch && cmd->queue->send_list_len) - flags |= MSG_MORE; + flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; else flags |= MSG_EOR; @@ -614,7 +614,7 @@ static int nvmet_try_send_r2t(struct nvmet_tcp_cmd *cmd, bool last_in_batch) int ret; if (!last_in_batch && cmd->queue->send_list_len) - flags |= MSG_MORE; + flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; else flags |= MSG_EOR; From f381ab1f26aa33f14412cb9b7cb7f947b88a55b2 Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Tue, 12 May 2020 18:01:43 -0700 Subject: [PATCH 0831/1043] nvmet-tcp: set MSG_EOR if we send last payload in the batch when trying to send the pdu data digest, we should set this flag. Reported-by: Christoph Hellwig Signed-off-by: Sagi Grimberg Signed-off-by: Christoph Hellwig --- drivers/nvme/target/tcp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index c08aec62115e..f8dd69f3b657 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -644,6 +644,8 @@ static int nvmet_try_send_ddgst(struct nvmet_tcp_cmd *cmd, bool last_in_batch) if (!last_in_batch && cmd->queue->send_list_len) msg.msg_flags |= MSG_MORE; + else + msg.msg_flags |= MSG_EOR; ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); if (unlikely(ret <= 0)) From 0236d3437909ff888e5c79228e2d5a851651c4c6 Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Mon, 18 May 2020 10:47:48 -0700 Subject: [PATCH 0832/1043] nvmet-tcp: move send/recv error handling in the send/recv methods instead of call-sites Have routines handle errors and just bail out of the poll loop. This simplifies the code and will help as we may enhance the poll loop logic and these are somewhat in the way. Signed-off-by: Sagi Grimberg Signed-off-by: Christoph Hellwig --- drivers/nvme/target/tcp.c | 43 ++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index f8dd69f3b657..6f557db0320d 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -325,6 +325,14 @@ static void nvmet_tcp_fatal_error(struct nvmet_tcp_queue *queue) kernel_sock_shutdown(queue->sock, SHUT_RDWR); } +static void nvmet_tcp_socket_error(struct nvmet_tcp_queue *queue, int status) +{ + if (status == -EPIPE || status == -ECONNRESET) + kernel_sock_shutdown(queue->sock, SHUT_RDWR); + else + nvmet_tcp_fatal_error(queue); +} + static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd) { struct nvme_sgl_desc *sgl = &cmd->req.cmd->common.dptr.sgl; @@ -718,11 +726,15 @@ static int nvmet_tcp_try_send(struct nvmet_tcp_queue *queue, for (i = 0; i < budget; i++) { ret = nvmet_tcp_try_send_one(queue, i == budget - 1); - if (ret <= 0) + if (unlikely(ret < 0)) { + nvmet_tcp_socket_error(queue, ret); + goto done; + } else if (ret == 0) { break; + } (*sends)++; } - +done: return ret; } @@ -1159,11 +1171,15 @@ static int nvmet_tcp_try_recv(struct nvmet_tcp_queue *queue, for (i = 0; i < budget; i++) { ret = nvmet_tcp_try_recv_one(queue); - if (ret <= 0) + if (unlikely(ret < 0)) { + nvmet_tcp_socket_error(queue, ret); + goto done; + } else if (ret == 0) { break; + } (*recvs)++; } - +done: return ret; } @@ -1188,27 +1204,16 @@ static void nvmet_tcp_io_work(struct work_struct *w) pending = false; ret = nvmet_tcp_try_recv(queue, NVMET_TCP_RECV_BUDGET, &ops); - if (ret > 0) { + if (ret > 0) pending = true; - } else if (ret < 0) { - if (ret == -EPIPE || ret == -ECONNRESET) - kernel_sock_shutdown(queue->sock, SHUT_RDWR); - else - nvmet_tcp_fatal_error(queue); + else if (ret < 0) return; - } ret = nvmet_tcp_try_send(queue, NVMET_TCP_SEND_BUDGET, &ops); - if (ret > 0) { - /* transmitted message/data */ + if (ret > 0) pending = true; - } else if (ret < 0) { - if (ret == -EPIPE || ret == -ECONNRESET) - kernel_sock_shutdown(queue->sock, SHUT_RDWR); - else - nvmet_tcp_fatal_error(queue); + else if (ret < 0) return; - } } while (pending && ops < NVMET_TCP_IO_WORK_BUDGET); From 9c9e76d5792b121f10c3b8ddbb639617e49197f7 Mon Sep 17 00:00:00 2001 From: Weiping Zhang Date: Sat, 9 May 2020 14:22:08 +0800 Subject: [PATCH 0833/1043] nvme-pci: make sure write/poll_queues less or equal then cpu count Check module parameter write/poll_queues before using it to catch too large values. Reproducer: modprobe -r nvme modprobe nvme write_queues=`nproc` echo $((`nproc`+1)) > /sys/module/nvme/parameters/write_queues echo 1 > /sys/block/nvme0n1/device/reset_controller [ 657.069000] ------------[ cut here ]------------ [ 657.069022] WARNING: CPU: 10 PID: 1163 at kernel/irq/affinity.c:390 irq_create_affinity_masks+0x47c/0x4a0 [ 657.069056] dm_region_hash dm_log dm_mod [ 657.069059] CPU: 10 PID: 1163 Comm: kworker/u193:9 Kdump: loaded Tainted: G W 5.6.0+ #8 [ 657.069060] Hardware name: Inspur SA5212M5/YZMB-00882-104, BIOS 4.0.9 08/27/2019 [ 657.069064] Workqueue: nvme-reset-wq nvme_reset_work [nvme] [ 657.069066] RIP: 0010:irq_create_affinity_masks+0x47c/0x4a0 [ 657.069067] Code: fe ff ff 48 c7 c0 b0 89 14 95 48 89 46 20 e9 e9 fb ff ff 31 c0 e9 90 fc ff ff 0f 0b 48 c7 44 24 08 00 00 00 00 e9 e9 fc ff ff <0f> 0b e9 87 fe ff ff 48 8b 7c 24 28 e8 33 a0 80 00 e9 b6 fc ff ff [ 657.069068] RSP: 0018:ffffb505ce1ffc78 EFLAGS: 00010202 [ 657.069069] RAX: 0000000000000060 RBX: ffff9b97921fe5c0 RCX: 0000000000000000 [ 657.069069] RDX: ffff9b67bad80000 RSI: 00000000ffffffa0 RDI: 0000000000000000 [ 657.069070] RBP: 0000000000000000 R08: 0000000000000000 R09: ffff9b97921fe718 [ 657.069070] R10: ffff9b97921fe710 R11: 0000000000000001 R12: 0000000000000064 [ 657.069070] R13: 0000000000000060 R14: 0000000000000000 R15: 0000000000000001 [ 657.069071] FS: 0000000000000000(0000) GS:ffff9b67c0880000(0000) knlGS:0000000000000000 [ 657.069072] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 657.069072] CR2: 0000559eac6fc238 CR3: 000000057860a002 CR4: 00000000007606e0 [ 657.069073] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 657.069073] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 657.069073] PKRU: 55555554 [ 657.069074] Call Trace: [ 657.069080] __pci_enable_msix_range+0x233/0x5a0 [ 657.069085] ? kernfs_put+0xec/0x190 [ 657.069086] pci_alloc_irq_vectors_affinity+0xbb/0x130 [ 657.069089] nvme_reset_work+0x6e6/0xeab [nvme] [ 657.069093] ? __switch_to_asm+0x34/0x70 [ 657.069094] ? __switch_to_asm+0x40/0x70 [ 657.069095] ? nvme_irq_check+0x30/0x30 [nvme] [ 657.069098] process_one_work+0x1a7/0x370 [ 657.069101] worker_thread+0x1c9/0x380 [ 657.069102] ? max_active_store+0x80/0x80 [ 657.069103] kthread+0x112/0x130 [ 657.069104] ? __kthread_parkme+0x70/0x70 [ 657.069105] ret_from_fork+0x35/0x40 [ 657.069106] ---[ end trace f4f06b7d24513d06 ]--- [ 657.077110] nvme nvme0: 95/1/0 default/read/poll queues Signed-off-by: Weiping Zhang Signed-off-by: Christoph Hellwig --- drivers/nvme/host/pci.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index b0978ac554d5..6cc4630ddd6d 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -68,14 +68,30 @@ static int io_queue_depth = 1024; module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644); MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2"); +static int io_queue_count_set(const char *val, const struct kernel_param *kp) +{ + unsigned int n; + int ret; + + ret = kstrtouint(val, 10, &n); + if (ret != 0 || n > num_possible_cpus()) + return -EINVAL; + return param_set_uint(val, kp); +} + +static const struct kernel_param_ops io_queue_count_ops = { + .set = io_queue_count_set, + .get = param_get_uint, +}; + static unsigned int write_queues; -module_param(write_queues, uint, 0644); +module_param_cb(write_queues, &io_queue_count_ops, &write_queues, 0644); MODULE_PARM_DESC(write_queues, "Number of queues to use for writes. If not set, reads and writes " "will share a queue set."); static unsigned int poll_queues; -module_param(poll_queues, uint, 0644); +module_param_cb(poll_queues, &io_queue_count_ops, &poll_queues, 0644); MODULE_PARM_DESC(poll_queues, "Number of queues to use for polled IO."); struct nvme_dev; @@ -3118,8 +3134,6 @@ static int __init nvme_init(void) BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64); BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2); - write_queues = min(write_queues, num_possible_cpus()); - poll_queues = min(poll_queues, num_possible_cpus()); return pci_register_driver(&nvme_driver); } From 614fc1c0d980423a131bfb5c93d8d53e5272f587 Mon Sep 17 00:00:00 2001 From: Martin George Date: Tue, 12 May 2020 22:17:04 +0530 Subject: [PATCH 0834/1043] nvme-fc: print proper nvme-fc devloss_tmo value The nvme-fc devloss_tmo is computed as the min of either the ctrl_loss_tmo (max_retries * reconnect_delay) or the remote port's devloss_tmo. But what gets printed as the nvme-fc devloss_tmo in nvme_fc_reconnect_or_delete() is always the remote port's devloss_tmo value. So correct this by printing the min value instead. Signed-off-by: Martin George Reviewed-by: James Smart Signed-off-by: Christoph Hellwig --- drivers/nvme/host/fc.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 0b3ab3355e25..8aabc056c754 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -3246,7 +3246,9 @@ nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status) dev_warn(ctrl->ctrl.device, "NVME-FC{%d}: dev_loss_tmo (%d) expired " "while waiting for remoteport connectivity.\n", - ctrl->cnum, portptr->dev_loss_tmo); + ctrl->cnum, min_t(int, portptr->dev_loss_tmo, + (ctrl->ctrl.opts->max_reconnects * + ctrl->ctrl.opts->reconnect_delay))); WARN_ON(nvme_delete_ctrl(&ctrl->ctrl)); } } From 84e4c204b6a0e81f56bd7d1254123390ef0498c8 Mon Sep 17 00:00:00 2001 From: Wu Bo Date: Wed, 13 May 2020 16:18:13 +0800 Subject: [PATCH 0835/1043] nvme: disable streams when get stream params failed Disable streams again if getting the stream params fails. Signed-off-by: Wu Bo Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 805d289e6cd9..43a6336d6264 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -552,19 +552,22 @@ static int nvme_configure_directives(struct nvme_ctrl *ctrl) ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL); if (ret) - return ret; + goto out_disable_stream; ctrl->nssa = le16_to_cpu(s.nssa); if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) { dev_info(ctrl->device, "too few streams (%u) available\n", ctrl->nssa); - nvme_disable_streams(ctrl); - return 0; + goto out_disable_stream; } ctrl->nr_streams = min_t(unsigned, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1); dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams); return 0; + +out_disable_stream: + nvme_disable_streams(ctrl); + return ret; } /* From 68ab60ca2d6bd8e6b1ecc2857a174c1c2b9451e9 Mon Sep 17 00:00:00 2001 From: Damien Le Moal Date: Thu, 14 May 2020 14:56:26 +0900 Subject: [PATCH 0836/1043] nvme: fix io_opt limit setting Currently, a namespace io_opt queue limit is set by default to the physical sector size of the namespace and to the the write optimal size (NOWS) when the namespace reports optimal IO sizes. This causes problems with block limits stacking in blk_stack_limits() when a namespace block device is combined with an HDD which generally do not report any optimal transfer size (io_opt limit is 0). The code: /* Optimal I/O a multiple of the physical block size? */ if (t->io_opt & (t->physical_block_size - 1)) { t->io_opt = 0; t->misaligned = 1; ret = -1; } in blk_stack_limits() results in an error return for this function when the combined devices have different but compatible physical sector sizes (e.g. 512B sector SSD with 4KB sector disks). Fix this by not setting the optimal IO size queue limit if the namespace does not report an optimal write size value. Signed-off-by: Damien Le Moal Reviewed-by: Bart van Assche Reviewed-by: Hannes Reinecke Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 43a6336d6264..4c194dcabd12 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1845,7 +1845,7 @@ static void nvme_update_disk_info(struct gendisk *disk, { sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze)); unsigned short bs = 1 << ns->lba_shift; - u32 atomic_bs, phys_bs, io_opt; + u32 atomic_bs, phys_bs, io_opt = 0; if (ns->lba_shift > PAGE_SHIFT) { /* unsupported block size, set capacity to 0 later */ @@ -1854,7 +1854,7 @@ static void nvme_update_disk_info(struct gendisk *disk, blk_mq_freeze_queue(disk->queue); blk_integrity_unregister(disk); - atomic_bs = phys_bs = io_opt = bs; + atomic_bs = phys_bs = bs; nvme_setup_streams_ns(ns->ctrl, ns, &phys_bs, &io_opt); if (id->nabo == 0) { /* From f1e71d75f04792721d411170168e68019ccb7de3 Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 7 May 2020 14:04:52 -0500 Subject: [PATCH 0837/1043] nvme: replace zero-length array with flexible-array The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] sizeof(flexible-array-member) triggers a warning because flexible array members have incomplete type[1]. There are some instances of code in which the sizeof operator is being incorrectly/erroneously applied to zero-length arrays and the result is zero. Such instances may be hiding some bugs. So, this work (flexible-array member conversions) will also help to get completely rid of those sorts of issues. This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Signed-off-by: Christoph Hellwig --- drivers/nvme/host/fc.c | 2 +- drivers/nvme/host/lightnvm.c | 2 +- include/linux/nvme.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 8aabc056c754..cb0007592c12 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -108,7 +108,7 @@ struct nvme_fc_fcp_op { struct nvme_fcp_op_w_sgl { struct nvme_fc_fcp_op op; struct scatterlist sgl[NVME_INLINE_SG_CNT]; - uint8_t priv[0]; + uint8_t priv[]; }; struct nvme_fc_lport { diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c index ec46693f6b64..3002bf972c6b 100644 --- a/drivers/nvme/host/lightnvm.c +++ b/drivers/nvme/host/lightnvm.c @@ -171,7 +171,7 @@ struct nvme_nvm_bb_tbl { __le32 tdresv; __le32 thresv; __le32 rsvd2[8]; - __u8 blk[0]; + __u8 blk[]; }; struct nvme_nvm_id20_addrf { diff --git a/include/linux/nvme.h b/include/linux/nvme.h index b235a48eac8c..e2993e6a9d7c 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -1185,7 +1185,7 @@ struct nvmf_disc_rsp_page_hdr { __le64 numrec; __le16 recfmt; __u8 resv14[1006]; - struct nvmf_disc_rsp_page_entry entries[0]; + struct nvmf_disc_rsp_page_entry entries[]; }; enum { From ec0862ac5aa03961fd2027f861689beccd91c5d1 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 15 May 2020 15:06:59 +0300 Subject: [PATCH 0838/1043] nvme: delete an unnecessary declaration The nvme_put_ctrl() is implemented earlier as an inline function so this declaration isn't required. Signed-off-by: Dan Carpenter Reviewed-by: Sagi Grimberg Reviewed-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig --- drivers/nvme/host/nvme.h | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index f3ab17778349..86f152e777bc 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -497,7 +497,6 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, void nvme_uninit_ctrl(struct nvme_ctrl *ctrl); void nvme_start_ctrl(struct nvme_ctrl *ctrl); void nvme_stop_ctrl(struct nvme_ctrl *ctrl); -void nvme_put_ctrl(struct nvme_ctrl *ctrl); int nvme_init_identify(struct nvme_ctrl *ctrl); void nvme_remove_namespaces(struct nvme_ctrl *ctrl); From 696ece751366e7a02a81fa0228125fe25a47969d Mon Sep 17 00:00:00 2001 From: Chaitanya Kulkarni Date: Tue, 19 May 2020 01:06:30 -0700 Subject: [PATCH 0839/1043] nvmet: add async event tracing support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a new tracepoint for the target to trace async event. This is helpful in debugging and comparing host and target side async events especially when host is connected to different targets on different machines and now that we rely on userspace components to generate AEN.  Signed-off-by: Chaitanya Kulkarni Reviewed-by: Christoph Hellwig Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig --- drivers/nvme/target/core.c | 1 + drivers/nvme/target/trace.h | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index b685f99d56a1..7ce3d3b0f5d6 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -151,6 +151,7 @@ static void nvmet_async_events_process(struct nvmet_ctrl *ctrl, u16 status) kfree(aen); mutex_unlock(&ctrl->lock); + trace_nvmet_async_event(ctrl, req->cqe->result.u32); nvmet_req_complete(req, status); } } diff --git a/drivers/nvme/target/trace.h b/drivers/nvme/target/trace.h index e645caa882dd..0458046d6501 100644 --- a/drivers/nvme/target/trace.h +++ b/drivers/nvme/target/trace.h @@ -130,6 +130,34 @@ TRACE_EVENT(nvmet_req_complete, ); +#define aer_name(aer) { aer, #aer } + +TRACE_EVENT(nvmet_async_event, + TP_PROTO(struct nvmet_ctrl *ctrl, __le32 result), + TP_ARGS(ctrl, result), + TP_STRUCT__entry( + __field(int, ctrl_id) + __field(u32, result) + ), + TP_fast_assign( + __entry->ctrl_id = ctrl->cntlid; + __entry->result = (le32_to_cpu(result) & 0xff00) >> 8; + ), + TP_printk("nvmet%d: NVME_AEN=%#08x [%s]", + __entry->ctrl_id, __entry->result, + __print_symbolic(__entry->result, + aer_name(NVME_AER_NOTICE_NS_CHANGED), + aer_name(NVME_AER_NOTICE_ANA), + aer_name(NVME_AER_NOTICE_FW_ACT_STARTING), + aer_name(NVME_AER_NOTICE_DISC_CHANGED), + aer_name(NVME_AER_ERROR), + aer_name(NVME_AER_SMART), + aer_name(NVME_AER_CSS), + aer_name(NVME_AER_VS)) + ) +); +#undef aer_name + #endif /* _TRACE_NVMET_H */ #undef TRACE_INCLUDE_PATH From 463c5fabb8dfca4941c4c50365bb7749ac5c9916 Mon Sep 17 00:00:00 2001 From: Chaitanya Kulkarni Date: Tue, 19 May 2020 01:06:27 -0700 Subject: [PATCH 0840/1043] nvmet: add helper to revalidate bdev and file ns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch adds a wrapper helper to indicate size change in the bdev & file-backed namespace when revalidating ns. This helper is needed in order to minimize code repetition in the next patch for configfs.c and existing admin-cmd.c.   Signed-off-by: Chaitanya Kulkarni Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig --- drivers/nvme/target/admin-cmd.c | 5 +---- drivers/nvme/target/core.c | 8 ++++++++ drivers/nvme/target/nvmet.h | 1 + 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 4c79aa804887..f544a14e8b5c 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -486,10 +486,7 @@ static void nvmet_execute_identify_ns(struct nvmet_req *req) if (!ns) goto done; - if (ns->bdev) - nvmet_bdev_ns_revalidate(ns); - else - nvmet_file_ns_revalidate(ns); + nvmet_ns_revalidate(ns); /* * nuse = ncap = nsze isn't always true, but we have no way to find diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 7ce3d3b0f5d6..ee5865568588 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -515,6 +515,14 @@ static void nvmet_p2pmem_ns_add_p2p(struct nvmet_ctrl *ctrl, ns->nsid); } +void nvmet_ns_revalidate(struct nvmet_ns *ns) +{ + if (ns->bdev) + nvmet_bdev_ns_revalidate(ns); + else + nvmet_file_ns_revalidate(ns); +} + int nvmet_ns_enable(struct nvmet_ns *ns) { struct nvmet_subsys *subsys = ns->subsys; diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 3d981eb6e100..93e0c2aa3e71 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -500,6 +500,7 @@ u16 nvmet_file_flush(struct nvmet_req *req); void nvmet_ns_changed(struct nvmet_subsys *subsys, u32 nsid); void nvmet_bdev_ns_revalidate(struct nvmet_ns *ns); int nvmet_file_ns_revalidate(struct nvmet_ns *ns); +void nvmet_ns_revalidate(struct nvmet_ns *ns); static inline u32 nvmet_rw_len(struct nvmet_req *req) { From de124f427347b0a1eb9708b1007251fc4c8a222f Mon Sep 17 00:00:00 2001 From: Chaitanya Kulkarni Date: Tue, 19 May 2020 01:06:28 -0700 Subject: [PATCH 0841/1043] nvmet: generate AEN for ns revalidate size change The newly added function nvmet_ns_revalidate() does update the ns size in the identify namespace in-core target data structure when host issues id-ns command. This can lead to host having inconsistencies between size of the namespace present in the id-ns command result and size of the corresponding block device until host scans the namespaces explicitly. To avoid this scenario generate AEN if old size is not same as the new one in nvmet_ns_revalidate(). This will allow automatic AEN generation when host calls id-ns command and also allows target to install userspace rules so that it can trigger nvmet_ns_revalidate() (using configfs interface with the help of next patch) resulting in appropriate AEN generation when underlying namespace size change is detected. Signed-off-by: Chaitanya Kulkarni Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig --- drivers/nvme/target/core.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index ee5865568588..fb78b165c123 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -517,10 +517,15 @@ static void nvmet_p2pmem_ns_add_p2p(struct nvmet_ctrl *ctrl, void nvmet_ns_revalidate(struct nvmet_ns *ns) { + loff_t oldsize = ns->size; + if (ns->bdev) nvmet_bdev_ns_revalidate(ns); else nvmet_file_ns_revalidate(ns); + + if (oldsize != ns->size) + nvmet_ns_changed(ns->subsys, ns->nsid); } int nvmet_ns_enable(struct nvmet_ns *ns) From 1f357548ec79a34e069716716b71194ce8b53e10 Mon Sep 17 00:00:00 2001 From: Chaitanya Kulkarni Date: Tue, 19 May 2020 01:06:29 -0700 Subject: [PATCH 0842/1043] nvmet: revalidate-ns & generate AEN from configfs Add a new attribute "revalidate_size" for the namespace which allows user to revalidate and generate the AEN if needed. This attribute is needed so that we can install userspace rules with systemd service based on inotify/fsnotify/uevent. The registered callback for such a service will end up writing to this attribute to generate AEN if needed. Signed-off-by: Chaitanya Kulkarni Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig --- drivers/nvme/target/configfs.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 24eb4cf53b4f..17c529260e12 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -540,6 +540,31 @@ static ssize_t nvmet_ns_buffered_io_store(struct config_item *item, CONFIGFS_ATTR(nvmet_ns_, buffered_io); +static ssize_t nvmet_ns_revalidate_size_store(struct config_item *item, + const char *page, size_t count) +{ + struct nvmet_ns *ns = to_nvmet_ns(item); + bool val; + + if (strtobool(page, &val)) + return -EINVAL; + + if (!val) + return -EINVAL; + + mutex_lock(&ns->subsys->lock); + if (!ns->enabled) { + pr_err("enable ns before revalidate.\n"); + mutex_unlock(&ns->subsys->lock); + return -EINVAL; + } + nvmet_ns_revalidate(ns); + mutex_unlock(&ns->subsys->lock); + return count; +} + +CONFIGFS_ATTR_WO(nvmet_ns_, revalidate_size); + static struct configfs_attribute *nvmet_ns_attrs[] = { &nvmet_ns_attr_device_path, &nvmet_ns_attr_device_nguid, @@ -547,6 +572,7 @@ static struct configfs_attribute *nvmet_ns_attrs[] = { &nvmet_ns_attr_ana_grpid, &nvmet_ns_attr_enable, &nvmet_ns_attr_buffered_io, + &nvmet_ns_attr_revalidate_size, #ifdef CONFIG_PCI_P2PDMA &nvmet_ns_attr_p2pmem, #endif From c295ee4742fda49de598a304ef9a95cf8da6b1f5 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Tue, 19 May 2020 17:05:48 +0300 Subject: [PATCH 0843/1043] block: always define struct blk_integrity in genhd.h This will reduce the amount of ifdefs inside the source code for various drivers and also will reduce the amount of stub functions that were created for the !CONFIG_BLK_DEV_INTEGRITY case. Suggested-by: Christoph Hellwig Signed-off-by: Max Gurtovoy Reviewed-by: Israel Rukshin Reviewed-by: Martin K. Petersen Signed-off-by: Christoph Hellwig --- include/linux/genhd.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/include/linux/genhd.h b/include/linux/genhd.h index f9c226f9546a..2590bed6e6b3 100644 --- a/include/linux/genhd.h +++ b/include/linux/genhd.h @@ -169,8 +169,6 @@ struct disk_part_tbl { struct disk_events; struct badblocks; -#if defined(CONFIG_BLK_DEV_INTEGRITY) - struct blk_integrity { const struct blk_integrity_profile *profile; unsigned char flags; @@ -179,8 +177,6 @@ struct blk_integrity { unsigned char tag_size; }; -#endif /* CONFIG_BLK_DEV_INTEGRITY */ - struct gendisk { /* major, first_minor and minors are input parameters only, * don't use directly. Use disk_devt() and disk_max_parts(). From ffc89b1d3ca45669e8d2226f5fd4dde756f7ad17 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Tue, 19 May 2020 17:05:49 +0300 Subject: [PATCH 0844/1043] nvme: introduce namespace features flag Replace the specific ext boolean (that implies on extended LBA format) with a feature in the new namespace features flag. This is a preparation for adding more namespace features (such as metadata specific features). Signed-off-by: Max Gurtovoy Reviewed-by: Israel Rukshin Reviewed-by: Martin K. Petersen Reviewed-by: James Smart Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 8 +++++--- drivers/nvme/host/lightnvm.c | 5 ++++- drivers/nvme/host/nvme.h | 6 +++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 4c194dcabd12..fecf484e7b08 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1305,7 +1305,7 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) meta_len = (io.nblocks + 1) * ns->ms; metadata = nvme_to_user_ptr(io.metadata); - if (ns->ext) { + if (ns->features & NVME_NS_EXT_LBAS) { length += meta_len; meta_len = 0; } else if (meta_len) { @@ -1885,7 +1885,7 @@ static void nvme_update_disk_info(struct gendisk *disk, blk_queue_io_min(disk->queue, phys_bs); blk_queue_io_opt(disk->queue, io_opt); - if (ns->ms && !ns->ext && + if (ns->ms && !(ns->features & NVME_NS_EXT_LBAS) && (ns->ctrl->ops->flags & NVME_F_METADATA_SUPPORTED)) nvme_init_integrity(disk, ns->ms, ns->pi_type); if ((ns->ms && !nvme_ns_has_pi(ns) && !blk_get_integrity(disk)) || @@ -1924,8 +1924,10 @@ static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) else iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob)); + ns->features = 0; ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms); - ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT); + if (ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT)) + ns->features |= NVME_NS_EXT_LBAS; /* the PI implementation requires metadata equal t10 pi tuple size */ if (ns->ms == sizeof(struct t10_pi_tuple)) ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK; diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c index 3002bf972c6b..69608755d415 100644 --- a/drivers/nvme/host/lightnvm.c +++ b/drivers/nvme/host/lightnvm.c @@ -961,7 +961,10 @@ int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node) geo = &dev->geo; geo->csecs = 1 << ns->lba_shift; geo->sos = ns->ms; - geo->ext = ns->ext; + if (ns->features & NVME_NS_EXT_LBAS) + geo->ext = true; + else + geo->ext = false; geo->mdts = ns->ctrl->max_hw_sectors; dev->q = q; diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 86f152e777bc..58ae6ebdc63a 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -364,6 +364,10 @@ struct nvme_ns_head { #endif }; +enum nvme_ns_features { + NVME_NS_EXT_LBAS = 1 << 0, /* support extended LBA format */ +}; + struct nvme_ns { struct list_head list; @@ -383,8 +387,8 @@ struct nvme_ns { u16 ms; u16 sgs; u32 sws; - bool ext; u8 pi_type; + unsigned long features; unsigned long flags; #define NVME_NS_REMOVING 0 #define NVME_NS_DEAD 1 From b29f84857a0f1cb4355363d0307d2b83897e8955 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Tue, 19 May 2020 17:05:50 +0300 Subject: [PATCH 0845/1043] nvme: introduce NVME_NS_METADATA_SUPPORTED flag This is a preparation for adding support for metadata in fabric controllers. New flag will imply that NVMe namespace supports getting metadata that was originally generated by host's block layer. Signed-off-by: Max Gurtovoy Reviewed-by: Israel Rukshin Reviewed-by: Martin K. Petersen Reviewed-by: James Smart Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 41 +++++++++++++++++++++++++++++++++------- drivers/nvme/host/nvme.h | 1 + 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index fecf484e7b08..aa168dd58f31 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1885,13 +1885,27 @@ static void nvme_update_disk_info(struct gendisk *disk, blk_queue_io_min(disk->queue, phys_bs); blk_queue_io_opt(disk->queue, io_opt); - if (ns->ms && !(ns->features & NVME_NS_EXT_LBAS) && - (ns->ctrl->ops->flags & NVME_F_METADATA_SUPPORTED)) - nvme_init_integrity(disk, ns->ms, ns->pi_type); - if ((ns->ms && !nvme_ns_has_pi(ns) && !blk_get_integrity(disk)) || - ns->lba_shift > PAGE_SHIFT) + /* + * The block layer can't support LBA sizes larger than the page size + * yet, so catch this early and don't allow block I/O. + */ + if (ns->lba_shift > PAGE_SHIFT) capacity = 0; + /* + * Register a metadata profile for PI, or the plain non-integrity NVMe + * metadata masquerading as Type 0 if supported, otherwise reject block + * I/O to namespaces with metadata except when the namespace supports + * PI, as it can strip/insert in that case. + */ + if (ns->ms) { + if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) && + (ns->features & NVME_NS_METADATA_SUPPORTED)) + nvme_init_integrity(disk, ns->ms, ns->pi_type); + else if (!nvme_ns_has_pi(ns)) + capacity = 0; + } + set_capacity_revalidate_and_notify(disk, capacity, false); nvme_config_discard(disk, ns); @@ -1926,14 +1940,27 @@ static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) ns->features = 0; ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms); - if (ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT)) - ns->features |= NVME_NS_EXT_LBAS; /* the PI implementation requires metadata equal t10 pi tuple size */ if (ns->ms == sizeof(struct t10_pi_tuple)) ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK; else ns->pi_type = 0; + if (ns->ms) { + if (id->flbas & NVME_NS_FLBAS_META_EXT) + ns->features |= NVME_NS_EXT_LBAS; + + /* + * For PCI, Extended logical block will be generated by the + * controller. Non-extended format can be generated by the + * block layer. + */ + if (ns->ctrl->ops->flags & NVME_F_METADATA_SUPPORTED) { + if (!(ns->features & NVME_NS_EXT_LBAS)) + ns->features |= NVME_NS_METADATA_SUPPORTED; + } + } + if (iob) blk_queue_chunk_sectors(ns->queue, rounddown_pow_of_two(iob)); nvme_update_disk_info(disk, ns, id); diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 58ae6ebdc63a..9ed6a3dac537 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -366,6 +366,7 @@ struct nvme_ns_head { enum nvme_ns_features { NVME_NS_EXT_LBAS = 1 << 0, /* support extended LBA format */ + NVME_NS_METADATA_SUPPORTED = 1 << 1, /* support getting generated md */ }; struct nvme_ns { From 4d2ce68835649afebbc5e8816b79426fb04c639f Mon Sep 17 00:00:00 2001 From: James Smart Date: Tue, 19 May 2020 17:05:51 +0300 Subject: [PATCH 0846/1043] nvme: make nvme_ns_has_pi accessible to transports Move the nvme_ns_has_pi() inline from core.c to the nvme.h header. This allows use by the transports. Signed-off-by: James Smart [maxg: added a comment for nvme_ns_has_pi()] Signed-off-by: Max Gurtovoy Reviewed-by: Israel Rukshin Reviewed-by: Martin K. Petersen Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 6 ------ drivers/nvme/host/nvme.h | 7 +++++++ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index aa168dd58f31..beea013f7b73 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include @@ -204,11 +203,6 @@ static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl) nvme_put_ctrl(ctrl); } -static inline bool nvme_ns_has_pi(struct nvme_ns *ns) -{ - return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple); -} - static blk_status_t nvme_error_status(u16 status) { switch (status & 0x7ff) { diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 9ed6a3dac537..c83ff69338d8 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -399,6 +400,12 @@ struct nvme_ns { }; +/* NVMe ns supports metadata actions by the controller (generate/strip) */ +static inline bool nvme_ns_has_pi(struct nvme_ns *ns) +{ + return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple); +} + struct nvme_ctrl_ops { const char *name; struct module *module; From 95093350394a394e7c4e778176194b14b76ec5d8 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Tue, 19 May 2020 17:05:52 +0300 Subject: [PATCH 0847/1043] nvme: introduce max_integrity_segments ctrl attribute This patch doesn't change any logic, and is needed as a preparation for adding PI support for fabrics drivers that will use an extended LBA format for metadata and will support more than 1 integrity segment. Signed-off-by: Max Gurtovoy Signed-off-by: Israel Rukshin Reviewed-by: Sagi Grimberg Reviewed-by: Martin K. Petersen Reviewed-by: James Smart Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 11 +++++++---- drivers/nvme/host/nvme.h | 1 + drivers/nvme/host/pci.c | 6 ++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index beea013f7b73..404edceb84cb 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1693,7 +1693,8 @@ static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo) } #ifdef CONFIG_BLK_DEV_INTEGRITY -static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type) +static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type, + u32 max_integrity_segments) { struct blk_integrity integrity; @@ -1716,10 +1717,11 @@ static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type) } integrity.tuple_size = ms; blk_integrity_register(disk, &integrity); - blk_queue_max_integrity_segments(disk->queue, 1); + blk_queue_max_integrity_segments(disk->queue, max_integrity_segments); } #else -static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type) +static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type, + u32 max_integrity_segments) { } #endif /* CONFIG_BLK_DEV_INTEGRITY */ @@ -1895,7 +1897,8 @@ static void nvme_update_disk_info(struct gendisk *disk, if (ns->ms) { if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) && (ns->features & NVME_NS_METADATA_SUPPORTED)) - nvme_init_integrity(disk, ns->ms, ns->pi_type); + nvme_init_integrity(disk, ns->ms, ns->pi_type, + ns->ctrl->max_integrity_segments); else if (!nvme_ns_has_pi(ns)) capacity = 0; } diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index c83ff69338d8..5b5ed128fd9e 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -229,6 +229,7 @@ struct nvme_ctrl { u32 page_size; u32 max_hw_sectors; u32 max_segments; + u32 max_integrity_segments; u16 crdt[3]; u16 oncs; u16 oacs; diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 6cc4630ddd6d..b307c06a783d 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -2565,6 +2565,12 @@ static void nvme_reset_work(struct work_struct *work) goto out; } + /* + * We do not support an SGL for metadata (yet), so we are limited to a + * single integrity segment for the separate metadata pointer. + */ + dev->ctrl.max_integrity_segments = 1; + result = nvme_init_identify(&dev->ctrl); if (result) goto out; From 33cfdc2aa6969829f42640f758357e4b015e9f7d Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Tue, 19 May 2020 17:05:53 +0300 Subject: [PATCH 0848/1043] nvme: enforce extended LBA format for fabrics metadata An extended LBA is a larger LBA that is created when metadata associated with the LBA is transferred contiguously with the LBA data (AKA interleaved). The metadata may be either transferred as part of the LBA (creating an extended LBA) or it may be transferred as a separate contiguous buffer of data. According to the NVMeoF spec, a fabrics ctrl supports only an Extended LBA format. Fail revalidation in case we have a spec violation. Also add a flag that will imply on capable transports and controllers as part of a preparation for allowing end-to-end protection information for fabric controllers. Suggested-by: Christoph Hellwig Signed-off-by: Max Gurtovoy Signed-off-by: Israel Rukshin Reviewed-by: James Smart Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 41 ++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 404edceb84cb..a3a4dbc59af1 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1916,9 +1916,10 @@ static void nvme_update_disk_info(struct gendisk *disk, blk_mq_unfreeze_queue(disk->queue); } -static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) +static int __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) { struct nvme_ns *ns = disk->private_data; + struct nvme_ctrl *ctrl = ns->ctrl; u32 iob; /* @@ -1929,9 +1930,9 @@ static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) if (ns->lba_shift == 0) ns->lba_shift = 9; - if ((ns->ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) && - is_power_of_2(ns->ctrl->max_hw_sectors)) - iob = ns->ctrl->max_hw_sectors; + if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) && + is_power_of_2(ctrl->max_hw_sectors)) + iob = ctrl->max_hw_sectors; else iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob)); @@ -1944,16 +1945,24 @@ static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) ns->pi_type = 0; if (ns->ms) { - if (id->flbas & NVME_NS_FLBAS_META_EXT) - ns->features |= NVME_NS_EXT_LBAS; - /* - * For PCI, Extended logical block will be generated by the - * controller. Non-extended format can be generated by the - * block layer. + * For PCIe only the separate metadata pointer is supported, + * as the block layer supplies metadata in a separate bio_vec + * chain. For Fabrics, only metadata as part of extended data + * LBA is supported on the wire per the Fabrics specification, + * but the HBA/HCA will do the remapping from the separate + * metadata buffers for us. */ - if (ns->ctrl->ops->flags & NVME_F_METADATA_SUPPORTED) { - if (!(ns->features & NVME_NS_EXT_LBAS)) + if (id->flbas & NVME_NS_FLBAS_META_EXT) { + ns->features |= NVME_NS_EXT_LBAS; + if ((ctrl->ops->flags & NVME_F_FABRICS) && + (ctrl->ops->flags & NVME_F_METADATA_SUPPORTED) && + ctrl->max_integrity_segments) + ns->features |= NVME_NS_METADATA_SUPPORTED; + } else { + if (WARN_ON_ONCE(ctrl->ops->flags & NVME_F_FABRICS)) + return -EINVAL; + if (ctrl->ops->flags & NVME_F_METADATA_SUPPORTED) ns->features |= NVME_NS_METADATA_SUPPORTED; } } @@ -1968,6 +1977,7 @@ static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id) revalidate_disk(ns->head->disk); } #endif + return 0; } static int nvme_revalidate_disk(struct gendisk *disk) @@ -2003,7 +2013,7 @@ static int nvme_revalidate_disk(struct gendisk *disk) goto free_id; } - __nvme_revalidate_disk(disk, id); + ret = __nvme_revalidate_disk(disk, id); free_id: kfree(id); out: @@ -3657,7 +3667,8 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid) memcpy(disk->disk_name, disk_name, DISK_NAME_LEN); ns->disk = disk; - __nvme_revalidate_disk(disk, id); + if (__nvme_revalidate_disk(disk, id)) + goto out_free_disk; if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) { ret = nvme_nvm_register(ns, disk_name, node); @@ -3684,6 +3695,8 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid) /* prevent double queue cleanup */ ns->disk->queue = NULL; put_disk(ns->disk); + out_free_disk: + del_gendisk(ns->disk); out_unlink_ns: mutex_lock(&ctrl->subsys->lock); list_del_rcu(&ns->siblings); From ba7ca2ae029607c7eb2c18e37e8bc0d2252d3d12 Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Tue, 19 May 2020 17:05:54 +0300 Subject: [PATCH 0849/1043] nvme: introduce NVME_INLINE_METADATA_SG_CNT SGL size of metadata is usually small. Thus, 1 inline sg should cover most cases. The macro will be used for pre-allocate a single SGL entry for metadata. The preallocation of small inline SGLs depends on SG_CHAIN capability so if the ARCH doesn't support SG_CHAIN, use the runtime allocation for the SGL. This patch is a preparation for adding metadata (T10-PI) over fabric support. Signed-off-by: Israel Rukshin Signed-off-by: Max Gurtovoy Reviewed-by: Martin K. Petersen Reviewed-by: James Smart Signed-off-by: Christoph Hellwig --- drivers/nvme/host/nvme.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 5b5ed128fd9e..fa5c75501049 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -31,8 +31,10 @@ extern unsigned int admin_timeout; #ifdef CONFIG_ARCH_NO_SG_CHAIN #define NVME_INLINE_SG_CNT 0 +#define NVME_INLINE_METADATA_SG_CNT 0 #else #define NVME_INLINE_SG_CNT 2 +#define NVME_INLINE_METADATA_SG_CNT 1 #endif extern struct workqueue_struct *nvme_wq; From 324d9e7814dd9c76bb3aebf2529b02149c340d48 Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Tue, 19 May 2020 17:05:55 +0300 Subject: [PATCH 0850/1043] nvme-rdma: introduce nvme_rdma_sgl structure Remove first_sgl pointer from struct nvme_rdma_request and use pointer arithmetic instead. The inline scatterlist, if exists, will be located right after the nvme_rdma_request. This patch is needed as a preparation for adding PI support. Signed-off-by: Israel Rukshin Reviewed-by: Max Gurtovoy Signed-off-by: Christoph Hellwig --- drivers/nvme/host/rdma.c | 41 +++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index cac8a930396a..40868749547a 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -48,6 +48,11 @@ struct nvme_rdma_qe { u64 dma; }; +struct nvme_rdma_sgl { + int nents; + struct sg_table sg_table; +}; + struct nvme_rdma_queue; struct nvme_rdma_request { struct nvme_request req; @@ -58,12 +63,10 @@ struct nvme_rdma_request { refcount_t ref; struct ib_sge sge[1 + NVME_RDMA_MAX_INLINE_SEGMENTS]; u32 num_sge; - int nents; struct ib_reg_wr reg_wr; struct ib_cqe reg_cqe; struct nvme_rdma_queue *queue; - struct sg_table sg_table; - struct scatterlist first_sgl[]; + struct nvme_rdma_sgl data_sgl; }; enum nvme_rdma_queue_flags { @@ -1158,8 +1161,9 @@ static void nvme_rdma_unmap_data(struct nvme_rdma_queue *queue, req->mr = NULL; } - ib_dma_unmap_sg(ibdev, req->sg_table.sgl, req->nents, rq_dma_dir(rq)); - sg_free_table_chained(&req->sg_table, NVME_INLINE_SG_CNT); + ib_dma_unmap_sg(ibdev, req->data_sgl.sg_table.sgl, req->data_sgl.nents, + rq_dma_dir(rq)); + sg_free_table_chained(&req->data_sgl.sg_table, NVME_INLINE_SG_CNT); } static int nvme_rdma_set_sg_null(struct nvme_command *c) @@ -1178,7 +1182,7 @@ static int nvme_rdma_map_sg_inline(struct nvme_rdma_queue *queue, int count) { struct nvme_sgl_desc *sg = &c->common.dptr.sgl; - struct scatterlist *sgl = req->sg_table.sgl; + struct scatterlist *sgl = req->data_sgl.sg_table.sgl; struct ib_sge *sge = &req->sge[1]; u32 len = 0; int i; @@ -1203,8 +1207,8 @@ static int nvme_rdma_map_sg_single(struct nvme_rdma_queue *queue, { struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl; - sg->addr = cpu_to_le64(sg_dma_address(req->sg_table.sgl)); - put_unaligned_le24(sg_dma_len(req->sg_table.sgl), sg->length); + sg->addr = cpu_to_le64(sg_dma_address(req->data_sgl.sg_table.sgl)); + put_unaligned_le24(sg_dma_len(req->data_sgl.sg_table.sgl), sg->length); put_unaligned_le32(queue->device->pd->unsafe_global_rkey, sg->key); sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4; return 0; @@ -1225,7 +1229,8 @@ static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue *queue, * Align the MR to a 4K page size to match the ctrl page size and * the block virtual boundary. */ - nr = ib_map_mr_sg(req->mr, req->sg_table.sgl, count, NULL, SZ_4K); + nr = ib_map_mr_sg(req->mr, req->data_sgl.sg_table.sgl, count, NULL, + SZ_4K); if (unlikely(nr < count)) { ib_mr_pool_put(queue->qp, &queue->qp->rdma_mrs, req->mr); req->mr = NULL; @@ -1272,17 +1277,18 @@ static int nvme_rdma_map_data(struct nvme_rdma_queue *queue, if (!blk_rq_nr_phys_segments(rq)) return nvme_rdma_set_sg_null(c); - req->sg_table.sgl = req->first_sgl; - ret = sg_alloc_table_chained(&req->sg_table, - blk_rq_nr_phys_segments(rq), req->sg_table.sgl, + req->data_sgl.sg_table.sgl = (struct scatterlist *)(req + 1); + ret = sg_alloc_table_chained(&req->data_sgl.sg_table, + blk_rq_nr_phys_segments(rq), req->data_sgl.sg_table.sgl, NVME_INLINE_SG_CNT); if (ret) return -ENOMEM; - req->nents = blk_rq_map_sg(rq->q, rq, req->sg_table.sgl); + req->data_sgl.nents = blk_rq_map_sg(rq->q, rq, + req->data_sgl.sg_table.sgl); - count = ib_dma_map_sg(ibdev, req->sg_table.sgl, req->nents, - rq_dma_dir(rq)); + count = ib_dma_map_sg(ibdev, req->data_sgl.sg_table.sgl, + req->data_sgl.nents, rq_dma_dir(rq)); if (unlikely(count <= 0)) { ret = -EIO; goto out_free_table; @@ -1311,9 +1317,10 @@ out: return 0; out_unmap_sg: - ib_dma_unmap_sg(ibdev, req->sg_table.sgl, req->nents, rq_dma_dir(rq)); + ib_dma_unmap_sg(ibdev, req->data_sgl.sg_table.sgl, req->data_sgl.nents, + rq_dma_dir(rq)); out_free_table: - sg_free_table_chained(&req->sg_table, NVME_INLINE_SG_CNT); + sg_free_table_chained(&req->data_sgl.sg_table, NVME_INLINE_SG_CNT); return ret; } From 5ec5d3bddc6b912b7de9e3eb6c1f2397faeca2bc Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Tue, 19 May 2020 17:05:56 +0300 Subject: [PATCH 0851/1043] nvme-rdma: add metadata/T10-PI support For capable HCAs (e.g. ConnectX-5/ConnectX-6) this will allow end-to-end protection information passthrough and validation for NVMe over RDMA transport. Metadata offload support was implemented over the new RDMA signature verbs API and it is enabled for capable controllers. Signed-off-by: Max Gurtovoy Signed-off-by: Israel Rukshin Signed-off-by: Christoph Hellwig --- drivers/nvme/host/rdma.c | 280 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 270 insertions(+), 10 deletions(-) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 40868749547a..f8f856dc0c67 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -34,6 +34,11 @@ #define NVME_RDMA_MAX_INLINE_SEGMENTS 4 +#define NVME_RDMA_DATA_SGL_SIZE \ + (sizeof(struct scatterlist) * NVME_INLINE_SG_CNT) +#define NVME_RDMA_METADATA_SGL_SIZE \ + (sizeof(struct scatterlist) * NVME_INLINE_METADATA_SG_CNT) + struct nvme_rdma_device { struct ib_device *dev; struct ib_pd *pd; @@ -67,6 +72,8 @@ struct nvme_rdma_request { struct ib_cqe reg_cqe; struct nvme_rdma_queue *queue; struct nvme_rdma_sgl data_sgl; + struct nvme_rdma_sgl *metadata_sgl; + bool use_sig_mr; }; enum nvme_rdma_queue_flags { @@ -88,6 +95,7 @@ struct nvme_rdma_queue { struct rdma_cm_id *cm_id; int cm_error; struct completion cm_done; + bool pi_support; }; struct nvme_rdma_ctrl { @@ -264,6 +272,8 @@ static int nvme_rdma_create_qp(struct nvme_rdma_queue *queue, const int factor) init_attr.qp_type = IB_QPT_RC; init_attr.send_cq = queue->ib_cq; init_attr.recv_cq = queue->ib_cq; + if (queue->pi_support) + init_attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN; ret = rdma_create_qp(queue->cm_id, dev->pd, &init_attr); @@ -293,6 +303,12 @@ static int nvme_rdma_init_request(struct blk_mq_tag_set *set, if (!req->sqe.data) return -ENOMEM; + /* metadata nvme_rdma_sgl struct is located after command's data SGL */ + if (queue->pi_support) + req->metadata_sgl = (void *)nvme_req(rq) + + sizeof(struct nvme_rdma_request) + + NVME_RDMA_DATA_SGL_SIZE; + req->queue = queue; return 0; @@ -403,6 +419,8 @@ static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue *queue) dev = queue->device; ibdev = dev->dev; + if (queue->pi_support) + ib_mr_pool_destroy(queue->qp, &queue->qp->sig_mrs); ib_mr_pool_destroy(queue->qp, &queue->qp->rdma_mrs); /* @@ -419,10 +437,16 @@ static void nvme_rdma_destroy_queue_ib(struct nvme_rdma_queue *queue) nvme_rdma_dev_put(dev); } -static int nvme_rdma_get_max_fr_pages(struct ib_device *ibdev) +static int nvme_rdma_get_max_fr_pages(struct ib_device *ibdev, bool pi_support) { - return min_t(u32, NVME_RDMA_MAX_SEGMENTS, - ibdev->attrs.max_fast_reg_page_list_len - 1); + u32 max_page_list_len; + + if (pi_support) + max_page_list_len = ibdev->attrs.max_pi_fast_reg_page_list_len; + else + max_page_list_len = ibdev->attrs.max_fast_reg_page_list_len; + + return min_t(u32, NVME_RDMA_MAX_SEGMENTS, max_page_list_len - 1); } static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue) @@ -479,7 +503,7 @@ static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue) * misaligned we'll end up using two entries for a single data page, * so one additional entry is required. */ - pages_per_mr = nvme_rdma_get_max_fr_pages(ibdev) + 1; + pages_per_mr = nvme_rdma_get_max_fr_pages(ibdev, queue->pi_support) + 1; ret = ib_mr_pool_init(queue->qp, &queue->qp->rdma_mrs, queue->queue_size, IB_MR_TYPE_MEM_REG, @@ -491,10 +515,24 @@ static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue) goto out_destroy_ring; } + if (queue->pi_support) { + ret = ib_mr_pool_init(queue->qp, &queue->qp->sig_mrs, + queue->queue_size, IB_MR_TYPE_INTEGRITY, + pages_per_mr, pages_per_mr); + if (ret) { + dev_err(queue->ctrl->ctrl.device, + "failed to initialize PI MR pool sized %d for QID %d\n", + queue->queue_size, idx); + goto out_destroy_mr_pool; + } + } + set_bit(NVME_RDMA_Q_TR_READY, &queue->flags); return 0; +out_destroy_mr_pool: + ib_mr_pool_destroy(queue->qp, &queue->qp->rdma_mrs); out_destroy_ring: nvme_rdma_free_ring(ibdev, queue->rsp_ring, queue->queue_size, sizeof(struct nvme_completion), DMA_FROM_DEVICE); @@ -516,6 +554,10 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl, queue = &ctrl->queues[idx]; queue->ctrl = ctrl; + if (idx && ctrl->ctrl.max_integrity_segments) + queue->pi_support = true; + else + queue->pi_support = false; init_completion(&queue->cm_done); if (idx > 0) @@ -726,7 +768,7 @@ static struct blk_mq_tag_set *nvme_rdma_alloc_tagset(struct nvme_ctrl *nctrl, set->reserved_tags = 2; /* connect + keep-alive */ set->numa_node = nctrl->numa_node; set->cmd_size = sizeof(struct nvme_rdma_request) + - NVME_INLINE_SG_CNT * sizeof(struct scatterlist); + NVME_RDMA_DATA_SGL_SIZE; set->driver_data = ctrl; set->nr_hw_queues = 1; set->timeout = ADMIN_TIMEOUT; @@ -740,7 +782,10 @@ static struct blk_mq_tag_set *nvme_rdma_alloc_tagset(struct nvme_ctrl *nctrl, set->numa_node = nctrl->numa_node; set->flags = BLK_MQ_F_SHOULD_MERGE; set->cmd_size = sizeof(struct nvme_rdma_request) + - NVME_INLINE_SG_CNT * sizeof(struct scatterlist); + NVME_RDMA_DATA_SGL_SIZE; + if (nctrl->max_integrity_segments) + set->cmd_size += sizeof(struct nvme_rdma_sgl) + + NVME_RDMA_METADATA_SGL_SIZE; set->driver_data = ctrl; set->nr_hw_queues = nctrl->queue_count - 1; set->timeout = NVME_IO_TIMEOUT; @@ -773,6 +818,7 @@ static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl, static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, bool new) { + bool pi_capable = false; int error; error = nvme_rdma_alloc_queue(ctrl, 0, NVME_AQ_DEPTH); @@ -782,7 +828,13 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, ctrl->device = ctrl->queues[0].device; ctrl->ctrl.numa_node = dev_to_node(ctrl->device->dev->dma_device); - ctrl->max_fr_pages = nvme_rdma_get_max_fr_pages(ctrl->device->dev); + /* T10-PI support */ + if (ctrl->device->dev->attrs.device_cap_flags & + IB_DEVICE_INTEGRITY_HANDOVER) + pi_capable = true; + + ctrl->max_fr_pages = nvme_rdma_get_max_fr_pages(ctrl->device->dev, + pi_capable); /* * Bind the async event SQE DMA mapping to the admin queue lifetime. @@ -824,6 +876,10 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, ctrl->ctrl.max_segments = ctrl->max_fr_pages; ctrl->ctrl.max_hw_sectors = ctrl->max_fr_pages << (ilog2(SZ_4K) - 9); + if (pi_capable) + ctrl->ctrl.max_integrity_segments = ctrl->max_fr_pages; + else + ctrl->ctrl.max_integrity_segments = 0; blk_mq_unquiesce_queue(ctrl->ctrl.admin_q); @@ -1152,12 +1208,23 @@ static void nvme_rdma_unmap_data(struct nvme_rdma_queue *queue, struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); struct nvme_rdma_device *dev = queue->device; struct ib_device *ibdev = dev->dev; + struct list_head *pool = &queue->qp->rdma_mrs; if (!blk_rq_nr_phys_segments(rq)) return; + if (blk_integrity_rq(rq)) { + ib_dma_unmap_sg(ibdev, req->metadata_sgl->sg_table.sgl, + req->metadata_sgl->nents, rq_dma_dir(rq)); + sg_free_table_chained(&req->metadata_sgl->sg_table, + NVME_INLINE_METADATA_SG_CNT); + } + + if (req->use_sig_mr) + pool = &queue->qp->sig_mrs; + if (req->mr) { - ib_mr_pool_put(queue->qp, &queue->qp->rdma_mrs, req->mr); + ib_mr_pool_put(queue->qp, pool, req->mr); req->mr = NULL; } @@ -1261,12 +1328,125 @@ static int nvme_rdma_map_sg_fr(struct nvme_rdma_queue *queue, return 0; } +static void nvme_rdma_set_sig_domain(struct blk_integrity *bi, + struct nvme_command *cmd, struct ib_sig_domain *domain, + u16 control, u8 pi_type) +{ + domain->sig_type = IB_SIG_TYPE_T10_DIF; + domain->sig.dif.bg_type = IB_T10DIF_CRC; + domain->sig.dif.pi_interval = 1 << bi->interval_exp; + domain->sig.dif.ref_tag = le32_to_cpu(cmd->rw.reftag); + if (control & NVME_RW_PRINFO_PRCHK_REF) + domain->sig.dif.ref_remap = true; + + domain->sig.dif.app_tag = le16_to_cpu(cmd->rw.apptag); + domain->sig.dif.apptag_check_mask = le16_to_cpu(cmd->rw.appmask); + domain->sig.dif.app_escape = true; + if (pi_type == NVME_NS_DPS_PI_TYPE3) + domain->sig.dif.ref_escape = true; +} + +static void nvme_rdma_set_sig_attrs(struct blk_integrity *bi, + struct nvme_command *cmd, struct ib_sig_attrs *sig_attrs, + u8 pi_type) +{ + u16 control = le16_to_cpu(cmd->rw.control); + + memset(sig_attrs, 0, sizeof(*sig_attrs)); + if (control & NVME_RW_PRINFO_PRACT) { + /* for WRITE_INSERT/READ_STRIP no memory domain */ + sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; + nvme_rdma_set_sig_domain(bi, cmd, &sig_attrs->wire, control, + pi_type); + /* Clear the PRACT bit since HCA will generate/verify the PI */ + control &= ~NVME_RW_PRINFO_PRACT; + cmd->rw.control = cpu_to_le16(control); + } else { + /* for WRITE_PASS/READ_PASS both wire/memory domains exist */ + nvme_rdma_set_sig_domain(bi, cmd, &sig_attrs->wire, control, + pi_type); + nvme_rdma_set_sig_domain(bi, cmd, &sig_attrs->mem, control, + pi_type); + } +} + +static void nvme_rdma_set_prot_checks(struct nvme_command *cmd, u8 *mask) +{ + *mask = 0; + if (le16_to_cpu(cmd->rw.control) & NVME_RW_PRINFO_PRCHK_REF) + *mask |= IB_SIG_CHECK_REFTAG; + if (le16_to_cpu(cmd->rw.control) & NVME_RW_PRINFO_PRCHK_GUARD) + *mask |= IB_SIG_CHECK_GUARD; +} + +static void nvme_rdma_sig_done(struct ib_cq *cq, struct ib_wc *wc) +{ + if (unlikely(wc->status != IB_WC_SUCCESS)) + nvme_rdma_wr_error(cq, wc, "SIG"); +} + +static int nvme_rdma_map_sg_pi(struct nvme_rdma_queue *queue, + struct nvme_rdma_request *req, struct nvme_command *c, + int count, int pi_count) +{ + struct nvme_rdma_sgl *sgl = &req->data_sgl; + struct ib_reg_wr *wr = &req->reg_wr; + struct request *rq = blk_mq_rq_from_pdu(req); + struct nvme_ns *ns = rq->q->queuedata; + struct bio *bio = rq->bio; + struct nvme_keyed_sgl_desc *sg = &c->common.dptr.ksgl; + int nr; + + req->mr = ib_mr_pool_get(queue->qp, &queue->qp->sig_mrs); + if (WARN_ON_ONCE(!req->mr)) + return -EAGAIN; + + nr = ib_map_mr_sg_pi(req->mr, sgl->sg_table.sgl, count, NULL, + req->metadata_sgl->sg_table.sgl, pi_count, NULL, + SZ_4K); + if (unlikely(nr)) + goto mr_put; + + nvme_rdma_set_sig_attrs(blk_get_integrity(bio->bi_disk), c, + req->mr->sig_attrs, ns->pi_type); + nvme_rdma_set_prot_checks(c, &req->mr->sig_attrs->check_mask); + + ib_update_fast_reg_key(req->mr, ib_inc_rkey(req->mr->rkey)); + + req->reg_cqe.done = nvme_rdma_sig_done; + memset(wr, 0, sizeof(*wr)); + wr->wr.opcode = IB_WR_REG_MR_INTEGRITY; + wr->wr.wr_cqe = &req->reg_cqe; + wr->wr.num_sge = 0; + wr->wr.send_flags = 0; + wr->mr = req->mr; + wr->key = req->mr->rkey; + wr->access = IB_ACCESS_LOCAL_WRITE | + IB_ACCESS_REMOTE_READ | + IB_ACCESS_REMOTE_WRITE; + + sg->addr = cpu_to_le64(req->mr->iova); + put_unaligned_le24(req->mr->length, sg->length); + put_unaligned_le32(req->mr->rkey, sg->key); + sg->type = NVME_KEY_SGL_FMT_DATA_DESC << 4; + + return 0; + +mr_put: + ib_mr_pool_put(queue->qp, &queue->qp->sig_mrs, req->mr); + req->mr = NULL; + if (nr < 0) + return nr; + return -EINVAL; +} + static int nvme_rdma_map_data(struct nvme_rdma_queue *queue, struct request *rq, struct nvme_command *c) { struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); struct nvme_rdma_device *dev = queue->device; struct ib_device *ibdev = dev->dev; + int pi_count = 0; int count, ret; req->num_sge = 1; @@ -1294,6 +1474,35 @@ static int nvme_rdma_map_data(struct nvme_rdma_queue *queue, goto out_free_table; } + if (blk_integrity_rq(rq)) { + req->metadata_sgl->sg_table.sgl = + (struct scatterlist *)(req->metadata_sgl + 1); + ret = sg_alloc_table_chained(&req->metadata_sgl->sg_table, + blk_rq_count_integrity_sg(rq->q, rq->bio), + req->metadata_sgl->sg_table.sgl, + NVME_INLINE_METADATA_SG_CNT); + if (unlikely(ret)) { + ret = -ENOMEM; + goto out_unmap_sg; + } + + req->metadata_sgl->nents = blk_rq_map_integrity_sg(rq->q, + rq->bio, req->metadata_sgl->sg_table.sgl); + pi_count = ib_dma_map_sg(ibdev, + req->metadata_sgl->sg_table.sgl, + req->metadata_sgl->nents, + rq_dma_dir(rq)); + if (unlikely(pi_count <= 0)) { + ret = -EIO; + goto out_free_pi_table; + } + } + + if (req->use_sig_mr) { + ret = nvme_rdma_map_sg_pi(queue, req, c, count, pi_count); + goto out; + } + if (count <= dev->num_inline_segments) { if (rq_data_dir(rq) == WRITE && nvme_rdma_queue_idx(queue) && queue->ctrl->use_inline_data && @@ -1312,10 +1521,18 @@ static int nvme_rdma_map_data(struct nvme_rdma_queue *queue, ret = nvme_rdma_map_sg_fr(queue, req, c, count); out: if (unlikely(ret)) - goto out_unmap_sg; + goto out_unmap_pi_sg; return 0; +out_unmap_pi_sg: + if (blk_integrity_rq(rq)) + ib_dma_unmap_sg(ibdev, req->metadata_sgl->sg_table.sgl, + req->metadata_sgl->nents, rq_dma_dir(rq)); +out_free_pi_table: + if (blk_integrity_rq(rq)) + sg_free_table_chained(&req->metadata_sgl->sg_table, + NVME_INLINE_METADATA_SG_CNT); out_unmap_sg: ib_dma_unmap_sg(ibdev, req->data_sgl.sg_table.sgl, req->data_sgl.nents, rq_dma_dir(rq)); @@ -1768,6 +1985,15 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx, blk_mq_start_request(rq); + if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) && + queue->pi_support && + (c->common.opcode == nvme_cmd_write || + c->common.opcode == nvme_cmd_read) && + nvme_ns_has_pi(ns)) + req->use_sig_mr = true; + else + req->use_sig_mr = false; + err = nvme_rdma_map_data(queue, rq, c); if (unlikely(err < 0)) { dev_err(queue->ctrl->ctrl.device, @@ -1808,12 +2034,46 @@ static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx) return ib_process_cq_direct(queue->ib_cq, -1); } +static void nvme_rdma_check_pi_status(struct nvme_rdma_request *req) +{ + struct request *rq = blk_mq_rq_from_pdu(req); + struct ib_mr_status mr_status; + int ret; + + ret = ib_check_mr_status(req->mr, IB_MR_CHECK_SIG_STATUS, &mr_status); + if (ret) { + pr_err("ib_check_mr_status failed, ret %d\n", ret); + nvme_req(rq)->status = NVME_SC_INVALID_PI; + return; + } + + if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { + switch (mr_status.sig_err.err_type) { + case IB_SIG_BAD_GUARD: + nvme_req(rq)->status = NVME_SC_GUARD_CHECK; + break; + case IB_SIG_BAD_REFTAG: + nvme_req(rq)->status = NVME_SC_REFTAG_CHECK; + break; + case IB_SIG_BAD_APPTAG: + nvme_req(rq)->status = NVME_SC_APPTAG_CHECK; + break; + } + pr_err("PI error found type %d expected 0x%x vs actual 0x%x\n", + mr_status.sig_err.err_type, mr_status.sig_err.expected, + mr_status.sig_err.actual); + } +} + static void nvme_rdma_complete_rq(struct request *rq) { struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); struct nvme_rdma_queue *queue = req->queue; struct ib_device *ibdev = queue->device->dev; + if (req->use_sig_mr) + nvme_rdma_check_pi_status(req); + nvme_rdma_unmap_data(queue, rq); ib_dma_unmap_single(ibdev, req->sqe.dma, sizeof(struct nvme_command), DMA_TO_DEVICE); @@ -1933,7 +2193,7 @@ out_fail: static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = { .name = "rdma", .module = THIS_MODULE, - .flags = NVME_F_FABRICS, + .flags = NVME_F_FABRICS | NVME_F_METADATA_SUPPORTED, .reg_read32 = nvmf_reg_read32, .reg_read64 = nvmf_reg_read64, .reg_write32 = nvmf_reg_write32, From d2d1c454a4a44010cac627fd63945ff5e7dd3b4c Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Tue, 19 May 2020 17:05:57 +0300 Subject: [PATCH 0852/1043] nvmet: add metadata characteristics for a namespace Fill those namespace fields from the block device format for adding metadata (T10-PI) over fabric support with block devices. Signed-off-by: Israel Rukshin Signed-off-by: Max Gurtovoy Reviewed-by: Martin K. Petersen Signed-off-by: Christoph Hellwig --- drivers/nvme/target/Kconfig | 1 + drivers/nvme/target/io-cmd-bdev.c | 22 ++++++++++++++++++++++ drivers/nvme/target/nvmet.h | 3 +++ 3 files changed, 26 insertions(+) diff --git a/drivers/nvme/target/Kconfig b/drivers/nvme/target/Kconfig index d7f48c0fb311..4474952d64c6 100644 --- a/drivers/nvme/target/Kconfig +++ b/drivers/nvme/target/Kconfig @@ -4,6 +4,7 @@ config NVME_TARGET tristate "NVMe Target support" depends on BLOCK depends on CONFIGFS_FS + select BLK_DEV_INTEGRITY_T10 if BLK_DEV_INTEGRITY select SGL_ALLOC help This enabled target side support for the NVMe protocol, that is diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index 0427e040e3dd..e518bb9bf718 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -47,6 +47,22 @@ void nvmet_bdev_set_limits(struct block_device *bdev, struct nvme_id_ns *id) id->nows = to0based(ql->io_opt / ql->logical_block_size); } +static void nvmet_bdev_ns_enable_integrity(struct nvmet_ns *ns) +{ + struct blk_integrity *bi = bdev_get_integrity(ns->bdev); + + if (bi) { + ns->metadata_size = bi->tuple_size; + if (bi->profile == &t10_pi_type1_crc) + ns->pi_type = NVME_NS_DPS_PI_TYPE1; + else if (bi->profile == &t10_pi_type3_crc) + ns->pi_type = NVME_NS_DPS_PI_TYPE3; + else + /* Unsupported metadata type */ + ns->metadata_size = 0; + } +} + int nvmet_bdev_ns_enable(struct nvmet_ns *ns) { int ret; @@ -64,6 +80,12 @@ int nvmet_bdev_ns_enable(struct nvmet_ns *ns) } ns->size = i_size_read(ns->bdev->bd_inode); ns->blksize_shift = blksize_bits(bdev_logical_block_size(ns->bdev)); + + ns->pi_type = 0; + ns->metadata_size = 0; + if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY_T10)) + nvmet_bdev_ns_enable_integrity(ns); + return 0; } diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 93e0c2aa3e71..daef06a3aff4 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -19,6 +19,7 @@ #include #include #include +#include #define NVMET_ASYNC_EVENTS 4 #define NVMET_ERROR_LOG_SLOTS 128 @@ -77,6 +78,8 @@ struct nvmet_ns { int use_p2pmem; struct pci_dev *p2p_dev; + int pi_type; + int metadata_size; }; static inline struct nvmet_ns *to_nvmet_ns(struct config_item *item) From 26af180c1bd9cdd6f9b96d8df58b51d5900a2978 Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Tue, 19 May 2020 17:05:58 +0300 Subject: [PATCH 0853/1043] nvmet: rename nvmet_rw_len to nvmet_rw_data_len The function doesn't add the metadata length (only data length is calculated). This is preparation for adding metadata (T10-PI) support. Signed-off-by: Israel Rukshin Reviewed-by: Max Gurtovoy Reviewed-by: Sagi Grimberg Reviewed-by: Martin K. Petersen Reviewed-by: James Smart Signed-off-by: Christoph Hellwig --- drivers/nvme/target/io-cmd-bdev.c | 2 +- drivers/nvme/target/io-cmd-file.c | 2 +- drivers/nvme/target/nvmet.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index e518bb9bf718..628e81a74444 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -178,7 +178,7 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) sector_t sector; int op, i; - if (!nvmet_check_data_len(req, nvmet_rw_len(req))) + if (!nvmet_check_data_len(req, nvmet_rw_data_len(req))) return; if (!req->sg_cnt) { diff --git a/drivers/nvme/target/io-cmd-file.c b/drivers/nvme/target/io-cmd-file.c index f0bd08d86ac0..b31717c5d75f 100644 --- a/drivers/nvme/target/io-cmd-file.c +++ b/drivers/nvme/target/io-cmd-file.c @@ -241,7 +241,7 @@ static void nvmet_file_execute_rw(struct nvmet_req *req) { ssize_t nr_bvec = req->sg_cnt; - if (!nvmet_check_data_len(req, nvmet_rw_len(req))) + if (!nvmet_check_data_len(req, nvmet_rw_data_len(req))) return; if (!req->sg_cnt || !nr_bvec) { diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index daef06a3aff4..8bed8a61345e 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -505,7 +505,7 @@ void nvmet_bdev_ns_revalidate(struct nvmet_ns *ns); int nvmet_file_ns_revalidate(struct nvmet_ns *ns); void nvmet_ns_revalidate(struct nvmet_ns *ns); -static inline u32 nvmet_rw_len(struct nvmet_req *req) +static inline u32 nvmet_rw_data_len(struct nvmet_req *req) { return ((u32)le16_to_cpu(req->cmd->rw.length) + 1) << req->ns->blksize_shift; From 136cc1ffcf0a3309c59d844cb1a4ddad964ea3d8 Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Tue, 19 May 2020 17:05:59 +0300 Subject: [PATCH 0854/1043] nvmet: rename nvmet_check_data_len to nvmet_check_transfer_len The function doesn't check only the data length, because the transfer length includes also the metadata length in some cases. This is preparation for adding metadata (T10-PI) support. Signed-off-by: Israel Rukshin Signed-off-by: Max Gurtovoy Reviewed-by: Sagi Grimberg Reviewed-by: Martin K. Petersen Reviewed-by: James Smart Signed-off-by: Christoph Hellwig --- drivers/nvme/target/admin-cmd.c | 14 +++++++------- drivers/nvme/target/core.c | 6 +++--- drivers/nvme/target/discovery.c | 8 ++++---- drivers/nvme/target/fabrics-cmd.c | 8 ++++---- drivers/nvme/target/io-cmd-bdev.c | 6 +++--- drivers/nvme/target/io-cmd-file.c | 6 +++--- drivers/nvme/target/nvmet.h | 2 +- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index f544a14e8b5c..7b6b0395eaca 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -295,7 +295,7 @@ out: static void nvmet_execute_get_log_page(struct nvmet_req *req) { - if (!nvmet_check_data_len(req, nvmet_get_log_page_len(req->cmd))) + if (!nvmet_check_transfer_len(req, nvmet_get_log_page_len(req->cmd))) return; switch (req->cmd->get_log_page.lid) { @@ -627,7 +627,7 @@ out: static void nvmet_execute_identify(struct nvmet_req *req) { - if (!nvmet_check_data_len(req, NVME_IDENTIFY_DATA_SIZE)) + if (!nvmet_check_transfer_len(req, NVME_IDENTIFY_DATA_SIZE)) return; switch (req->cmd->identify.cns) { @@ -656,7 +656,7 @@ static void nvmet_execute_identify(struct nvmet_req *req) */ static void nvmet_execute_abort(struct nvmet_req *req) { - if (!nvmet_check_data_len(req, 0)) + if (!nvmet_check_transfer_len(req, 0)) return; nvmet_set_result(req, 1); nvmet_req_complete(req, 0); @@ -745,7 +745,7 @@ static void nvmet_execute_set_features(struct nvmet_req *req) u16 nsqr; u16 ncqr; - if (!nvmet_check_data_len(req, 0)) + if (!nvmet_check_transfer_len(req, 0)) return; switch (cdw10 & 0xff) { @@ -817,7 +817,7 @@ static void nvmet_execute_get_features(struct nvmet_req *req) u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); u16 status = 0; - if (!nvmet_check_data_len(req, nvmet_feat_data_len(req, cdw10))) + if (!nvmet_check_transfer_len(req, nvmet_feat_data_len(req, cdw10))) return; switch (cdw10 & 0xff) { @@ -884,7 +884,7 @@ void nvmet_execute_async_event(struct nvmet_req *req) { struct nvmet_ctrl *ctrl = req->sq->ctrl; - if (!nvmet_check_data_len(req, 0)) + if (!nvmet_check_transfer_len(req, 0)) return; mutex_lock(&ctrl->lock); @@ -903,7 +903,7 @@ void nvmet_execute_keep_alive(struct nvmet_req *req) { struct nvmet_ctrl *ctrl = req->sq->ctrl; - if (!nvmet_check_data_len(req, 0)) + if (!nvmet_check_transfer_len(req, 0)) return; pr_debug("ctrl %d update keep-alive timer for %d secs\n", diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index fb78b165c123..a4ed70e4afe3 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -950,9 +950,9 @@ void nvmet_req_uninit(struct nvmet_req *req) } EXPORT_SYMBOL_GPL(nvmet_req_uninit); -bool nvmet_check_data_len(struct nvmet_req *req, size_t data_len) +bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len) { - if (unlikely(data_len != req->transfer_len)) { + if (unlikely(len != req->transfer_len)) { req->error_loc = offsetof(struct nvme_common_command, dptr); nvmet_req_complete(req, NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR); return false; @@ -960,7 +960,7 @@ bool nvmet_check_data_len(struct nvmet_req *req, size_t data_len) return true; } -EXPORT_SYMBOL_GPL(nvmet_check_data_len); +EXPORT_SYMBOL_GPL(nvmet_check_transfer_len); bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len) { diff --git a/drivers/nvme/target/discovery.c b/drivers/nvme/target/discovery.c index 0c2274b21e15..40cf0b6e6c9d 100644 --- a/drivers/nvme/target/discovery.c +++ b/drivers/nvme/target/discovery.c @@ -171,7 +171,7 @@ static void nvmet_execute_disc_get_log_page(struct nvmet_req *req) u16 status = 0; void *buffer; - if (!nvmet_check_data_len(req, data_len)) + if (!nvmet_check_transfer_len(req, data_len)) return; if (req->cmd->get_log_page.lid != NVME_LOG_DISC) { @@ -244,7 +244,7 @@ static void nvmet_execute_disc_identify(struct nvmet_req *req) const char model[] = "Linux"; u16 status = 0; - if (!nvmet_check_data_len(req, NVME_IDENTIFY_DATA_SIZE)) + if (!nvmet_check_transfer_len(req, NVME_IDENTIFY_DATA_SIZE)) return; if (req->cmd->identify.cns != NVME_ID_CNS_CTRL) { @@ -298,7 +298,7 @@ static void nvmet_execute_disc_set_features(struct nvmet_req *req) u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); u16 stat; - if (!nvmet_check_data_len(req, 0)) + if (!nvmet_check_transfer_len(req, 0)) return; switch (cdw10 & 0xff) { @@ -324,7 +324,7 @@ static void nvmet_execute_disc_get_features(struct nvmet_req *req) u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); u16 stat = 0; - if (!nvmet_check_data_len(req, 0)) + if (!nvmet_check_transfer_len(req, 0)) return; switch (cdw10 & 0xff) { diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c index feef15c38ec9..52a6f70c9d80 100644 --- a/drivers/nvme/target/fabrics-cmd.c +++ b/drivers/nvme/target/fabrics-cmd.c @@ -12,7 +12,7 @@ static void nvmet_execute_prop_set(struct nvmet_req *req) u64 val = le64_to_cpu(req->cmd->prop_set.value); u16 status = 0; - if (!nvmet_check_data_len(req, 0)) + if (!nvmet_check_transfer_len(req, 0)) return; if (req->cmd->prop_set.attrib & 1) { @@ -41,7 +41,7 @@ static void nvmet_execute_prop_get(struct nvmet_req *req) u16 status = 0; u64 val = 0; - if (!nvmet_check_data_len(req, 0)) + if (!nvmet_check_transfer_len(req, 0)) return; if (req->cmd->prop_get.attrib & 1) { @@ -156,7 +156,7 @@ static void nvmet_execute_admin_connect(struct nvmet_req *req) struct nvmet_ctrl *ctrl = NULL; u16 status = 0; - if (!nvmet_check_data_len(req, sizeof(struct nvmf_connect_data))) + if (!nvmet_check_transfer_len(req, sizeof(struct nvmf_connect_data))) return; d = kmalloc(sizeof(*d), GFP_KERNEL); @@ -223,7 +223,7 @@ static void nvmet_execute_io_connect(struct nvmet_req *req) u16 qid = le16_to_cpu(c->qid); u16 status = 0; - if (!nvmet_check_data_len(req, sizeof(struct nvmf_connect_data))) + if (!nvmet_check_transfer_len(req, sizeof(struct nvmf_connect_data))) return; d = kmalloc(sizeof(*d), GFP_KERNEL); diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index 628e81a74444..e065c2430bfa 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -178,7 +178,7 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) sector_t sector; int op, i; - if (!nvmet_check_data_len(req, nvmet_rw_data_len(req))) + if (!nvmet_check_transfer_len(req, nvmet_rw_data_len(req))) return; if (!req->sg_cnt) { @@ -239,7 +239,7 @@ static void nvmet_bdev_execute_flush(struct nvmet_req *req) { struct bio *bio = &req->b.inline_bio; - if (!nvmet_check_data_len(req, 0)) + if (!nvmet_check_transfer_len(req, 0)) return; bio_init(bio, req->inline_bvec, ARRAY_SIZE(req->inline_bvec)); @@ -331,7 +331,7 @@ static void nvmet_bdev_execute_write_zeroes(struct nvmet_req *req) sector_t nr_sector; int ret; - if (!nvmet_check_data_len(req, 0)) + if (!nvmet_check_transfer_len(req, 0)) return; sector = le64_to_cpu(write_zeroes->slba) << diff --git a/drivers/nvme/target/io-cmd-file.c b/drivers/nvme/target/io-cmd-file.c index b31717c5d75f..0abbefd9925e 100644 --- a/drivers/nvme/target/io-cmd-file.c +++ b/drivers/nvme/target/io-cmd-file.c @@ -241,7 +241,7 @@ static void nvmet_file_execute_rw(struct nvmet_req *req) { ssize_t nr_bvec = req->sg_cnt; - if (!nvmet_check_data_len(req, nvmet_rw_data_len(req))) + if (!nvmet_check_transfer_len(req, nvmet_rw_data_len(req))) return; if (!req->sg_cnt || !nr_bvec) { @@ -285,7 +285,7 @@ static void nvmet_file_flush_work(struct work_struct *w) static void nvmet_file_execute_flush(struct nvmet_req *req) { - if (!nvmet_check_data_len(req, 0)) + if (!nvmet_check_transfer_len(req, 0)) return; INIT_WORK(&req->f.work, nvmet_file_flush_work); schedule_work(&req->f.work); @@ -375,7 +375,7 @@ static void nvmet_file_write_zeroes_work(struct work_struct *w) static void nvmet_file_execute_write_zeroes(struct nvmet_req *req) { - if (!nvmet_check_data_len(req, 0)) + if (!nvmet_check_transfer_len(req, 0)) return; INIT_WORK(&req->f.work, nvmet_file_write_zeroes_work); schedule_work(&req->f.work); diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 8bed8a61345e..29ab0b96bd32 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -387,7 +387,7 @@ u16 nvmet_parse_fabrics_cmd(struct nvmet_req *req); bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq, struct nvmet_sq *sq, const struct nvmet_fabrics_ops *ops); void nvmet_req_uninit(struct nvmet_req *req); -bool nvmet_check_data_len(struct nvmet_req *req, size_t data_len); +bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len); bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len); void nvmet_req_complete(struct nvmet_req *req, u16 status); int nvmet_req_alloc_sgl(struct nvmet_req *req); From 39481fbd14ee272edd419d73a98bc637e2a3fd35 Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Tue, 19 May 2020 17:06:00 +0300 Subject: [PATCH 0855/1043] nvme: add Metadata Capabilities enumerations The enumerations will be used to expose the namespace metadata format by the target. Suggested-by: Christoph Hellwig Signed-off-by: Israel Rukshin Signed-off-by: Max Gurtovoy Reviewed-by: James Smart Reviewed-by: Martin K. Petersen Signed-off-by: Christoph Hellwig --- include/linux/nvme.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/linux/nvme.h b/include/linux/nvme.h index e2993e6a9d7c..5ce51ab4c50e 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -420,6 +420,12 @@ enum { NVME_NS_DPS_PI_TYPE3 = 3, }; +/* Identify Namespace Metadata Capabilities (MC): */ +enum { + NVME_MC_EXTENDED_LBA = (1 << 0), + NVME_MC_METADATA_PTR = (1 << 1), +}; + struct nvme_ns_id_desc { __u8 nidt; __u8 nidl; From ea52ac1c6605fbd25347fabf46233e260dd92eb2 Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Tue, 19 May 2020 17:06:01 +0300 Subject: [PATCH 0856/1043] nvmet: add metadata/T10-PI support Expose the namespace metadata format when PI is enabled. The user needs to enable the capability per subsystem and per port. The other metadata properties are taken from the namespace/bdev. Usage example: echo 1 > /config/nvmet/subsystems/${NAME}/attr_pi_enable echo 1 > /config/nvmet/ports/${PORT_NUM}/param_pi_enable Signed-off-by: Israel Rukshin Signed-off-by: Max Gurtovoy Reviewed-by: James Smart Signed-off-by: Christoph Hellwig --- drivers/nvme/target/admin-cmd.c | 26 +++++++++++--- drivers/nvme/target/configfs.c | 58 +++++++++++++++++++++++++++++++ drivers/nvme/target/core.c | 21 ++++++++--- drivers/nvme/target/fabrics-cmd.c | 7 ++-- drivers/nvme/target/nvmet.h | 19 ++++++++++ 5 files changed, 121 insertions(+), 10 deletions(-) diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 7b6b0395eaca..1db8c0498668 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -341,6 +341,7 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req) { struct nvmet_ctrl *ctrl = req->sq->ctrl; struct nvme_id_ctrl *id; + u32 cmd_capsule_size; u16 status = 0; id = kzalloc(sizeof(*id), GFP_KERNEL); @@ -433,9 +434,15 @@ static void nvmet_execute_identify_ctrl(struct nvmet_req *req) strlcpy(id->subnqn, ctrl->subsys->subsysnqn, sizeof(id->subnqn)); - /* Max command capsule size is sqe + single page of in-capsule data */ - id->ioccsz = cpu_to_le32((sizeof(struct nvme_command) + - req->port->inline_data_size) / 16); + /* + * Max command capsule size is sqe + in-capsule data size. + * Disable in-capsule data for Metadata capable controllers. + */ + cmd_capsule_size = sizeof(struct nvme_command); + if (!ctrl->pi_support) + cmd_capsule_size += req->port->inline_data_size; + id->ioccsz = cpu_to_le32(cmd_capsule_size / 16); + /* Max response capsule size is cqe */ id->iorcsz = cpu_to_le32(sizeof(struct nvme_completion) / 16); @@ -465,6 +472,7 @@ out: static void nvmet_execute_identify_ns(struct nvmet_req *req) { + struct nvmet_ctrl *ctrl = req->sq->ctrl; struct nvmet_ns *ns; struct nvme_id_ns *id; u16 status = 0; @@ -482,7 +490,7 @@ static void nvmet_execute_identify_ns(struct nvmet_req *req) } /* return an all zeroed buffer if we can't find an active namespace */ - ns = nvmet_find_namespace(req->sq->ctrl, req->cmd->identify.nsid); + ns = nvmet_find_namespace(ctrl, req->cmd->identify.nsid); if (!ns) goto done; @@ -523,6 +531,16 @@ static void nvmet_execute_identify_ns(struct nvmet_req *req) id->lbaf[0].ds = ns->blksize_shift; + if (ctrl->pi_support && nvmet_ns_has_pi(ns)) { + id->dpc = NVME_NS_DPC_PI_FIRST | NVME_NS_DPC_PI_LAST | + NVME_NS_DPC_PI_TYPE1 | NVME_NS_DPC_PI_TYPE2 | + NVME_NS_DPC_PI_TYPE3; + id->mc = NVME_MC_EXTENDED_LBA; + id->dps = ns->pi_type; + id->flbas = NVME_NS_FLBAS_META_EXT; + id->lbaf[0].ms = cpu_to_le16(ns->metadata_size); + } + if (ns->readonly) id->nsattr |= (1 << 0); nvmet_put_namespace(ns); diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 17c529260e12..419e0d4ce79b 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -248,6 +248,36 @@ static ssize_t nvmet_param_inline_data_size_store(struct config_item *item, CONFIGFS_ATTR(nvmet_, param_inline_data_size); +#ifdef CONFIG_BLK_DEV_INTEGRITY +static ssize_t nvmet_param_pi_enable_show(struct config_item *item, + char *page) +{ + struct nvmet_port *port = to_nvmet_port(item); + + return snprintf(page, PAGE_SIZE, "%d\n", port->pi_enable); +} + +static ssize_t nvmet_param_pi_enable_store(struct config_item *item, + const char *page, size_t count) +{ + struct nvmet_port *port = to_nvmet_port(item); + bool val; + + if (strtobool(page, &val)) + return -EINVAL; + + if (port->enabled) { + pr_err("Disable port before setting pi_enable value.\n"); + return -EACCES; + } + + port->pi_enable = val; + return count; +} + +CONFIGFS_ATTR(nvmet_, param_pi_enable); +#endif + static ssize_t nvmet_addr_trtype_show(struct config_item *item, char *page) { @@ -1010,6 +1040,28 @@ static ssize_t nvmet_subsys_attr_model_store(struct config_item *item, } CONFIGFS_ATTR(nvmet_subsys_, attr_model); +#ifdef CONFIG_BLK_DEV_INTEGRITY +static ssize_t nvmet_subsys_attr_pi_enable_show(struct config_item *item, + char *page) +{ + return snprintf(page, PAGE_SIZE, "%d\n", to_subsys(item)->pi_support); +} + +static ssize_t nvmet_subsys_attr_pi_enable_store(struct config_item *item, + const char *page, size_t count) +{ + struct nvmet_subsys *subsys = to_subsys(item); + bool pi_enable; + + if (strtobool(page, &pi_enable)) + return -EINVAL; + + subsys->pi_support = pi_enable; + return count; +} +CONFIGFS_ATTR(nvmet_subsys_, attr_pi_enable); +#endif + static struct configfs_attribute *nvmet_subsys_attrs[] = { &nvmet_subsys_attr_attr_allow_any_host, &nvmet_subsys_attr_attr_version, @@ -1017,6 +1069,9 @@ static struct configfs_attribute *nvmet_subsys_attrs[] = { &nvmet_subsys_attr_attr_cntlid_min, &nvmet_subsys_attr_attr_cntlid_max, &nvmet_subsys_attr_attr_model, +#ifdef CONFIG_BLK_DEV_INTEGRITY + &nvmet_subsys_attr_attr_pi_enable, +#endif NULL, }; @@ -1316,6 +1371,9 @@ static struct configfs_attribute *nvmet_port_attrs[] = { &nvmet_attr_addr_trsvcid, &nvmet_attr_addr_trtype, &nvmet_attr_param_inline_data_size, +#ifdef CONFIG_BLK_DEV_INTEGRITY + &nvmet_attr_param_pi_enable, +#endif NULL, }; diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index a4ed70e4afe3..f9172d38aff5 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -323,12 +323,21 @@ int nvmet_enable_port(struct nvmet_port *port) if (!try_module_get(ops->owner)) return -EINVAL; - ret = ops->add_port(port); - if (ret) { - module_put(ops->owner); - return ret; + /* + * If the user requested PI support and the transport isn't pi capable, + * don't enable the port. + */ + if (port->pi_enable && !ops->metadata_support) { + pr_err("T10-PI is not supported by transport type %d\n", + port->disc_addr.trtype); + ret = -EINVAL; + goto out_put; } + ret = ops->add_port(port); + if (ret) + goto out_put; + /* If the transport didn't set inline_data_size, then disable it. */ if (port->inline_data_size < 0) port->inline_data_size = 0; @@ -336,6 +345,10 @@ int nvmet_enable_port(struct nvmet_port *port) port->enabled = true; port->tr_ops = ops; return 0; + +out_put: + module_put(ops->owner); + return ret; } void nvmet_disable_port(struct nvmet_port *port) diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c index 52a6f70c9d80..42bd12b8bf00 100644 --- a/drivers/nvme/target/fabrics-cmd.c +++ b/drivers/nvme/target/fabrics-cmd.c @@ -197,6 +197,8 @@ static void nvmet_execute_admin_connect(struct nvmet_req *req) goto out; } + ctrl->pi_support = ctrl->port->pi_enable && ctrl->subsys->pi_support; + uuid_copy(&ctrl->hostid, &d->hostid); status = nvmet_install_queue(ctrl, req); @@ -205,8 +207,9 @@ static void nvmet_execute_admin_connect(struct nvmet_req *req) goto out; } - pr_info("creating controller %d for subsystem %s for NQN %s.\n", - ctrl->cntlid, ctrl->subsys->subsysnqn, ctrl->hostnqn); + pr_info("creating controller %d for subsystem %s for NQN %s%s.\n", + ctrl->cntlid, ctrl->subsys->subsysnqn, ctrl->hostnqn, + ctrl->pi_support ? " T10-PI is enabled" : ""); req->cqe->result.u16 = cpu_to_le16(ctrl->cntlid); out: diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 29ab0b96bd32..46b5d2b4ca0a 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -145,6 +145,7 @@ struct nvmet_port { bool enabled; int inline_data_size; const struct nvmet_fabrics_ops *tr_ops; + bool pi_enable; }; static inline struct nvmet_port *to_nvmet_port(struct config_item *item) @@ -204,6 +205,7 @@ struct nvmet_ctrl { spinlock_t error_lock; u64 err_counter; struct nvme_error_slot slots[NVMET_ERROR_LOG_SLOTS]; + bool pi_support; }; struct nvmet_subsys_model { @@ -233,6 +235,7 @@ struct nvmet_subsys { u64 ver; u64 serial; char *subsysnqn; + bool pi_support; struct config_group group; @@ -284,6 +287,7 @@ struct nvmet_fabrics_ops { unsigned int type; unsigned int msdbd; bool has_keyed_sgls : 1; + bool metadata_support : 1; void (*queue_response)(struct nvmet_req *req); int (*add_port)(struct nvmet_port *port); void (*remove_port)(struct nvmet_port *port); @@ -511,6 +515,14 @@ static inline u32 nvmet_rw_data_len(struct nvmet_req *req) req->ns->blksize_shift; } +static inline u32 nvmet_rw_metadata_len(struct nvmet_req *req) +{ + if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY)) + return 0; + return ((u32)le16_to_cpu(req->cmd->rw.length) + 1) * + req->ns->metadata_size; +} + static inline u32 nvmet_dsm_len(struct nvmet_req *req) { return (le32_to_cpu(req->cmd->dsm.nr) + 1) * @@ -525,4 +537,11 @@ static inline __le16 to0based(u32 a) return cpu_to_le16(max(1U, min(1U << 16, a)) - 1); } +static inline bool nvmet_ns_has_pi(struct nvmet_ns *ns) +{ + if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY)) + return false; + return ns->pi_type && ns->metadata_size == sizeof(struct t10_pi_tuple); +} + #endif /* _NVMET_H */ From c6e3f13398123a008cd2ee28f93510b113a32791 Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Tue, 19 May 2020 17:06:02 +0300 Subject: [PATCH 0857/1043] nvmet: add metadata support for block devices Allocate the metadata SGL buffers and set metadata fields for the request. Then create a block IO request for the metadata from the protection SG list. Signed-off-by: Israel Rukshin Signed-off-by: Max Gurtovoy Signed-off-by: Christoph Hellwig --- drivers/nvme/target/core.c | 95 ++++++++++++++++++++++--------- drivers/nvme/target/io-cmd-bdev.c | 87 +++++++++++++++++++++++++++- drivers/nvme/target/nvmet.h | 7 ++- drivers/nvme/target/rdma.c | 4 +- 4 files changed, 161 insertions(+), 32 deletions(-) diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index f9172d38aff5..edf54d9957b7 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -900,8 +900,11 @@ bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq, req->sq = sq; req->ops = ops; req->sg = NULL; + req->metadata_sg = NULL; req->sg_cnt = 0; + req->metadata_sg_cnt = 0; req->transfer_len = 0; + req->metadata_len = 0; req->cqe->status = 0; req->cqe->sq_head = 0; req->ns = NULL; @@ -986,50 +989,90 @@ bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len) return true; } -int nvmet_req_alloc_sgl(struct nvmet_req *req) +static unsigned int nvmet_data_transfer_len(struct nvmet_req *req) { - struct pci_dev *p2p_dev = NULL; + return req->transfer_len - req->metadata_len; +} - if (IS_ENABLED(CONFIG_PCI_P2PDMA)) { - if (req->sq->ctrl && req->ns) - p2p_dev = radix_tree_lookup(&req->sq->ctrl->p2p_ns_map, - req->ns->nsid); +static int nvmet_req_alloc_p2pmem_sgls(struct nvmet_req *req) +{ + req->sg = pci_p2pmem_alloc_sgl(req->p2p_dev, &req->sg_cnt, + nvmet_data_transfer_len(req)); + if (!req->sg) + goto out_err; - req->p2p_dev = NULL; - if (req->sq->qid && p2p_dev) { - req->sg = pci_p2pmem_alloc_sgl(p2p_dev, &req->sg_cnt, - req->transfer_len); - if (req->sg) { - req->p2p_dev = p2p_dev; - return 0; - } - } + if (req->metadata_len) { + req->metadata_sg = pci_p2pmem_alloc_sgl(req->p2p_dev, + &req->metadata_sg_cnt, req->metadata_len); + if (!req->metadata_sg) + goto out_free_sg; + } + return 0; +out_free_sg: + pci_p2pmem_free_sgl(req->p2p_dev, req->sg); +out_err: + return -ENOMEM; +} - /* - * If no P2P memory was available we fallback to using - * regular memory - */ +static bool nvmet_req_find_p2p_dev(struct nvmet_req *req) +{ + if (!IS_ENABLED(CONFIG_PCI_P2PDMA)) + return false; + + if (req->sq->ctrl && req->sq->qid && req->ns) { + req->p2p_dev = radix_tree_lookup(&req->sq->ctrl->p2p_ns_map, + req->ns->nsid); + if (req->p2p_dev) + return true; } - req->sg = sgl_alloc(req->transfer_len, GFP_KERNEL, &req->sg_cnt); + req->p2p_dev = NULL; + return false; +} + +int nvmet_req_alloc_sgls(struct nvmet_req *req) +{ + if (nvmet_req_find_p2p_dev(req) && !nvmet_req_alloc_p2pmem_sgls(req)) + return 0; + + req->sg = sgl_alloc(nvmet_data_transfer_len(req), GFP_KERNEL, + &req->sg_cnt); if (unlikely(!req->sg)) - return -ENOMEM; + goto out; + + if (req->metadata_len) { + req->metadata_sg = sgl_alloc(req->metadata_len, GFP_KERNEL, + &req->metadata_sg_cnt); + if (unlikely(!req->metadata_sg)) + goto out_free; + } return 0; +out_free: + sgl_free(req->sg); +out: + return -ENOMEM; } -EXPORT_SYMBOL_GPL(nvmet_req_alloc_sgl); +EXPORT_SYMBOL_GPL(nvmet_req_alloc_sgls); -void nvmet_req_free_sgl(struct nvmet_req *req) +void nvmet_req_free_sgls(struct nvmet_req *req) { - if (req->p2p_dev) + if (req->p2p_dev) { pci_p2pmem_free_sgl(req->p2p_dev, req->sg); - else + if (req->metadata_sg) + pci_p2pmem_free_sgl(req->p2p_dev, req->metadata_sg); + } else { sgl_free(req->sg); + if (req->metadata_sg) + sgl_free(req->metadata_sg); + } req->sg = NULL; + req->metadata_sg = NULL; req->sg_cnt = 0; + req->metadata_sg_cnt = 0; } -EXPORT_SYMBOL_GPL(nvmet_req_free_sgl); +EXPORT_SYMBOL_GPL(nvmet_req_free_sgls); static inline bool nvmet_cc_en(u32 cc) { diff --git a/drivers/nvme/target/io-cmd-bdev.c b/drivers/nvme/target/io-cmd-bdev.c index e065c2430bfa..07055f7ac398 100644 --- a/drivers/nvme/target/io-cmd-bdev.c +++ b/drivers/nvme/target/io-cmd-bdev.c @@ -169,6 +169,61 @@ static void nvmet_bio_done(struct bio *bio) bio_put(bio); } +#ifdef CONFIG_BLK_DEV_INTEGRITY +static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio, + struct sg_mapping_iter *miter) +{ + struct blk_integrity *bi; + struct bio_integrity_payload *bip; + struct block_device *bdev = req->ns->bdev; + int rc; + size_t resid, len; + + bi = bdev_get_integrity(bdev); + if (unlikely(!bi)) { + pr_err("Unable to locate bio_integrity\n"); + return -ENODEV; + } + + bip = bio_integrity_alloc(bio, GFP_NOIO, + min_t(unsigned int, req->metadata_sg_cnt, BIO_MAX_PAGES)); + if (IS_ERR(bip)) { + pr_err("Unable to allocate bio_integrity_payload\n"); + return PTR_ERR(bip); + } + + bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio)); + /* virtual start sector must be in integrity interval units */ + bip_set_seed(bip, bio->bi_iter.bi_sector >> + (bi->interval_exp - SECTOR_SHIFT)); + + resid = bip->bip_iter.bi_size; + while (resid > 0 && sg_miter_next(miter)) { + len = min_t(size_t, miter->length, resid); + rc = bio_integrity_add_page(bio, miter->page, len, + offset_in_page(miter->addr)); + if (unlikely(rc != len)) { + pr_err("bio_integrity_add_page() failed; %d\n", rc); + sg_miter_stop(miter); + return -ENOMEM; + } + + resid -= len; + if (len < miter->length) + miter->consumed -= miter->length - len; + } + sg_miter_stop(miter); + + return 0; +} +#else +static int nvmet_bdev_alloc_bip(struct nvmet_req *req, struct bio *bio, + struct sg_mapping_iter *miter) +{ + return -EINVAL; +} +#endif /* CONFIG_BLK_DEV_INTEGRITY */ + static void nvmet_bdev_execute_rw(struct nvmet_req *req) { int sg_cnt = req->sg_cnt; @@ -176,9 +231,12 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) struct scatterlist *sg; struct blk_plug plug; sector_t sector; - int op, i; + int op, i, rc; + struct sg_mapping_iter prot_miter; + unsigned int iter_flags; + unsigned int total_len = nvmet_rw_data_len(req) + req->metadata_len; - if (!nvmet_check_transfer_len(req, nvmet_rw_data_len(req))) + if (!nvmet_check_transfer_len(req, total_len)) return; if (!req->sg_cnt) { @@ -190,8 +248,10 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA)) op |= REQ_FUA; + iter_flags = SG_MITER_TO_SG; } else { op = REQ_OP_READ; + iter_flags = SG_MITER_FROM_SG; } if (is_pci_p2pdma_page(sg_page(req->sg))) @@ -213,11 +273,24 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) bio->bi_opf = op; blk_start_plug(&plug); + if (req->metadata_len) + sg_miter_start(&prot_miter, req->metadata_sg, + req->metadata_sg_cnt, iter_flags); + for_each_sg(req->sg, sg, req->sg_cnt, i) { while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset) != sg->length) { struct bio *prev = bio; + if (req->metadata_len) { + rc = nvmet_bdev_alloc_bip(req, bio, + &prot_miter); + if (unlikely(rc)) { + bio_io_error(bio); + return; + } + } + bio = bio_alloc(GFP_KERNEL, min(sg_cnt, BIO_MAX_PAGES)); bio_set_dev(bio, req->ns->bdev); bio->bi_iter.bi_sector = sector; @@ -231,6 +304,14 @@ static void nvmet_bdev_execute_rw(struct nvmet_req *req) sg_cnt--; } + if (req->metadata_len) { + rc = nvmet_bdev_alloc_bip(req, bio, &prot_miter); + if (unlikely(rc)) { + bio_io_error(bio); + return; + } + } + submit_bio(bio); blk_finish_plug(&plug); } @@ -358,6 +439,8 @@ u16 nvmet_bdev_parse_io_cmd(struct nvmet_req *req) case nvme_cmd_read: case nvme_cmd_write: req->execute = nvmet_bdev_execute_rw; + if (req->sq->ctrl->pi_support && nvmet_ns_has_pi(req->ns)) + req->metadata_len = nvmet_rw_metadata_len(req); return 0; case nvme_cmd_flush: req->execute = nvmet_bdev_execute_flush; diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 46b5d2b4ca0a..809691291e73 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -309,6 +309,7 @@ struct nvmet_req { struct nvmet_cq *cq; struct nvmet_ns *ns; struct scatterlist *sg; + struct scatterlist *metadata_sg; struct bio_vec inline_bvec[NVMET_MAX_INLINE_BIOVEC]; union { struct { @@ -322,8 +323,10 @@ struct nvmet_req { } f; }; int sg_cnt; + int metadata_sg_cnt; /* data length as parsed from the SGL descriptor: */ size_t transfer_len; + size_t metadata_len; struct nvmet_port *port; @@ -394,8 +397,8 @@ void nvmet_req_uninit(struct nvmet_req *req); bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len); bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len); void nvmet_req_complete(struct nvmet_req *req, u16 status); -int nvmet_req_alloc_sgl(struct nvmet_req *req); -void nvmet_req_free_sgl(struct nvmet_req *req); +int nvmet_req_alloc_sgls(struct nvmet_req *req); +void nvmet_req_free_sgls(struct nvmet_req *req); void nvmet_execute_keep_alive(struct nvmet_req *req); diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index 7a90b10359bb..178dbffa8c41 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -546,7 +546,7 @@ static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) } if (rsp->req.sg != rsp->cmd->inline_sg) - nvmet_req_free_sgl(&rsp->req); + nvmet_req_free_sgls(&rsp->req); if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list))) nvmet_rdma_process_wr_wait_list(queue); @@ -708,7 +708,7 @@ static u16 nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp *rsp, if (!rsp->req.transfer_len) return 0; - ret = nvmet_req_alloc_sgl(&rsp->req); + ret = nvmet_req_alloc_sgls(&rsp->req); if (unlikely(ret < 0)) goto error_out; From b09160c3996c11d62a08f9534c755103a10a89b4 Mon Sep 17 00:00:00 2001 From: Israel Rukshin Date: Tue, 19 May 2020 17:06:03 +0300 Subject: [PATCH 0858/1043] nvmet-rdma: add metadata/T10-PI support For capable HCAs (e.g. ConnectX-5/ConnectX-6) this will allow end-to-end protection information passthrough and validation for NVMe over RDMA transport. Metadata support was implemented over the new RDMA signature verbs API. Signed-off-by: Israel Rukshin Signed-off-by: Max Gurtovoy Signed-off-by: Christoph Hellwig --- drivers/nvme/target/rdma.c | 234 ++++++++++++++++++++++++++++++++++--- 1 file changed, 215 insertions(+), 19 deletions(-) diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index 178dbffa8c41..d5141780592e 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -33,6 +33,7 @@ /* Assume mpsmin == device_page_size == 4KB */ #define NVMET_RDMA_MAX_MDTS 8 +#define NVMET_RDMA_MAX_METADATA_MDTS 5 struct nvmet_rdma_srq; @@ -60,6 +61,7 @@ struct nvmet_rdma_rsp { struct nvmet_rdma_queue *queue; struct ib_cqe read_cqe; + struct ib_cqe write_cqe; struct rdma_rw_ctx rw; struct nvmet_req req; @@ -161,6 +163,7 @@ static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp); static void nvmet_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc); static void nvmet_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc); static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc); +static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc); static void nvmet_rdma_qp_event(struct ib_event *event, void *priv); static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue); static void nvmet_rdma_free_rsp(struct nvmet_rdma_device *ndev, @@ -423,6 +426,9 @@ static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device *ndev, /* Data In / RDMA READ */ r->read_cqe.done = nvmet_rdma_read_data_done; + /* Data Out / RDMA WRITE */ + r->write_cqe.done = nvmet_rdma_write_data_done; + return 0; out_free_rsp: @@ -532,6 +538,129 @@ static void nvmet_rdma_process_wr_wait_list(struct nvmet_rdma_queue *queue) spin_unlock(&queue->rsp_wr_wait_lock); } +static u16 nvmet_rdma_check_pi_status(struct ib_mr *sig_mr) +{ + struct ib_mr_status mr_status; + int ret; + u16 status = 0; + + ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); + if (ret) { + pr_err("ib_check_mr_status failed, ret %d\n", ret); + return NVME_SC_INVALID_PI; + } + + if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { + switch (mr_status.sig_err.err_type) { + case IB_SIG_BAD_GUARD: + status = NVME_SC_GUARD_CHECK; + break; + case IB_SIG_BAD_REFTAG: + status = NVME_SC_REFTAG_CHECK; + break; + case IB_SIG_BAD_APPTAG: + status = NVME_SC_APPTAG_CHECK; + break; + } + pr_err("PI error found type %d expected 0x%x vs actual 0x%x\n", + mr_status.sig_err.err_type, + mr_status.sig_err.expected, + mr_status.sig_err.actual); + } + + return status; +} + +static void nvmet_rdma_set_sig_domain(struct blk_integrity *bi, + struct nvme_command *cmd, struct ib_sig_domain *domain, + u16 control, u8 pi_type) +{ + domain->sig_type = IB_SIG_TYPE_T10_DIF; + domain->sig.dif.bg_type = IB_T10DIF_CRC; + domain->sig.dif.pi_interval = 1 << bi->interval_exp; + domain->sig.dif.ref_tag = le32_to_cpu(cmd->rw.reftag); + if (control & NVME_RW_PRINFO_PRCHK_REF) + domain->sig.dif.ref_remap = true; + + domain->sig.dif.app_tag = le16_to_cpu(cmd->rw.apptag); + domain->sig.dif.apptag_check_mask = le16_to_cpu(cmd->rw.appmask); + domain->sig.dif.app_escape = true; + if (pi_type == NVME_NS_DPS_PI_TYPE3) + domain->sig.dif.ref_escape = true; +} + +static void nvmet_rdma_set_sig_attrs(struct nvmet_req *req, + struct ib_sig_attrs *sig_attrs) +{ + struct nvme_command *cmd = req->cmd; + u16 control = le16_to_cpu(cmd->rw.control); + u8 pi_type = req->ns->pi_type; + struct blk_integrity *bi; + + bi = bdev_get_integrity(req->ns->bdev); + + memset(sig_attrs, 0, sizeof(*sig_attrs)); + + if (control & NVME_RW_PRINFO_PRACT) { + /* for WRITE_INSERT/READ_STRIP no wire domain */ + sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; + nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->mem, control, + pi_type); + /* Clear the PRACT bit since HCA will generate/verify the PI */ + control &= ~NVME_RW_PRINFO_PRACT; + cmd->rw.control = cpu_to_le16(control); + /* PI is added by the HW */ + req->transfer_len += req->metadata_len; + } else { + /* for WRITE_PASS/READ_PASS both wire/memory domains exist */ + nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->wire, control, + pi_type); + nvmet_rdma_set_sig_domain(bi, cmd, &sig_attrs->mem, control, + pi_type); + } + + if (control & NVME_RW_PRINFO_PRCHK_REF) + sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG; + if (control & NVME_RW_PRINFO_PRCHK_GUARD) + sig_attrs->check_mask |= IB_SIG_CHECK_GUARD; + if (control & NVME_RW_PRINFO_PRCHK_APP) + sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG; +} + +static int nvmet_rdma_rw_ctx_init(struct nvmet_rdma_rsp *rsp, u64 addr, u32 key, + struct ib_sig_attrs *sig_attrs) +{ + struct rdma_cm_id *cm_id = rsp->queue->cm_id; + struct nvmet_req *req = &rsp->req; + int ret; + + if (req->metadata_len) + ret = rdma_rw_ctx_signature_init(&rsp->rw, cm_id->qp, + cm_id->port_num, req->sg, req->sg_cnt, + req->metadata_sg, req->metadata_sg_cnt, sig_attrs, + addr, key, nvmet_data_dir(req)); + else + ret = rdma_rw_ctx_init(&rsp->rw, cm_id->qp, cm_id->port_num, + req->sg, req->sg_cnt, 0, addr, key, + nvmet_data_dir(req)); + + return ret; +} + +static void nvmet_rdma_rw_ctx_destroy(struct nvmet_rdma_rsp *rsp) +{ + struct rdma_cm_id *cm_id = rsp->queue->cm_id; + struct nvmet_req *req = &rsp->req; + + if (req->metadata_len) + rdma_rw_ctx_destroy_signature(&rsp->rw, cm_id->qp, + cm_id->port_num, req->sg, req->sg_cnt, + req->metadata_sg, req->metadata_sg_cnt, + nvmet_data_dir(req)); + else + rdma_rw_ctx_destroy(&rsp->rw, cm_id->qp, cm_id->port_num, + req->sg, req->sg_cnt, nvmet_data_dir(req)); +} static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) { @@ -539,11 +668,8 @@ static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); - if (rsp->n_rdma) { - rdma_rw_ctx_destroy(&rsp->rw, queue->qp, - queue->cm_id->port_num, rsp->req.sg, - rsp->req.sg_cnt, nvmet_data_dir(&rsp->req)); - } + if (rsp->n_rdma) + nvmet_rdma_rw_ctx_destroy(rsp); if (rsp->req.sg != rsp->cmd->inline_sg) nvmet_req_free_sgls(&rsp->req); @@ -598,11 +724,16 @@ static void nvmet_rdma_queue_response(struct nvmet_req *req) rsp->send_wr.opcode = IB_WR_SEND; } - if (nvmet_rdma_need_data_out(rsp)) - first_wr = rdma_rw_ctx_wrs(&rsp->rw, cm_id->qp, - cm_id->port_num, NULL, &rsp->send_wr); - else + if (nvmet_rdma_need_data_out(rsp)) { + if (rsp->req.metadata_len) + first_wr = rdma_rw_ctx_wrs(&rsp->rw, cm_id->qp, + cm_id->port_num, &rsp->write_cqe, NULL); + else + first_wr = rdma_rw_ctx_wrs(&rsp->rw, cm_id->qp, + cm_id->port_num, NULL, &rsp->send_wr); + } else { first_wr = &rsp->send_wr; + } nvmet_rdma_post_recv(rsp->queue->dev, rsp->cmd); @@ -621,15 +752,14 @@ static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc) struct nvmet_rdma_rsp *rsp = container_of(wc->wr_cqe, struct nvmet_rdma_rsp, read_cqe); struct nvmet_rdma_queue *queue = cq->cq_context; + u16 status = 0; WARN_ON(rsp->n_rdma <= 0); atomic_add(rsp->n_rdma, &queue->sq_wr_avail); - rdma_rw_ctx_destroy(&rsp->rw, queue->qp, - queue->cm_id->port_num, rsp->req.sg, - rsp->req.sg_cnt, nvmet_data_dir(&rsp->req)); rsp->n_rdma = 0; if (unlikely(wc->status != IB_WC_SUCCESS)) { + nvmet_rdma_rw_ctx_destroy(rsp); nvmet_req_uninit(&rsp->req); nvmet_rdma_release_rsp(rsp); if (wc->status != IB_WC_WR_FLUSH_ERR) { @@ -640,7 +770,58 @@ static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc) return; } - rsp->req.execute(&rsp->req); + if (rsp->req.metadata_len) + status = nvmet_rdma_check_pi_status(rsp->rw.reg->mr); + nvmet_rdma_rw_ctx_destroy(rsp); + + if (unlikely(status)) + nvmet_req_complete(&rsp->req, status); + else + rsp->req.execute(&rsp->req); +} + +static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct nvmet_rdma_rsp *rsp = + container_of(wc->wr_cqe, struct nvmet_rdma_rsp, write_cqe); + struct nvmet_rdma_queue *queue = cq->cq_context; + struct rdma_cm_id *cm_id = rsp->queue->cm_id; + u16 status; + + if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY)) + return; + + WARN_ON(rsp->n_rdma <= 0); + atomic_add(rsp->n_rdma, &queue->sq_wr_avail); + rsp->n_rdma = 0; + + if (unlikely(wc->status != IB_WC_SUCCESS)) { + nvmet_rdma_rw_ctx_destroy(rsp); + nvmet_req_uninit(&rsp->req); + nvmet_rdma_release_rsp(rsp); + if (wc->status != IB_WC_WR_FLUSH_ERR) { + pr_info("RDMA WRITE for CQE 0x%p failed with status %s (%d).\n", + wc->wr_cqe, ib_wc_status_msg(wc->status), + wc->status); + nvmet_rdma_error_comp(queue); + } + return; + } + + /* + * Upon RDMA completion check the signature status + * - if succeeded send good NVMe response + * - if failed send bad NVMe response with appropriate error + */ + status = nvmet_rdma_check_pi_status(rsp->rw.reg->mr); + if (unlikely(status)) + rsp->req.cqe->status = cpu_to_le16(status << 1); + nvmet_rdma_rw_ctx_destroy(rsp); + + if (unlikely(ib_post_send(cm_id->qp, &rsp->send_wr, NULL))) { + pr_err("sending cmd response failed\n"); + nvmet_rdma_release_rsp(rsp); + } } static void nvmet_rdma_use_inline_sg(struct nvmet_rdma_rsp *rsp, u32 len, @@ -697,9 +878,9 @@ static u16 nvmet_rdma_map_sgl_inline(struct nvmet_rdma_rsp *rsp) static u16 nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp *rsp, struct nvme_keyed_sgl_desc *sgl, bool invalidate) { - struct rdma_cm_id *cm_id = rsp->queue->cm_id; u64 addr = le64_to_cpu(sgl->addr); u32 key = get_unaligned_le32(sgl->key); + struct ib_sig_attrs sig_attrs; int ret; rsp->req.transfer_len = get_unaligned_le24(sgl->length); @@ -708,13 +889,14 @@ static u16 nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp *rsp, if (!rsp->req.transfer_len) return 0; + if (rsp->req.metadata_len) + nvmet_rdma_set_sig_attrs(&rsp->req, &sig_attrs); + ret = nvmet_req_alloc_sgls(&rsp->req); if (unlikely(ret < 0)) goto error_out; - ret = rdma_rw_ctx_init(&rsp->rw, cm_id->qp, cm_id->port_num, - rsp->req.sg, rsp->req.sg_cnt, 0, addr, key, - nvmet_data_dir(&rsp->req)); + ret = nvmet_rdma_rw_ctx_init(rsp, addr, key, &sig_attrs); if (unlikely(ret < 0)) goto error_out; rsp->n_rdma += ret; @@ -1108,6 +1290,9 @@ static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue *queue) qp_attr.cap.max_recv_sge = 1 + ndev->inline_page_count; } + if (queue->port->pi_enable && queue->host_qid) + qp_attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN; + ret = rdma_create_qp(queue->cm_id, ndev->pd, &qp_attr); if (ret) { pr_err("failed to create_qp ret= %d\n", ret); @@ -1226,6 +1411,7 @@ nvmet_rdma_alloc_queue(struct nvmet_rdma_device *ndev, struct rdma_cm_id *cm_id, struct rdma_cm_event *event) { + struct nvmet_rdma_port *port = cm_id->context; struct nvmet_rdma_queue *queue; int ret; @@ -1252,6 +1438,7 @@ nvmet_rdma_alloc_queue(struct nvmet_rdma_device *ndev, INIT_WORK(&queue->release_work, nvmet_rdma_release_queue_work); queue->dev = ndev; queue->cm_id = cm_id; + queue->port = port->nport; spin_lock_init(&queue->state_lock); queue->state = NVMET_RDMA_Q_CONNECTING; @@ -1369,7 +1556,6 @@ static int nvmet_rdma_cm_accept(struct rdma_cm_id *cm_id, static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id, struct rdma_cm_event *event) { - struct nvmet_rdma_port *port = cm_id->context; struct nvmet_rdma_device *ndev; struct nvmet_rdma_queue *queue; int ret = -EINVAL; @@ -1385,7 +1571,6 @@ static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id, ret = -ENOMEM; goto put_device; } - queue->port = port->nport; if (queue->host_qid == 0) { /* Let inflight controller teardown complete */ @@ -1657,6 +1842,14 @@ static int nvmet_rdma_enable_port(struct nvmet_rdma_port *port) goto out_destroy_id; } + if (port->nport->pi_enable && + !(cm_id->device->attrs.device_cap_flags & + IB_DEVICE_INTEGRITY_HANDOVER)) { + pr_err("T10-PI is not supported for %pISpcs\n", addr); + ret = -EINVAL; + goto out_destroy_id; + } + port->cm_id = cm_id; return 0; @@ -1766,6 +1959,8 @@ static void nvmet_rdma_disc_port_addr(struct nvmet_req *req, static u8 nvmet_rdma_get_mdts(const struct nvmet_ctrl *ctrl) { + if (ctrl->pi_support) + return NVMET_RDMA_MAX_METADATA_MDTS; return NVMET_RDMA_MAX_MDTS; } @@ -1774,6 +1969,7 @@ static const struct nvmet_fabrics_ops nvmet_rdma_ops = { .type = NVMF_TRTYPE_RDMA, .msdbd = 1, .has_keyed_sgls = 1, + .metadata_support = 1, .add_port = nvmet_rdma_add_port, .remove_port = nvmet_rdma_remove_port, .queue_response = nvmet_rdma_queue_response, From 64f5e9cdd711b030b05062c17b2ecfbce890cf4c Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Wed, 20 May 2020 12:48:12 -0700 Subject: [PATCH 0859/1043] nvmet: fix memory leak when removing namespaces and controllers concurrently When removing a namespace, we add an NS_CHANGE async event, however if the controller admin queue is removed after the event was added but not yet processed, we won't free the aens, resulting in the below memory leak [1]. Fix that by moving nvmet_async_event_free to the final controller release after it is detached from subsys->ctrls ensuring no async events are added, and modify it to simply remove all pending aens. -- $ cat /sys/kernel/debug/kmemleak unreferenced object 0xffff888c1af2c000 (size 32): comm "nvmetcli", pid 5164, jiffies 4295220864 (age 6829.924s) hex dump (first 32 bytes): 28 01 82 3b 8b 88 ff ff 28 01 82 3b 8b 88 ff ff (..;....(..;.... 02 00 04 65 76 65 6e 74 5f 66 69 6c 65 00 00 00 ...event_file... backtrace: [<00000000217ae580>] nvmet_add_async_event+0x57/0x290 [nvmet] [<0000000012aa2ea9>] nvmet_ns_changed+0x206/0x300 [nvmet] [<00000000bb3fd52e>] nvmet_ns_disable+0x367/0x4f0 [nvmet] [<00000000e91ca9ec>] nvmet_ns_free+0x15/0x180 [nvmet] [<00000000a15deb52>] config_item_release+0xf1/0x1c0 [<000000007e148432>] configfs_rmdir+0x555/0x7c0 [<00000000f4506ea6>] vfs_rmdir+0x142/0x3c0 [<0000000000acaaf0>] do_rmdir+0x2b2/0x340 [<0000000034d1aa52>] do_syscall_64+0xa5/0x4d0 [<00000000211f13bc>] entry_SYSCALL_64_after_hwframe+0x6a/0xdf Fixes: a07b4970f464 ("nvmet: add a generic NVMe target") Reported-by: David Milburn Signed-off-by: Sagi Grimberg Tested-by: David Milburn Signed-off-by: Christoph Hellwig --- drivers/nvme/target/core.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index edf54d9957b7..48f5123d875b 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -158,14 +158,12 @@ static void nvmet_async_events_process(struct nvmet_ctrl *ctrl, u16 status) static void nvmet_async_events_free(struct nvmet_ctrl *ctrl) { - struct nvmet_req *req; + struct nvmet_async_event *aen, *tmp; mutex_lock(&ctrl->lock); - while (ctrl->nr_async_event_cmds) { - req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; - mutex_unlock(&ctrl->lock); - nvmet_req_complete(req, NVME_SC_INTERNAL | NVME_SC_DNR); - mutex_lock(&ctrl->lock); + list_for_each_entry_safe(aen, tmp, &ctrl->async_events, entry) { + list_del(&aen->entry); + kfree(aen); } mutex_unlock(&ctrl->lock); } @@ -791,10 +789,8 @@ void nvmet_sq_destroy(struct nvmet_sq *sq) * If this is the admin queue, complete all AERs so that our * queue doesn't have outstanding requests on it. */ - if (ctrl && ctrl->sqs && ctrl->sqs[0] == sq) { + if (ctrl && ctrl->sqs && ctrl->sqs[0] == sq) nvmet_async_events_process(ctrl, status); - nvmet_async_events_free(ctrl); - } percpu_ref_kill_and_confirm(&sq->ref, nvmet_confirm_sq); wait_for_completion(&sq->confirm_done); wait_for_completion(&sq->free_done); @@ -1427,6 +1423,7 @@ static void nvmet_ctrl_free(struct kref *ref) ida_simple_remove(&cntlid_ida, ctrl->cntlid); + nvmet_async_events_free(ctrl); kfree(ctrl->sqs); kfree(ctrl->cqs); kfree(ctrl->changed_ns_list); From 1cdf9f7670a7d74e27177d5c390c2f8b3b9ba338 Mon Sep 17 00:00:00 2001 From: David Milburn Date: Mon, 18 May 2020 13:59:55 -0500 Subject: [PATCH 0860/1043] nvmet: cleanups the loop in nvmet_async_events_process Based-on-a-patch-by: Christoph Hellwig Tested-by: Yi Zhang Signed-off-by: David Milburn Signed-off-by: Christoph Hellwig --- drivers/nvme/target/core.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 48f5123d875b..6392bcd30bd7 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -134,15 +134,10 @@ static void nvmet_async_events_process(struct nvmet_ctrl *ctrl, u16 status) struct nvmet_async_event *aen; struct nvmet_req *req; - while (1) { - mutex_lock(&ctrl->lock); - aen = list_first_entry_or_null(&ctrl->async_events, - struct nvmet_async_event, entry); - if (!aen || !ctrl->nr_async_event_cmds) { - mutex_unlock(&ctrl->lock); - break; - } - + mutex_lock(&ctrl->lock); + while (ctrl->nr_async_event_cmds && !list_empty(&ctrl->async_events)) { + aen = list_first_entry(&ctrl->async_events, + struct nvmet_async_event, entry); req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; if (status == 0) nvmet_set_result(req, nvmet_async_event_result(aen)); @@ -153,7 +148,9 @@ static void nvmet_async_events_process(struct nvmet_ctrl *ctrl, u16 status) mutex_unlock(&ctrl->lock); trace_nvmet_async_event(ctrl, req->cqe->result.u32); nvmet_req_complete(req, status); + mutex_lock(&ctrl->lock); } + mutex_unlock(&ctrl->lock); } static void nvmet_async_events_free(struct nvmet_ctrl *ctrl) From 3b2a1ebceba3e03b17ef0970bb7757a3a64cdc8b Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Wed, 20 May 2020 19:22:53 -0700 Subject: [PATCH 0861/1043] nvme: set dma alignment to qword The default dma alignment mask is 511, which is much larger than any nvme controller requires. NVMe controllers accept qword aligned DMA addresses, so set the request_queue constraints to that. This can help avoid bounce buffers on user passthrough commands. Signed-off-by: Keith Busch Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index a3a4dbc59af1..569671e264b5 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2322,6 +2322,7 @@ static void nvme_set_queue_limits(struct nvme_ctrl *ctrl, blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX)); } blk_queue_virt_boundary(q, ctrl->page_size - 1); + blk_queue_dma_alignment(q, 7); if (ctrl->vwc & NVME_CTRL_VWC_PRESENT) vwc = true; blk_queue_write_cache(q, vwc, vwc); From fcdd14b86f6b891b5e894bf1dbeaf02cc79bdbce Mon Sep 17 00:00:00 2001 From: James Smart Date: Wed, 20 May 2020 11:59:27 -0700 Subject: [PATCH 0862/1043] lpfc: Fix pointer checks and comments in LS receive refactoring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Additional testing encountered null pointers that weren't fully qualified in lpfc_nvmet_xmt_ls_abort_cmp() and lpfc_nvmet_unsol_issue_abort(). The same error was detected and reported by static checker reporting: drivers/scsi/lpfc/lpfc_sli.c:2905 lpfc_nvme_unsol_ls_handler() error: we previously assumed 'phba->targetport' could be null (see line 2837) Fix by making phba->nvmet_support and phba->targetport validity checks in lpfc_nvmet_xmt_ls_abort_cmp() and lpfc_nvmet_unsol_issue_abort(). Fixes: 3a8070c567aaa (“lpfc: Refactor NVME LS receive handling”) Reported-by: Dan Carpenter Signed-off-by: Paul Ely Signed-off-by: James Smart Reviewed-by: Dan Carpenter Signed-off-by: Christoph Hellwig --- drivers/scsi/lpfc/lpfc_nvmet.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index 1c6bbbba70b5..bccf9da302ee 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -3207,8 +3207,10 @@ lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, ctxp = cmdwqe->context2; result = wcqe->parameter; - tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; - atomic_inc(&tgtp->xmt_ls_abort_cmpl); + if (phba->nvmet_support) { + tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; + atomic_inc(&tgtp->xmt_ls_abort_cmpl); + } lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, "6083 Abort cmpl: ctx x%px WCQE:%08x %08x %08x %08x\n", @@ -3244,7 +3246,7 @@ lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba, struct lpfc_async_xchg_ctx *ctxp, uint32_t sid, uint16_t xri) { - struct lpfc_nvmet_tgtport *tgtp; + struct lpfc_nvmet_tgtport *tgtp = NULL; struct lpfc_iocbq *abts_wqeq; union lpfc_wqe128 *wqe_abts; struct lpfc_nodelist *ndlp; @@ -3253,13 +3255,15 @@ lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba, "6067 ABTS: sid %x xri x%x/x%x\n", sid, xri, ctxp->wqeq->sli4_xritag); - tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; + if (phba->nvmet_support && phba->targetport) + tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; ndlp = lpfc_findnode_did(phba->pport, sid); if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) || ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) && (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) { - atomic_inc(&tgtp->xmt_abort_rsp_error); + if (tgtp) + atomic_inc(&tgtp->xmt_abort_rsp_error); lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, "6134 Drop ABTS - wrong NDLP state x%x.\n", (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE); @@ -3538,7 +3542,7 @@ lpfc_nvme_unsol_ls_issue_abort(struct lpfc_hba *phba, struct lpfc_async_xchg_ctx *ctxp, uint32_t sid, uint16_t xri) { - struct lpfc_nvmet_tgtport *tgtp; + struct lpfc_nvmet_tgtport *tgtp = NULL; struct lpfc_iocbq *abts_wqeq; unsigned long flags; int rc; @@ -3555,7 +3559,9 @@ lpfc_nvme_unsol_ls_issue_abort(struct lpfc_hba *phba, ctxp->state = LPFC_NVME_STE_LS_ABORT; } - tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; + if (phba->nvmet_support && phba->targetport) + tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private; + if (!ctxp->wqeq) { /* Issue ABTS for this WQE based on iotag */ ctxp->wqeq = lpfc_sli_get_iocbq(phba); @@ -3582,11 +3588,13 @@ lpfc_nvme_unsol_ls_issue_abort(struct lpfc_hba *phba, rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq); spin_unlock_irqrestore(&phba->hbalock, flags); if (rc == WQE_SUCCESS) { - atomic_inc(&tgtp->xmt_abort_unsol); + if (tgtp) + atomic_inc(&tgtp->xmt_abort_unsol); return 0; } out: - atomic_inc(&tgtp->xmt_abort_rsp_error); + if (tgtp) + atomic_inc(&tgtp->xmt_abort_rsp_error); abts_wqeq->context2 = NULL; abts_wqeq->context3 = NULL; lpfc_sli_release_iocbq(phba, abts_wqeq); From 4e57e0b9f343fd14497ab04b2bc08c1784830b9d Mon Sep 17 00:00:00 2001 From: James Smart Date: Wed, 20 May 2020 11:59:28 -0700 Subject: [PATCH 0863/1043] lpfc: fix axchg pointer reference after free and double frees The axchg structure is a structure allocated early in the lpfc_nvme_unsol_ls_handler() to represent the newly received exchange. Upon error, the out_fail path in the routine unconditionally frees the pointer, yet subsequently passes the pointer to the abort routine. Additionally, the abort routine, lpfc_nvme_unsol_ls_issue_abort(), also has a failure path that will attempt to delete the pointer on error. Fix these errors by: - Removing the unconditional free so that it stays valid if passed to the abort routine. - Revise the abort routine to not free the pointer. Instead, return a success/failure status. Note: if success, the later completion of the abort frees the structure. - Back in the unsol_ls_handler() error path, if the abort routine was skipped (thus no possible reference) or the abort routine returned error, free the pointer. Fixes: 3a8070c567aa ("lpfc: Refactor NVME LS receive handling") Reported-by: Dan Carpenter Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Dan Carpenter Signed-off-by: Christoph Hellwig --- drivers/scsi/lpfc/lpfc_nvmet.c | 3 +-- drivers/scsi/lpfc/lpfc_sli.c | 10 ++++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/scsi/lpfc/lpfc_nvmet.c b/drivers/scsi/lpfc/lpfc_nvmet.c index bccf9da302ee..32eb5e873e9b 100644 --- a/drivers/scsi/lpfc/lpfc_nvmet.c +++ b/drivers/scsi/lpfc/lpfc_nvmet.c @@ -3598,10 +3598,9 @@ out: abts_wqeq->context2 = NULL; abts_wqeq->context3 = NULL; lpfc_sli_release_iocbq(phba, abts_wqeq); - kfree(ctxp); lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS, "6056 Failed to Issue ABTS. Status x%x\n", rc); - return 0; + return 1; } /** diff --git a/drivers/scsi/lpfc/lpfc_sli.c b/drivers/scsi/lpfc/lpfc_sli.c index 1aaf40081e21..9e21c4f3b009 100644 --- a/drivers/scsi/lpfc/lpfc_sli.c +++ b/drivers/scsi/lpfc/lpfc_sli.c @@ -2813,7 +2813,7 @@ lpfc_nvme_unsol_ls_handler(struct lpfc_hba *phba, struct lpfc_iocbq *piocb) struct lpfc_async_xchg_ctx *axchg = NULL; char *failwhy = NULL; uint32_t oxid, sid, did, fctl, size; - int ret; + int ret = 1; d_buf = piocb->context2; @@ -2897,14 +2897,16 @@ lpfc_nvme_unsol_ls_handler(struct lpfc_hba *phba, struct lpfc_iocbq *piocb) (phba->nvmet_support) ? "T" : "I", ret); out_fail: - kfree(axchg); /* recycle receive buffer */ lpfc_in_buf_free(phba, &nvmebuf->dbuf); /* If start of new exchange, abort it */ - if (fctl & FC_FC_FIRST_SEQ && !(fctl & FC_FC_EX_CTX)) - lpfc_nvme_unsol_ls_issue_abort(phba, axchg, sid, oxid); + if (axchg && (fctl & FC_FC_FIRST_SEQ && !(fctl & FC_FC_EX_CTX))) + ret = lpfc_nvme_unsol_ls_issue_abort(phba, axchg, sid, oxid); + + if (ret) + kfree(axchg); } /** From 6b6e89636f51581895922780c3c4fd51bb9e1483 Mon Sep 17 00:00:00 2001 From: James Smart Date: Wed, 20 May 2020 11:59:29 -0700 Subject: [PATCH 0864/1043] lpfc: Fix return value in __lpfc_nvme_ls_abort A static checker reported the following issue: drivers/scsi/lpfc/lpfc_nvmet.c:1366 lpfc_nvmet_ls_abort() warn: 'ret' can be either negative or positive The comment indicates a non-zero value indicates error in the form of -Exxx, but the code is returning "1". Fix the code to return -EINVAL to be compliant to comment. Fixes: e96a22b0b7c2 ("lpfc: Refactor Send LS Abort support") Reported-by: Dan Carpenter Signed-off-by: Dick Kennedy Signed-off-by: James Smart Reviewed-by: Dan Carpenter Signed-off-by: Christoph Hellwig --- drivers/scsi/lpfc/lpfc_nvme.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/scsi/lpfc/lpfc_nvme.c b/drivers/scsi/lpfc/lpfc_nvme.c index 21bbccf0dc31..b46ba70f78da 100644 --- a/drivers/scsi/lpfc/lpfc_nvme.c +++ b/drivers/scsi/lpfc/lpfc_nvme.c @@ -895,7 +895,7 @@ __lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, "6213 NVMEx LS REQ Abort: Unable to locate req x%p\n", pnvme_lsreq); - return 1; + return -EINVAL; } static int From e7cda1ee94f464e02e356319aec7d83ccba8cab4 Mon Sep 17 00:00:00 2001 From: Chengguang Xu Date: Tue, 26 May 2020 17:03:43 +0800 Subject: [PATCH 0865/1043] erofs: code cleanup by removing ifdef macro surrounding Define erofs_listxattr and erofs_xattr_handlers to NULL when CONFIG_EROFS_FS_XATTR is not enabled, then we can remove many ugly ifdef macros in the code. Signed-off-by: Chengguang Xu Reviewed-by: Gao Xiang Reviewed-by: Chao Yu Link: https://lore.kernel.org/r/20200526090343.22794-1-cgxu519@mykernel.net Signed-off-by: Gao Xiang --- fs/erofs/inode.c | 6 ------ fs/erofs/namei.c | 2 -- fs/erofs/super.c | 4 +--- fs/erofs/xattr.h | 7 ++----- 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c index 3350ab65d892..7dd4bbe9674f 100644 --- a/fs/erofs/inode.c +++ b/fs/erofs/inode.c @@ -311,27 +311,21 @@ int erofs_getattr(const struct path *path, struct kstat *stat, const struct inode_operations erofs_generic_iops = { .getattr = erofs_getattr, -#ifdef CONFIG_EROFS_FS_XATTR .listxattr = erofs_listxattr, -#endif .get_acl = erofs_get_acl, }; const struct inode_operations erofs_symlink_iops = { .get_link = page_get_link, .getattr = erofs_getattr, -#ifdef CONFIG_EROFS_FS_XATTR .listxattr = erofs_listxattr, -#endif .get_acl = erofs_get_acl, }; const struct inode_operations erofs_fast_symlink_iops = { .get_link = simple_get_link, .getattr = erofs_getattr, -#ifdef CONFIG_EROFS_FS_XATTR .listxattr = erofs_listxattr, -#endif .get_acl = erofs_get_acl, }; diff --git a/fs/erofs/namei.c b/fs/erofs/namei.c index 3abbecbf73de..52f201e03c62 100644 --- a/fs/erofs/namei.c +++ b/fs/erofs/namei.c @@ -244,9 +244,7 @@ static struct dentry *erofs_lookup(struct inode *dir, const struct inode_operations erofs_dir_iops = { .lookup = erofs_lookup, .getattr = erofs_getattr, -#ifdef CONFIG_EROFS_FS_XATTR .listxattr = erofs_listxattr, -#endif .get_acl = erofs_get_acl, }; diff --git a/fs/erofs/super.c b/fs/erofs/super.c index b514c67e5fc2..8e46d204a0c2 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -408,10 +408,8 @@ static int erofs_fill_super(struct super_block *sb, void *data, int silent) sb->s_time_gran = 1; sb->s_op = &erofs_sops; - -#ifdef CONFIG_EROFS_FS_XATTR sb->s_xattr = erofs_xattr_handlers; -#endif + /* set erofs default mount options */ erofs_default_options(sbi); diff --git a/fs/erofs/xattr.h b/fs/erofs/xattr.h index 50966f1c676e..e4e5093f012c 100644 --- a/fs/erofs/xattr.h +++ b/fs/erofs/xattr.h @@ -76,11 +76,8 @@ static inline int erofs_getxattr(struct inode *inode, int index, return -EOPNOTSUPP; } -static inline ssize_t erofs_listxattr(struct dentry *dentry, - char *buffer, size_t buffer_size) -{ - return -EOPNOTSUPP; -} +#define erofs_listxattr (NULL) +#define erofs_xattr_handlers (NULL) #endif /* !CONFIG_EROFS_FS_XATTR */ #ifdef CONFIG_EROFS_FS_POSIX_ACL From 2038e0416518b30bb40857fbafa3733a6bae93ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro=20Fern=C3=A1ndez=20Rojas?= Date: Tue, 26 May 2020 13:03:24 +0200 Subject: [PATCH 0866/1043] MIPS: BCM63xx: fix 6328 boot selection bit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MISC_STRAP_BUS_BOOT_SEL_SHIFT is 18 according to Broadcom's GPL source code. Signed-off-by: Álvaro Fernández Rojas Acked-by: Florian Fainelli Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/mach-bcm63xx/bcm63xx_regs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/mips/include/asm/mach-bcm63xx/bcm63xx_regs.h b/arch/mips/include/asm/mach-bcm63xx/bcm63xx_regs.h index bc3444cd4ef2..9ceb5e72889f 100644 --- a/arch/mips/include/asm/mach-bcm63xx/bcm63xx_regs.h +++ b/arch/mips/include/asm/mach-bcm63xx/bcm63xx_regs.h @@ -1367,8 +1367,8 @@ #define MISC_STRAPBUS_6328_REG 0x240 #define STRAPBUS_6328_FCVO_SHIFT 7 #define STRAPBUS_6328_FCVO_MASK (0x1f << STRAPBUS_6328_FCVO_SHIFT) -#define STRAPBUS_6328_BOOT_SEL_SERIAL (1 << 28) -#define STRAPBUS_6328_BOOT_SEL_NAND (0 << 28) +#define STRAPBUS_6328_BOOT_SEL_SERIAL (1 << 18) +#define STRAPBUS_6328_BOOT_SEL_NAND (0 << 18) /************************************************************************* * _REG relative to RSET_PCIE From b6caa1d8c80cb71b6162cb1f1ec13aa655026c9f Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Tue, 26 May 2020 17:21:12 +0800 Subject: [PATCH 0867/1043] PCI: Don't disable decoding when mmio_always_on is set Don't disable MEM/IO decoding when a device have both non_compliant_bars and mmio_always_on. That would allow us quirk devices with junk in BARs but can't disable their decoding. Signed-off-by: Jiaxun Yang Acked-by: Bjorn Helgaas Signed-off-by: Thomas Bogendoerfer --- drivers/pci/probe.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 77b8a145c39b..d9c2c3301a8a 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -1822,7 +1822,7 @@ int pci_setup_device(struct pci_dev *dev) /* Device class may be changed after fixup */ class = dev->class >> 8; - if (dev->non_compliant_bars) { + if (dev->non_compliant_bars && !dev->mmio_always_on) { pci_read_config_word(dev, PCI_COMMAND, &cmd); if (cmd & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY)) { pci_info(dev, "device has non-compliant BARs; disabling IO/MEM decoding\n"); From 1f58cca5cf2ba27ab448cc95df48070e069d5451 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Tue, 26 May 2020 17:21:13 +0800 Subject: [PATCH 0868/1043] PCI: Add Loongson PCI Controller support This controller can be found on Loongson-2K SoC, Loongson-3 systems with RS780E/LS7A PCH. The RS780E part of code was previously located at arch/mips/pci/ops-loongson3.c and now it can use generic PCI driver implementation. Signed-off-by: Jiaxun Yang Reviewed-by: Rob Herring Acked-by: Lorenzo Pieralisi Signed-off-by: Thomas Bogendoerfer --- drivers/pci/controller/Kconfig | 10 ++ drivers/pci/controller/Makefile | 1 + drivers/pci/controller/pci-loongson.c | 247 ++++++++++++++++++++++++++ 3 files changed, 258 insertions(+) create mode 100644 drivers/pci/controller/pci-loongson.c diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig index 91bfdb784829..ae36edb1d7db 100644 --- a/drivers/pci/controller/Kconfig +++ b/drivers/pci/controller/Kconfig @@ -258,6 +258,16 @@ config PCI_HYPERV_INTERFACE The Hyper-V PCI Interface is a helper driver allows other drivers to have a common interface with the Hyper-V PCI frontend driver. +config PCI_LOONGSON + bool "LOONGSON PCI Controller" + depends on MACH_LOONGSON64 || COMPILE_TEST + depends on OF + depends on PCI_QUIRKS + default MACH_LOONGSON64 + help + Say Y here if you want to enable PCI controller support on + Loongson systems. + source "drivers/pci/controller/dwc/Kconfig" source "drivers/pci/controller/mobiveil/Kconfig" source "drivers/pci/controller/cadence/Kconfig" diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile index 158c59771824..fbac4b0190a0 100644 --- a/drivers/pci/controller/Makefile +++ b/drivers/pci/controller/Makefile @@ -28,6 +28,7 @@ obj-$(CONFIG_PCIE_MEDIATEK) += pcie-mediatek.o obj-$(CONFIG_PCIE_TANGO_SMP8759) += pcie-tango.o obj-$(CONFIG_VMD) += vmd.o obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb.o +obj-$(CONFIG_PCI_LOONGSON) += pci-loongson.o # pcie-hisi.o quirks are needed even without CONFIG_PCIE_DW obj-y += dwc/ obj-y += mobiveil/ diff --git a/drivers/pci/controller/pci-loongson.c b/drivers/pci/controller/pci-loongson.c new file mode 100644 index 000000000000..459009c8a4a0 --- /dev/null +++ b/drivers/pci/controller/pci-loongson.c @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Loongson PCI Host Controller Driver + * + * Copyright (C) 2020 Jiaxun Yang + */ + +#include +#include +#include +#include + +#include "../pci.h" + +/* Device IDs */ +#define DEV_PCIE_PORT_0 0x7a09 +#define DEV_PCIE_PORT_1 0x7a19 +#define DEV_PCIE_PORT_2 0x7a29 + +#define DEV_LS2K_APB 0x7a02 +#define DEV_LS7A_CONF 0x7a10 +#define DEV_LS7A_LPC 0x7a0c + +#define FLAG_CFG0 BIT(0) +#define FLAG_CFG1 BIT(1) +#define FLAG_DEV_FIX BIT(2) + +struct loongson_pci { + void __iomem *cfg0_base; + void __iomem *cfg1_base; + struct platform_device *pdev; + u32 flags; +}; + +/* Fixup wrong class code in PCIe bridges */ +static void bridge_class_quirk(struct pci_dev *dev) +{ + dev->class = PCI_CLASS_BRIDGE_PCI << 8; +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LOONGSON, + DEV_PCIE_PORT_0, bridge_class_quirk); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LOONGSON, + DEV_PCIE_PORT_1, bridge_class_quirk); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LOONGSON, + DEV_PCIE_PORT_2, bridge_class_quirk); + +static void system_bus_quirk(struct pci_dev *pdev) +{ + /* + * The address space consumed by these devices is outside the + * resources of the host bridge. + */ + pdev->mmio_always_on = 1; + pdev->non_compliant_bars = 1; +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS2K_APB, system_bus_quirk); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_CONF, system_bus_quirk); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_LPC, system_bus_quirk); + +static void loongson_mrrs_quirk(struct pci_dev *dev) +{ + struct pci_bus *bus = dev->bus; + struct pci_dev *bridge; + static const struct pci_device_id bridge_devids[] = { + { PCI_VDEVICE(LOONGSON, DEV_PCIE_PORT_0) }, + { PCI_VDEVICE(LOONGSON, DEV_PCIE_PORT_1) }, + { PCI_VDEVICE(LOONGSON, DEV_PCIE_PORT_2) }, + { 0, }, + }; + + /* look for the matching bridge */ + while (!pci_is_root_bus(bus)) { + bridge = bus->self; + bus = bus->parent; + /* + * Some Loongson PCIe ports have a h/w limitation of + * 256 bytes maximum read request size. They can't handle + * anything larger than this. So force this limit on + * any devices attached under these ports. + */ + if (pci_match_id(bridge_devids, bridge)) { + if (pcie_get_readrq(dev) > 256) { + pci_info(dev, "limiting MRRS to 256\n"); + pcie_set_readrq(dev, 256); + } + break; + } + } +} +DECLARE_PCI_FIXUP_ENABLE(PCI_ANY_ID, PCI_ANY_ID, loongson_mrrs_quirk); + +static void __iomem *cfg1_map(struct loongson_pci *priv, int bus, + unsigned int devfn, int where) +{ + unsigned long addroff = 0x0; + + if (bus != 0) + addroff |= BIT(28); /* Type 1 Access */ + addroff |= (where & 0xff) | ((where & 0xf00) << 16); + addroff |= (bus << 16) | (devfn << 8); + return priv->cfg1_base + addroff; +} + +static void __iomem *cfg0_map(struct loongson_pci *priv, int bus, + unsigned int devfn, int where) +{ + unsigned long addroff = 0x0; + + if (bus != 0) + addroff |= BIT(24); /* Type 1 Access */ + addroff |= (bus << 16) | (devfn << 8) | where; + return priv->cfg0_base + addroff; +} + +static void __iomem *pci_loongson_map_bus(struct pci_bus *bus, unsigned int devfn, + int where) +{ + unsigned char busnum = bus->number; + struct pci_host_bridge *bridge = pci_find_host_bridge(bus); + struct loongson_pci *priv = pci_host_bridge_priv(bridge); + + /* + * Do not read more than one device on the bus other than + * the host bus. For our hardware the root bus is always bus 0. + */ + if (priv->flags & FLAG_DEV_FIX && busnum != 0 && + PCI_SLOT(devfn) > 0) + return NULL; + + /* CFG0 can only access standard space */ + if (where < PCI_CFG_SPACE_SIZE && priv->cfg0_base) + return cfg0_map(priv, busnum, devfn, where); + + /* CFG1 can access extended space */ + if (where < PCI_CFG_SPACE_EXP_SIZE && priv->cfg1_base) + return cfg1_map(priv, busnum, devfn, where); + + return NULL; +} + +static int loongson_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +{ + int irq; + u8 val; + + irq = of_irq_parse_and_map_pci(dev, slot, pin); + if (irq > 0) + return irq; + + /* Care i8259 legacy systems */ + pci_read_config_byte(dev, PCI_INTERRUPT_LINE, &val); + /* i8259 only have 15 IRQs */ + if (val > 15) + return 0; + + return val; +} + +/* H/w only accept 32-bit PCI operations */ +static struct pci_ops loongson_pci_ops = { + .map_bus = pci_loongson_map_bus, + .read = pci_generic_config_read32, + .write = pci_generic_config_write32, +}; + +static const struct of_device_id loongson_pci_of_match[] = { + { .compatible = "loongson,ls2k-pci", + .data = (void *)(FLAG_CFG0 | FLAG_CFG1 | FLAG_DEV_FIX), }, + { .compatible = "loongson,ls7a-pci", + .data = (void *)(FLAG_CFG0 | FLAG_CFG1 | FLAG_DEV_FIX), }, + { .compatible = "loongson,rs780e-pci", + .data = (void *)(FLAG_CFG0), }, + {} +}; + +static int loongson_pci_probe(struct platform_device *pdev) +{ + struct loongson_pci *priv; + struct device *dev = &pdev->dev; + struct device_node *node = dev->of_node; + struct pci_host_bridge *bridge; + struct resource *regs; + int err; + + if (!node) + return -ENODEV; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*priv)); + if (!bridge) + return -ENODEV; + + priv = pci_host_bridge_priv(bridge); + priv->pdev = pdev; + priv->flags = (unsigned long)of_device_get_match_data(dev); + + regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!regs) { + dev_err(dev, "missing mem resources for cfg0\n"); + return -EINVAL; + } + + priv->cfg0_base = devm_pci_remap_cfg_resource(dev, regs); + if (IS_ERR(priv->cfg0_base)) + return PTR_ERR(priv->cfg0_base); + + /* CFG1 is optional */ + if (priv->flags & FLAG_CFG1) { + regs = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (!regs) + dev_info(dev, "missing mem resource for cfg1\n"); + else { + priv->cfg1_base = devm_pci_remap_cfg_resource(dev, regs); + if (IS_ERR(priv->cfg1_base)) + priv->cfg1_base = NULL; + } + } + + err = pci_parse_request_of_pci_ranges(dev, &bridge->windows, + &bridge->dma_ranges, NULL); + if (err) { + dev_err(dev, "failed to get bridge resources\n"); + return err; + } + + bridge->dev.parent = dev; + bridge->sysdata = priv; + bridge->ops = &loongson_pci_ops; + bridge->map_irq = loongson_map_irq; + + err = pci_host_probe(bridge); + if (err) + return err; + + return 0; +} + +static struct platform_driver loongson_pci_driver = { + .driver = { + .name = "loongson-pci", + .of_match_table = loongson_pci_of_match, + }, + .probe = loongson_pci_probe, +}; +builtin_platform_driver(loongson_pci_driver); From 83e757ecfd5d73a33d30d546b2463851854c8628 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Tue, 26 May 2020 17:21:14 +0800 Subject: [PATCH 0869/1043] dt-bindings: Document Loongson PCI Host Controller PCI host controller found on Loongson PCHs and SoCs. Signed-off-by: Jiaxun Yang Reviewed-by: Rob Herring Signed-off-by: Thomas Bogendoerfer --- .../devicetree/bindings/pci/loongson.yaml | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 Documentation/devicetree/bindings/pci/loongson.yaml diff --git a/Documentation/devicetree/bindings/pci/loongson.yaml b/Documentation/devicetree/bindings/pci/loongson.yaml new file mode 100644 index 000000000000..30e7cf1aeb87 --- /dev/null +++ b/Documentation/devicetree/bindings/pci/loongson.yaml @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/pci/loongson.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Loongson PCI Host Controller + +maintainers: + - Jiaxun Yang + +description: |+ + PCI host controller found on Loongson PCHs and SoCs. + +allOf: + - $ref: /schemas/pci/pci-bus.yaml# + +properties: + compatible: + oneOf: + - const: loongson,ls2k-pci + - const: loongson,ls7a-pci + - const: loongson,rs780e-pci + + reg: + minItems: 1 + maxItems: 2 + items: + - description: CFG0 standard config space register + - description: CFG1 extended config space register + + ranges: + minItems: 1 + maxItems: 3 + + +required: + - compatible + - reg + - ranges + +examples: + - | + + bus { + #address-cells = <2>; + #size-cells = <2>; + pcie@1a000000 { + compatible = "loongson,rs780e-pci"; + device_type = "pci"; + #address-cells = <3>; + #size-cells = <2>; + + // CPU_PHYSICAL(2) SIZE(2) + reg = <0x0 0x1a000000 0x0 0x2000000>; + + // BUS_ADDRESS(3) CPU_PHYSICAL(2) SIZE(2) + ranges = <0x01000000 0x0 0x00004000 0x0 0x00004000 0x0 0x00004000>, + <0x02000000 0x0 0x40000000 0x0 0x40000000 0x0 0x40000000>; + }; + }; +... From d8242e6a71bac37b6cde52c0add533615eef8c5e Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Tue, 26 May 2020 17:21:15 +0800 Subject: [PATCH 0870/1043] MIPS: DTS: Loongson64: Add PCI Controller Node Add PCI Host controller node for Loongson64 with RS780E PCH dts. Note that PCI interrupts are probed via legacy way, as different machine have different interrupt arrangement, we can't cover all of them in dt. Signed-off-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/loongson/rs780e-pch.dtsi | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/mips/boot/dts/loongson/rs780e-pch.dtsi b/arch/mips/boot/dts/loongson/rs780e-pch.dtsi index 8687c4f7370a..d0d5d60a8697 100644 --- a/arch/mips/boot/dts/loongson/rs780e-pch.dtsi +++ b/arch/mips/boot/dts/loongson/rs780e-pch.dtsi @@ -9,6 +9,18 @@ 0 0x40000000 0 0x40000000 0 0x40000000 0xfd 0xfe000000 0xfd 0xfe000000 0 0x2000000 /* PCI Config Space */>; + pci@1a000000 { + compatible = "loongson,rs780e-pci"; + device_type = "pci"; + #address-cells = <3>; + #size-cells = <2>; + + reg = <0 0x1a000000 0 0x02000000>; + + ranges = <0x01000000 0 0x00004000 0 0x18004000 0 0x00004000>, + <0x02000000 0 0x40000000 0 0x40000000 0 0x40000000>; + }; + isa { compatible = "isa"; #address-cells = <2>; From 6423e59a64e7b490499d9d1fb1c2323eea5f134c Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Tue, 26 May 2020 17:21:16 +0800 Subject: [PATCH 0871/1043] MIPS: Loongson64: Switch to generic PCI driver We can now enable generic PCI driver in Kconfig, and remove legacy PCI driver code. Radeon vbios quirk is moved to the platform folder to fit the new structure. Signed-off-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kconfig | 1 + arch/mips/loongson64/Makefile | 2 +- arch/mips/loongson64/vbios_quirk.c | 29 ++++++++ arch/mips/pci/Makefile | 1 - arch/mips/pci/fixup-loongson3.c | 71 ------------------ arch/mips/pci/ops-loongson3.c | 116 ----------------------------- 6 files changed, 31 insertions(+), 189 deletions(-) create mode 100644 arch/mips/loongson64/vbios_quirk.c delete mode 100644 arch/mips/pci/fixup-loongson3.c delete mode 100644 arch/mips/pci/ops-loongson3.c diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index c5fe30c78402..c0405999677a 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -464,6 +464,7 @@ config MACH_LOONGSON64 select IRQ_MIPS_CPU select NR_CPUS_DEFAULT_64 select USE_GENERIC_EARLY_PRINTK_8250 + select PCI_DRIVERS_GENERIC select SYS_HAS_CPU_LOONGSON64 select SYS_HAS_EARLY_PRINTK select SYS_SUPPORTS_SMP diff --git a/arch/mips/loongson64/Makefile b/arch/mips/loongson64/Makefile index 61f6add20530..39c06f52b08f 100644 --- a/arch/mips/loongson64/Makefile +++ b/arch/mips/loongson64/Makefile @@ -8,6 +8,6 @@ obj-$(CONFIG_MACH_LOONGSON64) += cop2-ex.o platform.o dma.o \ obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_RS780_HPET) += hpet.o -obj-$(CONFIG_PCI) += pci.o obj-$(CONFIG_SUSPEND) += pm.o +obj-$(CONFIG_PCI_QUIRKS) += vbios_quirk.o obj-$(CONFIG_CPU_LOONGSON3_CPUCFG_EMULATION) += cpucfg-emul.o diff --git a/arch/mips/loongson64/vbios_quirk.c b/arch/mips/loongson64/vbios_quirk.c new file mode 100644 index 000000000000..9a29e94d3db1 --- /dev/null +++ b/arch/mips/loongson64/vbios_quirk.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include +#include + +static void pci_fixup_radeon(struct pci_dev *pdev) +{ + struct resource *res = &pdev->resource[PCI_ROM_RESOURCE]; + + if (res->start) + return; + + if (!loongson_sysconf.vgabios_addr) + return; + + pci_disable_rom(pdev); + if (res->parent) + release_resource(res); + + res->start = virt_to_phys((void *) loongson_sysconf.vgabios_addr); + res->end = res->start + 256*1024 - 1; + res->flags = IORESOURCE_MEM | IORESOURCE_ROM_SHADOW | + IORESOURCE_PCI_FIXED; + + dev_info(&pdev->dev, "BAR %d: assigned %pR for Radeon ROM\n", + PCI_ROM_RESOURCE, res); +} +DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_ATI, 0x9615, + PCI_CLASS_DISPLAY_VGA, 8, pci_fixup_radeon); diff --git a/arch/mips/pci/Makefile b/arch/mips/pci/Makefile index f238d7b9e0b2..0f68d6849978 100644 --- a/arch/mips/pci/Makefile +++ b/arch/mips/pci/Makefile @@ -32,7 +32,6 @@ obj-$(CONFIG_ATH79) += fixup-ath79.o obj-$(CONFIG_MIPS_COBALT) += fixup-cobalt.o obj-$(CONFIG_LEMOTE_FULOONG2E) += fixup-fuloong2e.o ops-loongson2.o obj-$(CONFIG_LEMOTE_MACH2F) += fixup-lemote2f.o ops-loongson2.o -obj-$(CONFIG_MACH_LOONGSON64) += fixup-loongson3.o ops-loongson3.o obj-$(CONFIG_MIPS_MALTA) += fixup-malta.o pci-malta.o obj-$(CONFIG_SGI_IP27) += pci-ip27.o obj-$(CONFIG_SGI_IP32) += fixup-ip32.o ops-mace.o pci-ip32.o diff --git a/arch/mips/pci/fixup-loongson3.c b/arch/mips/pci/fixup-loongson3.c deleted file mode 100644 index 8a741c2c6685..000000000000 --- a/arch/mips/pci/fixup-loongson3.c +++ /dev/null @@ -1,71 +0,0 @@ -/* - * fixup-loongson3.c - * - * Copyright (C) 2012 Lemote, Inc. - * Author: Xiang Yu, xiangy@lemote.com - * Chen Huacai, chenhc@lemote.com - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include -#include - -static void print_fixup_info(const struct pci_dev *pdev) -{ - dev_info(&pdev->dev, "Device %x:%x, irq %d\n", - pdev->vendor, pdev->device, pdev->irq); -} - -int pcibios_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) -{ - print_fixup_info(dev); - return dev->irq; -} - -static void pci_fixup_radeon(struct pci_dev *pdev) -{ - struct resource *res = &pdev->resource[PCI_ROM_RESOURCE]; - - if (res->start) - return; - - if (!loongson_sysconf.vgabios_addr) - return; - - pci_disable_rom(pdev); - if (res->parent) - release_resource(res); - - res->start = virt_to_phys((void *) loongson_sysconf.vgabios_addr); - res->end = res->start + 256*1024 - 1; - res->flags = IORESOURCE_MEM | IORESOURCE_ROM_SHADOW | - IORESOURCE_PCI_FIXED; - - dev_info(&pdev->dev, "BAR %d: assigned %pR for Radeon ROM\n", - PCI_ROM_RESOURCE, res); -} - -DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_ATI, PCI_ANY_ID, - PCI_CLASS_DISPLAY_VGA, 8, pci_fixup_radeon); - -/* Do platform specific device initialization at pci_enable_device() time */ -int pcibios_plat_dev_init(struct pci_dev *dev) -{ - return 0; -} diff --git a/arch/mips/pci/ops-loongson3.c b/arch/mips/pci/ops-loongson3.c deleted file mode 100644 index 2f6ad36bdea6..000000000000 --- a/arch/mips/pci/ops-loongson3.c +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include - -#include - -#include - -#define PCI_ACCESS_READ 0 -#define PCI_ACCESS_WRITE 1 - -#define HT1LO_PCICFG_BASE 0x1a000000 -#define HT1LO_PCICFG_BASE_TP1 0x1b000000 - -static int loongson3_pci_config_access(unsigned char access_type, - struct pci_bus *bus, unsigned int devfn, - int where, u32 *data) -{ - unsigned char busnum = bus->number; - int function = PCI_FUNC(devfn); - int device = PCI_SLOT(devfn); - int reg = where & ~3; - void *addrp; - u64 addr; - - if (where < PCI_CFG_SPACE_SIZE) { /* standard config */ - addr = (busnum << 16) | (device << 11) | (function << 8) | reg; - if (busnum == 0) { - if (device > 31) - return PCIBIOS_DEVICE_NOT_FOUND; - addrp = (void *)TO_UNCAC(HT1LO_PCICFG_BASE | addr); - } else { - addrp = (void *)TO_UNCAC(HT1LO_PCICFG_BASE_TP1 | addr); - } - } else if (where < PCI_CFG_SPACE_EXP_SIZE) { /* extended config */ - struct pci_dev *rootdev; - - rootdev = pci_get_domain_bus_and_slot(0, 0, 0); - if (!rootdev) - return PCIBIOS_DEVICE_NOT_FOUND; - - addr = pci_resource_start(rootdev, 3); - if (!addr) - return PCIBIOS_DEVICE_NOT_FOUND; - - addr |= busnum << 20 | device << 15 | function << 12 | reg; - addrp = (void *)TO_UNCAC(addr); - } else { - return PCIBIOS_DEVICE_NOT_FOUND; - } - - if (access_type == PCI_ACCESS_WRITE) - writel(*data, addrp); - else { - *data = readl(addrp); - if (*data == 0xffffffff) { - *data = -1; - return PCIBIOS_DEVICE_NOT_FOUND; - } - } - return PCIBIOS_SUCCESSFUL; -} - -static int loongson3_pci_pcibios_read(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 *val) -{ - u32 data = 0; - int ret = loongson3_pci_config_access(PCI_ACCESS_READ, - bus, devfn, where, &data); - - if (ret != PCIBIOS_SUCCESSFUL) - return ret; - - if (size == 1) - *val = (data >> ((where & 3) << 3)) & 0xff; - else if (size == 2) - *val = (data >> ((where & 3) << 3)) & 0xffff; - else - *val = data; - - return PCIBIOS_SUCCESSFUL; -} - -static int loongson3_pci_pcibios_write(struct pci_bus *bus, unsigned int devfn, - int where, int size, u32 val) -{ - u32 data = 0; - int ret; - - if (size == 4) - data = val; - else { - ret = loongson3_pci_config_access(PCI_ACCESS_READ, - bus, devfn, where, &data); - if (ret != PCIBIOS_SUCCESSFUL) - return ret; - - if (size == 1) - data = (data & ~(0xff << ((where & 3) << 3))) | - (val << ((where & 3) << 3)); - else if (size == 2) - data = (data & ~(0xffff << ((where & 3) << 3))) | - (val << ((where & 3) << 3)); - } - - ret = loongson3_pci_config_access(PCI_ACCESS_WRITE, - bus, devfn, where, &data); - - return ret; -} - -struct pci_ops loongson_pci_ops = { - .read = loongson3_pci_pcibios_read, - .write = loongson3_pci_pcibios_write -}; From 963287e48314d334d3c5b377dca8ea0dec4ac718 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Mon, 25 May 2020 19:41:53 +0200 Subject: [PATCH 0872/1043] MIPS: ingenic: DTS: Add memory info of GCW Zero Add memory info of the GCW Zero in its devicetree. The bootloader generally provides this information, but since it is fixed to 512 MiB, it doesn't hurt to have it in devicetree. It allows the kernel to boot without any parameter passed as argument. Signed-off-by: Paul Cercueil Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ingenic/gcw0.dts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/mips/boot/dts/ingenic/gcw0.dts b/arch/mips/boot/dts/ingenic/gcw0.dts index c2396b5ca175..8d22828787d8 100644 --- a/arch/mips/boot/dts/ingenic/gcw0.dts +++ b/arch/mips/boot/dts/ingenic/gcw0.dts @@ -19,6 +19,12 @@ serial3 = &uart3; }; + memory: memory { + device_type = "memory"; + reg = <0x0 0x10000000>, + <0x30000000 0x10000000>; + }; + chosen { stdout-path = "serial2:57600n8"; }; From d653d1ffba8ecd472a78a5f0210a04672060b27b Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Mon, 25 May 2020 19:41:54 +0200 Subject: [PATCH 0873/1043] MIPS: ingenic: Add support for GCW Zero prototype Add support for the GCW Zero prototype. The only (?) difference is that it only has 256 MiB of RAM, compared to the 512 MiB of RAM of the retail device. Signed-off-by: Paul Cercueil Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/ingenic/gcw0_proto.dts | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 arch/mips/boot/dts/ingenic/gcw0_proto.dts diff --git a/arch/mips/boot/dts/ingenic/gcw0_proto.dts b/arch/mips/boot/dts/ingenic/gcw0_proto.dts new file mode 100644 index 000000000000..02df22f8ae0f --- /dev/null +++ b/arch/mips/boot/dts/ingenic/gcw0_proto.dts @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 +/dts-v1/; + +#include "gcw0.dts" + +/ { + model = "GCW Zero Prototype"; +}; + +&memory { + /* Prototype has only 256 MiB of RAM */ + reg = <0x0 0x10000000>; +}; From 62249209a77222e1cfb38dde310bd5a877524f7d Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Mon, 25 May 2020 19:41:55 +0200 Subject: [PATCH 0874/1043] MIPS: ingenic: Default to a generic board Having a generic board option makes it possible to create a kernel that will run on various Ingenic SoCs, as long as the right devicetree is provided. Signed-off-by: Paul Cercueil Signed-off-by: Thomas Bogendoerfer --- arch/mips/jz4740/Kconfig | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/mips/jz4740/Kconfig b/arch/mips/jz4740/Kconfig index 412d2faa3cdf..9c2e8c15bb97 100644 --- a/arch/mips/jz4740/Kconfig +++ b/arch/mips/jz4740/Kconfig @@ -2,7 +2,14 @@ choice prompt "Machine type" depends on MACH_INGENIC - default JZ4740_QI_LB60 + default INGENIC_GENERIC_BOARD + +config INGENIC_GENERIC_BOARD + bool "Generic board" + select MACH_JZ4740 + select MACH_JZ4770 + select MACH_JZ4780 + select MACH_X1000 config JZ4740_QI_LB60 bool "Qi Hardware Ben NanoNote" From 4dd7683ea1d66975fb258d02f2b74a7e9a32b131 Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Wed, 27 May 2020 10:25:17 +0800 Subject: [PATCH 0875/1043] MIPS: Do not flush tlb page when updating PTE entry It is not necessary to flush tlb page on all CPUs if suitable PTE entry exists already during page fault handling, just updating TLB is fine. Here redefine flush_tlb_fix_spurious_fault as empty on MIPS system. Signed-off-by: Bibo Mao Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/pgtable.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h index 9b01d2d78753..f8f48fc361d5 100644 --- a/arch/mips/include/asm/pgtable.h +++ b/arch/mips/include/asm/pgtable.h @@ -478,6 +478,11 @@ static inline pgprot_t pgprot_writecombine(pgprot_t _prot) return __pgprot(prot); } +static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, + unsigned long address) +{ +} + /* * Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. From 7df676974359f927056b882e10a5b24d2033169b Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Wed, 27 May 2020 10:25:18 +0800 Subject: [PATCH 0876/1043] mm/memory.c: Update local TLB if PTE entry exists If two threads concurrently fault at the same page, the thread that won the race updates the PTE and its local TLB. For now, the other thread gives up, simply does nothing, and continues. It could happen that this second thread triggers another fault, whereby it only updates its local TLB while handling the fault. Instead of triggering another fault, let's directly update the local TLB of the second thread. Function update_mmu_tlb is used here to update local TLB on the second thread, and it is defined as empty on other arches. Signed-off-by: Bibo Mao Acked-by: Andrew Morton Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/pgtable.h | 23 +++++++++++++++++++++++ include/asm-generic/pgtable.h | 17 +++++++++++++++++ mm/memory.c | 27 +++++++++++++++++++-------- 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h index f8f48fc361d5..6f40612b3152 100644 --- a/arch/mips/include/asm/pgtable.h +++ b/arch/mips/include/asm/pgtable.h @@ -483,6 +483,26 @@ static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, { } +#define __HAVE_ARCH_PTE_SAME +static inline int pte_same(pte_t pte_a, pte_t pte_b) +{ + return pte_val(pte_a) == pte_val(pte_b); +} + +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +static inline int ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep, + pte_t entry, int dirty) +{ + if (!pte_same(*ptep, entry)) + set_pte_at(vma->vm_mm, address, ptep, entry); + /* + * update_mmu_cache will unconditionally execute, handling both + * the case that the PTE changed and the spurious fault case. + */ + return true; +} + /* * Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. @@ -526,6 +546,9 @@ static inline void update_mmu_cache(struct vm_area_struct *vma, __update_tlb(vma, address, pte); } +#define __HAVE_ARCH_UPDATE_MMU_TLB +#define update_mmu_tlb update_mmu_cache + static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) { diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h index 329b8c8ca703..fa5c73ff8226 100644 --- a/include/asm-generic/pgtable.h +++ b/include/asm-generic/pgtable.h @@ -188,6 +188,23 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, } #endif + +/* + * If two threads concurrently fault at the same page, the thread that + * won the race updates the PTE and its local TLB/Cache. The other thread + * gives up, simply does nothing, and continues; on architectures where + * software can update TLB, local TLB can be updated here to avoid next page + * fault. This function updates TLB only, do nothing with cache or others. + * It is the difference with function update_mmu_cache. + */ +#ifndef __HAVE_ARCH_UPDATE_MMU_TLB +static inline void update_mmu_tlb(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) +{ +} +#define __HAVE_ARCH_UPDATE_MMU_TLB +#endif + /* * Some architectures may be able to avoid expensive synchronization * primitives when modifications are made to PTE's which are already diff --git a/mm/memory.c b/mm/memory.c index f703fe8c8346..8bb31c4d94d0 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2436,10 +2436,9 @@ static inline bool cow_user_page(struct page *dst, struct page *src, if (!likely(pte_same(*vmf->pte, vmf->orig_pte))) { /* * Other thread has already handled the fault - * and we don't need to do anything. If it's - * not the case, the fault will be triggered - * again on the same address. + * and update local tlb only */ + update_mmu_tlb(vma, addr, vmf->pte); ret = false; goto pte_unlock; } @@ -2463,7 +2462,8 @@ static inline bool cow_user_page(struct page *dst, struct page *src, vmf->pte = pte_offset_map_lock(mm, vmf->pmd, addr, &vmf->ptl); locked = true; if (!likely(pte_same(*vmf->pte, vmf->orig_pte))) { - /* The PTE changed under us. Retry page fault. */ + /* The PTE changed under us, update local tlb */ + update_mmu_tlb(vma, addr, vmf->pte); ret = false; goto pte_unlock; } @@ -2752,6 +2752,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) new_page = old_page; page_copied = 1; } else { + update_mmu_tlb(vma, vmf->address, vmf->pte); mem_cgroup_cancel_charge(new_page, memcg, false); } @@ -2812,6 +2813,7 @@ vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf) * pte_offset_map_lock. */ if (!pte_same(*vmf->pte, vmf->orig_pte)) { + update_mmu_tlb(vmf->vma, vmf->address, vmf->pte); pte_unmap_unlock(vmf->pte, vmf->ptl); return VM_FAULT_NOPAGE; } @@ -2936,6 +2938,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf) vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); if (!pte_same(*vmf->pte, vmf->orig_pte)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); unlock_page(vmf->page); pte_unmap_unlock(vmf->pte, vmf->ptl); put_page(vmf->page); @@ -3341,8 +3344,10 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) vma->vm_page_prot)); vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); - if (!pte_none(*vmf->pte)) + if (!pte_none(*vmf->pte)) { + update_mmu_tlb(vma, vmf->address, vmf->pte); goto unlock; + } ret = check_stable_address_space(vma->vm_mm); if (ret) goto unlock; @@ -3378,8 +3383,10 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address, &vmf->ptl); - if (!pte_none(*vmf->pte)) + if (!pte_none(*vmf->pte)) { + update_mmu_cache(vma, vmf->address, vmf->pte); goto release; + } ret = check_stable_address_space(vma->vm_mm); if (ret) @@ -3646,8 +3653,10 @@ vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg, } /* Re-check under ptl */ - if (unlikely(!pte_none(*vmf->pte))) + if (unlikely(!pte_none(*vmf->pte))) { + update_mmu_tlb(vma, vmf->address, vmf->pte); return VM_FAULT_NOPAGE; + } flush_icache_page(vma, page); entry = mk_pte(page, vma->vm_page_prot); @@ -4224,8 +4233,10 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd); spin_lock(vmf->ptl); entry = vmf->orig_pte; - if (unlikely(!pte_same(*vmf->pte, entry))) + if (unlikely(!pte_same(*vmf->pte, entry))) { + update_mmu_tlb(vmf->vma, vmf->address, vmf->pte); goto unlock; + } if (vmf->flags & FAULT_FLAG_WRITE) { if (!pte_write(entry)) return do_wp_page(vmf); From 44bf431b47b4298d8aaba7b02d0ad29fc609a03c Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Wed, 27 May 2020 10:25:19 +0800 Subject: [PATCH 0877/1043] mm/memory.c: Add memory read privilege on page fault handling Here add pte_sw_mkyoung function to make page readable on MIPS platform during page fault handling. This patch improves page fault latency about 10% on my MIPS machine with lmbench lat_pagefault case. It is noop function on other arches, there is no negative influence on those architectures. Signed-off-by: Bibo Mao Acked-by: Andrew Morton Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/pgtable.h | 2 ++ include/asm-generic/pgtable.h | 16 ++++++++++++++++ mm/memory.c | 3 +++ 3 files changed, 21 insertions(+) diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h index 6f40612b3152..d9772aff2f02 100644 --- a/arch/mips/include/asm/pgtable.h +++ b/arch/mips/include/asm/pgtable.h @@ -414,6 +414,8 @@ static inline pte_t pte_mkyoung(pte_t pte) return pte; } +#define pte_sw_mkyoung pte_mkyoung + #ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT static inline int pte_huge(pte_t pte) { return pte_val(pte) & _PAGE_HUGE; } diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h index fa5c73ff8226..b5278ec627db 100644 --- a/include/asm-generic/pgtable.h +++ b/include/asm-generic/pgtable.h @@ -244,6 +244,22 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addres } #endif +/* + * On some architectures hardware does not set page access bit when accessing + * memory page, it is responsibilty of software setting this bit. It brings + * out extra page fault penalty to track page access bit. For optimization page + * access bit can be set during all page fault flow on these arches. + * To be differentiate with macro pte_mkyoung, this macro is used on platforms + * where software maintains page access bit. + */ +#ifndef pte_sw_mkyoung +static inline pte_t pte_sw_mkyoung(pte_t pte) +{ + return pte; +} +#define pte_sw_mkyoung pte_sw_mkyoung +#endif + #ifndef pte_savedwrite #define pte_savedwrite pte_write #endif diff --git a/mm/memory.c b/mm/memory.c index 8bb31c4d94d0..c7c8960bdd1e 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2704,6 +2704,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) } flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte)); entry = mk_pte(new_page, vma->vm_page_prot); + entry = pte_sw_mkyoung(entry); entry = maybe_mkwrite(pte_mkdirty(entry), vma); /* * Clear the pte entry and flush it first, before updating the @@ -3378,6 +3379,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) __SetPageUptodate(page); entry = mk_pte(page, vma->vm_page_prot); + entry = pte_sw_mkyoung(entry); if (vma->vm_flags & VM_WRITE) entry = pte_mkwrite(pte_mkdirty(entry)); @@ -3660,6 +3662,7 @@ vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg, flush_icache_page(vma, page); entry = mk_pte(page, vma->vm_page_prot); + entry = pte_sw_mkyoung(entry); if (write) entry = maybe_mkwrite(pte_mkdirty(entry), vma); /* copy-on-write page */ From 273b5fa00fadabd587fac858ea61a17b3653892d Mon Sep 17 00:00:00 2001 From: Bibo Mao Date: Wed, 27 May 2020 10:25:20 +0800 Subject: [PATCH 0878/1043] MIPS: mm: add page valid judgement in function pte_modify If original PTE has _PAGE_ACCESSED bit set, and new pte has no _PAGE_NO_READ bit set, we can add _PAGE_SILENT_READ bit to enable page valid bit. Signed-off-by: Bibo Mao Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/pgtable.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h index d9772aff2f02..85b39c9fd09e 100644 --- a/arch/mips/include/asm/pgtable.h +++ b/arch/mips/include/asm/pgtable.h @@ -532,8 +532,11 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) #else static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { - return __pte((pte_val(pte) & _PAGE_CHG_MASK) | - (pgprot_val(newprot) & ~_PAGE_CHG_MASK)); + pte_val(pte) &= _PAGE_CHG_MASK; + pte_val(pte) |= pgprot_val(newprot) & ~_PAGE_CHG_MASK; + if ((pte_val(pte) & _PAGE_ACCESSED) && !(pte_val(pte) & _PAGE_NO_READ)) + pte_val(pte) |= _PAGE_SILENT_READ; + return pte; } #endif From 31e1b3efa802f97a17628dde280006c4cee4ce5e Mon Sep 17 00:00:00 2001 From: YuanJunQing Date: Wed, 27 May 2020 14:11:30 +0800 Subject: [PATCH 0879/1043] MIPS: Fix IRQ tracing when call handle_fpe() and handle_msa_fpe() Register "a1" is unsaved in this function, when CONFIG_TRACE_IRQFLAGS is enabled, the TRACE_IRQS_OFF macro will call trace_hardirqs_off(), and this may change register "a1". The changed register "a1" as argument will be send to do_fpe() and do_msa_fpe(). Signed-off-by: YuanJunQing Signed-off-by: Thomas Bogendoerfer --- arch/mips/kernel/genex.S | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/mips/kernel/genex.S b/arch/mips/kernel/genex.S index 8236fb291e3f..a1b966f3578e 100644 --- a/arch/mips/kernel/genex.S +++ b/arch/mips/kernel/genex.S @@ -476,20 +476,20 @@ NESTED(nmi_handler, PT_SIZE, sp) .endm .macro __build_clear_fpe + CLI + TRACE_IRQS_OFF .set push /* gas fails to assemble cfc1 for some archs (octeon).*/ \ .set mips1 SET_HARDFLOAT cfc1 a1, fcr31 .set pop - CLI - TRACE_IRQS_OFF .endm .macro __build_clear_msa_fpe - _cfcmsa a1, MSA_CSR CLI TRACE_IRQS_OFF + _cfcmsa a1, MSA_CSR .endm .macro __build_clear_ade From 3b5b7b1f70e79fb060d485412b3c748f0a83111b Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 27 May 2020 12:01:51 +0800 Subject: [PATCH 0880/1043] bcache: remove redundant variables i and n Variables i and n are being assigned but are never used. They are redundant and can be removed. Signed-off-by: Colin Ian King Signed-off-by: Coly Li Addresses-Coverity: ("Unused value") Signed-off-by: Jens Axboe --- drivers/md/bcache/btree.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c index 72856e5f23a3..114d0d73d909 100644 --- a/drivers/md/bcache/btree.c +++ b/drivers/md/bcache/btree.c @@ -1907,10 +1907,8 @@ static int bch_btree_check_thread(void *arg) struct btree_iter iter; struct bkey *k, *p; int cur_idx, prev_idx, skip_nr; - int i, n; k = p = NULL; - i = n = 0; cur_idx = prev_idx = 0; ret = 0; From 46f5aa8806e34f2e48de852cc7db2c74c3a5cd8d Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Wed, 27 May 2020 12:01:52 +0800 Subject: [PATCH 0881/1043] bcache: Convert pr_ uses to a more typical style Remove the trailing newline from the define of pr_fmt and add newlines to the uses. Miscellanea: o Convert bch_bkey_dump from multiple uses of pr_err to pr_cont as the earlier conversion was inappropriate done causing multiple lines to be emitted where only a single output line was desired o Use vsprintf extension %pV in bch_cache_set_error to avoid multiple line output where only a single line output was desired o Coalesce formats Fixes: 6ae63e3501c4 ("bcache: replace printk() by pr_*() routines") Signed-off-by: Joe Perches Signed-off-by: Coly Li Signed-off-by: Jens Axboe --- drivers/md/bcache/bcache.h | 2 +- drivers/md/bcache/bset.c | 6 +- drivers/md/bcache/btree.c | 14 ++-- drivers/md/bcache/extents.c | 12 ++-- drivers/md/bcache/io.c | 8 +-- drivers/md/bcache/journal.c | 34 +++++----- drivers/md/bcache/request.c | 6 +- drivers/md/bcache/super.c | 123 +++++++++++++++++----------------- drivers/md/bcache/sysfs.c | 8 +-- drivers/md/bcache/writeback.c | 6 +- 10 files changed, 110 insertions(+), 109 deletions(-) diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h index 74a9849ea164..221e0191b687 100644 --- a/drivers/md/bcache/bcache.h +++ b/drivers/md/bcache/bcache.h @@ -176,7 +176,7 @@ * - updates to non leaf nodes just happen synchronously (see btree_split()). */ -#define pr_fmt(fmt) "bcache: %s() " fmt "\n", __func__ +#define pr_fmt(fmt) "bcache: %s() " fmt, __func__ #include #include diff --git a/drivers/md/bcache/bset.c b/drivers/md/bcache/bset.c index 4385303836d8..4995fcaefe29 100644 --- a/drivers/md/bcache/bset.c +++ b/drivers/md/bcache/bset.c @@ -6,7 +6,7 @@ * Copyright 2012 Google, Inc. */ -#define pr_fmt(fmt) "bcache: %s() " fmt "\n", __func__ +#define pr_fmt(fmt) "bcache: %s() " fmt, __func__ #include "util.h" #include "bset.h" @@ -31,7 +31,7 @@ void bch_dump_bset(struct btree_keys *b, struct bset *i, unsigned int set) if (b->ops->key_dump) b->ops->key_dump(b, k); else - pr_err("%llu:%llu\n", KEY_INODE(k), KEY_OFFSET(k)); + pr_cont("%llu:%llu\n", KEY_INODE(k), KEY_OFFSET(k)); if (next < bset_bkey_last(i) && bkey_cmp(k, b->ops->is_extents ? @@ -1225,7 +1225,7 @@ static void btree_mergesort(struct btree_keys *b, struct bset *out, out->keys = last ? (uint64_t *) bkey_next(last) - out->d : 0; - pr_debug("sorted %i keys", out->keys); + pr_debug("sorted %i keys\n", out->keys); } static void __btree_sort(struct btree_keys *b, struct btree_iter *iter, diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c index 114d0d73d909..39de94edd73a 100644 --- a/drivers/md/bcache/btree.c +++ b/drivers/md/bcache/btree.c @@ -619,7 +619,7 @@ retry: * and BTREE_NODE_journal_flush bit cleared by btree_flush_write(). */ if (btree_node_journal_flush(b)) { - pr_debug("bnode %p is flushing by journal, retry", b); + pr_debug("bnode %p is flushing by journal, retry\n", b); mutex_unlock(&b->write_lock); udelay(1); goto retry; @@ -802,7 +802,7 @@ int bch_btree_cache_alloc(struct cache_set *c) c->shrink.batch = c->btree_pages * 2; if (register_shrinker(&c->shrink)) - pr_warn("bcache: %s: could not register shrinker", + pr_warn("bcache: %s: could not register shrinker\n", __func__); return 0; @@ -1054,7 +1054,7 @@ retry: */ if (btree_node_journal_flush(b)) { mutex_unlock(&b->write_lock); - pr_debug("bnode %p journal_flush set, retry", b); + pr_debug("bnode %p journal_flush set, retry\n", b); udelay(1); goto retry; } @@ -1798,7 +1798,7 @@ static void bch_btree_gc(struct cache_set *c) schedule_timeout_interruptible(msecs_to_jiffies (GC_SLEEP_MS)); else if (ret) - pr_warn("gc failed!"); + pr_warn("gc failed!\n"); } while (ret && !test_bit(CACHE_SET_IO_DISABLE, &c->flags)); bch_btree_gc_finish(c); @@ -2043,7 +2043,7 @@ int bch_btree_check(struct cache_set *c) &check_state->infos[i], name); if (IS_ERR(check_state->infos[i].thread)) { - pr_err("fails to run thread bch_btrchk[%d]", i); + pr_err("fails to run thread bch_btrchk[%d]\n", i); for (--i; i >= 0; i--) kthread_stop(check_state->infos[i].thread); ret = -ENOMEM; @@ -2454,7 +2454,7 @@ int bch_btree_insert(struct cache_set *c, struct keylist *keys, if (ret) { struct bkey *k; - pr_err("error %i", ret); + pr_err("error %i\n", ret); while ((k = bch_keylist_pop(keys))) bkey_put(c, k); @@ -2742,7 +2742,7 @@ struct keybuf_key *bch_keybuf_next_rescan(struct cache_set *c, break; if (bkey_cmp(&buf->last_scanned, end) >= 0) { - pr_debug("scan finished"); + pr_debug("scan finished\n"); break; } diff --git a/drivers/md/bcache/extents.c b/drivers/md/bcache/extents.c index 886710043025..9162af5bb6ec 100644 --- a/drivers/md/bcache/extents.c +++ b/drivers/md/bcache/extents.c @@ -130,18 +130,18 @@ static void bch_bkey_dump(struct btree_keys *keys, const struct bkey *k) char buf[80]; bch_extent_to_text(buf, sizeof(buf), k); - pr_err(" %s", buf); + pr_cont(" %s", buf); for (j = 0; j < KEY_PTRS(k); j++) { size_t n = PTR_BUCKET_NR(b->c, k, j); - pr_err(" bucket %zu", n); + pr_cont(" bucket %zu", n); if (n >= b->c->sb.first_bucket && n < b->c->sb.nbuckets) - pr_err(" prio %i", - PTR_BUCKET(b->c, k, j)->prio); + pr_cont(" prio %i", + PTR_BUCKET(b->c, k, j)->prio); } - pr_err(" %s\n", bch_ptr_status(b->c, k)); + pr_cont(" %s\n", bch_ptr_status(b->c, k)); } /* Btree ptrs */ @@ -553,7 +553,7 @@ static bool bch_extent_bad(struct btree_keys *bk, const struct bkey *k) if (stale && KEY_DIRTY(k)) { bch_extent_to_text(buf, sizeof(buf), k); - pr_info("stale dirty pointer, stale %u, key: %s", + pr_info("stale dirty pointer, stale %u, key: %s\n", stale, buf); } diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c index 4d93f07f63e5..b25ee33b0d0b 100644 --- a/drivers/md/bcache/io.c +++ b/drivers/md/bcache/io.c @@ -65,14 +65,14 @@ void bch_count_backing_io_errors(struct cached_dev *dc, struct bio *bio) * we shouldn't count failed REQ_RAHEAD bio to dc->io_errors. */ if (bio->bi_opf & REQ_RAHEAD) { - pr_warn_ratelimited("%s: Read-ahead I/O failed on backing device, ignore", + pr_warn_ratelimited("%s: Read-ahead I/O failed on backing device, ignore\n", dc->backing_dev_name); return; } errors = atomic_add_return(1, &dc->io_errors); if (errors < dc->error_limit) - pr_err("%s: IO error on backing device, unrecoverable", + pr_err("%s: IO error on backing device, unrecoverable\n", dc->backing_dev_name); else bch_cached_dev_error(dc); @@ -123,12 +123,12 @@ void bch_count_io_errors(struct cache *ca, errors >>= IO_ERROR_SHIFT; if (errors < ca->set->error_limit) - pr_err("%s: IO error on %s%s", + pr_err("%s: IO error on %s%s\n", ca->cache_dev_name, m, is_read ? ", recovering." : "."); else bch_cache_set_error(ca->set, - "%s: too many IO errors %s", + "%s: too many IO errors %s\n", ca->cache_dev_name, m); } } diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c index 0e3ff9745ac7..90aac4e2333f 100644 --- a/drivers/md/bcache/journal.c +++ b/drivers/md/bcache/journal.c @@ -47,7 +47,7 @@ static int journal_read_bucket(struct cache *ca, struct list_head *list, closure_init_stack(&cl); - pr_debug("reading %u", bucket_index); + pr_debug("reading %u\n", bucket_index); while (offset < ca->sb.bucket_size) { reread: left = ca->sb.bucket_size - offset; @@ -78,13 +78,13 @@ reread: left = ca->sb.bucket_size - offset; size_t blocks, bytes = set_bytes(j); if (j->magic != jset_magic(&ca->sb)) { - pr_debug("%u: bad magic", bucket_index); + pr_debug("%u: bad magic\n", bucket_index); return ret; } if (bytes > left << 9 || bytes > PAGE_SIZE << JSET_BITS) { - pr_info("%u: too big, %zu bytes, offset %u", + pr_info("%u: too big, %zu bytes, offset %u\n", bucket_index, bytes, offset); return ret; } @@ -93,7 +93,7 @@ reread: left = ca->sb.bucket_size - offset; goto reread; if (j->csum != csum_set(j)) { - pr_info("%u: bad csum, %zu bytes, offset %u", + pr_info("%u: bad csum, %zu bytes, offset %u\n", bucket_index, bytes, offset); return ret; } @@ -190,7 +190,7 @@ int bch_journal_read(struct cache_set *c, struct list_head *list) uint64_t seq; bitmap_zero(bitmap, SB_JOURNAL_BUCKETS); - pr_debug("%u journal buckets", ca->sb.njournal_buckets); + pr_debug("%u journal buckets\n", ca->sb.njournal_buckets); /* * Read journal buckets ordered by golden ratio hash to quickly @@ -215,7 +215,7 @@ int bch_journal_read(struct cache_set *c, struct list_head *list) * If that fails, check all the buckets we haven't checked * already */ - pr_debug("falling back to linear search"); + pr_debug("falling back to linear search\n"); for (l = find_first_zero_bit(bitmap, ca->sb.njournal_buckets); l < ca->sb.njournal_buckets; @@ -233,7 +233,7 @@ bsearch: /* Binary search */ m = l; r = find_next_bit(bitmap, ca->sb.njournal_buckets, l + 1); - pr_debug("starting binary search, l %u r %u", l, r); + pr_debug("starting binary search, l %u r %u\n", l, r); while (l + 1 < r) { seq = list_entry(list->prev, struct journal_replay, @@ -253,7 +253,7 @@ bsearch: * Read buckets in reverse order until we stop finding more * journal entries */ - pr_debug("finishing up: m %u njournal_buckets %u", + pr_debug("finishing up: m %u njournal_buckets %u\n", m, ca->sb.njournal_buckets); l = m; @@ -370,10 +370,10 @@ int bch_journal_replay(struct cache_set *s, struct list_head *list) if (n != i->j.seq) { if (n == start && is_discard_enabled(s)) - pr_info("bcache: journal entries %llu-%llu may be discarded! (replaying %llu-%llu)", + pr_info("journal entries %llu-%llu may be discarded! (replaying %llu-%llu)\n", n, i->j.seq - 1, start, end); else { - pr_err("bcache: journal entries %llu-%llu missing! (replaying %llu-%llu)", + pr_err("journal entries %llu-%llu missing! (replaying %llu-%llu)\n", n, i->j.seq - 1, start, end); ret = -EIO; goto err; @@ -403,7 +403,7 @@ int bch_journal_replay(struct cache_set *s, struct list_head *list) entries++; } - pr_info("journal replay done, %i keys in %i entries, seq %llu", + pr_info("journal replay done, %i keys in %i entries, seq %llu\n", keys, entries, end); err: while (!list_empty(list)) { @@ -481,7 +481,7 @@ static void btree_flush_write(struct cache_set *c) break; if (btree_node_journal_flush(b)) - pr_err("BUG: flush_write bit should not be set here!"); + pr_err("BUG: flush_write bit should not be set here!\n"); mutex_lock(&b->write_lock); @@ -534,13 +534,13 @@ static void btree_flush_write(struct cache_set *c) for (i = 0; i < nr; i++) { b = btree_nodes[i]; if (!b) { - pr_err("BUG: btree_nodes[%d] is NULL", i); + pr_err("BUG: btree_nodes[%d] is NULL\n", i); continue; } /* safe to check without holding b->write_lock */ if (!btree_node_journal_flush(b)) { - pr_err("BUG: bnode %p: journal_flush bit cleaned", b); + pr_err("BUG: bnode %p: journal_flush bit cleaned\n", b); continue; } @@ -548,14 +548,14 @@ static void btree_flush_write(struct cache_set *c) if (!btree_current_write(b)->journal) { clear_bit(BTREE_NODE_journal_flush, &b->flags); mutex_unlock(&b->write_lock); - pr_debug("bnode %p: written by others", b); + pr_debug("bnode %p: written by others\n", b); continue; } if (!btree_node_dirty(b)) { clear_bit(BTREE_NODE_journal_flush, &b->flags); mutex_unlock(&b->write_lock); - pr_debug("bnode %p: dirty bit cleaned by others", b); + pr_debug("bnode %p: dirty bit cleaned by others\n", b); continue; } @@ -716,7 +716,7 @@ void bch_journal_next(struct journal *j) j->cur->data->keys = 0; if (fifo_full(&j->pin)) - pr_debug("journal_pin full (%zu)", fifo_used(&j->pin)); + pr_debug("journal_pin full (%zu)\n", fifo_used(&j->pin)); } static void journal_write_endio(struct bio *bio) diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c index 77d1a2697517..72672178970b 100644 --- a/drivers/md/bcache/request.c +++ b/drivers/md/bcache/request.c @@ -110,7 +110,7 @@ static void bch_data_invalidate(struct closure *cl) struct data_insert_op *op = container_of(cl, struct data_insert_op, cl); struct bio *bio = op->bio; - pr_debug("invalidating %i sectors from %llu", + pr_debug("invalidating %i sectors from %llu\n", bio_sectors(bio), (uint64_t) bio->bi_iter.bi_sector); while (bio_sectors(bio)) { @@ -396,7 +396,7 @@ static bool check_should_bypass(struct cached_dev *dc, struct bio *bio) if (bio->bi_iter.bi_sector & (c->sb.block_size - 1) || bio_sectors(bio) & (c->sb.block_size - 1)) { - pr_debug("skipping unaligned io"); + pr_debug("skipping unaligned io\n"); goto skip; } @@ -650,7 +650,7 @@ static void backing_request_endio(struct bio *bio) */ if (unlikely(s->iop.writeback && bio->bi_opf & REQ_PREFLUSH)) { - pr_err("Can't flush %s: returned bi_status %i", + pr_err("Can't flush %s: returned bi_status %i\n", dc->backing_dev_name, bio->bi_status); } else { /* set to orig_bio->bi_status in bio_complete() */ diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index d98354fa28e3..467149f3bcc5 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -89,7 +89,7 @@ static const char *read_super(struct cache_sb *sb, struct block_device *bdev, for (i = 0; i < SB_JOURNAL_BUCKETS; i++) sb->d[i] = le64_to_cpu(s->d[i]); - pr_debug("read sb version %llu, flags %llu, seq %llu, journal size %u", + pr_debug("read sb version %llu, flags %llu, seq %llu, journal size %u\n", sb->version, sb->flags, sb->seq, sb->keys); err = "Not a bcache superblock (bad offset)"; @@ -234,7 +234,7 @@ static void __write_super(struct cache_sb *sb, struct cache_sb_disk *out, out->csum = csum_set(out); - pr_debug("ver %llu, flags %llu, seq %llu", + pr_debug("ver %llu, flags %llu, seq %llu\n", sb->version, sb->flags, sb->seq); submit_bio(bio); @@ -365,11 +365,11 @@ static void uuid_io(struct cache_set *c, int op, unsigned long op_flags, } bch_extent_to_text(buf, sizeof(buf), k); - pr_debug("%s UUIDs at %s", op == REQ_OP_WRITE ? "wrote" : "read", buf); + pr_debug("%s UUIDs at %s\n", op == REQ_OP_WRITE ? "wrote" : "read", buf); for (u = c->uuids; u < c->uuids + c->nr_uuids; u++) if (!bch_is_zero(u->uuid, 16)) - pr_debug("Slot %zi: %pU: %s: 1st: %u last: %u inv: %u", + pr_debug("Slot %zi: %pU: %s: 1st: %u last: %u inv: %u\n", u - c->uuids, u->uuid, u->label, u->first_reg, u->last_reg, u->invalidated); @@ -534,7 +534,7 @@ int bch_prio_write(struct cache *ca, bool wait) struct bucket *b; struct closure cl; - pr_debug("free_prio=%zu, free_none=%zu, free_inc=%zu", + pr_debug("free_prio=%zu, free_none=%zu, free_inc=%zu\n", fifo_used(&ca->free[RESERVE_PRIO]), fifo_used(&ca->free[RESERVE_NONE]), fifo_used(&ca->free_inc)); @@ -629,12 +629,12 @@ static int prio_read(struct cache *ca, uint64_t bucket) if (p->csum != bch_crc64(&p->magic, bucket_bytes(ca) - 8)) { - pr_warn("bad csum reading priorities"); + pr_warn("bad csum reading priorities\n"); goto out; } if (p->magic != pset_magic(&ca->sb)) { - pr_warn("bad magic reading priorities"); + pr_warn("bad magic reading priorities\n"); goto out; } @@ -728,11 +728,11 @@ static void bcache_device_link(struct bcache_device *d, struct cache_set *c, ret = sysfs_create_link(&d->kobj, &c->kobj, "cache"); if (ret < 0) - pr_err("Couldn't create device -> cache set symlink"); + pr_err("Couldn't create device -> cache set symlink\n"); ret = sysfs_create_link(&c->kobj, &d->kobj, d->name); if (ret < 0) - pr_err("Couldn't create cache set -> device symlink"); + pr_err("Couldn't create cache set -> device symlink\n"); clear_bit(BCACHE_DEV_UNLINK_DONE, &d->flags); } @@ -789,9 +789,9 @@ static void bcache_device_free(struct bcache_device *d) lockdep_assert_held(&bch_register_lock); if (disk) - pr_info("%s stopped", disk->disk_name); + pr_info("%s stopped\n", disk->disk_name); else - pr_err("bcache device (NULL gendisk) stopped"); + pr_err("bcache device (NULL gendisk) stopped\n"); if (d->c) bcache_device_detach(d); @@ -830,7 +830,7 @@ static int bcache_device_init(struct bcache_device *d, unsigned int block_size, d->nr_stripes = DIV_ROUND_UP_ULL(sectors, d->stripe_size); if (!d->nr_stripes || d->nr_stripes > max_stripes) { - pr_err("nr_stripes too large or invalid: %u (start sector beyond end of disk?)", + pr_err("nr_stripes too large or invalid: %u (start sector beyond end of disk?)\n", (unsigned int)d->nr_stripes); return -ENOMEM; } @@ -928,11 +928,11 @@ static int cached_dev_status_update(void *arg) dc->offline_seconds = 0; if (dc->offline_seconds >= BACKING_DEV_OFFLINE_TIMEOUT) { - pr_err("%s: device offline for %d seconds", + pr_err("%s: device offline for %d seconds\n", dc->backing_dev_name, BACKING_DEV_OFFLINE_TIMEOUT); - pr_err("%s: disable I/O request due to backing " - "device offline", dc->disk.name); + pr_err("%s: disable I/O request due to backing device offline\n", + dc->disk.name); dc->io_disable = true; /* let others know earlier that io_disable is true */ smp_mb(); @@ -959,7 +959,7 @@ int bch_cached_dev_run(struct cached_dev *dc) }; if (dc->io_disable) { - pr_err("I/O disabled on cached dev %s", + pr_err("I/O disabled on cached dev %s\n", dc->backing_dev_name); kfree(env[1]); kfree(env[2]); @@ -971,7 +971,7 @@ int bch_cached_dev_run(struct cached_dev *dc) kfree(env[1]); kfree(env[2]); kfree(buf); - pr_info("cached dev %s is running already", + pr_info("cached dev %s is running already\n", dc->backing_dev_name); return -EBUSY; } @@ -1001,16 +1001,14 @@ int bch_cached_dev_run(struct cached_dev *dc) if (sysfs_create_link(&d->kobj, &disk_to_dev(d->disk)->kobj, "dev") || sysfs_create_link(&disk_to_dev(d->disk)->kobj, &d->kobj, "bcache")) { - pr_err("Couldn't create bcache dev <-> disk sysfs symlinks"); + pr_err("Couldn't create bcache dev <-> disk sysfs symlinks\n"); return -ENOMEM; } dc->status_update_thread = kthread_run(cached_dev_status_update, dc, "bcache_status_update"); if (IS_ERR(dc->status_update_thread)) { - pr_warn("failed to create bcache_status_update kthread, " - "continue to run without monitoring backing " - "device status"); + pr_warn("failed to create bcache_status_update kthread, continue to run without monitoring backing device status\n"); } return 0; @@ -1036,7 +1034,7 @@ static void cancel_writeback_rate_update_dwork(struct cached_dev *dc) } while (time_out > 0); if (time_out == 0) - pr_warn("give up waiting for dc->writeback_write_update to quit"); + pr_warn("give up waiting for dc->writeback_write_update to quit\n"); cancel_delayed_work_sync(&dc->writeback_rate_update); } @@ -1077,7 +1075,7 @@ static void cached_dev_detach_finish(struct work_struct *w) mutex_unlock(&bch_register_lock); - pr_info("Caching disabled for %s", dc->backing_dev_name); + pr_info("Caching disabled for %s\n", dc->backing_dev_name); /* Drop ref we took in cached_dev_detach() */ closure_put(&dc->disk.cl); @@ -1117,20 +1115,20 @@ int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c, return -ENOENT; if (dc->disk.c) { - pr_err("Can't attach %s: already attached", + pr_err("Can't attach %s: already attached\n", dc->backing_dev_name); return -EINVAL; } if (test_bit(CACHE_SET_STOPPING, &c->flags)) { - pr_err("Can't attach %s: shutting down", + pr_err("Can't attach %s: shutting down\n", dc->backing_dev_name); return -EINVAL; } if (dc->sb.block_size < c->sb.block_size) { /* Will die */ - pr_err("Couldn't attach %s: block size less than set's block size", + pr_err("Couldn't attach %s: block size less than set's block size\n", dc->backing_dev_name); return -EINVAL; } @@ -1138,7 +1136,7 @@ int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c, /* Check whether already attached */ list_for_each_entry_safe(exist_dc, t, &c->cached_devs, list) { if (!memcmp(dc->sb.uuid, exist_dc->sb.uuid, 16)) { - pr_err("Tried to attach %s but duplicate UUID already attached", + pr_err("Tried to attach %s but duplicate UUID already attached\n", dc->backing_dev_name); return -EINVAL; @@ -1157,14 +1155,14 @@ int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c, if (!u) { if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) { - pr_err("Couldn't find uuid for %s in set", + pr_err("Couldn't find uuid for %s in set\n", dc->backing_dev_name); return -ENOENT; } u = uuid_find_empty(c); if (!u) { - pr_err("Not caching %s, no room for UUID", + pr_err("Not caching %s, no room for UUID\n", dc->backing_dev_name); return -EINVAL; } @@ -1210,7 +1208,7 @@ int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c, down_write(&dc->writeback_lock); if (bch_cached_dev_writeback_start(dc)) { up_write(&dc->writeback_lock); - pr_err("Couldn't start writeback facilities for %s", + pr_err("Couldn't start writeback facilities for %s\n", dc->disk.disk->disk_name); return -ENOMEM; } @@ -1233,7 +1231,7 @@ int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c, */ kthread_stop(dc->writeback_thread); cancel_writeback_rate_update_dwork(dc); - pr_err("Couldn't run cached device %s", + pr_err("Couldn't run cached device %s\n", dc->backing_dev_name); return ret; } @@ -1244,7 +1242,7 @@ int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c, /* Allow the writeback thread to proceed */ up_write(&dc->writeback_lock); - pr_info("Caching %s as %s on set %pU", + pr_info("Caching %s as %s on set %pU\n", dc->backing_dev_name, dc->disk.disk->disk_name, dc->disk.c->sb.set_uuid); @@ -1384,7 +1382,7 @@ static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk, if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj)) goto err; - pr_info("registered backing device %s", dc->backing_dev_name); + pr_info("registered backing device %s\n", dc->backing_dev_name); list_add(&dc->list, &uncached_devices); /* attach to a matched cache set if it exists */ @@ -1401,7 +1399,7 @@ static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk, return 0; err: - pr_notice("error %s: %s", dc->backing_dev_name, err); + pr_notice("error %s: %s\n", dc->backing_dev_name, err); bcache_device_stop(&dc->disk); return ret; } @@ -1497,7 +1495,7 @@ int bch_flash_dev_create(struct cache_set *c, uint64_t size) u = uuid_find_empty(c); if (!u) { - pr_err("Can't create volume, no room for UUID"); + pr_err("Can't create volume, no room for UUID\n"); return -EINVAL; } @@ -1523,7 +1521,7 @@ bool bch_cached_dev_error(struct cached_dev *dc) smp_mb(); pr_err("stop %s: too many IO errors on backing device %s\n", - dc->disk.disk->disk_name, dc->backing_dev_name); + dc->disk.disk->disk_name, dc->backing_dev_name); bcache_device_stop(&dc->disk); return true; @@ -1534,6 +1532,7 @@ bool bch_cached_dev_error(struct cached_dev *dc) __printf(2, 3) bool bch_cache_set_error(struct cache_set *c, const char *fmt, ...) { + struct va_format vaf; va_list args; if (c->on_error != ON_ERROR_PANIC && @@ -1541,20 +1540,22 @@ bool bch_cache_set_error(struct cache_set *c, const char *fmt, ...) return false; if (test_and_set_bit(CACHE_SET_IO_DISABLE, &c->flags)) - pr_info("CACHE_SET_IO_DISABLE already set"); + pr_info("CACHE_SET_IO_DISABLE already set\n"); /* * XXX: we can be called from atomic context * acquire_console_sem(); */ - pr_err("bcache: error on %pU: ", c->sb.set_uuid); - va_start(args, fmt); - vprintk(fmt, args); - va_end(args); - pr_err(", disabling caching\n"); + vaf.fmt = fmt; + vaf.va = &args; + + pr_err("error on %pU: %pV, disabling caching\n", + c->sb.set_uuid, &vaf); + + va_end(args); if (c->on_error == ON_ERROR_PANIC) panic("panic forced after error\n"); @@ -1606,7 +1607,7 @@ static void cache_set_free(struct closure *cl) list_del(&c->list); mutex_unlock(&bch_register_lock); - pr_info("Cache set %pU unregistered", c->sb.set_uuid); + pr_info("Cache set %pU unregistered\n", c->sb.set_uuid); wake_up(&unregister_wait); closure_debug_destroy(&c->cl); @@ -1677,7 +1678,7 @@ static void conditional_stop_bcache_device(struct cache_set *c, struct cached_dev *dc) { if (dc->stop_when_cache_set_failed == BCH_CACHED_DEV_STOP_ALWAYS) { - pr_warn("stop_when_cache_set_failed of %s is \"always\", stop it for failed cache set %pU.", + pr_warn("stop_when_cache_set_failed of %s is \"always\", stop it for failed cache set %pU.\n", d->disk->disk_name, c->sb.set_uuid); bcache_device_stop(d); } else if (atomic_read(&dc->has_dirty)) { @@ -1685,7 +1686,7 @@ static void conditional_stop_bcache_device(struct cache_set *c, * dc->stop_when_cache_set_failed == BCH_CACHED_STOP_AUTO * and dc->has_dirty == 1 */ - pr_warn("stop_when_cache_set_failed of %s is \"auto\" and cache is dirty, stop it to avoid potential data corruption.", + pr_warn("stop_when_cache_set_failed of %s is \"auto\" and cache is dirty, stop it to avoid potential data corruption.\n", d->disk->disk_name); /* * There might be a small time gap that cache set is @@ -1707,7 +1708,7 @@ static void conditional_stop_bcache_device(struct cache_set *c, * dc->stop_when_cache_set_failed == BCH_CACHED_STOP_AUTO * and dc->has_dirty == 0 */ - pr_warn("stop_when_cache_set_failed of %s is \"auto\" and cache is clean, keep it alive.", + pr_warn("stop_when_cache_set_failed of %s is \"auto\" and cache is clean, keep it alive.\n", d->disk->disk_name); } } @@ -1874,7 +1875,7 @@ static int run_cache_set(struct cache_set *c) if (bch_journal_read(c, &journal)) goto err; - pr_debug("btree_journal_read() done"); + pr_debug("btree_journal_read() done\n"); err = "no journal entries found"; if (list_empty(&journal)) @@ -1920,7 +1921,7 @@ static int run_cache_set(struct cache_set *c) bch_journal_mark(c, &journal); bch_initial_gc_finish(c); - pr_debug("btree_check() done"); + pr_debug("btree_check() done\n"); /* * bcache_journal_next() can't happen sooner, or @@ -1951,7 +1952,7 @@ static int run_cache_set(struct cache_set *c) if (bch_journal_replay(c, &journal)) goto err; } else { - pr_notice("invalidating existing data"); + pr_notice("invalidating existing data\n"); for_each_cache(ca, c, i) { unsigned int j; @@ -2085,7 +2086,7 @@ found: memcpy(c->sb.set_uuid, ca->sb.set_uuid, 16); c->sb.flags = ca->sb.flags; c->sb.seq = ca->sb.seq; - pr_debug("set version = %llu", c->sb.version); + pr_debug("set version = %llu\n", c->sb.version); } kobject_get(&ca->kobj); @@ -2247,7 +2248,7 @@ err_btree_alloc: err_free: module_put(THIS_MODULE); if (err) - pr_notice("error %s: %s", ca->cache_dev_name, err); + pr_notice("error %s: %s\n", ca->cache_dev_name, err); return ret; } @@ -2301,14 +2302,14 @@ static int register_cache(struct cache_sb *sb, struct cache_sb_disk *sb_disk, goto out; } - pr_info("registered cache device %s", ca->cache_dev_name); + pr_info("registered cache device %s\n", ca->cache_dev_name); out: kobject_put(&ca->kobj); err: if (err) - pr_notice("error %s: %s", ca->cache_dev_name, err); + pr_notice("error %s: %s\n", ca->cache_dev_name, err); return ret; } @@ -2461,7 +2462,7 @@ out_free_path: out_module_put: module_put(THIS_MODULE); out: - pr_info("error %s: %s", path?path:"", err); + pr_info("error %s: %s\n", path?path:"", err); return ret; } @@ -2506,7 +2507,7 @@ static ssize_t bch_pending_bdevs_cleanup(struct kobject *k, mutex_unlock(&bch_register_lock); list_for_each_entry_safe(pdev, tpdev, &pending_devs, list) { - pr_info("delete pdev %p", pdev); + pr_info("delete pdev %p\n", pdev); list_del(&pdev->list); bcache_device_stop(&pdev->dc->disk); kfree(pdev); @@ -2549,7 +2550,7 @@ static int bcache_reboot(struct notifier_block *n, unsigned long code, void *x) mutex_unlock(&bch_register_lock); - pr_info("Stopping all devices:"); + pr_info("Stopping all devices:\n"); /* * The reason bch_register_lock is not held to call @@ -2599,9 +2600,9 @@ static int bcache_reboot(struct notifier_block *n, unsigned long code, void *x) finish_wait(&unregister_wait, &wait); if (stopped) - pr_info("All devices stopped"); + pr_info("All devices stopped\n"); else - pr_notice("Timeout waiting for devices to be closed"); + pr_notice("Timeout waiting for devices to be closed\n"); out: mutex_unlock(&bch_register_lock); } @@ -2637,7 +2638,7 @@ static void check_module_parameters(void) if (bch_cutoff_writeback_sync == 0) bch_cutoff_writeback_sync = CUTOFF_WRITEBACK_SYNC; else if (bch_cutoff_writeback_sync > CUTOFF_WRITEBACK_SYNC_MAX) { - pr_warn("set bch_cutoff_writeback_sync (%u) to max value %u", + pr_warn("set bch_cutoff_writeback_sync (%u) to max value %u\n", bch_cutoff_writeback_sync, CUTOFF_WRITEBACK_SYNC_MAX); bch_cutoff_writeback_sync = CUTOFF_WRITEBACK_SYNC_MAX; } @@ -2645,13 +2646,13 @@ static void check_module_parameters(void) if (bch_cutoff_writeback == 0) bch_cutoff_writeback = CUTOFF_WRITEBACK; else if (bch_cutoff_writeback > CUTOFF_WRITEBACK_MAX) { - pr_warn("set bch_cutoff_writeback (%u) to max value %u", + pr_warn("set bch_cutoff_writeback (%u) to max value %u\n", bch_cutoff_writeback, CUTOFF_WRITEBACK_MAX); bch_cutoff_writeback = CUTOFF_WRITEBACK_MAX; } if (bch_cutoff_writeback > bch_cutoff_writeback_sync) { - pr_warn("set bch_cutoff_writeback (%u) to %u", + pr_warn("set bch_cutoff_writeback (%u) to %u\n", bch_cutoff_writeback, bch_cutoff_writeback_sync); bch_cutoff_writeback = bch_cutoff_writeback_sync; } diff --git a/drivers/md/bcache/sysfs.c b/drivers/md/bcache/sysfs.c index 323276994aab..0dadec5a78f6 100644 --- a/drivers/md/bcache/sysfs.c +++ b/drivers/md/bcache/sysfs.c @@ -421,7 +421,7 @@ STORE(__cached_dev) return size; } if (v == -ENOENT) - pr_err("Can't attach %s: cache set not found", buf); + pr_err("Can't attach %s: cache set not found\n", buf); return v; } @@ -455,7 +455,7 @@ STORE(bch_cached_dev) */ if (dc->writeback_running) { dc->writeback_running = false; - pr_err("%s: failed to run non-existent writeback thread", + pr_err("%s: failed to run non-existent writeback thread\n", dc->disk.disk->disk_name); } } else @@ -872,11 +872,11 @@ STORE(__bch_cache_set) if (v) { if (test_and_set_bit(CACHE_SET_IO_DISABLE, &c->flags)) - pr_warn("CACHE_SET_IO_DISABLE already set"); + pr_warn("CACHE_SET_IO_DISABLE already set\n"); } else { if (!test_and_clear_bit(CACHE_SET_IO_DISABLE, &c->flags)) - pr_warn("CACHE_SET_IO_DISABLE already cleared"); + pr_warn("CACHE_SET_IO_DISABLE already cleared\n"); } } diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c index 3f7641fb28d5..1cf1e5016cb9 100644 --- a/drivers/md/bcache/writeback.c +++ b/drivers/md/bcache/writeback.c @@ -809,7 +809,7 @@ static int bch_root_node_dirty_init(struct cache_set *c, schedule_timeout_interruptible( msecs_to_jiffies(INIT_KEYS_SLEEP_MS)); else if (ret < 0) { - pr_warn("sectors dirty init failed, ret=%d!", ret); + pr_warn("sectors dirty init failed, ret=%d!\n", ret); break; } } while (ret == -EAGAIN); @@ -917,7 +917,7 @@ void bch_sectors_dirty_init(struct bcache_device *d) state = kzalloc(sizeof(struct bch_dirty_init_state), GFP_KERNEL); if (!state) { - pr_warn("sectors dirty init failed: cannot allocate memory"); + pr_warn("sectors dirty init failed: cannot allocate memory\n"); return; } @@ -945,7 +945,7 @@ void bch_sectors_dirty_init(struct bcache_device *d) &state->infos[i], name); if (IS_ERR(state->infos[i].thread)) { - pr_err("fails to run thread bch_dirty_init[%d]", i); + pr_err("fails to run thread bch_dirty_init[%d]\n", i); for (--i; i >= 0; i--) kthread_stop(state->infos[i].thread); goto out; From 86da9f736740eba602389908574dfbb0f517baa5 Mon Sep 17 00:00:00 2001 From: Coly Li Date: Wed, 27 May 2020 12:01:53 +0800 Subject: [PATCH 0882/1043] bcache: fix refcount underflow in bcache_device_free() The problematic code piece in bcache_device_free() is, 785 static void bcache_device_free(struct bcache_device *d) 786 { 787 struct gendisk *disk = d->disk; [snipped] 799 if (disk) { 800 if (disk->flags & GENHD_FL_UP) 801 del_gendisk(disk); 802 803 if (disk->queue) 804 blk_cleanup_queue(disk->queue); 805 806 ida_simple_remove(&bcache_device_idx, 807 first_minor_to_idx(disk->first_minor)); 808 put_disk(disk); 809 } [snipped] 816 } At line 808, put_disk(disk) may encounter kobject refcount of 'disk' being underflow. Here is how to reproduce the issue, - Attche the backing device to a cache device and do random write to make the cache being dirty. - Stop the bcache device while the cache device has dirty data of the backing device. - Only register the backing device back, NOT register cache device. - The bcache device node /dev/bcache0 won't show up, because backing device waits for the cache device shows up for the missing dirty data. - Now echo 1 into /sys/fs/bcache/pendings_cleanup, to stop the pending backing device. - After the pending backing device stopped, use 'dmesg' to check kernel message, a use-after-free warning from KASA reported the refcount of kobject linked to the 'disk' is underflow. The dropping refcount at line 808 in the above code piece is added by add_disk(d->disk) in bch_cached_dev_run(). But in the above condition the cache device is not registered, bch_cached_dev_run() has no chance to be called and the refcount is not added. The put_disk() for a non- added refcount of gendisk kobject triggers a underflow warning. This patch checks whether GENHD_FL_UP is set in disk->flags, if it is not set then the bcache device was not added, don't call put_disk() and the the underflow issue can be avoided. Signed-off-by: Coly Li Signed-off-by: Jens Axboe --- drivers/md/bcache/super.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 467149f3bcc5..a10a3c78f4ff 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -797,7 +797,9 @@ static void bcache_device_free(struct bcache_device *d) bcache_device_detach(d); if (disk) { - if (disk->flags & GENHD_FL_UP) + bool disk_added = (disk->flags & GENHD_FL_UP) != 0; + + if (disk_added) del_gendisk(disk); if (disk->queue) @@ -805,7 +807,8 @@ static void bcache_device_free(struct bcache_device *d) ida_simple_remove(&bcache_device_idx, first_minor_to_idx(disk->first_minor)); - put_disk(disk); + if (disk_added) + put_disk(disk); } bioset_exit(&d->bio_split); From 9e23ccf8f0a22e5b86a9e0d8ecbb49fe2fa73ae9 Mon Sep 17 00:00:00 2001 From: Coly Li Date: Wed, 27 May 2020 12:01:54 +0800 Subject: [PATCH 0883/1043] bcache: asynchronous devices registration When there is a lot of data cached on cache device, the bcach internal btree can take a very long to validate during the backing device and cache device registration. In my test, it may takes 55+ minutes to check all the internal btree nodes. The problem is that the registration is invoked by udev rules and the udevd has 180 seconds timeout by default. If the btree node checking time is longer than udevd timeout, the registering process will be killed by udevd with SIGKILL. If the registering process has pending sigal, creating kthread for bcache will fail and the device registration will fail. The result is, for bcache device which cached a lot of data on cache device, the bcache device node like /dev/bcache won't create always due to the very long btree checking time. A solution to avoid the udevd 180 seconds timeout is to register devices in an asynchronous way. Which is, after writing cache or backing device path into /sys/fs/bcache/register_async, the kernel code will create a kworker and move all the btree node checking (for cache device) or dirty data counting (for cached device) in the kwork context. Then the kworder is scheduled on system_wq and the registration code just returned to user space udev rule task. By this asynchronous way, the udev task for bcache rule will complete in seconds, no matter how long time spent in the kworker context, it won't be killed by udevd for a timeout. After all the checking and counting are done asynchronously in the kworker, the bcache device will eventually be created successfully. This patch does the above chagne and add a register sysfs file /sys/fs/bcache/register_async. Writing the registering device path into this sysfs file will do the asynchronous registration. The register_async interface is for very rare condition and won't be used for common users. In future I plan to make the asynchronous registration as default behavior, which depends on feedback for this patch. Signed-off-by: Coly Li Signed-off-by: Jens Axboe --- drivers/md/bcache/super.c | 100 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index a10a3c78f4ff..b971d8e916d5 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -2327,6 +2327,7 @@ static ssize_t bch_pending_bdevs_cleanup(struct kobject *k, kobj_attribute_write(register, register_bcache); kobj_attribute_write(register_quiet, register_bcache); +kobj_attribute_write(register_async, register_bcache); kobj_attribute_write(pendings_cleanup, bch_pending_bdevs_cleanup); static bool bch_is_open_backing(struct block_device *bdev) @@ -2362,6 +2363,83 @@ static bool bch_is_open(struct block_device *bdev) return bch_is_open_cache(bdev) || bch_is_open_backing(bdev); } +struct async_reg_args { + struct work_struct reg_work; + char *path; + struct cache_sb *sb; + struct cache_sb_disk *sb_disk; + struct block_device *bdev; +}; + +static void register_bdev_worker(struct work_struct *work) +{ + int fail = false; + struct async_reg_args *args = + container_of(work, struct async_reg_args, reg_work); + struct cached_dev *dc; + + dc = kzalloc(sizeof(*dc), GFP_KERNEL); + if (!dc) { + fail = true; + put_page(virt_to_page(args->sb_disk)); + blkdev_put(args->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); + goto out; + } + + mutex_lock(&bch_register_lock); + if (register_bdev(args->sb, args->sb_disk, args->bdev, dc) < 0) + fail = true; + mutex_unlock(&bch_register_lock); + +out: + if (fail) + pr_info("error %s: fail to register backing device\n", + args->path); + kfree(args->sb); + kfree(args->path); + kfree(args); + module_put(THIS_MODULE); +} + +static void register_cache_worker(struct work_struct *work) +{ + int fail = false; + struct async_reg_args *args = + container_of(work, struct async_reg_args, reg_work); + struct cache *ca; + + ca = kzalloc(sizeof(*ca), GFP_KERNEL); + if (!ca) { + fail = true; + put_page(virt_to_page(args->sb_disk)); + blkdev_put(args->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); + goto out; + } + + /* blkdev_put() will be called in bch_cache_release() */ + if (register_cache(args->sb, args->sb_disk, args->bdev, ca) != 0) + fail = true; + +out: + if (fail) + pr_info("error %s: fail to register cache device\n", + args->path); + kfree(args->sb); + kfree(args->path); + kfree(args); + module_put(THIS_MODULE); +} + +static void register_device_aync(struct async_reg_args *args) +{ + if (SB_IS_BDEV(args->sb)) + INIT_WORK(&args->reg_work, register_bdev_worker); + else + INIT_WORK(&args->reg_work, register_cache_worker); + + queue_work(system_wq, &args->reg_work); +} + static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr, const char *buffer, size_t size) { @@ -2424,6 +2502,26 @@ static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr, goto out_blkdev_put; err = "failed to register device"; + if (attr == &ksysfs_register_async) { + /* register in asynchronous way */ + struct async_reg_args *args = + kzalloc(sizeof(struct async_reg_args), GFP_KERNEL); + + if (!args) { + ret = -ENOMEM; + err = "cannot allocate memory"; + goto out_put_sb_page; + } + + args->path = path; + args->sb = sb; + args->sb_disk = sb_disk; + args->bdev = bdev; + register_device_aync(args); + /* No wait and returns to user space */ + goto async_done; + } + if (SB_IS_BDEV(sb)) { struct cached_dev *dc = kzalloc(sizeof(*dc), GFP_KERNEL); @@ -2451,6 +2549,7 @@ done: kfree(sb); kfree(path); module_put(THIS_MODULE); +async_done: return size; out_put_sb_page: @@ -2666,6 +2765,7 @@ static int __init bcache_init(void) static const struct attribute *files[] = { &ksysfs_register.attr, &ksysfs_register_quiet.attr, + &ksysfs_register_async.attr, &ksysfs_pendings_cleanup.attr, NULL }; From 0c8d3fceade2ab1bbac68bca013e62bfdb851d19 Mon Sep 17 00:00:00 2001 From: Coly Li Date: Wed, 27 May 2020 12:01:55 +0800 Subject: [PATCH 0884/1043] bcache: configure the asynchronous registertion to be experimental In order to avoid the experimental async registration interface to be treated as new kernel ABI for common users, this patch makes it as an experimental kernel configure BCACHE_ASYNC_REGISTRAION. This interface is for extreme large cached data situation, to make sure the bcache device can always created without the udev timeout issue. For normal users the async or sync registration does not make difference. In future when we decide to use the asynchronous registration as default behavior, this experimental interface may be removed. Signed-off-by: Coly Li Signed-off-by: Jens Axboe --- drivers/md/bcache/Kconfig | 9 +++++++++ drivers/md/bcache/super.c | 2 ++ 2 files changed, 11 insertions(+) diff --git a/drivers/md/bcache/Kconfig b/drivers/md/bcache/Kconfig index 6dfa653d30db..bf7dd96db9b3 100644 --- a/drivers/md/bcache/Kconfig +++ b/drivers/md/bcache/Kconfig @@ -26,3 +26,12 @@ config BCACHE_CLOSURES_DEBUG Keeps all active closures in a linked list and provides a debugfs interface to list them, which makes it possible to see asynchronous operations that get stuck. + +config BCACHE_ASYNC_REGISTRAION + bool "Asynchronous device registration (EXPERIMENTAL)" + depends on BCACHE + help + Add a sysfs file /sys/fs/bcache/register_async. Writing registering + device path into this file will returns immediately and the real + registration work is handled in kernel work queue in asynchronous + way. diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index b971d8e916d5..f9975c22bf7e 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -2765,7 +2765,9 @@ static int __init bcache_init(void) static const struct attribute *files[] = { &ksysfs_register.attr, &ksysfs_register_quiet.attr, +#ifdef CONFIG_BCACHE_ASYNC_REGISTRAION &ksysfs_register_async.attr, +#endif &ksysfs_pendings_cleanup.attr, NULL }; From 7d6d283777838462083412edb1e674a32eb6341c Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Wed, 27 May 2020 14:34:34 +0800 Subject: [PATCH 0885/1043] MIPS: Loongson64: select NO_EXCEPT_FILL Loongson64 load kernel at 0x82000000 and allocate exception vectors by ebase. So we don't need to reserve space for exception vectors at head of kernel. Signed-off-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index c0405999677a..b6338e806a4b 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -462,6 +462,7 @@ config MACH_LOONGSON64 select ISA select I8259 select IRQ_MIPS_CPU + select NO_EXCEPT_FILL select NR_CPUS_DEFAULT_64 select USE_GENERIC_EARLY_PRINTK_8250 select PCI_DRIVERS_GENERIC From dc3ffbb14060c943469d5e12900db3a60bc3fa64 Mon Sep 17 00:00:00 2001 From: Dave Chinner Date: Wed, 20 May 2020 13:17:10 -0700 Subject: [PATCH 0886/1043] xfs: gut error handling in xfs_trans_unreserve_and_mod_sb() xfs: gut error handling in xfs_trans_unreserve_and_mod_sb() From: Dave Chinner The error handling in xfs_trans_unreserve_and_mod_sb() is largely incorrect - rolling back the changes in the transaction if only one counter underruns makes all the other counters incorrect. We still allow the change to proceed and committing the transaction, except now we have multiple incorrect counters instead of a single underflow. Further, we don't actually report the error to the caller, so this is completely silent except on debug kernels that will assert on failure before we even get to the rollback code. Hence this error handling is broken, untested, and largely unnecessary complexity. Just remove it. Signed-off-by: Dave Chinner Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_trans.c | 163 ++++++--------------------------------------- 1 file changed, 20 insertions(+), 143 deletions(-) diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 28b983ff8b11..e4e29135ad1b 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -534,57 +534,9 @@ xfs_trans_apply_sb_deltas( sizeof(sbp->sb_frextents) - 1); } -STATIC int -xfs_sb_mod8( - uint8_t *field, - int8_t delta) -{ - int8_t counter = *field; - - counter += delta; - if (counter < 0) { - ASSERT(0); - return -EINVAL; - } - *field = counter; - return 0; -} - -STATIC int -xfs_sb_mod32( - uint32_t *field, - int32_t delta) -{ - int32_t counter = *field; - - counter += delta; - if (counter < 0) { - ASSERT(0); - return -EINVAL; - } - *field = counter; - return 0; -} - -STATIC int -xfs_sb_mod64( - uint64_t *field, - int64_t delta) -{ - int64_t counter = *field; - - counter += delta; - if (counter < 0) { - ASSERT(0); - return -EINVAL; - } - *field = counter; - return 0; -} - /* - * xfs_trans_unreserve_and_mod_sb() is called to release unused reservations - * and apply superblock counter changes to the in-core superblock. The + * xfs_trans_unreserve_and_mod_sb() is called to release unused reservations and + * apply superblock counter changes to the in-core superblock. The * t_res_fdblocks_delta and t_res_frextents_delta fields are explicitly NOT * applied to the in-core superblock. The idea is that that has already been * done. @@ -629,20 +581,17 @@ xfs_trans_unreserve_and_mod_sb( /* apply the per-cpu counters */ if (blkdelta) { error = xfs_mod_fdblocks(mp, blkdelta, rsvd); - if (error) - goto out; + ASSERT(!error); } if (idelta) { error = xfs_mod_icount(mp, idelta); - if (error) - goto out_undo_fdblocks; + ASSERT(!error); } if (ifreedelta) { error = xfs_mod_ifree(mp, ifreedelta); - if (error) - goto out_undo_icount; + ASSERT(!error); } if (rtxdelta == 0 && !(tp->t_flags & XFS_TRANS_SB_DIRTY)) @@ -650,95 +599,23 @@ xfs_trans_unreserve_and_mod_sb( /* apply remaining deltas */ spin_lock(&mp->m_sb_lock); - if (rtxdelta) { - error = xfs_sb_mod64(&mp->m_sb.sb_frextents, rtxdelta); - if (error) - goto out_undo_ifree; - } - - if (tp->t_dblocks_delta != 0) { - error = xfs_sb_mod64(&mp->m_sb.sb_dblocks, tp->t_dblocks_delta); - if (error) - goto out_undo_frextents; - } - if (tp->t_agcount_delta != 0) { - error = xfs_sb_mod32(&mp->m_sb.sb_agcount, tp->t_agcount_delta); - if (error) - goto out_undo_dblocks; - } - if (tp->t_imaxpct_delta != 0) { - error = xfs_sb_mod8(&mp->m_sb.sb_imax_pct, tp->t_imaxpct_delta); - if (error) - goto out_undo_agcount; - } - if (tp->t_rextsize_delta != 0) { - error = xfs_sb_mod32(&mp->m_sb.sb_rextsize, - tp->t_rextsize_delta); - if (error) - goto out_undo_imaxpct; - } - if (tp->t_rbmblocks_delta != 0) { - error = xfs_sb_mod32(&mp->m_sb.sb_rbmblocks, - tp->t_rbmblocks_delta); - if (error) - goto out_undo_rextsize; - } - if (tp->t_rblocks_delta != 0) { - error = xfs_sb_mod64(&mp->m_sb.sb_rblocks, tp->t_rblocks_delta); - if (error) - goto out_undo_rbmblocks; - } - if (tp->t_rextents_delta != 0) { - error = xfs_sb_mod64(&mp->m_sb.sb_rextents, - tp->t_rextents_delta); - if (error) - goto out_undo_rblocks; - } - if (tp->t_rextslog_delta != 0) { - error = xfs_sb_mod8(&mp->m_sb.sb_rextslog, - tp->t_rextslog_delta); - if (error) - goto out_undo_rextents; - } + mp->m_sb.sb_frextents += rtxdelta; + mp->m_sb.sb_dblocks += tp->t_dblocks_delta; + mp->m_sb.sb_agcount += tp->t_agcount_delta; + mp->m_sb.sb_imax_pct += tp->t_imaxpct_delta; + mp->m_sb.sb_rextsize += tp->t_rextsize_delta; + mp->m_sb.sb_rbmblocks += tp->t_rbmblocks_delta; + mp->m_sb.sb_rblocks += tp->t_rblocks_delta; + mp->m_sb.sb_rextents += tp->t_rextents_delta; + mp->m_sb.sb_rextslog += tp->t_rextslog_delta; spin_unlock(&mp->m_sb_lock); - return; -out_undo_rextents: - if (tp->t_rextents_delta) - xfs_sb_mod64(&mp->m_sb.sb_rextents, -tp->t_rextents_delta); -out_undo_rblocks: - if (tp->t_rblocks_delta) - xfs_sb_mod64(&mp->m_sb.sb_rblocks, -tp->t_rblocks_delta); -out_undo_rbmblocks: - if (tp->t_rbmblocks_delta) - xfs_sb_mod32(&mp->m_sb.sb_rbmblocks, -tp->t_rbmblocks_delta); -out_undo_rextsize: - if (tp->t_rextsize_delta) - xfs_sb_mod32(&mp->m_sb.sb_rextsize, -tp->t_rextsize_delta); -out_undo_imaxpct: - if (tp->t_rextsize_delta) - xfs_sb_mod8(&mp->m_sb.sb_imax_pct, -tp->t_imaxpct_delta); -out_undo_agcount: - if (tp->t_agcount_delta) - xfs_sb_mod32(&mp->m_sb.sb_agcount, -tp->t_agcount_delta); -out_undo_dblocks: - if (tp->t_dblocks_delta) - xfs_sb_mod64(&mp->m_sb.sb_dblocks, -tp->t_dblocks_delta); -out_undo_frextents: - if (rtxdelta) - xfs_sb_mod64(&mp->m_sb.sb_frextents, -rtxdelta); -out_undo_ifree: - spin_unlock(&mp->m_sb_lock); - if (ifreedelta) - xfs_mod_ifree(mp, -ifreedelta); -out_undo_icount: - if (idelta) - xfs_mod_icount(mp, -idelta); -out_undo_fdblocks: - if (blkdelta) - xfs_mod_fdblocks(mp, -blkdelta, rsvd); -out: - ASSERT(error == 0); + /* + * Debug checks outside of the spinlock so they don't lock up the + * machine if they fail. + */ + ASSERT(mp->m_sb.sb_imax_pct >= 0); + ASSERT(mp->m_sb.sb_rextslog >= 0); return; } From f18c9a9030972d892a244968c653aceb98e27c70 Mon Sep 17 00:00:00 2001 From: Dave Chinner Date: Wed, 20 May 2020 13:17:11 -0700 Subject: [PATCH 0887/1043] xfs: reduce free inode accounting overhead Shaokun Zhang reported that XFS was using substantial CPU time in percpu_count_sum() when running a single threaded benchmark on a high CPU count (128p) machine from xfs_mod_ifree(). The issue is that the filesystem is empty when the benchmark runs, so inode allocation is running with a very low inode free count. With the percpu counter batching, this means comparisons when the counter is less that 128 * 256 = 32768 use the slow path of adding up all the counters across the CPUs, and this is expensive on high CPU count machines. The summing in xfs_mod_ifree() is only used to fire an assert if an underrun occurs. The error is ignored by the higher level code. Hence this is really just debug code and we don't need to run it on production kernels, nor do we need such debug checks to return error values just to trigger an assert. Finally, xfs_mod_icount/xfs_mod_ifree are only called from xfs_trans_unreserve_and_mod_sb(), so get rid of them and just directly call the percpu_counter_add/percpu_counter_compare functions. The compare functions are now run only on debug builds as they are internal to ASSERT() checks and so only compiled in when ASSERTs are active (CONFIG_XFS_DEBUG=y or CONFIG_XFS_WARN=y). Reported-by: Shaokun Zhang Signed-off-by: Dave Chinner Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_mount.c | 33 --------------------------------- fs/xfs/xfs_mount.h | 2 -- fs/xfs/xfs_trans.c | 17 +++++++++++++---- 3 files changed, 13 insertions(+), 39 deletions(-) diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c index bb91f04266b9..d5dcf9869860 100644 --- a/fs/xfs/xfs_mount.c +++ b/fs/xfs/xfs_mount.c @@ -1189,39 +1189,6 @@ xfs_log_sbcount(xfs_mount_t *mp) return xfs_sync_sb(mp, true); } -/* - * Deltas for the inode count are +/-64, hence we use a large batch size - * of 128 so we don't need to take the counter lock on every update. - */ -#define XFS_ICOUNT_BATCH 128 -int -xfs_mod_icount( - struct xfs_mount *mp, - int64_t delta) -{ - percpu_counter_add_batch(&mp->m_icount, delta, XFS_ICOUNT_BATCH); - if (__percpu_counter_compare(&mp->m_icount, 0, XFS_ICOUNT_BATCH) < 0) { - ASSERT(0); - percpu_counter_add(&mp->m_icount, -delta); - return -EINVAL; - } - return 0; -} - -int -xfs_mod_ifree( - struct xfs_mount *mp, - int64_t delta) -{ - percpu_counter_add(&mp->m_ifree, delta); - if (percpu_counter_compare(&mp->m_ifree, 0) < 0) { - ASSERT(0); - percpu_counter_add(&mp->m_ifree, -delta); - return -EINVAL; - } - return 0; -} - /* * Deltas for the block count can vary from 1 to very large, but lock contention * only occurs on frequent small block count updates such as in the delayed diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index aba5a1579279..4835581f3eb0 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -392,8 +392,6 @@ extern int xfs_initialize_perag(xfs_mount_t *mp, xfs_agnumber_t agcount, xfs_agnumber_t *maxagi); extern void xfs_unmountfs(xfs_mount_t *); -extern int xfs_mod_icount(struct xfs_mount *mp, int64_t delta); -extern int xfs_mod_ifree(struct xfs_mount *mp, int64_t delta); extern int xfs_mod_fdblocks(struct xfs_mount *mp, int64_t delta, bool reserved); extern int xfs_mod_frextents(struct xfs_mount *mp, int64_t delta); diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index e4e29135ad1b..2222a0ed3155 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -545,7 +545,12 @@ xfs_trans_apply_sb_deltas( * used block counts are not updated in the on disk superblock. In this case, * XFS_TRANS_SB_DIRTY will not be set when the transaction is updated but we * still need to update the incore superblock with the changes. + * + * Deltas for the inode count are +/-64, hence we use a large batch size of 128 + * so we don't need to take the counter lock on every update. */ +#define XFS_ICOUNT_BATCH 128 + void xfs_trans_unreserve_and_mod_sb( struct xfs_trans *tp) @@ -585,13 +590,17 @@ xfs_trans_unreserve_and_mod_sb( } if (idelta) { - error = xfs_mod_icount(mp, idelta); - ASSERT(!error); + percpu_counter_add_batch(&mp->m_icount, idelta, + XFS_ICOUNT_BATCH); + if (idelta < 0) + ASSERT(__percpu_counter_compare(&mp->m_icount, 0, + XFS_ICOUNT_BATCH) >= 0); } if (ifreedelta) { - error = xfs_mod_ifree(mp, ifreedelta); - ASSERT(!error); + percpu_counter_add(&mp->m_ifree, ifreedelta); + if (ifreedelta < 0) + ASSERT(percpu_counter_compare(&mp->m_ifree, 0) >= 0); } if (rtxdelta == 0 && !(tp->t_flags & XFS_TRANS_SB_DIRTY)) From b0dff466c00975a3e3ec97e6b0266bfd3e4805d6 Mon Sep 17 00:00:00 2001 From: Dave Chinner Date: Wed, 20 May 2020 13:17:11 -0700 Subject: [PATCH 0888/1043] xfs: separate read-only variables in struct xfs_mount Seeing massive cpu usage from xfs_agino_range() on one machine; instruction level profiles look similar to another machine running the same workload, only one machine is consuming 10x as much CPU as the other and going much slower. The only real difference between the two machines is core count per socket. Both are running identical 16p/16GB virtual machine configurations Machine A: 25.83% [k] xfs_agino_range 12.68% [k] __xfs_dir3_data_check 6.95% [k] xfs_verify_ino 6.78% [k] xfs_dir2_data_entry_tag_p 3.56% [k] xfs_buf_find 2.31% [k] xfs_verify_dir_ino 2.02% [k] xfs_dabuf_map.constprop.0 1.65% [k] xfs_ag_block_count And takes around 13 minutes to remove 50 million inodes. Machine B: 13.90% [k] __pv_queued_spin_lock_slowpath 3.76% [k] do_raw_spin_lock 2.83% [k] xfs_dir3_leaf_check_int 2.75% [k] xfs_agino_range 2.51% [k] __raw_callee_save___pv_queued_spin_unlock 2.18% [k] __xfs_dir3_data_check 2.02% [k] xfs_log_commit_cil And takes around 5m30s to remove 50 million inodes. Suspect is cacheline contention on m_sectbb_log which is used in one of the macros in xfs_agino_range. This is a read-only variable but shares a cacheline with m_active_trans which is a global atomic that gets bounced all around the machine. The workload is trying to run hundreds of thousands of transactions per second and hence cacheline contention will be occurring on this atomic counter. Hence xfs_agino_range() is likely just be an innocent bystander as the cache coherency protocol fights over the cacheline between CPU cores and sockets. On machine A, this rearrangement of the struct xfs_mount results in the profile changing to: 9.77% [kernel] [k] xfs_agino_range 6.27% [kernel] [k] __xfs_dir3_data_check 5.31% [kernel] [k] __pv_queued_spin_lock_slowpath 4.54% [kernel] [k] xfs_buf_find 3.79% [kernel] [k] do_raw_spin_lock 3.39% [kernel] [k] xfs_verify_ino 2.73% [kernel] [k] __raw_callee_save___pv_queued_spin_unlock Vastly less CPU usage in xfs_agino_range(), but still 3x the amount of machine B and still runs substantially slower than it should. Current rm -rf of 50 million files: vanilla patched machine A 13m20s 6m42s machine B 5m30s 5m02s It's an improvement, hence indicating that separation and further optimisation of read-only global filesystem data is worthwhile, but it clearly isn't the underlying issue causing this specific performance degradation. Signed-off-by: Dave Chinner Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_mount.h | 152 +++++++++++++++++++++++++-------------------- 1 file changed, 84 insertions(+), 68 deletions(-) diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index 4835581f3eb0..c1f92c1847bb 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -55,61 +55,25 @@ struct xfs_error_cfg { long retry_timeout; /* in jiffies, -1 = infinite */ }; +/* + * The struct xfsmount layout is optimised to separate read-mostly variables + * from variables that are frequently modified. We put the read-mostly variables + * first, then place all the other variables at the end. + * + * Typically, read-mostly variables are those that are set at mount time and + * never changed again, or only change rarely as a result of things like sysfs + * knobs being tweaked. + */ typedef struct xfs_mount { - struct super_block *m_super; - - /* - * Bitsets of per-fs metadata that have been checked and/or are sick. - * Callers must hold m_sb_lock to access these two fields. - */ - uint8_t m_fs_checked; - uint8_t m_fs_sick; - /* - * Bitsets of rt metadata that have been checked and/or are sick. - * Callers must hold m_sb_lock to access this field. - */ - uint8_t m_rt_checked; - uint8_t m_rt_sick; - - struct xfs_ail *m_ail; /* fs active log item list */ - struct xfs_sb m_sb; /* copy of fs superblock */ - spinlock_t m_sb_lock; /* sb counter lock */ - struct percpu_counter m_icount; /* allocated inodes counter */ - struct percpu_counter m_ifree; /* free inodes counter */ - struct percpu_counter m_fdblocks; /* free block counter */ - /* - * Count of data device blocks reserved for delayed allocations, - * including indlen blocks. Does not include allocated CoW staging - * extents or anything related to the rt device. - */ - struct percpu_counter m_delalloc_blks; - + struct super_block *m_super; + struct xfs_ail *m_ail; /* fs active log item list */ struct xfs_buf *m_sb_bp; /* buffer for superblock */ char *m_rtname; /* realtime device name */ char *m_logname; /* external log device name */ - int m_bsize; /* fs logical block size */ - xfs_agnumber_t m_agfrotor; /* last ag where space found */ - xfs_agnumber_t m_agirotor; /* last ag dir inode alloced */ - spinlock_t m_agirotor_lock;/* .. and lock protecting it */ - xfs_agnumber_t m_maxagi; /* highest inode alloc group */ - uint m_allocsize_log;/* min write size log bytes */ - uint m_allocsize_blocks; /* min write size blocks */ struct xfs_da_geometry *m_dir_geo; /* directory block geometry */ struct xfs_da_geometry *m_attr_geo; /* attribute block geometry */ struct xlog *m_log; /* log specific stuff */ - struct xfs_ino_geometry m_ino_geo; /* inode geometry */ - int m_logbufs; /* number of log buffers */ - int m_logbsize; /* size of each log buffer */ - uint m_rsumlevels; /* rt summary levels */ - uint m_rsumsize; /* size of rt summary, bytes */ - /* - * Optional cache of rt summary level per bitmap block with the - * invariant that m_rsum_cache[bbno] <= the minimum i for which - * rsum[i][bbno] != 0. Reads and writes are serialized by the rsumip - * inode lock. - */ - uint8_t *m_rsum_cache; struct xfs_inode *m_rbmip; /* pointer to bitmap inode */ struct xfs_inode *m_rsumip; /* pointer to summary inode */ struct xfs_inode *m_rootip; /* pointer to root directory */ @@ -117,9 +81,26 @@ typedef struct xfs_mount { xfs_buftarg_t *m_ddev_targp; /* saves taking the address */ xfs_buftarg_t *m_logdev_targp;/* ptr to log device */ xfs_buftarg_t *m_rtdev_targp; /* ptr to rt device */ + /* + * Optional cache of rt summary level per bitmap block with the + * invariant that m_rsum_cache[bbno] <= the minimum i for which + * rsum[i][bbno] != 0. Reads and writes are serialized by the rsumip + * inode lock. + */ + uint8_t *m_rsum_cache; + struct xfs_mru_cache *m_filestream; /* per-mount filestream data */ + struct workqueue_struct *m_buf_workqueue; + struct workqueue_struct *m_unwritten_workqueue; + struct workqueue_struct *m_cil_workqueue; + struct workqueue_struct *m_reclaim_workqueue; + struct workqueue_struct *m_eofblocks_workqueue; + struct workqueue_struct *m_sync_workqueue; + + int m_bsize; /* fs logical block size */ uint8_t m_blkbit_log; /* blocklog + NBBY */ uint8_t m_blkbb_log; /* blocklog - BBSHIFT */ uint8_t m_agno_log; /* log #ag's */ + uint8_t m_sectbb_log; /* sectlog - BBSHIFT */ uint m_blockmask; /* sb_blocksize-1 */ uint m_blockwsize; /* sb_blocksize in words */ uint m_blockwmask; /* blockwsize-1 */ @@ -138,47 +119,83 @@ typedef struct xfs_mount { xfs_extlen_t m_ag_prealloc_blocks; /* reserved ag blocks */ uint m_alloc_set_aside; /* space we can't use */ uint m_ag_max_usable; /* max space per AG */ + int m_dalign; /* stripe unit */ + int m_swidth; /* stripe width */ + xfs_agnumber_t m_maxagi; /* highest inode alloc group */ + uint m_allocsize_log;/* min write size log bytes */ + uint m_allocsize_blocks; /* min write size blocks */ + int m_logbufs; /* number of log buffers */ + int m_logbsize; /* size of each log buffer */ + uint m_rsumlevels; /* rt summary levels */ + uint m_rsumsize; /* size of rt summary, bytes */ + int m_fixedfsid[2]; /* unchanged for life of FS */ + uint m_qflags; /* quota status flags */ + uint64_t m_flags; /* global mount flags */ + int64_t m_low_space[XFS_LOWSP_MAX]; + struct xfs_ino_geometry m_ino_geo; /* inode geometry */ + struct xfs_trans_resv m_resv; /* precomputed res values */ + /* low free space thresholds */ + bool m_always_cow; + bool m_fail_unmount; + bool m_finobt_nores; /* no per-AG finobt resv. */ + bool m_update_sb; /* sb needs update in mount */ + + /* + * Bitsets of per-fs metadata that have been checked and/or are sick. + * Callers must hold m_sb_lock to access these two fields. + */ + uint8_t m_fs_checked; + uint8_t m_fs_sick; + /* + * Bitsets of rt metadata that have been checked and/or are sick. + * Callers must hold m_sb_lock to access this field. + */ + uint8_t m_rt_checked; + uint8_t m_rt_sick; + + /* + * End of read-mostly variables. Frequently written variables and locks + * should be placed below this comment from now on. The first variable + * here is marked as cacheline aligned so they it is separated from + * the read-mostly variables. + */ + + spinlock_t ____cacheline_aligned m_sb_lock; /* sb counter lock */ + struct percpu_counter m_icount; /* allocated inodes counter */ + struct percpu_counter m_ifree; /* free inodes counter */ + struct percpu_counter m_fdblocks; /* free block counter */ + /* + * Count of data device blocks reserved for delayed allocations, + * including indlen blocks. Does not include allocated CoW staging + * extents or anything related to the rt device. + */ + struct percpu_counter m_delalloc_blks; + struct radix_tree_root m_perag_tree; /* per-ag accounting info */ spinlock_t m_perag_lock; /* lock for m_perag_tree */ - struct mutex m_growlock; /* growfs mutex */ - int m_fixedfsid[2]; /* unchanged for life of FS */ - uint64_t m_flags; /* global mount flags */ - bool m_finobt_nores; /* no per-AG finobt resv. */ - uint m_qflags; /* quota status flags */ - struct xfs_trans_resv m_resv; /* precomputed res values */ uint64_t m_resblks; /* total reserved blocks */ uint64_t m_resblks_avail;/* available reserved blocks */ uint64_t m_resblks_save; /* reserved blks @ remount,ro */ - int m_dalign; /* stripe unit */ - int m_swidth; /* stripe width */ - uint8_t m_sectbb_log; /* sectlog - BBSHIFT */ atomic_t m_active_trans; /* number trans frozen */ - struct xfs_mru_cache *m_filestream; /* per-mount filestream data */ struct delayed_work m_reclaim_work; /* background inode reclaim */ struct delayed_work m_eofblocks_work; /* background eof blocks trimming */ struct delayed_work m_cowblocks_work; /* background cow blocks trimming */ - bool m_update_sb; /* sb needs update in mount */ - int64_t m_low_space[XFS_LOWSP_MAX]; - /* low free space thresholds */ struct xfs_kobj m_kobj; struct xfs_kobj m_error_kobj; struct xfs_kobj m_error_meta_kobj; struct xfs_error_cfg m_error_cfg[XFS_ERR_CLASS_MAX][XFS_ERR_ERRNO_MAX]; struct xstats m_stats; /* per-fs stats */ + xfs_agnumber_t m_agfrotor; /* last ag where space found */ + xfs_agnumber_t m_agirotor; /* last ag dir inode alloced */ + spinlock_t m_agirotor_lock;/* .. and lock protecting it */ /* * Workqueue item so that we can coalesce multiple inode flush attempts * into a single flush. */ struct work_struct m_flush_inodes_work; - struct workqueue_struct *m_buf_workqueue; - struct workqueue_struct *m_unwritten_workqueue; - struct workqueue_struct *m_cil_workqueue; - struct workqueue_struct *m_reclaim_workqueue; - struct workqueue_struct *m_eofblocks_workqueue; - struct workqueue_struct *m_sync_workqueue; /* * Generation of the filesysyem layout. This is incremented by each @@ -190,9 +207,8 @@ typedef struct xfs_mount { * to various other kinds of pain inflicted on the pNFS server. */ uint32_t m_generation; + struct mutex m_growlock; /* growfs mutex */ - bool m_always_cow; - bool m_fail_unmount; #ifdef DEBUG /* * Frequency with which errors are injected. Replaces xfs_etest; the From b41b46c20c0bd32cd0a3795fcd2b892213cb6f5e Mon Sep 17 00:00:00 2001 From: Dave Chinner Date: Wed, 20 May 2020 13:17:11 -0700 Subject: [PATCH 0889/1043] xfs: remove the m_active_trans counter It's a global atomic counter, and we are hitting it at a rate of half a million transactions a second, so it's bouncing the counter cacheline all over the place on large machines. We don't actually need it anymore - it used to be required because the VFS freeze code could not track/prevent filesystem transactions that were running, but that problem no longer exists. Hence to remove the counter, we simply have to ensure that nothing calls xfs_sync_sb() while we are trying to quiesce the filesytem. That only happens if the log worker is still running when we call xfs_quiesce_attr(). The log worker is cancelled at the end of xfs_quiesce_attr() by calling xfs_log_quiesce(), so just call it early here and then we can remove the counter altogether. Concurrent create, 50 million inodes, identical 16p/16GB virtual machines on different physical hosts. Machine A has twice the CPU cores per socket of machine B: unpatched patched machine A: 3m16s 2m00s machine B: 4m04s 4m05s Create rates: unpatched patched machine A: 282k+/-31k 468k+/-21k machine B: 231k+/-8k 233k+/-11k Concurrent rm of same 50 million inodes: unpatched patched machine A: 6m42s 2m33s machine B: 4m47s 4m47s The transaction rate on the fast machine went from just under 300k/sec to 700k/sec, which indicates just how much of a bottleneck this atomic counter was. Signed-off-by: Dave Chinner Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_mount.h | 1 - fs/xfs/xfs_super.c | 17 +++++------------ fs/xfs/xfs_trans.c | 27 +++++++++++---------------- 3 files changed, 16 insertions(+), 29 deletions(-) diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h index c1f92c1847bb..3725d25ad97e 100644 --- a/fs/xfs/xfs_mount.h +++ b/fs/xfs/xfs_mount.h @@ -176,7 +176,6 @@ typedef struct xfs_mount { uint64_t m_resblks; /* total reserved blocks */ uint64_t m_resblks_avail;/* available reserved blocks */ uint64_t m_resblks_save; /* reserved blks @ remount,ro */ - atomic_t m_active_trans; /* number trans frozen */ struct delayed_work m_reclaim_work; /* background inode reclaim */ struct delayed_work m_eofblocks_work; /* background eof blocks trimming */ diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index aae469f73efe..fa58cb07c8fd 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -874,8 +874,10 @@ xfs_restore_resvblks(struct xfs_mount *mp) * there is no log replay required to write the inodes to disk - this is the * primary difference between a sync and a quiesce. * - * Note: xfs_log_quiesce() stops background log work - the callers must ensure - * it is started again when appropriate. + * We cancel log work early here to ensure all transactions the log worker may + * run have finished before we clean up and log the superblock and write an + * unmount record. The unfreeze process is responsible for restarting the log + * worker correctly. */ void xfs_quiesce_attr( @@ -883,9 +885,7 @@ xfs_quiesce_attr( { int error = 0; - /* wait for all modifications to complete */ - while (atomic_read(&mp->m_active_trans) > 0) - delay(100); + cancel_delayed_work_sync(&mp->m_log->l_work); /* force the log to unpin objects from the now complete transactions */ xfs_log_force(mp, XFS_LOG_SYNC); @@ -899,12 +899,6 @@ xfs_quiesce_attr( if (error) xfs_warn(mp, "xfs_attr_quiesce: failed to log sb changes. " "Frozen image may not be consistent."); - /* - * Just warn here till VFS can correctly support - * read-only remount without racing. - */ - WARN_ON(atomic_read(&mp->m_active_trans) != 0); - xfs_log_quiesce(mp); } @@ -1793,7 +1787,6 @@ static int xfs_init_fs_context( INIT_RADIX_TREE(&mp->m_perag_tree, GFP_ATOMIC); spin_lock_init(&mp->m_perag_lock); mutex_init(&mp->m_growlock); - atomic_set(&mp->m_active_trans, 0); INIT_WORK(&mp->m_flush_inodes_work, xfs_flush_inodes_worker); INIT_DELAYED_WORK(&mp->m_reclaim_work, xfs_reclaim_worker); INIT_DELAYED_WORK(&mp->m_eofblocks_work, xfs_eofblocks_worker); diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c index 2222a0ed3155..3c94e5ff4316 100644 --- a/fs/xfs/xfs_trans.c +++ b/fs/xfs/xfs_trans.c @@ -68,7 +68,6 @@ xfs_trans_free( xfs_extent_busy_clear(tp->t_mountp, &tp->t_busy, false); trace_xfs_trans_free(tp, _RET_IP_); - atomic_dec(&tp->t_mountp->m_active_trans); if (!(tp->t_flags & XFS_TRANS_NO_WRITECOUNT)) sb_end_intwrite(tp->t_mountp->m_super); xfs_trans_free_dqinfo(tp); @@ -125,8 +124,6 @@ xfs_trans_dup( xfs_defer_move(ntp, tp); xfs_trans_dup_dqinfo(tp, ntp); - - atomic_inc(&tp->t_mountp->m_active_trans); return ntp; } @@ -275,7 +272,6 @@ xfs_trans_alloc( */ WARN_ON(resp->tr_logres > 0 && mp->m_super->s_writers.frozen == SB_FREEZE_COMPLETE); - atomic_inc(&mp->m_active_trans); tp->t_magic = XFS_TRANS_HEADER_MAGIC; tp->t_flags = flags; @@ -299,20 +295,19 @@ xfs_trans_alloc( /* * Create an empty transaction with no reservation. This is a defensive - * mechanism for routines that query metadata without actually modifying - * them -- if the metadata being queried is somehow cross-linked (think a - * btree block pointer that points higher in the tree), we risk deadlock. - * However, blocks grabbed as part of a transaction can be re-grabbed. - * The verifiers will notice the corrupt block and the operation will fail - * back to userspace without deadlocking. + * mechanism for routines that query metadata without actually modifying them -- + * if the metadata being queried is somehow cross-linked (think a btree block + * pointer that points higher in the tree), we risk deadlock. However, blocks + * grabbed as part of a transaction can be re-grabbed. The verifiers will + * notice the corrupt block and the operation will fail back to userspace + * without deadlocking. * - * Note the zero-length reservation; this transaction MUST be cancelled - * without any dirty data. + * Note the zero-length reservation; this transaction MUST be cancelled without + * any dirty data. * - * Callers should obtain freeze protection to avoid two conflicts with fs - * freezing: (1) having active transactions trip the m_active_trans ASSERTs; - * and (2) grabbing buffers at the same time that freeze is trying to drain - * the buffer LRU list. + * Callers should obtain freeze protection to avoid a conflict with fs freezing + * where we can be grabbing buffers at the same time that freeze is trying to + * drain the buffer LRU list. */ int xfs_trans_alloc_empty( From c8d329f311c4d3d8f8e6dc5897ec235e37f48ae8 Mon Sep 17 00:00:00 2001 From: Eric Sandeen Date: Thu, 21 May 2020 13:06:59 -0700 Subject: [PATCH 0890/1043] xfs: group quota should return EDQUOT when prj quota enabled Long ago, group & project quota were mutually exclusive, and so when we turned on XFS_QMOPT_ENOSPC ("return ENOSPC if project quota is exceeded") when project quota was enabled, we only needed to disable it again for user quota. When group & project quota got separated, this got missed, and as a result if project quota is enabled and group quota is exceeded, the error code returned is incorrectly returned as ENOSPC not EDQUOT. Fix this by stripping XFS_QMOPT_ENOSPC out of flags for group quota when we try to reserve the space. Signed-off-by: Eric Sandeen Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_trans_dquot.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/xfs/xfs_trans_dquot.c b/fs/xfs/xfs_trans_dquot.c index d1b9869bc5fa..2c3557a80e69 100644 --- a/fs/xfs/xfs_trans_dquot.c +++ b/fs/xfs/xfs_trans_dquot.c @@ -758,7 +758,8 @@ xfs_trans_reserve_quota_bydquots( } if (gdqp) { - error = xfs_trans_dqresv(tp, mp, gdqp, nblks, ninos, flags); + error = xfs_trans_dqresv(tp, mp, gdqp, nblks, ninos, + (flags & ~XFS_QMOPT_ENOSPC)); if (error) goto unwind_usr; } From dcf1ccc99e6db06a3a3cc9f72161f7d084a38d40 Mon Sep 17 00:00:00 2001 From: Eric Sandeen Date: Thu, 21 May 2020 13:06:59 -0700 Subject: [PATCH 0891/1043] xfs: always return -ENOSPC on project quota reservation failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit XFS project quota treats project hierarchies as "mini filesysems" and so rather than -EDQUOT, the intent is to return -ENOSPC when a quota reservation fails, but this behavior is not consistent. The only place we make a decision between -EDQUOT and -ENOSPC returns based on quota type is in xfs_trans_dqresv(). This behavior is currently controlled by whether or not the XFS_QMOPT_ENOSPC flag gets passed into the quota reservation. However, its use is not consistent; paths such as xfs_create() and xfs_symlink() don't set the flag, so a reservation failure will return -EDQUOT for project quota reservation failures rather than -ENOSPC for these sorts of operations, even for project quota: # mkdir mnt/project # xfs_quota -x -c "project -s -p mnt/project 42" mnt # xfs_quota -x -c 'limit -p isoft=2 ihard=3 42' mnt # touch mnt/project/file{1,2,3} touch: cannot touch ‘mnt/project/file3’: Disk quota exceeded We can make this consistent by not requiring the flag to be set at the top of the callchain; instead we can simply test whether we are reserving a project quota with XFS_QM_ISPDQ in xfs_trans_dqresv and if so, return -ENOSPC for that failure. This removes the need for the XFS_QMOPT_ENOSPC altogether and simplifies the code a fair bit. Signed-off-by: Eric Sandeen Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_quota_defs.h | 1 - fs/xfs/xfs_qm.c | 9 +++------ fs/xfs/xfs_trans_dquot.c | 16 +++++----------- 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/fs/xfs/libxfs/xfs_quota_defs.h b/fs/xfs/libxfs/xfs_quota_defs.h index b2113b17e53c..56d9dd787e7b 100644 --- a/fs/xfs/libxfs/xfs_quota_defs.h +++ b/fs/xfs/libxfs/xfs_quota_defs.h @@ -100,7 +100,6 @@ typedef uint16_t xfs_qwarncnt_t; #define XFS_QMOPT_FORCE_RES 0x0000010 /* ignore quota limits */ #define XFS_QMOPT_SBVERSION 0x0000040 /* change superblock version num */ #define XFS_QMOPT_GQUOTA 0x0002000 /* group dquot requested */ -#define XFS_QMOPT_ENOSPC 0x0004000 /* enospc instead of edquot (prj) */ /* * flags to xfs_trans_mod_dquot to indicate which field needs to be diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index fc93f88a9926..27a907645631 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -1808,7 +1808,7 @@ xfs_qm_vop_chown_reserve( { struct xfs_mount *mp = ip->i_mount; uint64_t delblks; - unsigned int blkflags, prjflags = 0; + unsigned int blkflags; struct xfs_dquot *udq_unres = NULL; struct xfs_dquot *gdq_unres = NULL; struct xfs_dquot *pdq_unres = NULL; @@ -1849,7 +1849,6 @@ xfs_qm_vop_chown_reserve( if (XFS_IS_PQUOTA_ON(ip->i_mount) && pdqp && ip->i_d.di_projid != be32_to_cpu(pdqp->q_core.d_id)) { - prjflags = XFS_QMOPT_ENOSPC; pdq_delblks = pdqp; if (delblks) { ASSERT(ip->i_pdquot); @@ -1859,8 +1858,7 @@ xfs_qm_vop_chown_reserve( error = xfs_trans_reserve_quota_bydquots(tp, ip->i_mount, udq_delblks, gdq_delblks, pdq_delblks, - ip->i_d.di_nblocks, 1, - flags | blkflags | prjflags); + ip->i_d.di_nblocks, 1, flags | blkflags); if (error) return error; @@ -1878,8 +1876,7 @@ xfs_qm_vop_chown_reserve( ASSERT(udq_unres || gdq_unres || pdq_unres); error = xfs_trans_reserve_quota_bydquots(NULL, ip->i_mount, udq_delblks, gdq_delblks, pdq_delblks, - (xfs_qcnt_t)delblks, 0, - flags | blkflags | prjflags); + (xfs_qcnt_t)delblks, 0, flags | blkflags); if (error) return error; xfs_trans_reserve_quota_bydquots(NULL, ip->i_mount, diff --git a/fs/xfs/xfs_trans_dquot.c b/fs/xfs/xfs_trans_dquot.c index 2c3557a80e69..2c07897a3c37 100644 --- a/fs/xfs/xfs_trans_dquot.c +++ b/fs/xfs/xfs_trans_dquot.c @@ -711,7 +711,7 @@ xfs_trans_dqresv( error_return: xfs_dqunlock(dqp); - if (flags & XFS_QMOPT_ENOSPC) + if (XFS_QM_ISPDQ(dqp)) return -ENOSPC; return -EDQUOT; } @@ -751,15 +751,13 @@ xfs_trans_reserve_quota_bydquots( ASSERT(flags & XFS_QMOPT_RESBLK_MASK); if (udqp) { - error = xfs_trans_dqresv(tp, mp, udqp, nblks, ninos, - (flags & ~XFS_QMOPT_ENOSPC)); + error = xfs_trans_dqresv(tp, mp, udqp, nblks, ninos, flags); if (error) return error; } if (gdqp) { - error = xfs_trans_dqresv(tp, mp, gdqp, nblks, ninos, - (flags & ~XFS_QMOPT_ENOSPC)); + error = xfs_trans_dqresv(tp, mp, gdqp, nblks, ninos, flags); if (error) goto unwind_usr; } @@ -804,16 +802,12 @@ xfs_trans_reserve_quota_nblks( if (!XFS_IS_QUOTA_RUNNING(mp) || !XFS_IS_QUOTA_ON(mp)) return 0; - if (XFS_IS_PQUOTA_ON(mp)) - flags |= XFS_QMOPT_ENOSPC; ASSERT(!xfs_is_quota_inode(&mp->m_sb, ip->i_ino)); ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL)); - ASSERT((flags & ~(XFS_QMOPT_FORCE_RES | XFS_QMOPT_ENOSPC)) == - XFS_TRANS_DQ_RES_RTBLKS || - (flags & ~(XFS_QMOPT_FORCE_RES | XFS_QMOPT_ENOSPC)) == - XFS_TRANS_DQ_RES_BLKS); + ASSERT((flags & ~(XFS_QMOPT_FORCE_RES)) == XFS_TRANS_DQ_RES_RTBLKS || + (flags & ~(XFS_QMOPT_FORCE_RES)) == XFS_TRANS_DQ_RES_BLKS); /* * Reserve nblks against these dquots, with trans as the mediator. From 8d077f5bfc273a85ea4c665d4f15b7fd06b97c20 Mon Sep 17 00:00:00 2001 From: Eric Sandeen Date: Thu, 21 May 2020 13:07:00 -0700 Subject: [PATCH 0892/1043] xfs: fix up some whitespace in quota code There is a fair bit of whitespace damage in the quota code, so fix up enough of it that subsequent patches are restricted to functional change to aid review. Signed-off-by: Eric Sandeen Reviewed-by: Allison Collins Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_qm.h | 44 +++++++++++++++++++++---------------------- fs/xfs/xfs_quotaops.c | 8 ++++---- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/fs/xfs/xfs_qm.h b/fs/xfs/xfs_qm.h index 4e57edca8bce..3a850401b102 100644 --- a/fs/xfs/xfs_qm.h +++ b/fs/xfs/xfs_qm.h @@ -42,12 +42,12 @@ extern struct kmem_zone *xfs_qm_dqtrxzone; #define XFS_DQUOT_CLUSTER_SIZE_FSB (xfs_filblks_t)1 struct xfs_def_quota { - xfs_qcnt_t bhardlimit; /* default data blk hard limit */ - xfs_qcnt_t bsoftlimit; /* default data blk soft limit */ - xfs_qcnt_t ihardlimit; /* default inode count hard limit */ - xfs_qcnt_t isoftlimit; /* default inode count soft limit */ - xfs_qcnt_t rtbhardlimit; /* default realtime blk hard limit */ - xfs_qcnt_t rtbsoftlimit; /* default realtime blk soft limit */ + xfs_qcnt_t bhardlimit; /* default data blk hard limit */ + xfs_qcnt_t bsoftlimit; /* default data blk soft limit */ + xfs_qcnt_t ihardlimit; /* default inode count hard limit */ + xfs_qcnt_t isoftlimit; /* default inode count soft limit */ + xfs_qcnt_t rtbhardlimit; /* default realtime blk hard limit */ + xfs_qcnt_t rtbsoftlimit; /* default realtime blk soft limit */ }; /* @@ -55,28 +55,28 @@ struct xfs_def_quota { * The mount structure keeps a pointer to this. */ struct xfs_quotainfo { - struct radix_tree_root qi_uquota_tree; - struct radix_tree_root qi_gquota_tree; - struct radix_tree_root qi_pquota_tree; - struct mutex qi_tree_lock; + struct radix_tree_root qi_uquota_tree; + struct radix_tree_root qi_gquota_tree; + struct radix_tree_root qi_pquota_tree; + struct mutex qi_tree_lock; struct xfs_inode *qi_uquotaip; /* user quota inode */ struct xfs_inode *qi_gquotaip; /* group quota inode */ struct xfs_inode *qi_pquotaip; /* project quota inode */ - struct list_lru qi_lru; - int qi_dquots; - time64_t qi_btimelimit; /* limit for blks timer */ - time64_t qi_itimelimit; /* limit for inodes timer */ - time64_t qi_rtbtimelimit;/* limit for rt blks timer */ - xfs_qwarncnt_t qi_bwarnlimit; /* limit for blks warnings */ - xfs_qwarncnt_t qi_iwarnlimit; /* limit for inodes warnings */ - xfs_qwarncnt_t qi_rtbwarnlimit;/* limit for rt blks warnings */ - struct mutex qi_quotaofflock;/* to serialize quotaoff */ - xfs_filblks_t qi_dqchunklen; /* # BBs in a chunk of dqs */ - uint qi_dqperchunk; /* # ondisk dqs in above chunk */ + struct list_lru qi_lru; + int qi_dquots; + time64_t qi_btimelimit; /* limit for blks timer */ + time64_t qi_itimelimit; /* limit for inodes timer */ + time64_t qi_rtbtimelimit;/* limit for rt blks timer */ + xfs_qwarncnt_t qi_bwarnlimit; /* limit for blks warnings */ + xfs_qwarncnt_t qi_iwarnlimit; /* limit for inodes warnings */ + xfs_qwarncnt_t qi_rtbwarnlimit;/* limit for rt blks warnings */ + struct mutex qi_quotaofflock;/* to serialize quotaoff */ + xfs_filblks_t qi_dqchunklen; /* # BBs in a chunk of dqs */ + uint qi_dqperchunk; /* # ondisk dq in above chunk */ struct xfs_def_quota qi_usr_default; struct xfs_def_quota qi_grp_default; struct xfs_def_quota qi_prj_default; - struct shrinker qi_shrinker; + struct shrinker qi_shrinker; }; static inline struct radix_tree_root * diff --git a/fs/xfs/xfs_quotaops.c b/fs/xfs/xfs_quotaops.c index b5d10ecb5474..411eeefa2a5c 100644 --- a/fs/xfs/xfs_quotaops.c +++ b/fs/xfs/xfs_quotaops.c @@ -23,8 +23,8 @@ xfs_qm_fill_state( struct xfs_inode *ip, xfs_ino_t ino) { - struct xfs_quotainfo *q = mp->m_quotainfo; - bool tempqip = false; + struct xfs_quotainfo *q = mp->m_quotainfo; + bool tempqip = false; tstate->ino = ino; if (!ip && ino == NULLFSINO) @@ -109,8 +109,8 @@ xfs_fs_set_info( int type, struct qc_info *info) { - struct xfs_mount *mp = XFS_M(sb); - struct qc_dqblk newlim; + struct xfs_mount *mp = XFS_M(sb); + struct qc_dqblk newlim; if (sb_rdonly(sb)) return -EROFS; From 3dbb9aa310089702ac1023296d26672f36ea4096 Mon Sep 17 00:00:00 2001 From: Eric Sandeen Date: Thu, 21 May 2020 13:07:00 -0700 Subject: [PATCH 0893/1043] xfs: pass xfs_dquot to xfs_qm_adjust_dqtimers Pass xfs_dquot rather than xfs_disk_dquot to xfs_qm_adjust_dqtimers; this makes it symmetric with xfs_qm_adjust_dqlimits and will help the next patch. Signed-off-by: Eric Sandeen Reviewed-by: Allison Collins Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_dquot.c | 3 ++- fs/xfs/xfs_dquot.h | 2 +- fs/xfs/xfs_qm.c | 2 +- fs/xfs/xfs_qm_syscalls.c | 2 +- fs/xfs/xfs_trans_dquot.c | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 55b95d45303b..714eceacbab2 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -114,8 +114,9 @@ xfs_qm_adjust_dqlimits( void xfs_qm_adjust_dqtimers( struct xfs_mount *mp, - struct xfs_disk_dquot *d) + struct xfs_dquot *dq) { + struct xfs_disk_dquot *d = &dq->q_core; ASSERT(d->d_id); #ifdef DEBUG diff --git a/fs/xfs/xfs_dquot.h b/fs/xfs/xfs_dquot.h index fe3e46df604b..71e36c85e20b 100644 --- a/fs/xfs/xfs_dquot.h +++ b/fs/xfs/xfs_dquot.h @@ -154,7 +154,7 @@ void xfs_qm_dqdestroy(struct xfs_dquot *dqp); int xfs_qm_dqflush(struct xfs_dquot *dqp, struct xfs_buf **bpp); void xfs_qm_dqunpin_wait(struct xfs_dquot *dqp); void xfs_qm_adjust_dqtimers(struct xfs_mount *mp, - struct xfs_disk_dquot *d); + struct xfs_dquot *d); void xfs_qm_adjust_dqlimits(struct xfs_mount *mp, struct xfs_dquot *d); xfs_dqid_t xfs_qm_id_for_quotatype(struct xfs_inode *ip, uint type); diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index 27a907645631..6609b4bb1628 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -1117,7 +1117,7 @@ xfs_qm_quotacheck_dqadjust( */ if (dqp->q_core.d_id) { xfs_qm_adjust_dqlimits(mp, dqp); - xfs_qm_adjust_dqtimers(mp, &dqp->q_core); + xfs_qm_adjust_dqtimers(mp, dqp); } dqp->dq_flags |= XFS_DQ_DIRTY; diff --git a/fs/xfs/xfs_qm_syscalls.c b/fs/xfs/xfs_qm_syscalls.c index 9edf761eec73..bd0f005570af 100644 --- a/fs/xfs/xfs_qm_syscalls.c +++ b/fs/xfs/xfs_qm_syscalls.c @@ -588,7 +588,7 @@ xfs_qm_scall_setqlim( * is on or off. We don't really want to bother with iterating * over all ondisk dquots and turning the timers on/off. */ - xfs_qm_adjust_dqtimers(mp, ddq); + xfs_qm_adjust_dqtimers(mp, dqp); } dqp->dq_flags |= XFS_DQ_DIRTY; xfs_trans_log_dquot(tp, dqp); diff --git a/fs/xfs/xfs_trans_dquot.c b/fs/xfs/xfs_trans_dquot.c index 2c07897a3c37..20542076e32a 100644 --- a/fs/xfs/xfs_trans_dquot.c +++ b/fs/xfs/xfs_trans_dquot.c @@ -388,7 +388,7 @@ xfs_trans_apply_dquot_deltas( */ if (d->d_id) { xfs_qm_adjust_dqlimits(tp->t_mountp, dqp); - xfs_qm_adjust_dqtimers(tp->t_mountp, d); + xfs_qm_adjust_dqtimers(tp->t_mountp, dqp); } dqp->dq_flags |= XFS_DQ_DIRTY; From ce6e7e79ced35a8ba4576d70bb999e8835f95769 Mon Sep 17 00:00:00 2001 From: Eric Sandeen Date: Thu, 21 May 2020 13:07:00 -0700 Subject: [PATCH 0894/1043] xfs: switch xfs_get_defquota to take explicit type xfs_get_defquota() currently takes an xfs_dquot, and from that obtains the type of default quota we should get (user/group/project). But early in init, we don't have access to a fully set up quota, so that's not possible. The next patch needs go set up default quota timers early, so switch xfs_get_defquota to take an explicit type and add a helper function to obtain that type from an xfs_dquot for the existing callers. Signed-off-by: Eric Sandeen Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_dquot.c | 2 +- fs/xfs/xfs_qm.c | 2 +- fs/xfs/xfs_qm.h | 33 ++++++++++++++++++++++----------- fs/xfs/xfs_qm_syscalls.c | 2 +- fs/xfs/xfs_trans_dquot.c | 2 +- 5 files changed, 26 insertions(+), 15 deletions(-) diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 714eceacbab2..6196f7c52b24 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -75,7 +75,7 @@ xfs_qm_adjust_dqlimits( int prealloc = 0; ASSERT(d->d_id); - defq = xfs_get_defquota(dq, q); + defq = xfs_get_defquota(q, xfs_dquot_type(dq)); if (defq->bsoftlimit && !d->d_blk_softlimit) { d->d_blk_softlimit = cpu_to_be64(defq->bsoftlimit); diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index 6609b4bb1628..ac0b5e7f8522 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -558,7 +558,7 @@ xfs_qm_set_defquota( return; ddqp = &dqp->q_core; - defq = xfs_get_defquota(dqp, qinf); + defq = xfs_get_defquota(qinf, xfs_dquot_type(dqp)); /* * Timers and warnings have been already set, let's just set the diff --git a/fs/xfs/xfs_qm.h b/fs/xfs/xfs_qm.h index 3a850401b102..c6f83171357e 100644 --- a/fs/xfs/xfs_qm.h +++ b/fs/xfs/xfs_qm.h @@ -113,6 +113,17 @@ xfs_quota_inode(xfs_mount_t *mp, uint dq_flags) return NULL; } +static inline int +xfs_dquot_type(struct xfs_dquot *dqp) +{ + if (XFS_QM_ISUDQ(dqp)) + return XFS_DQ_USER; + if (XFS_QM_ISGDQ(dqp)) + return XFS_DQ_GROUP; + ASSERT(XFS_QM_ISPDQ(dqp)); + return XFS_DQ_PROJ; +} + extern void xfs_trans_mod_dquot(struct xfs_trans *tp, struct xfs_dquot *dqp, uint field, int64_t delta); extern void xfs_trans_dqjoin(struct xfs_trans *, struct xfs_dquot *); @@ -164,19 +175,19 @@ extern int xfs_qm_scall_quotaon(struct xfs_mount *, uint); extern int xfs_qm_scall_quotaoff(struct xfs_mount *, uint); static inline struct xfs_def_quota * -xfs_get_defquota(struct xfs_dquot *dqp, struct xfs_quotainfo *qi) +xfs_get_defquota(struct xfs_quotainfo *qi, int type) { - struct xfs_def_quota *defq; - - if (XFS_QM_ISUDQ(dqp)) - defq = &qi->qi_usr_default; - else if (XFS_QM_ISGDQ(dqp)) - defq = &qi->qi_grp_default; - else { - ASSERT(XFS_QM_ISPDQ(dqp)); - defq = &qi->qi_prj_default; + switch (type) { + case XFS_DQ_USER: + return &qi->qi_usr_default; + case XFS_DQ_GROUP: + return &qi->qi_grp_default; + case XFS_DQ_PROJ: + return &qi->qi_prj_default; + default: + ASSERT(0); + return NULL; } - return defq; } #endif /* __XFS_QM_H__ */ diff --git a/fs/xfs/xfs_qm_syscalls.c b/fs/xfs/xfs_qm_syscalls.c index bd0f005570af..6fa08ae0b5f5 100644 --- a/fs/xfs/xfs_qm_syscalls.c +++ b/fs/xfs/xfs_qm_syscalls.c @@ -479,7 +479,7 @@ xfs_qm_scall_setqlim( goto out_unlock; } - defq = xfs_get_defquota(dqp, q); + defq = xfs_get_defquota(q, xfs_dquot_type(dqp)); xfs_dqunlock(dqp); error = xfs_trans_alloc(mp, &M_RES(mp)->tr_qm_setqlim, 0, 0, 0, &tp); diff --git a/fs/xfs/xfs_trans_dquot.c b/fs/xfs/xfs_trans_dquot.c index 20542076e32a..edde366ca8e9 100644 --- a/fs/xfs/xfs_trans_dquot.c +++ b/fs/xfs/xfs_trans_dquot.c @@ -591,7 +591,7 @@ xfs_trans_dqresv( xfs_dqlock(dqp); - defq = xfs_get_defquota(dqp, q); + defq = xfs_get_defquota(q, xfs_dquot_type(dqp)); if (flags & XFS_TRANS_DQ_RES_BLKS) { hardlimit = be64_to_cpu(dqp->q_core.d_blk_hardlimit); From e850301f0981741c47f5ee423380a2a963fce563 Mon Sep 17 00:00:00 2001 From: Eric Sandeen Date: Thu, 21 May 2020 13:07:01 -0700 Subject: [PATCH 0895/1043] xfs: per-type quota timers and warn limits Move timers and warnings out of xfs_quotainfo and into xfs_def_quota so that we can utilize them on a per-type basis, rather than enforcing them based on the values found in the first enabled quota type. Signed-off-by: Eric Sandeen [zlang: new way to get defquota in xfs_qm_init_timelimits] [zlang: remove redundant defq assign] Signed-off-by: Zorro Lang Signed-off-by: Eric Sandeen Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_dquot.c | 10 ++++++--- fs/xfs/xfs_qm.c | 46 ++++++++++++++++++---------------------- fs/xfs/xfs_qm.h | 13 ++++++------ fs/xfs/xfs_qm_syscalls.c | 12 +++++------ fs/xfs/xfs_quotaops.c | 22 +++++++++---------- fs/xfs/xfs_trans_dquot.c | 6 +++--- 6 files changed, 55 insertions(+), 54 deletions(-) diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 6196f7c52b24..d5b7f03e93c8 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -116,8 +116,12 @@ xfs_qm_adjust_dqtimers( struct xfs_mount *mp, struct xfs_dquot *dq) { + struct xfs_quotainfo *qi = mp->m_quotainfo; struct xfs_disk_dquot *d = &dq->q_core; + struct xfs_def_quota *defq; + ASSERT(d->d_id); + defq = xfs_get_defquota(qi, xfs_dquot_type(dq)); #ifdef DEBUG if (d->d_blk_hardlimit) @@ -139,7 +143,7 @@ xfs_qm_adjust_dqtimers( (be64_to_cpu(d->d_bcount) > be64_to_cpu(d->d_blk_hardlimit)))) { d->d_btimer = cpu_to_be32(ktime_get_real_seconds() + - mp->m_quotainfo->qi_btimelimit); + defq->btimelimit); } else { d->d_bwarns = 0; } @@ -162,7 +166,7 @@ xfs_qm_adjust_dqtimers( (be64_to_cpu(d->d_icount) > be64_to_cpu(d->d_ino_hardlimit)))) { d->d_itimer = cpu_to_be32(ktime_get_real_seconds() + - mp->m_quotainfo->qi_itimelimit); + defq->itimelimit); } else { d->d_iwarns = 0; } @@ -185,7 +189,7 @@ xfs_qm_adjust_dqtimers( (be64_to_cpu(d->d_rtbcount) > be64_to_cpu(d->d_rtb_hardlimit)))) { d->d_rtbtimer = cpu_to_be32(ktime_get_real_seconds() + - mp->m_quotainfo->qi_rtbtimelimit); + defq->rtbtimelimit); } else { d->d_rtbwarns = 0; } diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c index ac0b5e7f8522..d6cd83317344 100644 --- a/fs/xfs/xfs_qm.c +++ b/fs/xfs/xfs_qm.c @@ -577,19 +577,22 @@ xfs_qm_set_defquota( static void xfs_qm_init_timelimits( struct xfs_mount *mp, - struct xfs_quotainfo *qinf) + uint type) { + struct xfs_quotainfo *qinf = mp->m_quotainfo; + struct xfs_def_quota *defq; struct xfs_disk_dquot *ddqp; struct xfs_dquot *dqp; - uint type; int error; - qinf->qi_btimelimit = XFS_QM_BTIMELIMIT; - qinf->qi_itimelimit = XFS_QM_ITIMELIMIT; - qinf->qi_rtbtimelimit = XFS_QM_RTBTIMELIMIT; - qinf->qi_bwarnlimit = XFS_QM_BWARNLIMIT; - qinf->qi_iwarnlimit = XFS_QM_IWARNLIMIT; - qinf->qi_rtbwarnlimit = XFS_QM_RTBWARNLIMIT; + defq = xfs_get_defquota(qinf, type); + + defq->btimelimit = XFS_QM_BTIMELIMIT; + defq->itimelimit = XFS_QM_ITIMELIMIT; + defq->rtbtimelimit = XFS_QM_RTBTIMELIMIT; + defq->bwarnlimit = XFS_QM_BWARNLIMIT; + defq->iwarnlimit = XFS_QM_IWARNLIMIT; + defq->rtbwarnlimit = XFS_QM_RTBWARNLIMIT; /* * We try to get the limits from the superuser's limits fields. @@ -597,39 +600,30 @@ xfs_qm_init_timelimits( * * Since we may not have done a quotacheck by this point, just read * the dquot without attaching it to any hashtables or lists. - * - * Timers and warnings are globally set by the first timer found in - * user/group/proj quota types, otherwise a default value is used. - * This should be split into different fields per quota type. */ - if (XFS_IS_UQUOTA_RUNNING(mp)) - type = XFS_DQ_USER; - else if (XFS_IS_GQUOTA_RUNNING(mp)) - type = XFS_DQ_GROUP; - else - type = XFS_DQ_PROJ; error = xfs_qm_dqget_uncached(mp, 0, type, &dqp); if (error) return; ddqp = &dqp->q_core; + /* * The warnings and timers set the grace period given to * a user or group before he or she can not perform any * more writing. If it is zero, a default is used. */ if (ddqp->d_btimer) - qinf->qi_btimelimit = be32_to_cpu(ddqp->d_btimer); + defq->btimelimit = be32_to_cpu(ddqp->d_btimer); if (ddqp->d_itimer) - qinf->qi_itimelimit = be32_to_cpu(ddqp->d_itimer); + defq->itimelimit = be32_to_cpu(ddqp->d_itimer); if (ddqp->d_rtbtimer) - qinf->qi_rtbtimelimit = be32_to_cpu(ddqp->d_rtbtimer); + defq->rtbtimelimit = be32_to_cpu(ddqp->d_rtbtimer); if (ddqp->d_bwarns) - qinf->qi_bwarnlimit = be16_to_cpu(ddqp->d_bwarns); + defq->bwarnlimit = be16_to_cpu(ddqp->d_bwarns); if (ddqp->d_iwarns) - qinf->qi_iwarnlimit = be16_to_cpu(ddqp->d_iwarns); + defq->iwarnlimit = be16_to_cpu(ddqp->d_iwarns); if (ddqp->d_rtbwarns) - qinf->qi_rtbwarnlimit = be16_to_cpu(ddqp->d_rtbwarns); + defq->rtbwarnlimit = be16_to_cpu(ddqp->d_rtbwarns); xfs_qm_dqdestroy(dqp); } @@ -675,7 +669,9 @@ xfs_qm_init_quotainfo( mp->m_qflags |= (mp->m_sb.sb_qflags & XFS_ALL_QUOTA_CHKD); - xfs_qm_init_timelimits(mp, qinf); + xfs_qm_init_timelimits(mp, XFS_DQ_USER); + xfs_qm_init_timelimits(mp, XFS_DQ_GROUP); + xfs_qm_init_timelimits(mp, XFS_DQ_PROJ); if (XFS_IS_UQUOTA_RUNNING(mp)) xfs_qm_set_defquota(mp, XFS_DQ_USER, qinf); diff --git a/fs/xfs/xfs_qm.h b/fs/xfs/xfs_qm.h index c6f83171357e..7b0e771fcbce 100644 --- a/fs/xfs/xfs_qm.h +++ b/fs/xfs/xfs_qm.h @@ -41,7 +41,14 @@ extern struct kmem_zone *xfs_qm_dqtrxzone; */ #define XFS_DQUOT_CLUSTER_SIZE_FSB (xfs_filblks_t)1 +/* Defaults for each quota type: time limits, warn limits, usage limits */ struct xfs_def_quota { + time64_t btimelimit; /* limit for blks timer */ + time64_t itimelimit; /* limit for inodes timer */ + time64_t rtbtimelimit; /* limit for rt blks timer */ + xfs_qwarncnt_t bwarnlimit; /* limit for blks warnings */ + xfs_qwarncnt_t iwarnlimit; /* limit for inodes warnings */ + xfs_qwarncnt_t rtbwarnlimit; /* limit for rt blks warnings */ xfs_qcnt_t bhardlimit; /* default data blk hard limit */ xfs_qcnt_t bsoftlimit; /* default data blk soft limit */ xfs_qcnt_t ihardlimit; /* default inode count hard limit */ @@ -64,12 +71,6 @@ struct xfs_quotainfo { struct xfs_inode *qi_pquotaip; /* project quota inode */ struct list_lru qi_lru; int qi_dquots; - time64_t qi_btimelimit; /* limit for blks timer */ - time64_t qi_itimelimit; /* limit for inodes timer */ - time64_t qi_rtbtimelimit;/* limit for rt blks timer */ - xfs_qwarncnt_t qi_bwarnlimit; /* limit for blks warnings */ - xfs_qwarncnt_t qi_iwarnlimit; /* limit for inodes warnings */ - xfs_qwarncnt_t qi_rtbwarnlimit;/* limit for rt blks warnings */ struct mutex qi_quotaofflock;/* to serialize quotaoff */ xfs_filblks_t qi_dqchunklen; /* # BBs in a chunk of dqs */ uint qi_dqperchunk; /* # ondisk dq in above chunk */ diff --git a/fs/xfs/xfs_qm_syscalls.c b/fs/xfs/xfs_qm_syscalls.c index 6fa08ae0b5f5..9b69ce16a540 100644 --- a/fs/xfs/xfs_qm_syscalls.c +++ b/fs/xfs/xfs_qm_syscalls.c @@ -563,23 +563,23 @@ xfs_qm_scall_setqlim( * for warnings. */ if (newlim->d_fieldmask & QC_SPC_TIMER) { - q->qi_btimelimit = newlim->d_spc_timer; + defq->btimelimit = newlim->d_spc_timer; ddq->d_btimer = cpu_to_be32(newlim->d_spc_timer); } if (newlim->d_fieldmask & QC_INO_TIMER) { - q->qi_itimelimit = newlim->d_ino_timer; + defq->itimelimit = newlim->d_ino_timer; ddq->d_itimer = cpu_to_be32(newlim->d_ino_timer); } if (newlim->d_fieldmask & QC_RT_SPC_TIMER) { - q->qi_rtbtimelimit = newlim->d_rt_spc_timer; + defq->rtbtimelimit = newlim->d_rt_spc_timer; ddq->d_rtbtimer = cpu_to_be32(newlim->d_rt_spc_timer); } if (newlim->d_fieldmask & QC_SPC_WARNS) - q->qi_bwarnlimit = newlim->d_spc_warns; + defq->bwarnlimit = newlim->d_spc_warns; if (newlim->d_fieldmask & QC_INO_WARNS) - q->qi_iwarnlimit = newlim->d_ino_warns; + defq->iwarnlimit = newlim->d_ino_warns; if (newlim->d_fieldmask & QC_RT_SPC_WARNS) - q->qi_rtbwarnlimit = newlim->d_rt_spc_warns; + defq->rtbwarnlimit = newlim->d_rt_spc_warns; } else { /* * If the user is now over quota, start the timelimit. diff --git a/fs/xfs/xfs_quotaops.c b/fs/xfs/xfs_quotaops.c index 411eeefa2a5c..bf809b77a316 100644 --- a/fs/xfs/xfs_quotaops.c +++ b/fs/xfs/xfs_quotaops.c @@ -21,9 +21,9 @@ xfs_qm_fill_state( struct qc_type_state *tstate, struct xfs_mount *mp, struct xfs_inode *ip, - xfs_ino_t ino) + xfs_ino_t ino, + struct xfs_def_quota *defq) { - struct xfs_quotainfo *q = mp->m_quotainfo; bool tempqip = false; tstate->ino = ino; @@ -37,12 +37,12 @@ xfs_qm_fill_state( tstate->flags |= QCI_SYSFILE; tstate->blocks = ip->i_d.di_nblocks; tstate->nextents = ip->i_df.if_nextents; - tstate->spc_timelimit = (u32)q->qi_btimelimit; - tstate->ino_timelimit = (u32)q->qi_itimelimit; - tstate->rt_spc_timelimit = (u32)q->qi_rtbtimelimit; - tstate->spc_warnlimit = q->qi_bwarnlimit; - tstate->ino_warnlimit = q->qi_iwarnlimit; - tstate->rt_spc_warnlimit = q->qi_rtbwarnlimit; + tstate->spc_timelimit = (u32)defq->btimelimit; + tstate->ino_timelimit = (u32)defq->itimelimit; + tstate->rt_spc_timelimit = (u32)defq->rtbtimelimit; + tstate->spc_warnlimit = defq->bwarnlimit; + tstate->ino_warnlimit = defq->iwarnlimit; + tstate->rt_spc_warnlimit = defq->rtbwarnlimit; if (tempqip) xfs_irele(ip); } @@ -77,11 +77,11 @@ xfs_fs_get_quota_state( state->s_state[PRJQUOTA].flags |= QCI_LIMITS_ENFORCED; xfs_qm_fill_state(&state->s_state[USRQUOTA], mp, q->qi_uquotaip, - mp->m_sb.sb_uquotino); + mp->m_sb.sb_uquotino, &q->qi_usr_default); xfs_qm_fill_state(&state->s_state[GRPQUOTA], mp, q->qi_gquotaip, - mp->m_sb.sb_gquotino); + mp->m_sb.sb_gquotino, &q->qi_grp_default); xfs_qm_fill_state(&state->s_state[PRJQUOTA], mp, q->qi_pquotaip, - mp->m_sb.sb_pquotino); + mp->m_sb.sb_pquotino, &q->qi_prj_default); return 0; } diff --git a/fs/xfs/xfs_trans_dquot.c b/fs/xfs/xfs_trans_dquot.c index edde366ca8e9..c0f73b82c055 100644 --- a/fs/xfs/xfs_trans_dquot.c +++ b/fs/xfs/xfs_trans_dquot.c @@ -602,7 +602,7 @@ xfs_trans_dqresv( softlimit = defq->bsoftlimit; timer = be32_to_cpu(dqp->q_core.d_btimer); warns = be16_to_cpu(dqp->q_core.d_bwarns); - warnlimit = dqp->q_mount->m_quotainfo->qi_bwarnlimit; + warnlimit = defq->bwarnlimit; resbcountp = &dqp->q_res_bcount; } else { ASSERT(flags & XFS_TRANS_DQ_RES_RTBLKS); @@ -614,7 +614,7 @@ xfs_trans_dqresv( softlimit = defq->rtbsoftlimit; timer = be32_to_cpu(dqp->q_core.d_rtbtimer); warns = be16_to_cpu(dqp->q_core.d_rtbwarns); - warnlimit = dqp->q_mount->m_quotainfo->qi_rtbwarnlimit; + warnlimit = defq->rtbwarnlimit; resbcountp = &dqp->q_res_rtbcount; } @@ -650,7 +650,7 @@ xfs_trans_dqresv( total_count = be64_to_cpu(dqp->q_core.d_icount) + ninos; timer = be32_to_cpu(dqp->q_core.d_itimer); warns = be16_to_cpu(dqp->q_core.d_iwarns); - warnlimit = dqp->q_mount->m_quotainfo->qi_iwarnlimit; + warnlimit = defq->iwarnlimit; hardlimit = be64_to_cpu(dqp->q_core.d_ino_hardlimit); if (!hardlimit) hardlimit = defq->ihardlimit; From df42ce64dc3eb7f01f8b601abd3881c6dddfbbaa Mon Sep 17 00:00:00 2001 From: Eric Sandeen Date: Thu, 21 May 2020 13:07:01 -0700 Subject: [PATCH 0896/1043] xfs: allow individual quota grace period extension The only grace period which can be set in the kernel today is for id 0, i.e. the default grace period for all users. However, setting an individual grace period is useful; for example: Alice has a soft quota of 100 inodes, and a hard quota of 200 inodes Alice uses 150 inodes, and enters a short grace period Alice really needs to use those 150 inodes past the grace period The administrator extends Alice's grace period until next Monday vfs quota users such as ext4 can do this today, with setquota -T To enable this for XFS, we simply move the timelimit assignment out from under the (id == 0) test. Default setting remains under (id == 0). Note that this now is consistent with how we set warnings. (Userspace requires updates to enable this as well; xfs_quota needs to parse new options, and setquota needs to set appropriate field flags.) Signed-off-by: Eric Sandeen Reviewed-by: Christoph Hellwig Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_qm_syscalls.c | 48 +++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/fs/xfs/xfs_qm_syscalls.c b/fs/xfs/xfs_qm_syscalls.c index 9b69ce16a540..362ccec2da99 100644 --- a/fs/xfs/xfs_qm_syscalls.c +++ b/fs/xfs/xfs_qm_syscalls.c @@ -555,32 +555,40 @@ xfs_qm_scall_setqlim( ddq->d_rtbwarns = cpu_to_be16(newlim->d_rt_spc_warns); if (id == 0) { - /* - * Timelimits for the super user set the relative time - * the other users can be over quota for this file system. - * If it is zero a default is used. Ditto for the default - * soft and hard limit values (already done, above), and - * for warnings. - */ - if (newlim->d_fieldmask & QC_SPC_TIMER) { - defq->btimelimit = newlim->d_spc_timer; - ddq->d_btimer = cpu_to_be32(newlim->d_spc_timer); - } - if (newlim->d_fieldmask & QC_INO_TIMER) { - defq->itimelimit = newlim->d_ino_timer; - ddq->d_itimer = cpu_to_be32(newlim->d_ino_timer); - } - if (newlim->d_fieldmask & QC_RT_SPC_TIMER) { - defq->rtbtimelimit = newlim->d_rt_spc_timer; - ddq->d_rtbtimer = cpu_to_be32(newlim->d_rt_spc_timer); - } if (newlim->d_fieldmask & QC_SPC_WARNS) defq->bwarnlimit = newlim->d_spc_warns; if (newlim->d_fieldmask & QC_INO_WARNS) defq->iwarnlimit = newlim->d_ino_warns; if (newlim->d_fieldmask & QC_RT_SPC_WARNS) defq->rtbwarnlimit = newlim->d_rt_spc_warns; - } else { + } + + /* + * Timelimits for the super user set the relative time the other users + * can be over quota for this file system. If it is zero a default is + * used. Ditto for the default soft and hard limit values (already + * done, above), and for warnings. + * + * For other IDs, userspace can bump out the grace period if over + * the soft limit. + */ + if (newlim->d_fieldmask & QC_SPC_TIMER) + ddq->d_btimer = cpu_to_be32(newlim->d_spc_timer); + if (newlim->d_fieldmask & QC_INO_TIMER) + ddq->d_itimer = cpu_to_be32(newlim->d_ino_timer); + if (newlim->d_fieldmask & QC_RT_SPC_TIMER) + ddq->d_rtbtimer = cpu_to_be32(newlim->d_rt_spc_timer); + + if (id == 0) { + if (newlim->d_fieldmask & QC_SPC_TIMER) + defq->btimelimit = newlim->d_spc_timer; + if (newlim->d_fieldmask & QC_INO_TIMER) + defq->itimelimit = newlim->d_ino_timer; + if (newlim->d_fieldmask & QC_RT_SPC_TIMER) + defq->rtbtimelimit = newlim->d_rt_spc_timer; + } + + if (id != 0) { /* * If the user is now over quota, start the timelimit. * The user will not be 'warned'. From 3737bb2c67770afea6362aeaf4f48a969d119777 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 21 May 2020 13:08:46 -0700 Subject: [PATCH 0897/1043] xfs: move eofblocks conversion function to xfs_ioctl.c Move xfs_fs_eofblocks_from_user into the only file that actually uses it, so that we don't have this function cluttering up the header file. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_icache.h | 35 ----------------------------------- fs/xfs/xfs_ioctl.c | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index 48f1fd2bb6ad..c13bc8a3e02f 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -81,41 +81,6 @@ int xfs_inode_ag_iterator_tag(struct xfs_mount *mp, int (*execute)(struct xfs_inode *ip, int flags, void *args), int flags, void *args, int tag); -static inline int -xfs_fs_eofblocks_from_user( - struct xfs_fs_eofblocks *src, - struct xfs_eofblocks *dst) -{ - if (src->eof_version != XFS_EOFBLOCKS_VERSION) - return -EINVAL; - - if (src->eof_flags & ~XFS_EOF_FLAGS_VALID) - return -EINVAL; - - if (memchr_inv(&src->pad32, 0, sizeof(src->pad32)) || - memchr_inv(src->pad64, 0, sizeof(src->pad64))) - return -EINVAL; - - dst->eof_flags = src->eof_flags; - dst->eof_prid = src->eof_prid; - dst->eof_min_file_size = src->eof_min_file_size; - - dst->eof_uid = INVALID_UID; - if (src->eof_flags & XFS_EOF_FLAGS_UID) { - dst->eof_uid = make_kuid(current_user_ns(), src->eof_uid); - if (!uid_valid(dst->eof_uid)) - return -EINVAL; - } - - dst->eof_gid = INVALID_GID; - if (src->eof_flags & XFS_EOF_FLAGS_GID) { - dst->eof_gid = make_kgid(current_user_ns(), src->eof_gid); - if (!gid_valid(dst->eof_gid)) - return -EINVAL; - } - return 0; -} - int xfs_icache_inode_is_allocated(struct xfs_mount *mp, struct xfs_trans *tp, xfs_ino_t ino, bool *inuse); diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 7a71c03e9022..a40f88cf3ab7 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -2039,6 +2039,41 @@ out: return error; } +static inline int +xfs_fs_eofblocks_from_user( + struct xfs_fs_eofblocks *src, + struct xfs_eofblocks *dst) +{ + if (src->eof_version != XFS_EOFBLOCKS_VERSION) + return -EINVAL; + + if (src->eof_flags & ~XFS_EOF_FLAGS_VALID) + return -EINVAL; + + if (memchr_inv(&src->pad32, 0, sizeof(src->pad32)) || + memchr_inv(src->pad64, 0, sizeof(src->pad64))) + return -EINVAL; + + dst->eof_flags = src->eof_flags; + dst->eof_prid = src->eof_prid; + dst->eof_min_file_size = src->eof_min_file_size; + + dst->eof_uid = INVALID_UID; + if (src->eof_flags & XFS_EOF_FLAGS_UID) { + dst->eof_uid = make_kuid(current_user_ns(), src->eof_uid); + if (!uid_valid(dst->eof_uid)) + return -EINVAL; + } + + dst->eof_gid = INVALID_GID; + if (src->eof_flags & XFS_EOF_FLAGS_GID) { + dst->eof_gid = make_kgid(current_user_ns(), src->eof_gid); + if (!gid_valid(dst->eof_gid)) + return -EINVAL; + } + return 0; +} + /* * Note: some of the ioctl's return positive numbers as a * byte count indicating success, such as readlink_by_handle. From fc96be95e6c612eb77c0c0306cef2da1b8a243f9 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 21 May 2020 13:08:47 -0700 Subject: [PATCH 0898/1043] xfs: replace open-coded XFS_ICI_NO_TAG Use XFS_ICI_NO_TAG instead of -1 when appropriate. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_icache.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index d806d3bfa893..83a3f2c8167f 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -835,7 +835,7 @@ restart: rcu_read_lock(); - if (tag == -1) + if (tag == XFS_ICI_NO_TAG) nr_found = radix_tree_gang_lookup(&pag->pag_ici_root, (void **)batch, first_index, XFS_LOOKUP_BATCH); @@ -993,8 +993,8 @@ xfs_inode_ag_iterator_flags( ag = 0; while ((pag = xfs_perag_get(mp, ag))) { ag = pag->pag_agno + 1; - error = xfs_inode_ag_walk(mp, pag, execute, flags, args, -1, - iter_flags); + error = xfs_inode_ag_walk(mp, pag, execute, flags, args, + XFS_ICI_NO_TAG, iter_flags); xfs_perag_put(pag); if (error) { last_error = error; From 43d24bcf19d139b904752b5727def241920cff37 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 21 May 2020 13:08:47 -0700 Subject: [PATCH 0899/1043] xfs: remove unused xfs_inode_ag_iterator function Not used by anyone, so get rid of it. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_icache.c | 11 ----------- fs/xfs/xfs_icache.h | 3 --- 2 files changed, 14 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 83a3f2c8167f..89c935b29021 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1005,17 +1005,6 @@ xfs_inode_ag_iterator_flags( return last_error; } -int -xfs_inode_ag_iterator( - struct xfs_mount *mp, - int (*execute)(struct xfs_inode *ip, int flags, - void *args), - int flags, - void *args) -{ - return xfs_inode_ag_iterator_flags(mp, execute, flags, args, 0); -} - int xfs_inode_ag_iterator_tag( struct xfs_mount *mp, diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index c13bc8a3e02f..0556fa32074f 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -71,9 +71,6 @@ int xfs_inode_free_quota_cowblocks(struct xfs_inode *ip); void xfs_cowblocks_worker(struct work_struct *); void xfs_queue_cowblocks(struct xfs_mount *); -int xfs_inode_ag_iterator(struct xfs_mount *mp, - int (*execute)(struct xfs_inode *ip, int flags, void *args), - int flags, void *args); int xfs_inode_ag_iterator_flags(struct xfs_mount *mp, int (*execute)(struct xfs_inode *ip, int flags, void *args), int flags, void *args, int iter_flags); From 9be0590453cdae1b7aa2eb08515cc140ecc850c3 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 21 May 2020 13:08:47 -0700 Subject: [PATCH 0900/1043] xfs: remove xfs_inode_ag_iterator_flags Combine xfs_inode_ag_iterator_flags and xfs_inode_ag_iterator_tag into a single wrapper function since there's only one caller of the _flags variant. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_icache.c | 43 ++++++++++++---------------------------- fs/xfs/xfs_icache.h | 5 +---- fs/xfs/xfs_qm_syscalls.c | 4 ++-- 3 files changed, 16 insertions(+), 36 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 89c935b29021..77dfbcfff06c 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -976,38 +976,22 @@ xfs_cowblocks_worker( xfs_queue_cowblocks(mp); } -int -xfs_inode_ag_iterator_flags( +/* Fetch the next (possibly tagged) per-AG structure. */ +static inline struct xfs_perag * +xfs_inode_walk_get_perag( struct xfs_mount *mp, - int (*execute)(struct xfs_inode *ip, int flags, - void *args), - int flags, - void *args, - int iter_flags) + xfs_agnumber_t agno, + int tag) { - struct xfs_perag *pag; - int error = 0; - int last_error = 0; - xfs_agnumber_t ag; - - ag = 0; - while ((pag = xfs_perag_get(mp, ag))) { - ag = pag->pag_agno + 1; - error = xfs_inode_ag_walk(mp, pag, execute, flags, args, - XFS_ICI_NO_TAG, iter_flags); - xfs_perag_put(pag); - if (error) { - last_error = error; - if (error == -EFSCORRUPTED) - break; - } - } - return last_error; + if (tag == XFS_ICI_NO_TAG) + return xfs_perag_get(mp, agno); + return xfs_perag_get_tag(mp, agno, tag); } int -xfs_inode_ag_iterator_tag( +xfs_inode_ag_iterator( struct xfs_mount *mp, + int iter_flags, int (*execute)(struct xfs_inode *ip, int flags, void *args), int flags, @@ -1020,10 +1004,10 @@ xfs_inode_ag_iterator_tag( xfs_agnumber_t ag; ag = 0; - while ((pag = xfs_perag_get_tag(mp, ag, tag))) { + while ((pag = xfs_inode_walk_get_perag(mp, ag, tag))) { ag = pag->pag_agno + 1; error = xfs_inode_ag_walk(mp, pag, execute, flags, args, tag, - 0); + iter_flags); xfs_perag_put(pag); if (error) { last_error = error; @@ -1543,8 +1527,7 @@ __xfs_icache_free_eofblocks( if (eofb && (eofb->eof_flags & XFS_EOF_FLAGS_SYNC)) flags = SYNC_WAIT; - return xfs_inode_ag_iterator_tag(mp, execute, flags, - eofb, tag); + return xfs_inode_ag_iterator(mp, 0, execute, flags, eofb, tag); } int diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index 0556fa32074f..2d5ab9957d9f 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -71,10 +71,7 @@ int xfs_inode_free_quota_cowblocks(struct xfs_inode *ip); void xfs_cowblocks_worker(struct work_struct *); void xfs_queue_cowblocks(struct xfs_mount *); -int xfs_inode_ag_iterator_flags(struct xfs_mount *mp, - int (*execute)(struct xfs_inode *ip, int flags, void *args), - int flags, void *args, int iter_flags); -int xfs_inode_ag_iterator_tag(struct xfs_mount *mp, +int xfs_inode_ag_iterator(struct xfs_mount *mp, int iter_flags, int (*execute)(struct xfs_inode *ip, int flags, void *args), int flags, void *args, int tag); diff --git a/fs/xfs/xfs_qm_syscalls.c b/fs/xfs/xfs_qm_syscalls.c index 362ccec2da99..6bcfc8fc23e9 100644 --- a/fs/xfs/xfs_qm_syscalls.c +++ b/fs/xfs/xfs_qm_syscalls.c @@ -780,6 +780,6 @@ xfs_qm_dqrele_all_inodes( uint flags) { ASSERT(mp->m_quotainfo); - xfs_inode_ag_iterator_flags(mp, xfs_dqrele_inode, flags, NULL, - XFS_AGITER_INEW_WAIT); + xfs_inode_ag_iterator(mp, XFS_AGITER_INEW_WAIT, xfs_dqrele_inode, + flags, NULL, XFS_ICI_NO_TAG); } From 390600f811f1adfb9158e9b670b81219a9d94d72 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 21 May 2020 13:08:48 -0700 Subject: [PATCH 0901/1043] xfs: remove flags argument from xfs_inode_ag_walk The incore inode walk code passes a flags argument and a pointer from the xfs_inode_ag_iterator caller all the way to the iteration function. We can reduce the function complexity by passing flags through the private pointer. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_icache.c | 43 ++++++++++++++++------------------------ fs/xfs/xfs_icache.h | 4 ++-- fs/xfs/xfs_qm_syscalls.c | 15 +++++++------- 3 files changed, 27 insertions(+), 35 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 77dfbcfff06c..323fe9a77600 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -810,9 +810,7 @@ STATIC int xfs_inode_ag_walk( struct xfs_mount *mp, struct xfs_perag *pag, - int (*execute)(struct xfs_inode *ip, int flags, - void *args), - int flags, + int (*execute)(struct xfs_inode *ip, void *args), void *args, int tag, int iter_flags) @@ -888,7 +886,7 @@ restart: if ((iter_flags & XFS_AGITER_INEW_WAIT) && xfs_iflags_test(batch[i], XFS_INEW)) xfs_inew_wait(batch[i]); - error = execute(batch[i], flags, args); + error = execute(batch[i], args); xfs_irele(batch[i]); if (error == -EAGAIN) { skipped++; @@ -992,9 +990,7 @@ int xfs_inode_ag_iterator( struct xfs_mount *mp, int iter_flags, - int (*execute)(struct xfs_inode *ip, int flags, - void *args), - int flags, + int (*execute)(struct xfs_inode *ip, void *args), void *args, int tag) { @@ -1006,7 +1002,7 @@ xfs_inode_ag_iterator( ag = 0; while ((pag = xfs_inode_walk_get_perag(mp, ag, tag))) { ag = pag->pag_agno + 1; - error = xfs_inode_ag_walk(mp, pag, execute, flags, args, tag, + error = xfs_inode_ag_walk(mp, pag, execute, args, tag, iter_flags); xfs_perag_put(pag); if (error) { @@ -1463,12 +1459,14 @@ xfs_inode_match_id_union( STATIC int xfs_inode_free_eofblocks( struct xfs_inode *ip, - int flags, void *args) { - int ret = 0; - struct xfs_eofblocks *eofb = args; - int match; + struct xfs_eofblocks *eofb = args; + bool wait; + int match; + int ret; + + wait = eofb && (eofb->eof_flags & XFS_EOF_FLAGS_SYNC); if (!xfs_can_free_eofblocks(ip, false)) { /* inode could be preallocated or append-only */ @@ -1481,8 +1479,7 @@ xfs_inode_free_eofblocks( * If the mapping is dirty the operation can block and wait for some * time. Unless we are waiting, skip it. */ - if (!(flags & SYNC_WAIT) && - mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY)) + if (!wait && mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY)) return 0; if (eofb) { @@ -1504,10 +1501,11 @@ xfs_inode_free_eofblocks( * scanner moving and revisit the inode in a subsequent pass. */ if (!xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) { - if (flags & SYNC_WAIT) - ret = -EAGAIN; - return ret; + if (wait) + return -EAGAIN; + return 0; } + ret = xfs_free_eofblocks(ip); xfs_iunlock(ip, XFS_IOLOCK_EXCL); @@ -1518,16 +1516,10 @@ static int __xfs_icache_free_eofblocks( struct xfs_mount *mp, struct xfs_eofblocks *eofb, - int (*execute)(struct xfs_inode *ip, int flags, - void *args), + int (*execute)(struct xfs_inode *ip, void *args), int tag) { - int flags = SYNC_TRYLOCK; - - if (eofb && (eofb->eof_flags & XFS_EOF_FLAGS_SYNC)) - flags = SYNC_WAIT; - - return xfs_inode_ag_iterator(mp, 0, execute, flags, eofb, tag); + return xfs_inode_ag_iterator(mp, 0, execute, eofb, tag); } int @@ -1752,7 +1744,6 @@ xfs_prep_free_cowblocks( STATIC int xfs_inode_free_cowblocks( struct xfs_inode *ip, - int flags, void *args) { struct xfs_eofblocks *eofb = args; diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index 2d5ab9957d9f..e7f86ebd7b22 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -72,8 +72,8 @@ void xfs_cowblocks_worker(struct work_struct *); void xfs_queue_cowblocks(struct xfs_mount *); int xfs_inode_ag_iterator(struct xfs_mount *mp, int iter_flags, - int (*execute)(struct xfs_inode *ip, int flags, void *args), - int flags, void *args, int tag); + int (*execute)(struct xfs_inode *ip, void *args), + void *args, int tag); int xfs_icache_inode_is_allocated(struct xfs_mount *mp, struct xfs_trans *tp, xfs_ino_t ino, bool *inuse); diff --git a/fs/xfs/xfs_qm_syscalls.c b/fs/xfs/xfs_qm_syscalls.c index 6bcfc8fc23e9..f7db1a05f7b8 100644 --- a/fs/xfs/xfs_qm_syscalls.c +++ b/fs/xfs/xfs_qm_syscalls.c @@ -737,9 +737,10 @@ xfs_qm_scall_getquota_next( STATIC int xfs_dqrele_inode( struct xfs_inode *ip, - int flags, void *args) { + uint *flags = args; + /* skip quota inodes */ if (ip == ip->i_mount->m_quotainfo->qi_uquotaip || ip == ip->i_mount->m_quotainfo->qi_gquotaip || @@ -751,15 +752,15 @@ xfs_dqrele_inode( } xfs_ilock(ip, XFS_ILOCK_EXCL); - if ((flags & XFS_UQUOTA_ACCT) && ip->i_udquot) { + if ((*flags & XFS_UQUOTA_ACCT) && ip->i_udquot) { xfs_qm_dqrele(ip->i_udquot); ip->i_udquot = NULL; } - if ((flags & XFS_GQUOTA_ACCT) && ip->i_gdquot) { + if ((*flags & XFS_GQUOTA_ACCT) && ip->i_gdquot) { xfs_qm_dqrele(ip->i_gdquot); ip->i_gdquot = NULL; } - if ((flags & XFS_PQUOTA_ACCT) && ip->i_pdquot) { + if ((*flags & XFS_PQUOTA_ACCT) && ip->i_pdquot) { xfs_qm_dqrele(ip->i_pdquot); ip->i_pdquot = NULL; } @@ -776,10 +777,10 @@ xfs_dqrele_inode( */ void xfs_qm_dqrele_all_inodes( - struct xfs_mount *mp, - uint flags) + struct xfs_mount *mp, + uint flags) { ASSERT(mp->m_quotainfo); xfs_inode_ag_iterator(mp, XFS_AGITER_INEW_WAIT, xfs_dqrele_inode, - flags, NULL, XFS_ICI_NO_TAG); + &flags, XFS_ICI_NO_TAG); } From 8921a0fda54adf0a45ef15a9507594829e4ec6f5 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 21 May 2020 13:08:48 -0700 Subject: [PATCH 0902/1043] xfs: remove __xfs_icache_free_eofblocks This is now a pointless wrapper, so kill it. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_icache.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 323fe9a77600..bd06edbac5ce 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1512,22 +1512,12 @@ xfs_inode_free_eofblocks( return ret; } -static int -__xfs_icache_free_eofblocks( - struct xfs_mount *mp, - struct xfs_eofblocks *eofb, - int (*execute)(struct xfs_inode *ip, void *args), - int tag) -{ - return xfs_inode_ag_iterator(mp, 0, execute, eofb, tag); -} - int xfs_icache_free_eofblocks( struct xfs_mount *mp, struct xfs_eofblocks *eofb) { - return __xfs_icache_free_eofblocks(mp, eofb, xfs_inode_free_eofblocks, + return xfs_inode_ag_iterator(mp, 0, xfs_inode_free_eofblocks, eofb, XFS_ICI_EOFBLOCKS_TAG); } @@ -1789,7 +1779,7 @@ xfs_icache_free_cowblocks( struct xfs_mount *mp, struct xfs_eofblocks *eofb) { - return __xfs_icache_free_eofblocks(mp, eofb, xfs_inode_free_cowblocks, + return xfs_inode_ag_iterator(mp, 0, xfs_inode_free_cowblocks, eofb, XFS_ICI_COWBLOCKS_TAG); } From a91bf9928e1e16dd175ff363efa54ebc9e38d53d Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 21 May 2020 13:08:48 -0700 Subject: [PATCH 0903/1043] xfs: refactor eofb matching into a single helper Refactor the two eofb-matching logics into a single helper so that we don't repeat ourselves. Signed-off-by: Darrick J. Wong Reviewed-by: Brian Foster Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_icache.c | 62 +++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index bd06edbac5ce..d10424a0015a 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -1456,6 +1456,36 @@ xfs_inode_match_id_union( return 0; } +/* + * Is this inode @ip eligible for eof/cow block reclamation, given some + * filtering parameters @eofb? The inode is eligible if @eofb is null or + * if the predicate functions match. + */ +static bool +xfs_inode_matches_eofb( + struct xfs_inode *ip, + struct xfs_eofblocks *eofb) +{ + int match; + + if (!eofb) + return true; + + if (eofb->eof_flags & XFS_EOF_FLAGS_UNION) + match = xfs_inode_match_id_union(ip, eofb); + else + match = xfs_inode_match_id(ip, eofb); + if (!match) + return false; + + /* skip the inode if the file size is too small */ + if ((eofb->eof_flags & XFS_EOF_FLAGS_MINFILESIZE) && + XFS_ISIZE(ip) < eofb->eof_min_file_size) + return false; + + return true; +} + STATIC int xfs_inode_free_eofblocks( struct xfs_inode *ip, @@ -1463,7 +1493,6 @@ xfs_inode_free_eofblocks( { struct xfs_eofblocks *eofb = args; bool wait; - int match; int ret; wait = eofb && (eofb->eof_flags & XFS_EOF_FLAGS_SYNC); @@ -1482,19 +1511,8 @@ xfs_inode_free_eofblocks( if (!wait && mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY)) return 0; - if (eofb) { - if (eofb->eof_flags & XFS_EOF_FLAGS_UNION) - match = xfs_inode_match_id_union(ip, eofb); - else - match = xfs_inode_match_id(ip, eofb); - if (!match) - return 0; - - /* skip the inode if the file size is too small */ - if (eofb->eof_flags & XFS_EOF_FLAGS_MINFILESIZE && - XFS_ISIZE(ip) < eofb->eof_min_file_size) - return 0; - } + if (!xfs_inode_matches_eofb(ip, eofb)) + return 0; /* * If the caller is waiting, return -EAGAIN to keep the background @@ -1737,25 +1755,13 @@ xfs_inode_free_cowblocks( void *args) { struct xfs_eofblocks *eofb = args; - int match; int ret = 0; if (!xfs_prep_free_cowblocks(ip)) return 0; - if (eofb) { - if (eofb->eof_flags & XFS_EOF_FLAGS_UNION) - match = xfs_inode_match_id_union(ip, eofb); - else - match = xfs_inode_match_id(ip, eofb); - if (!match) - return 0; - - /* skip the inode if the file size is too small */ - if (eofb->eof_flags & XFS_EOF_FLAGS_MINFILESIZE && - XFS_ISIZE(ip) < eofb->eof_min_file_size) - return 0; - } + if (!xfs_inode_matches_eofb(ip, eofb)) + return 0; /* Free the CoW blocks */ xfs_ilock(ip, XFS_IOLOCK_EXCL); From 39b1cfd75b278f5cb1e58517f6ec1ac9ef16d9f4 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 21 May 2020 13:08:49 -0700 Subject: [PATCH 0904/1043] xfs: fix inode ag walk predicate function return values There are a number of predicate functions that help the incore inode walking code decide if we really want to apply the iteration function to the inode. These are boolean decisions, so change the return types to boolean to match. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_icache.c | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index d10424a0015a..0f2edda5b08d 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -761,7 +761,12 @@ xfs_icache_inode_is_allocated( */ #define XFS_LOOKUP_BATCH 32 -STATIC int +/* + * Decide if the given @ip is eligible to be a part of the inode walk, and + * grab it if so. Returns true if it's ready to go or false if we should just + * ignore it. + */ +STATIC bool xfs_inode_ag_walk_grab( struct xfs_inode *ip, int flags) @@ -792,18 +797,18 @@ xfs_inode_ag_walk_grab( /* nothing to sync during shutdown */ if (XFS_FORCED_SHUTDOWN(ip->i_mount)) - return -EFSCORRUPTED; + return false; /* If we can't grab the inode, it must on it's way to reclaim. */ if (!igrab(inode)) - return -ENOENT; + return false; /* inode is valid */ - return 0; + return true; out_unlock_noent: spin_unlock(&ip->i_flags_lock); - return -ENOENT; + return false; } STATIC int @@ -855,7 +860,7 @@ restart: for (i = 0; i < nr_found; i++) { struct xfs_inode *ip = batch[i]; - if (done || xfs_inode_ag_walk_grab(ip, iter_flags)) + if (done || !xfs_inode_ag_walk_grab(ip, iter_flags)) batch[i] = NULL; /* @@ -1412,48 +1417,48 @@ xfs_reclaim_inodes_count( return reclaimable; } -STATIC int +STATIC bool xfs_inode_match_id( struct xfs_inode *ip, struct xfs_eofblocks *eofb) { if ((eofb->eof_flags & XFS_EOF_FLAGS_UID) && !uid_eq(VFS_I(ip)->i_uid, eofb->eof_uid)) - return 0; + return false; if ((eofb->eof_flags & XFS_EOF_FLAGS_GID) && !gid_eq(VFS_I(ip)->i_gid, eofb->eof_gid)) - return 0; + return false; if ((eofb->eof_flags & XFS_EOF_FLAGS_PRID) && ip->i_d.di_projid != eofb->eof_prid) - return 0; + return false; - return 1; + return true; } /* * A union-based inode filtering algorithm. Process the inode if any of the * criteria match. This is for global/internal scans only. */ -STATIC int +STATIC bool xfs_inode_match_id_union( struct xfs_inode *ip, struct xfs_eofblocks *eofb) { if ((eofb->eof_flags & XFS_EOF_FLAGS_UID) && uid_eq(VFS_I(ip)->i_uid, eofb->eof_uid)) - return 1; + return true; if ((eofb->eof_flags & XFS_EOF_FLAGS_GID) && gid_eq(VFS_I(ip)->i_gid, eofb->eof_gid)) - return 1; + return true; if ((eofb->eof_flags & XFS_EOF_FLAGS_PRID) && ip->i_d.di_projid == eofb->eof_prid) - return 1; + return true; - return 0; + return false; } /* @@ -1466,7 +1471,7 @@ xfs_inode_matches_eofb( struct xfs_inode *ip, struct xfs_eofblocks *eofb) { - int match; + bool match; if (!eofb) return true; From 7e88d31423e2874be03f8ebc6274c917f674743a Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 21 May 2020 13:08:49 -0700 Subject: [PATCH 0905/1043] xfs: use bool for done in xfs_inode_ag_walk This is a boolean variable, so use the bool type. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_icache.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 0f2edda5b08d..0adc6cc0b37e 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -823,11 +823,11 @@ xfs_inode_ag_walk( uint32_t first_index; int last_error = 0; int skipped; - int done; + bool done; int nr_found; restart: - done = 0; + done = false; skipped = 0; first_index = 0; nr_found = 0; @@ -879,7 +879,7 @@ restart: continue; first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1); if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino)) - done = 1; + done = true; } /* unlock now we've grabbed the inodes. */ From 5662d38ccdbd9e5c816f5c0b7f490a29729217b0 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 21 May 2020 13:08:50 -0700 Subject: [PATCH 0906/1043] xfs: move xfs_inode_ag_iterator to be closer to the perag walking code Move the xfs_inode_ag_iterator function to be nearer xfs_inode_ag_walk so that we don't have to scroll back and forth to figure out how the incore inode walking function works. No functional changes. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_icache.c | 88 ++++++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 0adc6cc0b37e..3a4b97669c7d 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -811,6 +811,10 @@ out_unlock_noent: return false; } +/* + * For a given per-AG structure @pag, grab, @execute, and rele all incore + * inodes with the given radix tree @tag. + */ STATIC int xfs_inode_ag_walk( struct xfs_mount *mp, @@ -916,6 +920,50 @@ restart: return last_error; } +/* Fetch the next (possibly tagged) per-AG structure. */ +static inline struct xfs_perag * +xfs_inode_walk_get_perag( + struct xfs_mount *mp, + xfs_agnumber_t agno, + int tag) +{ + if (tag == XFS_ICI_NO_TAG) + return xfs_perag_get(mp, agno); + return xfs_perag_get_tag(mp, agno, tag); +} + +/* + * Call the @execute function on all incore inodes matching the radix tree + * @tag. + */ +int +xfs_inode_ag_iterator( + struct xfs_mount *mp, + int iter_flags, + int (*execute)(struct xfs_inode *ip, void *args), + void *args, + int tag) +{ + struct xfs_perag *pag; + int error = 0; + int last_error = 0; + xfs_agnumber_t ag; + + ag = 0; + while ((pag = xfs_inode_walk_get_perag(mp, ag, tag))) { + ag = pag->pag_agno + 1; + error = xfs_inode_ag_walk(mp, pag, execute, args, tag, + iter_flags); + xfs_perag_put(pag); + if (error) { + last_error = error; + if (error == -EFSCORRUPTED) + break; + } + } + return last_error; +} + /* * Background scanning to trim post-EOF preallocated space. This is queued * based on the 'speculative_prealloc_lifetime' tunable (5m by default). @@ -979,46 +1027,6 @@ xfs_cowblocks_worker( xfs_queue_cowblocks(mp); } -/* Fetch the next (possibly tagged) per-AG structure. */ -static inline struct xfs_perag * -xfs_inode_walk_get_perag( - struct xfs_mount *mp, - xfs_agnumber_t agno, - int tag) -{ - if (tag == XFS_ICI_NO_TAG) - return xfs_perag_get(mp, agno); - return xfs_perag_get_tag(mp, agno, tag); -} - -int -xfs_inode_ag_iterator( - struct xfs_mount *mp, - int iter_flags, - int (*execute)(struct xfs_inode *ip, void *args), - void *args, - int tag) -{ - struct xfs_perag *pag; - int error = 0; - int last_error = 0; - xfs_agnumber_t ag; - - ag = 0; - while ((pag = xfs_inode_walk_get_perag(mp, ag, tag))) { - ag = pag->pag_agno + 1; - error = xfs_inode_ag_walk(mp, pag, execute, args, tag, - iter_flags); - xfs_perag_put(pag); - if (error) { - last_error = error; - if (error == -EFSCORRUPTED) - break; - } - } - return last_error; -} - /* * Grab the inode for reclaim exclusively. * Return 0 if we grabbed it, non-zero otherwise. From 042f65f4a79c819d74309c2607959fccacc4094a Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 21 May 2020 13:08:50 -0700 Subject: [PATCH 0907/1043] xfs: straighten out all the naming around incore inode tree walks We're not very consistent about function names for the incore inode iteration function. Turn them all into xfs_inode_walk* variants. Signed-off-by: Darrick J. Wong Reviewed-by: Brian Foster Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_icache.c | 18 +++++++++--------- fs/xfs/xfs_icache.h | 6 +++--- fs/xfs/xfs_qm_syscalls.c | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 3a4b97669c7d..82e70b9f2560 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -767,12 +767,12 @@ xfs_icache_inode_is_allocated( * ignore it. */ STATIC bool -xfs_inode_ag_walk_grab( +xfs_inode_walk_ag_grab( struct xfs_inode *ip, int flags) { struct inode *inode = VFS_I(ip); - bool newinos = !!(flags & XFS_AGITER_INEW_WAIT); + bool newinos = !!(flags & XFS_INODE_WALK_INEW_WAIT); ASSERT(rcu_read_lock_held()); @@ -816,7 +816,7 @@ out_unlock_noent: * inodes with the given radix tree @tag. */ STATIC int -xfs_inode_ag_walk( +xfs_inode_walk_ag( struct xfs_mount *mp, struct xfs_perag *pag, int (*execute)(struct xfs_inode *ip, void *args), @@ -864,7 +864,7 @@ restart: for (i = 0; i < nr_found; i++) { struct xfs_inode *ip = batch[i]; - if (done || !xfs_inode_ag_walk_grab(ip, iter_flags)) + if (done || !xfs_inode_walk_ag_grab(ip, iter_flags)) batch[i] = NULL; /* @@ -892,7 +892,7 @@ restart: for (i = 0; i < nr_found; i++) { if (!batch[i]) continue; - if ((iter_flags & XFS_AGITER_INEW_WAIT) && + if ((iter_flags & XFS_INODE_WALK_INEW_WAIT) && xfs_iflags_test(batch[i], XFS_INEW)) xfs_inew_wait(batch[i]); error = execute(batch[i], args); @@ -937,7 +937,7 @@ xfs_inode_walk_get_perag( * @tag. */ int -xfs_inode_ag_iterator( +xfs_inode_walk( struct xfs_mount *mp, int iter_flags, int (*execute)(struct xfs_inode *ip, void *args), @@ -952,7 +952,7 @@ xfs_inode_ag_iterator( ag = 0; while ((pag = xfs_inode_walk_get_perag(mp, ag, tag))) { ag = pag->pag_agno + 1; - error = xfs_inode_ag_walk(mp, pag, execute, args, tag, + error = xfs_inode_walk_ag(mp, pag, execute, args, tag, iter_flags); xfs_perag_put(pag); if (error) { @@ -1548,7 +1548,7 @@ xfs_icache_free_eofblocks( struct xfs_mount *mp, struct xfs_eofblocks *eofb) { - return xfs_inode_ag_iterator(mp, 0, xfs_inode_free_eofblocks, eofb, + return xfs_inode_walk(mp, 0, xfs_inode_free_eofblocks, eofb, XFS_ICI_EOFBLOCKS_TAG); } @@ -1798,7 +1798,7 @@ xfs_icache_free_cowblocks( struct xfs_mount *mp, struct xfs_eofblocks *eofb) { - return xfs_inode_ag_iterator(mp, 0, xfs_inode_free_cowblocks, eofb, + return xfs_inode_walk(mp, 0, xfs_inode_free_cowblocks, eofb, XFS_ICI_COWBLOCKS_TAG); } diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h index e7f86ebd7b22..93b54e7d55f0 100644 --- a/fs/xfs/xfs_icache.h +++ b/fs/xfs/xfs_icache.h @@ -24,7 +24,7 @@ struct xfs_eofblocks { * tags for inode radix tree */ #define XFS_ICI_NO_TAG (-1) /* special flag for an untagged lookup - in xfs_inode_ag_iterator */ + in xfs_inode_walk */ #define XFS_ICI_RECLAIM_TAG 0 /* inode is to be reclaimed */ #define XFS_ICI_EOFBLOCKS_TAG 1 /* inode has blocks beyond EOF */ #define XFS_ICI_COWBLOCKS_TAG 2 /* inode can have cow blocks to gc */ @@ -40,7 +40,7 @@ struct xfs_eofblocks { /* * flags for AG inode iterator */ -#define XFS_AGITER_INEW_WAIT 0x1 /* wait on new inodes */ +#define XFS_INODE_WALK_INEW_WAIT 0x1 /* wait on new inodes */ int xfs_iget(struct xfs_mount *mp, struct xfs_trans *tp, xfs_ino_t ino, uint flags, uint lock_flags, xfs_inode_t **ipp); @@ -71,7 +71,7 @@ int xfs_inode_free_quota_cowblocks(struct xfs_inode *ip); void xfs_cowblocks_worker(struct work_struct *); void xfs_queue_cowblocks(struct xfs_mount *); -int xfs_inode_ag_iterator(struct xfs_mount *mp, int iter_flags, +int xfs_inode_walk(struct xfs_mount *mp, int iter_flags, int (*execute)(struct xfs_inode *ip, void *args), void *args, int tag); diff --git a/fs/xfs/xfs_qm_syscalls.c b/fs/xfs/xfs_qm_syscalls.c index f7db1a05f7b8..7effd7a28136 100644 --- a/fs/xfs/xfs_qm_syscalls.c +++ b/fs/xfs/xfs_qm_syscalls.c @@ -781,6 +781,6 @@ xfs_qm_dqrele_all_inodes( uint flags) { ASSERT(mp->m_quotainfo); - xfs_inode_ag_iterator(mp, XFS_AGITER_INEW_WAIT, xfs_dqrele_inode, + xfs_inode_walk(mp, XFS_INODE_WALK_INEW_WAIT, xfs_dqrele_inode, &flags, XFS_ICI_NO_TAG); } From 964176bd32da9847112b505f57ee6e602ee5c84d Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 21 May 2020 13:08:50 -0700 Subject: [PATCH 0908/1043] xfs: rearrange xfs_inode_walk_ag parameters The perag structure already has a pointer to the xfs_mount, so we don't need to pass that separately and can drop it. Having done that, move iter_flags so that the argument order is the same between xfs_inode_walk and xfs_inode_walk_ag. The latter will make things less confusing for a future patch that enables background scanning work to be done in parallel. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_icache.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c index 82e70b9f2560..0a5ac6f9a583 100644 --- a/fs/xfs/xfs_icache.c +++ b/fs/xfs/xfs_icache.c @@ -817,13 +817,13 @@ out_unlock_noent: */ STATIC int xfs_inode_walk_ag( - struct xfs_mount *mp, struct xfs_perag *pag, + int iter_flags, int (*execute)(struct xfs_inode *ip, void *args), void *args, - int tag, - int iter_flags) + int tag) { + struct xfs_mount *mp = pag->pag_mount; uint32_t first_index; int last_error = 0; int skipped; @@ -952,8 +952,7 @@ xfs_inode_walk( ag = 0; while ((pag = xfs_inode_walk_get_perag(mp, ag, tag))) { ag = pag->pag_agno + 1; - error = xfs_inode_walk_ag(mp, pag, execute, args, tag, - iter_flags); + error = xfs_inode_walk_ag(pag, iter_flags, execute, args, tag); xfs_perag_put(pag); if (error) { last_error = error; From 1edd2c055dff9710b1e29d4df01902abb0a55f1f Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Sat, 23 May 2020 09:43:30 -0700 Subject: [PATCH 0909/1043] xfs: don't fail unwritten extent conversion on writeback due to edquot During writeback, it's possible for the quota block reservation in xfs_iomap_write_unwritten to fail with EDQUOT because we hit the quota limit. This causes writeback errors for data that was already written to disk, when it's not even guaranteed that the bmbt will expand to exceed the quota limit. Irritatingly, this condition is reported to userspace as EIO by fsync, which is confusing. We wrote the data, so allow the reservation. That might put us slightly above the hard limit, but it's better than losing data after a write. Signed-off-by: Darrick J. Wong Reviewed-by: Brian Foster Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_iomap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 6ae3a2457777..7d8966ce630a 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -563,7 +563,7 @@ xfs_iomap_write_unwritten( xfs_trans_ijoin(tp, ip, 0); error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0, - XFS_QMOPT_RES_REGBLKS); + XFS_QMOPT_RES_REGBLKS | XFS_QMOPT_FORCE_RES); if (error) goto error_on_bmapi_transaction; From f0322c7cc05eb23ef034775f9b39254cbd4f3678 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Sat, 23 May 2020 09:43:30 -0700 Subject: [PATCH 0910/1043] xfs: measure all contiguous previous extents for prealloc size When we're estimating a new speculative preallocation length for an extending write, we should walk backwards through the extent list to determine the number of number of blocks that are physically and logically contiguous with the write offset, and use that as an input to the preallocation size computation. This way, preallocation length is truly measured by the effectiveness of the allocator in giving us contiguous allocations without being influenced by the state of a given extent. This fixes both the problem where ZERO_RANGE within an EOF can reduce preallocation, and prevents the unnecessary shrinkage of preallocation when delalloc extents are turned into unwritten extents. This was found as a regression in xfs/014 after changing delalloc writes to create unwritten extents during writeback. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_iomap.c | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index 7d8966ce630a..e74a8c2c94ce 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -377,15 +377,17 @@ xfs_iomap_prealloc_size( loff_t count, struct xfs_iext_cursor *icur) { + struct xfs_iext_cursor ncur = *icur; + struct xfs_bmbt_irec prev, got; struct xfs_mount *mp = ip->i_mount; struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork); xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); - struct xfs_bmbt_irec prev; - int shift = 0; int64_t freesp; xfs_fsblock_t qblocks; - int qshift = 0; xfs_fsblock_t alloc_blocks = 0; + xfs_extlen_t plen; + int shift = 0; + int qshift = 0; if (offset + count <= XFS_ISIZE(ip)) return 0; @@ -400,7 +402,7 @@ xfs_iomap_prealloc_size( */ if ((mp->m_flags & XFS_MOUNT_ALLOCSIZE) || XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_dalign) || - !xfs_iext_peek_prev_extent(ifp, icur, &prev) || + !xfs_iext_prev_extent(ifp, &ncur, &prev) || prev.br_startoff + prev.br_blockcount < offset_fsb) return mp->m_allocsize_blocks; @@ -413,16 +415,28 @@ xfs_iomap_prealloc_size( * preallocation size. * * If the extent is a hole, then preallocation is essentially disabled. - * Otherwise we take the size of the preceding data extent as the basis - * for the preallocation size. If the size of the extent is greater than - * half the maximum extent length, then use the current offset as the - * basis. This ensures that for large files the preallocation size - * always extends to MAXEXTLEN rather than falling short due to things - * like stripe unit/width alignment of real extents. + * Otherwise we take the size of the preceding data extents as the basis + * for the preallocation size. Note that we don't care if the previous + * extents are written or not. + * + * If the size of the extents is greater than half the maximum extent + * length, then use the current offset as the basis. This ensures that + * for large files the preallocation size always extends to MAXEXTLEN + * rather than falling short due to things like stripe unit/width + * alignment of real extents. */ - if (prev.br_blockcount <= (MAXEXTLEN >> 1)) - alloc_blocks = prev.br_blockcount << 1; - else + plen = prev.br_blockcount; + while (xfs_iext_prev_extent(ifp, &ncur, &got)) { + if (plen > MAXEXTLEN / 2 || + isnullstartblock(got.br_startblock) || + got.br_startoff + got.br_blockcount != prev.br_startoff || + got.br_startblock + got.br_blockcount != prev.br_startblock) + break; + plen += got.br_blockcount; + prev = got; + } + alloc_blocks = plen * 2; + if (alloc_blocks > MAXEXTLEN) alloc_blocks = XFS_B_TO_FSB(mp, offset); if (!alloc_blocks) goto check_writeio; From 590b16516ef38e2e88674fe0a0cea39927a8ee2e Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Sat, 23 May 2020 09:43:30 -0700 Subject: [PATCH 0911/1043] xfs: refactor xfs_iomap_prealloc_size Refactor xfs_iomap_prealloc_size to be the function that dynamically computes the per-file preallocation size by moving the allocsize= case to the caller. Break up the huge comment preceding the function to annotate the relevant parts of the code, and remove the impossible check_writeio case. Suggested-by: Christoph Hellwig Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/xfs_iomap.c | 83 +++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 48 deletions(-) diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index e74a8c2c94ce..b9a8c3798e08 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -352,22 +352,10 @@ xfs_quota_calc_throttle( } /* - * If we are doing a write at the end of the file and there are no allocations - * past this one, then extend the allocation out to the file system's write - * iosize. - * * If we don't have a user specified preallocation size, dynamically increase * the preallocation size as the size of the file grows. Cap the maximum size * at a single extent or less if the filesystem is near full. The closer the - * filesystem is to full, the smaller the maximum prealocation. - * - * As an exception we don't do any preallocation at all if the file is smaller - * than the minimum preallocation and we are using the default dynamic - * preallocation scheme, as it is likely this is the only write to the file that - * is going to be done. - * - * We clean up any extra space left over when the file is closed in - * xfs_inactive(). + * filesystem is to being full, the smaller the maximum preallocation. */ STATIC xfs_fsblock_t xfs_iomap_prealloc_size( @@ -389,41 +377,28 @@ xfs_iomap_prealloc_size( int shift = 0; int qshift = 0; - if (offset + count <= XFS_ISIZE(ip)) - return 0; - - if (!(mp->m_flags & XFS_MOUNT_ALLOCSIZE) && - (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_allocsize_blocks))) + /* + * As an exception we don't do any preallocation at all if the file is + * smaller than the minimum preallocation and we are using the default + * dynamic preallocation scheme, as it is likely this is the only write + * to the file that is going to be done. + */ + if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_allocsize_blocks)) return 0; /* - * If an explicit allocsize is set, the file is small, or we - * are writing behind a hole, then use the minimum prealloc: + * Use the minimum preallocation size for small files or if we are + * writing right after a hole. */ - if ((mp->m_flags & XFS_MOUNT_ALLOCSIZE) || - XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_dalign) || + if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_dalign) || !xfs_iext_prev_extent(ifp, &ncur, &prev) || prev.br_startoff + prev.br_blockcount < offset_fsb) return mp->m_allocsize_blocks; /* - * Determine the initial size of the preallocation. We are beyond the - * current EOF here, but we need to take into account whether this is - * a sparse write or an extending write when determining the - * preallocation size. Hence we need to look up the extent that ends - * at the current write offset and use the result to determine the - * preallocation size. - * - * If the extent is a hole, then preallocation is essentially disabled. - * Otherwise we take the size of the preceding data extents as the basis - * for the preallocation size. Note that we don't care if the previous - * extents are written or not. - * - * If the size of the extents is greater than half the maximum extent - * length, then use the current offset as the basis. This ensures that - * for large files the preallocation size always extends to MAXEXTLEN - * rather than falling short due to things like stripe unit/width - * alignment of real extents. + * Take the size of the preceding data extents as the basis for the + * preallocation size. Note that we don't care if the previous extents + * are written or not. */ plen = prev.br_blockcount; while (xfs_iext_prev_extent(ifp, &ncur, &got)) { @@ -435,19 +410,25 @@ xfs_iomap_prealloc_size( plen += got.br_blockcount; prev = got; } + + /* + * If the size of the extents is greater than half the maximum extent + * length, then use the current offset as the basis. This ensures that + * for large files the preallocation size always extends to MAXEXTLEN + * rather than falling short due to things like stripe unit/width + * alignment of real extents. + */ alloc_blocks = plen * 2; if (alloc_blocks > MAXEXTLEN) alloc_blocks = XFS_B_TO_FSB(mp, offset); - if (!alloc_blocks) - goto check_writeio; qblocks = alloc_blocks; /* * MAXEXTLEN is not a power of two value but we round the prealloc down * to the nearest power of two value after throttling. To prevent the - * round down from unconditionally reducing the maximum supported prealloc - * size, we round up first, apply appropriate throttling, round down and - * cap the value to MAXEXTLEN. + * round down from unconditionally reducing the maximum supported + * prealloc size, we round up first, apply appropriate throttling, + * round down and cap the value to MAXEXTLEN. */ alloc_blocks = XFS_FILEOFF_MIN(roundup_pow_of_two(MAXEXTLEN), alloc_blocks); @@ -508,7 +489,6 @@ xfs_iomap_prealloc_size( */ while (alloc_blocks && alloc_blocks >= freesp) alloc_blocks >>= 4; -check_writeio: if (alloc_blocks < mp->m_allocsize_blocks) alloc_blocks = mp->m_allocsize_blocks; trace_xfs_iomap_prealloc_size(ip, alloc_blocks, shift, @@ -975,9 +955,16 @@ xfs_buffered_write_iomap_begin( if (error) goto out_unlock; - if (eof) { - prealloc_blocks = xfs_iomap_prealloc_size(ip, allocfork, offset, - count, &icur); + if (eof && offset + count > XFS_ISIZE(ip)) { + /* + * Determine the initial size of the preallocation. + * We clean up any extra preallocation when the file is closed. + */ + if (mp->m_flags & XFS_MOUNT_ALLOCSIZE) + prealloc_blocks = mp->m_allocsize_blocks; + else + prealloc_blocks = xfs_iomap_prealloc_size(ip, allocfork, + offset, count, &icur); if (prealloc_blocks) { xfs_extlen_t align; xfs_off_t end_offset; From a5949d3faedf492fa7863b914da408047ab46eb0 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Sat, 23 May 2020 09:43:31 -0700 Subject: [PATCH 0912/1043] xfs: force writes to delalloc regions to unwritten When writing to a delalloc region in the data fork, commit the new allocations (of the da reservation) as unwritten so that the mappings are only marked written once writeback completes successfully. This fixes the problem of stale data exposure if the system goes down during targeted writeback of a specific region of a file, as tested by generic/042. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig Reviewed-by: Brian Foster --- fs/xfs/libxfs/xfs_bmap.c | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c index edc63dba007f..667cdd0dfdf4 100644 --- a/fs/xfs/libxfs/xfs_bmap.c +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -4145,17 +4145,7 @@ xfs_bmapi_allocate( bma->got.br_blockcount = bma->length; bma->got.br_state = XFS_EXT_NORM; - /* - * In the data fork, a wasdelay extent has been initialized, so - * shouldn't be flagged as unwritten. - * - * For the cow fork, however, we convert delalloc reservations - * (extents allocated for speculative preallocation) to - * allocated unwritten extents, and only convert the unwritten - * extents to real extents when we're about to write the data. - */ - if ((!bma->wasdel || (bma->flags & XFS_BMAPI_COWFORK)) && - (bma->flags & XFS_BMAPI_PREALLOC)) + if (bma->flags & XFS_BMAPI_PREALLOC) bma->got.br_state = XFS_EXT_UNWRITTEN; if (bma->wasdel) @@ -4563,8 +4553,23 @@ xfs_bmapi_convert_delalloc( bma.offset = bma.got.br_startoff; bma.length = max_t(xfs_filblks_t, bma.got.br_blockcount, MAXEXTLEN); bma.minleft = xfs_bmapi_minleft(tp, ip, whichfork); + + /* + * When we're converting the delalloc reservations backing dirty pages + * in the page cache, we must be careful about how we create the new + * extents: + * + * New CoW fork extents are created unwritten, turned into real extents + * when we're about to write the data to disk, and mapped into the data + * fork after the write finishes. End of story. + * + * New data fork extents must be mapped in as unwritten and converted + * to real extents after the write succeeds to avoid exposing stale + * disk contents if we crash. + */ + bma.flags = XFS_BMAPI_PREALLOC; if (whichfork == XFS_COW_FORK) - bma.flags = XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC; + bma.flags |= XFS_BMAPI_COWFORK; if (!xfs_iext_peek_prev_extent(ifp, &bma.icur, &bma.prev)) bma.prev.br_startoff = NULLFILEOFF; From 6dcde60efd946e38fac8d276a6ca47492103e856 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Tue, 26 May 2020 09:33:11 -0700 Subject: [PATCH 0913/1043] xfs: more lockdep whackamole with kmem_alloc* Dave Airlie reported the following lockdep complaint: > ====================================================== > WARNING: possible circular locking dependency detected > 5.7.0-0.rc5.20200515git1ae7efb38854.1.fc33.x86_64 #1 Not tainted > ------------------------------------------------------ > kswapd0/159 is trying to acquire lock: > ffff9b38d01a4470 (&xfs_nondir_ilock_class){++++}-{3:3}, > at: xfs_ilock+0xde/0x2c0 [xfs] > > but task is already holding lock: > ffffffffbbb8bd00 (fs_reclaim){+.+.}-{0:0}, at: > __fs_reclaim_acquire+0x5/0x30 > > which lock already depends on the new lock. > > > the existing dependency chain (in reverse order) is: > > -> #1 (fs_reclaim){+.+.}-{0:0}: > fs_reclaim_acquire+0x34/0x40 > __kmalloc+0x4f/0x270 > kmem_alloc+0x93/0x1d0 [xfs] > kmem_alloc_large+0x4c/0x130 [xfs] > xfs_attr_copy_value+0x74/0xa0 [xfs] > xfs_attr_get+0x9d/0xc0 [xfs] > xfs_get_acl+0xb6/0x200 [xfs] > get_acl+0x81/0x160 > posix_acl_xattr_get+0x3f/0xd0 > vfs_getxattr+0x148/0x170 > getxattr+0xa7/0x240 > path_getxattr+0x52/0x80 > do_syscall_64+0x5c/0xa0 > entry_SYSCALL_64_after_hwframe+0x49/0xb3 > > -> #0 (&xfs_nondir_ilock_class){++++}-{3:3}: > __lock_acquire+0x1257/0x20d0 > lock_acquire+0xb0/0x310 > down_write_nested+0x49/0x120 > xfs_ilock+0xde/0x2c0 [xfs] > xfs_reclaim_inode+0x3f/0x400 [xfs] > xfs_reclaim_inodes_ag+0x20b/0x410 [xfs] > xfs_reclaim_inodes_nr+0x31/0x40 [xfs] > super_cache_scan+0x190/0x1e0 > do_shrink_slab+0x184/0x420 > shrink_slab+0x182/0x290 > shrink_node+0x174/0x680 > balance_pgdat+0x2d0/0x5f0 > kswapd+0x21f/0x510 > kthread+0x131/0x150 > ret_from_fork+0x3a/0x50 > > other info that might help us debug this: > > Possible unsafe locking scenario: > > CPU0 CPU1 > ---- ---- > lock(fs_reclaim); > lock(&xfs_nondir_ilock_class); > lock(fs_reclaim); > lock(&xfs_nondir_ilock_class); > > *** DEADLOCK *** > > 4 locks held by kswapd0/159: > #0: ffffffffbbb8bd00 (fs_reclaim){+.+.}-{0:0}, at: > __fs_reclaim_acquire+0x5/0x30 > #1: ffffffffbbb7cef8 (shrinker_rwsem){++++}-{3:3}, at: > shrink_slab+0x115/0x290 > #2: ffff9b39f07a50e8 > (&type->s_umount_key#56){++++}-{3:3}, at: super_cache_scan+0x38/0x1e0 > #3: ffff9b39f077f258 > (&pag->pag_ici_reclaim_lock){+.+.}-{3:3}, at: > xfs_reclaim_inodes_ag+0x82/0x410 [xfs] This is a known false positive because inodes cannot simultaneously be getting reclaimed and the target of a getxattr operation, but lockdep doesn't know that. We can (selectively) shut up lockdep until either it gets smarter or we change inode reclaim not to require the ILOCK by applying a stupid GFP_NOLOCKDEP bandaid. Reported-by: Dave Airlie Signed-off-by: Darrick J. Wong Tested-by: Dave Airlie Reviewed-by: Brian Foster --- fs/xfs/kmem.h | 6 +++++- fs/xfs/libxfs/xfs_attr_leaf.c | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/fs/xfs/kmem.h b/fs/xfs/kmem.h index fc87ea9f6843..34cbcfde9228 100644 --- a/fs/xfs/kmem.h +++ b/fs/xfs/kmem.h @@ -19,6 +19,7 @@ typedef unsigned __bitwise xfs_km_flags_t; #define KM_NOFS ((__force xfs_km_flags_t)0x0004u) #define KM_MAYFAIL ((__force xfs_km_flags_t)0x0008u) #define KM_ZERO ((__force xfs_km_flags_t)0x0010u) +#define KM_NOLOCKDEP ((__force xfs_km_flags_t)0x0020u) /* * We use a special process flag to avoid recursive callbacks into @@ -30,7 +31,7 @@ kmem_flags_convert(xfs_km_flags_t flags) { gfp_t lflags; - BUG_ON(flags & ~(KM_NOFS|KM_MAYFAIL|KM_ZERO)); + BUG_ON(flags & ~(KM_NOFS | KM_MAYFAIL | KM_ZERO | KM_NOLOCKDEP)); lflags = GFP_KERNEL | __GFP_NOWARN; if (flags & KM_NOFS) @@ -49,6 +50,9 @@ kmem_flags_convert(xfs_km_flags_t flags) if (flags & KM_ZERO) lflags |= __GFP_ZERO; + if (flags & KM_NOLOCKDEP) + lflags |= __GFP_NOLOCKDEP; + return lflags; } diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index f3d18a1f5b20..2f7e89e4be3e 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -488,7 +488,7 @@ xfs_attr_copy_value( } if (!args->value) { - args->value = kmem_alloc_large(valuelen, 0); + args->value = kmem_alloc_large(valuelen, KM_NOLOCKDEP); if (!args->value) return -ENOMEM; } From 6129ed877d409037b79866327102c9dc59a302fe Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 27 May 2020 01:49:09 -0700 Subject: [PATCH 0914/1043] KVM: x86/mmu: Set mmio_value to '0' if reserved #PF can't be generated Set the mmio_value to '0' instead of simply clearing the present bit to squash a benign warning in kvm_mmu_set_mmio_spte_mask() that complains about the mmio_value overlapping the lower GFN mask on systems with 52 bits of PA space. Opportunistically clean up the code and comments. Cc: stable@vger.kernel.org Fixes: d43e2675e96fc ("KVM: x86: only do L1TF workaround on affected processors") Signed-off-by: Sean Christopherson Message-Id: <20200527084909.23492-1-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu/mmu.c | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 86619631ff6a..92d056954194 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -6143,25 +6143,16 @@ static void kvm_set_mmio_spte_mask(void) u64 mask; /* - * Set the reserved bits and the present bit of an paging-structure - * entry to generate page fault with PFER.RSV = 1. + * Set a reserved PA bit in MMIO SPTEs to generate page faults with + * PFEC.RSVD=1 on MMIO accesses. 64-bit PTEs (PAE, x86-64, and EPT + * paging) support a maximum of 52 bits of PA, i.e. if the CPU supports + * 52-bit physical addresses then there are no reserved PA bits in the + * PTEs and so the reserved PA approach must be disabled. */ - - /* - * Mask the uppermost physical address bit, which would be reserved as - * long as the supported physical address width is less than 52. - */ - mask = 1ull << 51; - - /* Set the present bit. */ - mask |= 1ull; - - /* - * If reserved bit is not supported, clear the present bit to disable - * mmio page fault. - */ - if (shadow_phys_bits == 52) - mask &= ~1ull; + if (shadow_phys_bits < 52) + mask = BIT_ULL(51) | PT_PRESENT_MASK; + else + mask = 0; kvm_mmu_set_mmio_spte_mask(mask, mask, ACC_WRITE_MASK | ACC_USER_MASK); } From 0abcc8f65cc23b65bc8d1614cc64b02b1641ed7c Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Sat, 23 May 2020 19:14:54 +0300 Subject: [PATCH 0915/1043] KVM: VMX: enable X86_FEATURE_WAITPKG in KVM capabilities Even though we might not allow the guest to use WAITPKG's new instructions, we should tell KVM that the feature is supported by the host CPU. Note that vmx_waitpkg_supported checks that WAITPKG _can_ be set in secondary execution controls as specified by VMX capability MSR, rather that we actually enable it for a guest. Cc: stable@vger.kernel.org Fixes: e69e72faa3a0 ("KVM: x86: Add support for user wait instructions") Suggested-by: Paolo Bonzini Signed-off-by: Maxim Levitsky Message-Id: <20200523161455.3940-2-mlevitsk@redhat.com> Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 89c766fad889..9b63ac8d97ee 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -7138,6 +7138,9 @@ static __init void vmx_set_cpu_caps(void) /* CPUID 0x80000001 */ if (!cpu_has_vmx_rdtscp()) kvm_cpu_cap_clear(X86_FEATURE_RDTSCP); + + if (vmx_waitpkg_supported()) + kvm_cpu_cap_check_and_set(X86_FEATURE_WAITPKG); } static void vmx_request_immediate_exit(struct kvm_vcpu *vcpu) From f4cfcd2d5aea4e96c5d483c476f3057b6b7baf6a Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Sat, 23 May 2020 19:14:55 +0300 Subject: [PATCH 0916/1043] KVM: x86: don't expose MSR_IA32_UMWAIT_CONTROL unconditionally This msr is only available when the host supports WAITPKG feature. This breaks a nested guest, if the L1 hypervisor is set to ignore unknown msrs, because the only other safety check that the kernel does is that it attempts to read the msr and rejects it if it gets an exception. Cc: stable@vger.kernel.org Fixes: 6e3ba4abce ("KVM: vmx: Emulate MSR IA32_UMWAIT_CONTROL") Signed-off-by: Maxim Levitsky Message-Id: <20200523161455.3940-3-mlevitsk@redhat.com> Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c17e6eb9ad43..e0083a08da9e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -5242,6 +5242,10 @@ static void kvm_init_msr_list(void) if (!kvm_cpu_cap_has(X86_FEATURE_RDTSCP)) continue; break; + case MSR_IA32_UMWAIT_CONTROL: + if (!kvm_cpu_cap_has(X86_FEATURE_WAITPKG)) + continue; + break; case MSR_IA32_RTIT_CTL: case MSR_IA32_RTIT_STATUS: if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT)) From e7581caca4c105d81a490a3e15cf46d6407e3fa7 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 19 May 2020 05:04:49 -0400 Subject: [PATCH 0917/1043] KVM: x86: simplify is_mmio_spte We can simply look at bits 52-53 to identify MMIO entries in KVM's page tables. Therefore, there is no need to pass a mask to kvm_mmu_set_mmio_spte_mask. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu.h | 2 +- arch/x86/kvm/mmu/mmu.c | 10 +++------- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/vmx/vmx.c | 3 +-- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index 8a3b1bce722a..048e865ad485 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -51,7 +51,7 @@ static inline u64 rsvd_bits(int s, int e) return ((1ULL << (e - s + 1)) - 1) << s; } -void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value, u64 access_mask); +void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 access_mask); void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 92d056954194..bef05a437a89 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -244,7 +244,6 @@ static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */ static u64 __read_mostly shadow_user_mask; static u64 __read_mostly shadow_accessed_mask; static u64 __read_mostly shadow_dirty_mask; -static u64 __read_mostly shadow_mmio_mask; static u64 __read_mostly shadow_mmio_value; static u64 __read_mostly shadow_mmio_access_mask; static u64 __read_mostly shadow_present_mask; @@ -331,21 +330,19 @@ static void kvm_flush_remote_tlbs_with_address(struct kvm *kvm, kvm_flush_remote_tlbs_with_range(kvm, &range); } -void kvm_mmu_set_mmio_spte_mask(u64 mmio_mask, u64 mmio_value, u64 access_mask) +void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 access_mask) { BUG_ON((u64)(unsigned)access_mask != access_mask); - BUG_ON((mmio_mask & mmio_value) != mmio_value); WARN_ON(mmio_value & (shadow_nonpresent_or_rsvd_mask << shadow_nonpresent_or_rsvd_mask_len)); WARN_ON(mmio_value & shadow_nonpresent_or_rsvd_lower_gfn_mask); shadow_mmio_value = mmio_value | SPTE_MMIO_MASK; - shadow_mmio_mask = mmio_mask | SPTE_SPECIAL_MASK; shadow_mmio_access_mask = access_mask; } EXPORT_SYMBOL_GPL(kvm_mmu_set_mmio_spte_mask); static bool is_mmio_spte(u64 spte) { - return (spte & shadow_mmio_mask) == shadow_mmio_value; + return (spte & SPTE_SPECIAL_MASK) == SPTE_MMIO_MASK; } static inline bool sp_ad_disabled(struct kvm_mmu_page *sp) @@ -568,7 +565,6 @@ static void kvm_mmu_reset_all_pte_masks(void) shadow_dirty_mask = 0; shadow_nx_mask = 0; shadow_x_mask = 0; - shadow_mmio_mask = 0; shadow_present_mask = 0; shadow_acc_track_mask = 0; @@ -6154,7 +6150,7 @@ static void kvm_set_mmio_spte_mask(void) else mask = 0; - kvm_mmu_set_mmio_spte_mask(mask, mask, ACC_WRITE_MASK | ACC_USER_MASK); + kvm_mmu_set_mmio_spte_mask(mask, ACC_WRITE_MASK | ACC_USER_MASK); } static bool get_nx_auto_mode(void) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index a862c768fd54..3c9b321a420a 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -780,7 +780,7 @@ static __init void svm_adjust_mmio_mask(void) */ mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0; - kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK); + kvm_mmu_set_mmio_spte_mask(mask, PT_WRITABLE_MASK | PT_USER_MASK); } static void svm_hardware_teardown(void) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 9b63ac8d97ee..65ff16f00c08 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4147,8 +4147,7 @@ static void ept_set_mmio_spte_mask(void) * EPT Misconfigurations can be generated if the value of bits 2:0 * of an EPT paging-structure entry is 110b (write/execute). */ - kvm_mmu_set_mmio_spte_mask(VMX_EPT_RWX_MASK, - VMX_EPT_MISCONFIG_WX_VALUE, 0); + kvm_mmu_set_mmio_spte_mask(VMX_EPT_MISCONFIG_WX_VALUE, 0); } #define VMX_XSS_EXIT_BITMAP 0 From df2a69af85bef169ab6810cc57f6b6b943941e7e Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 19 May 2020 12:51:32 -0400 Subject: [PATCH 0918/1043] KVM: x86: allow KVM_STATE_NESTED_MTF_PENDING in kvm_state flags The migration functionality was left incomplete in commit 5ef8acbdd687 ("KVM: nVMX: Emulate MTF when performing instruction emulation", 2020-02-23), fix it. Fixes: 5ef8acbdd687 ("KVM: nVMX: Emulate MTF when performing instruction emulation") Cc: stable@vger.kernel.org Reviewed-by: Oliver Upton Reviewed-by: Vitaly Kuznetsov Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 8ac69c16f153..0621a63afee7 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -4626,7 +4626,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, if (kvm_state.flags & ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE - | KVM_STATE_NESTED_EVMCS)) + | KVM_STATE_NESTED_EVMCS | KVM_STATE_NESTED_MTF_PENDING)) break; /* nested_run_pending implies guest_mode. */ From 7cb85fc465fdb91e88eff02b496150b1270952cf Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 18 Feb 2020 15:40:10 -0800 Subject: [PATCH 0919/1043] KVM: x86: Remove superfluous brackets from case statement Remove unnecessary brackets from a case statement that unintentionally encapsulates unrelated case statements in the same switch statement. While technically legal and functionally correct syntax, the brackets are visually confusing and potentially dangerous, e.g. the last of the encapsulated case statements has an undocumented fall-through that isn't flagged by compilers due the encapsulation. Reviewed-by: Vitaly Kuznetsov Signed-off-by: Sean Christopherson Message-Id: <20200218234012.7110-2-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 0621a63afee7..43f9b63399f0 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -5303,7 +5303,7 @@ static void kvm_init_msr_list(void) !intel_pt_validate_hw_cap(PT_CAP_single_range_output))) continue; break; - case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: { + case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B: if (!kvm_cpu_cap_has(X86_FEATURE_INTEL_PT) || msrs_to_save_all[i] - MSR_IA32_RTIT_ADDR0_A >= intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2) @@ -5318,7 +5318,7 @@ static void kvm_init_msr_list(void) if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_EVENTSEL0 >= min(INTEL_PMC_MAX_GENERIC, x86_pmu.num_counters_gp)) continue; - } + break; default: break; } From cb97c2d680dd9cefd2b36cafd2426f52d85b27e7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 18 Feb 2020 15:40:11 -0800 Subject: [PATCH 0920/1043] KVM: x86: Take an unsigned 32-bit int for has_emulated_msr()'s index Take a u32 for the index in has_emulated_msr() to match hardware, which treats MSR indices as unsigned 32-bit values. Functionally, taking a signed int doesn't cause problems with the current code base, but could theoretically cause problems with 32-bit KVM, e.g. if the index were checked via a less-than statement, which would evaluate incorrectly for MSR indices with bit 31 set. Reviewed-by: Vitaly Kuznetsov Signed-off-by: Sean Christopherson Message-Id: <20200218234012.7110-3-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/vmx/vmx.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index fd78bd44b2d6..db261da578f3 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1064,7 +1064,7 @@ struct kvm_x86_ops { void (*hardware_disable)(void); void (*hardware_unsetup)(void); bool (*cpu_has_accelerated_tpr)(void); - bool (*has_emulated_msr)(int index); + bool (*has_emulated_msr)(u32 index); void (*cpuid_update)(struct kvm_vcpu *vcpu); unsigned int vm_size; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index e9c0fb68387d..d877a0f70cac 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3525,7 +3525,7 @@ static bool svm_cpu_has_accelerated_tpr(void) return false; } -static bool svm_has_emulated_msr(int index) +static bool svm_has_emulated_msr(u32 index) { switch (index) { case MSR_IA32_MCG_EXT_CTL: diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 0ea5a225a579..ab31033121f9 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -6434,7 +6434,7 @@ static void vmx_handle_exit_irqoff(struct kvm_vcpu *vcpu) handle_exception_nmi_irqoff(vmx); } -static bool vmx_has_emulated_msr(int index) +static bool vmx_has_emulated_msr(u32 index) { switch (index) { case MSR_IA32_SMBASE: From a8cfbae59284ef20f38936a135f82a4d8d9105d2 Mon Sep 17 00:00:00 2001 From: Miaohe Lin Date: Wed, 19 Feb 2020 10:45:48 +0800 Subject: [PATCH 0921/1043] KVM: VMX: replace "fall through" with "return" to indicate different case The second "/* fall through */" in rmode_exception() makes code harder to read. Replace it with "return" to indicate they are different cases, only the #DB and #BP check vcpu->guest_debug, while others don't care. And this also improves the readability. Suggested-by: Vitaly Kuznetsov Reviewed-by: Vitaly Kuznetsov Signed-off-by: Miaohe Lin Message-Id: <1582080348-20827-1-git-send-email-linmiaohe@huawei.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index ab31033121f9..4e76e30b661c 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4626,10 +4626,8 @@ static bool rmode_exception(struct kvm_vcpu *vcpu, int vec) return false; /* fall through */ case DB_VECTOR: - if (vcpu->guest_debug & - (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) - return false; - /* fall through */ + return !(vcpu->guest_debug & + (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)); case DE_VECTOR: case OF_VECTOR: case BR_VECTOR: From 80bc97f2d84dbec6df6b36d6bbc0715c14f80c61 Mon Sep 17 00:00:00 2001 From: Haiwei Li Date: Mon, 18 May 2020 09:31:38 +0800 Subject: [PATCH 0922/1043] KVM: Fix the indentation to match coding style There is a bad indentation in next&queue branch. The patch looks like fixes nothing though it fixes the indentation. Before fixing: if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) { kvm_skip_emulated_instruction(vcpu); ret = EXIT_FASTPATH_EXIT_HANDLED; } break; case MSR_IA32_TSCDEADLINE: After fixing: if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) { kvm_skip_emulated_instruction(vcpu); ret = EXIT_FASTPATH_EXIT_HANDLED; } break; case MSR_IA32_TSCDEADLINE: Signed-off-by: Haiwei Li Message-Id: <2f78457e-f3a7-3bc9-e237-3132ee87f71e@gmail.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 43f9b63399f0..375d3fc0a4a6 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1631,7 +1631,7 @@ fastpath_t handle_fastpath_set_msr_irqoff(struct kvm_vcpu *vcpu) if (!handle_fastpath_set_x2apic_icr_irqoff(vcpu, data)) { kvm_skip_emulated_instruction(vcpu); ret = EXIT_FASTPATH_EXIT_HANDLED; - } + } break; case MSR_IA32_TSCDEADLINE: data = kvm_read_edx_eax(vcpu); From 88197e6ab33a909e54f683b5e3a2e45cc307108a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BD=AD=E6=B5=A9=28Richard=29?= Date: Thu, 21 May 2020 05:57:49 +0000 Subject: [PATCH 0923/1043] kvm/x86: Remove redundant function implementations pic_in_kernel(), ioapic_in_kernel() and irqchip_kernel() have the same implementation. Signed-off-by: Peng Hao Message-Id: Signed-off-by: Paolo Bonzini --- arch/x86/kvm/ioapic.h | 8 ++------ arch/x86/kvm/irq.h | 15 +++++---------- arch/x86/kvm/lapic.c | 1 + arch/x86/kvm/mmu/mmu.c | 1 + arch/x86/kvm/x86.c | 1 + 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/arch/x86/kvm/ioapic.h b/arch/x86/kvm/ioapic.h index 2fb2e3c80724..660401700075 100644 --- a/arch/x86/kvm/ioapic.h +++ b/arch/x86/kvm/ioapic.h @@ -3,8 +3,8 @@ #define __KVM_IO_APIC_H #include - #include +#include "irq.h" struct kvm; struct kvm_vcpu; @@ -108,11 +108,7 @@ do { \ static inline int ioapic_in_kernel(struct kvm *kvm) { - int mode = kvm->arch.irqchip_mode; - - /* Matches smp_wmb() when setting irqchip_mode */ - smp_rmb(); - return mode == KVM_IRQCHIP_KERNEL; + return irqchip_kernel(kvm); } void kvm_rtc_eoi_tracking_restore_one(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h index f173ab6b407e..9b64abf9b3f1 100644 --- a/arch/x86/kvm/irq.h +++ b/arch/x86/kvm/irq.h @@ -16,7 +16,6 @@ #include #include -#include "ioapic.h" #include "lapic.h" #define PIC_NUM_PINS 16 @@ -66,15 +65,6 @@ void kvm_pic_destroy(struct kvm *kvm); int kvm_pic_read_irq(struct kvm *kvm); void kvm_pic_update_irq(struct kvm_pic *s); -static inline int pic_in_kernel(struct kvm *kvm) -{ - int mode = kvm->arch.irqchip_mode; - - /* Matches smp_wmb() when setting irqchip_mode */ - smp_rmb(); - return mode == KVM_IRQCHIP_KERNEL; -} - static inline int irqchip_split(struct kvm *kvm) { int mode = kvm->arch.irqchip_mode; @@ -93,6 +83,11 @@ static inline int irqchip_kernel(struct kvm *kvm) return mode == KVM_IRQCHIP_KERNEL; } +static inline int pic_in_kernel(struct kvm *kvm) +{ + return irqchip_kernel(kvm); +} + static inline int irqchip_in_kernel(struct kvm *kvm) { int mode = kvm->arch.irqchip_mode; diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 2a3b57401a68..34a7e0533dad 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -36,6 +36,7 @@ #include #include "kvm_cache_regs.h" #include "irq.h" +#include "ioapic.h" #include "trace.h" #include "x86.h" #include "cpuid.h" diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index f63258183b8f..fd1c9145505c 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -16,6 +16,7 @@ */ #include "irq.h" +#include "ioapic.h" #include "mmu.h" #include "x86.h" #include "kvm_cache_regs.h" diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 375d3fc0a4a6..6d13e9d1bb75 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -18,6 +18,7 @@ #include #include "irq.h" +#include "ioapic.h" #include "mmu.h" #include "i8254.h" #include "tss.h" From a3535be731c2a343912578465021f50937f7b099 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Sat, 16 May 2020 09:19:06 -0400 Subject: [PATCH 0924/1043] KVM: nSVM: fix condition for filtering async PF Async page faults have to be trapped in the host (L1 in this case), since the APF reason was passed from L0 to L1 and stored in the L1 APF data page. This was completely reversed: the page faults were passed to the guest, a L2 hypervisor. Cc: stable@vger.kernel.org Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index a89a166d1cb8..f4cd2d0cc360 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -880,8 +880,8 @@ int nested_svm_exit_special(struct vcpu_svm *svm) return NESTED_EXIT_HOST; break; case SVM_EXIT_EXCP_BASE + PF_VECTOR: - /* When we're shadowing, trap PFs, but not async PF */ - if (!npt_enabled && svm->vcpu.arch.apf.host_apf_reason == 0) + /* Trap async PF even if not shadowing */ + if (!npt_enabled || svm->vcpu.arch.apf.host_apf_reason) return NESTED_EXIT_HOST; break; default: From 6c0238c4a62b3a0b1201aeb7e33a4636d552a436 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 20 May 2020 08:02:17 -0400 Subject: [PATCH 0925/1043] KVM: nSVM: leave ASID aside in copy_vmcb_control_area Restoring the ASID from the hsave area on VMEXIT is wrong, because its value depends on the handling of TLB flushes. Just skipping the field in copy_vmcb_control_area will do. Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index f4cd2d0cc360..d544cce4f964 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -150,7 +150,7 @@ static void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb dst->iopm_base_pa = from->iopm_base_pa; dst->msrpm_base_pa = from->msrpm_base_pa; dst->tsc_offset = from->tsc_offset; - dst->asid = from->asid; + /* asid not copied, it is handled manually for svm->vmcb. */ dst->tlb_ctl = from->tlb_ctl; dst->int_ctl = from->int_ctl; dst->int_vector = from->int_vector; From 7d2e8748af62b0de7c7bbcb0d62f937e88fd7027 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Wed, 27 May 2020 01:54:00 -0700 Subject: [PATCH 0926/1043] KVM: x86: Initialize tdp_level during vCPU creation Initialize vcpu->arch.tdp_level during vCPU creation to avoid consuming garbage if userspace calls KVM_RUN without first calling KVM_SET_CPUID. Fixes: e93fd3b3e89e9 ("KVM: x86/mmu: Capture TDP level when updating CPUID") Reported-by: syzbot+904752567107eefb728c@syzkaller.appspotmail.com Signed-off-by: Sean Christopherson Message-Id: <20200527085400.23759-1-sean.j.christopherson@intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 6d13e9d1bb75..329bdd2eb2cf 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -9418,6 +9418,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) fx_init(vcpu); vcpu->arch.maxphyaddr = cpuid_query_maxphyaddr(vcpu); + vcpu->arch.tdp_level = kvm_x86_ops.get_tdp_level(vcpu); vcpu->arch.pat = MSR_IA32_CR_PAT_DEFAULT; From b6162e82aef19fee9c32cb3fe9ac30d9116a8c73 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Wed, 27 May 2020 11:01:02 +0200 Subject: [PATCH 0927/1043] KVM: nSVM: Preserve registers modifications done before nested_svm_vmexit() L2 guest hang is observed after 'exit_required' was dropped and nSVM switched to check_nested_events() completely. The hang is a busy loop when e.g. KVM is emulating an instruction (e.g. L2 is accessing MMIO space and we drop to userspace). After nested_svm_vmexit() and when L1 is doing VMRUN nested guest's RIP is not advanced so KVM goes into emulating the same instruction which caused nested_svm_vmexit() and the loop continues. nested_svm_vmexit() is not new, however, with check_nested_events() we're now calling it later than before. In case by that time KVM has modified register state we may pick stale values from VMCB when trying to save nested guest state to nested VMCB. nVMX code handles this case correctly: sync_vmcs02_to_vmcs12() called from nested_vmx_vmexit() does e.g 'vmcs12->guest_rip = kvm_rip_read(vcpu)' and this ensures KVM-made modifications are preserved. Do the same for nSVM. Generally, nested_vmx_vmexit()/nested_svm_vmexit() need to pick up all nested guest state modifications done by KVM after vmexit. It would be great to find a way to express this in a way which would not require to manually track these changes, e.g. nested_{vmcb,vmcs}_get_field(). Co-debugged-with: Paolo Bonzini Signed-off-by: Vitaly Kuznetsov Message-Id: <20200527090102.220647-1-vkuznets@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index d544cce4f964..f39e0d578b9b 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -491,9 +491,9 @@ int nested_svm_vmexit(struct vcpu_svm *svm) nested_vmcb->save.cr2 = vmcb->save.cr2; nested_vmcb->save.cr4 = svm->vcpu.arch.cr4; nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu); - nested_vmcb->save.rip = vmcb->save.rip; - nested_vmcb->save.rsp = vmcb->save.rsp; - nested_vmcb->save.rax = vmcb->save.rax; + nested_vmcb->save.rip = kvm_rip_read(&svm->vcpu); + nested_vmcb->save.rsp = kvm_rsp_read(&svm->vcpu); + nested_vmcb->save.rax = kvm_rax_read(&svm->vcpu); nested_vmcb->save.dr7 = vmcb->save.dr7; nested_vmcb->save.dr6 = svm->vcpu.arch.dr6; nested_vmcb->save.cpl = vmcb->save.cpl; From c6b22f59d694d0caf61aefb262d9639b3d9661d5 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 26 May 2020 09:05:27 -0400 Subject: [PATCH 0928/1043] KVM: x86: track manually whether an event has been injected Instead of calling kvm_event_needs_reinjection, track its future return value in a variable. This will be useful in the next patch. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 329bdd2eb2cf..77b9b4e66673 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7717,11 +7717,14 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu) static int inject_pending_event(struct kvm_vcpu *vcpu) { int r; + bool can_inject = true; /* try to reinject previous events if any */ - if (vcpu->arch.exception.injected) + if (vcpu->arch.exception.injected) { kvm_x86_ops.queue_exception(vcpu); + can_inject = false; + } /* * Do not inject an NMI or interrupt if there is a pending * exception. Exceptions and interrupts are recognized at @@ -7737,10 +7740,13 @@ static int inject_pending_event(struct kvm_vcpu *vcpu) * fully complete the previous instruction. */ else if (!vcpu->arch.exception.pending) { - if (vcpu->arch.nmi_injected) + if (vcpu->arch.nmi_injected) { kvm_x86_ops.set_nmi(vcpu); - else if (vcpu->arch.interrupt.injected) + can_inject = false; + } else if (vcpu->arch.interrupt.injected) { kvm_x86_ops.set_irq(vcpu); + can_inject = false; + } } WARN_ON_ONCE(vcpu->arch.exception.injected && @@ -7790,10 +7796,11 @@ static int inject_pending_event(struct kvm_vcpu *vcpu) } kvm_x86_ops.queue_exception(vcpu); + can_inject = false; } - /* Don't consider new event if we re-injected an event */ - if (kvm_event_needs_reinjection(vcpu)) + /* Finish re-injection before considering new events */ + if (!can_inject) return 0; if (vcpu->arch.smi_pending && From 9c43783c702fd68c96342e83ed3bb8828484a5e0 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Thu, 28 May 2020 09:59:01 +0800 Subject: [PATCH 0929/1043] MIPS: DTS: Fix build errors used with various configs If CONFIG_MIPS_MALTA is not set but CONFIG_LEGACY_BOARD_SEAD3 is set, the subdir arch/mips/boot/dts/mti will not be built, so the sead3.dts which depends on CONFIG_LEGACY_BOARD_SEAD3 in this subdir is also not built, and then there exists the following build error, fix it. LD .tmp_vmlinux.kallsyms1 arch/mips/generic/board-sead3.o:(.mips.machines.init+0x4): undefined reference to `__dtb_sead3_begin' Makefile:1106: recipe for target 'vmlinux' failed make: *** [vmlinux] Error 1 Additionally, add CONFIG_FIT_IMAGE_FDT_BOSTON check for subdir img to fix the following build error when CONFIG_MACH_PISTACHIO is not set but CONFIG_FIT_IMAGE_FDT_BOSTON is set. FATAL ERROR: Couldn't open "boot/dts/img/boston.dtb": No such file or directory Reported-by: kbuild test robot Reported-by: Guenter Roeck Fixes: 41528ba6afe6 ("MIPS: DTS: Only build subdir of current platform") Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/boot/dts/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/mips/boot/dts/Makefile b/arch/mips/boot/dts/Makefile index dce32d16be73..19027129add8 100644 --- a/arch/mips/boot/dts/Makefile +++ b/arch/mips/boot/dts/Makefile @@ -2,11 +2,13 @@ subdir-$(CONFIG_BMIPS_GENERIC) += brcm subdir-$(CONFIG_CAVIUM_OCTEON_SOC) += cavium-octeon subdir-$(CONFIG_MACH_PISTACHIO) += img +subdir-$(CONFIG_FIT_IMAGE_FDT_BOSTON) += img subdir-$(CONFIG_MACH_INGENIC) += ingenic subdir-$(CONFIG_LANTIQ) += lantiq subdir-$(CONFIG_MACH_LOONGSON64) += loongson subdir-$(CONFIG_MSCC_OCELOT) += mscc subdir-$(CONFIG_MIPS_MALTA) += mti +subdir-$(CONFIG_LEGACY_BOARD_SEAD3) += mti subdir-$(CONFIG_NLM_XLP_BOARD) += netlogic subdir-$(CONFIG_FIT_IMAGE_FDT_NI169445) += ni subdir-$(CONFIG_MACH_PIC32) += pic32 From a202bf71f08b3ef15356db30535e30b03cf23aec Mon Sep 17 00:00:00 2001 From: Lichao Liu Date: Thu, 28 May 2020 09:10:31 +0800 Subject: [PATCH 0930/1043] MIPS: CPU_LOONGSON2EF need software to maintain cache consistency CPU_LOONGSON2EF need software to maintain cache consistency, so modify the 'cpu_needs_post_dma_flush' function to return true when the cpu type is CPU_LOONGSON2EF. Cc: stable@vger.kernel.org Signed-off-by: Lichao Liu Reviewed-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/mm/dma-noncoherent.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c index fcea92d95d86..563c2c0d0c81 100644 --- a/arch/mips/mm/dma-noncoherent.c +++ b/arch/mips/mm/dma-noncoherent.c @@ -33,6 +33,7 @@ static inline bool cpu_needs_post_dma_flush(void) case CPU_R10000: case CPU_R12000: case CPU_BMIPS5000: + case CPU_LOONGSON2EF: return true; default: /* From 482cd90cd781c9b5607ed9c9f8641dc95a9d4cce Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Wed, 27 May 2020 21:17:21 +0800 Subject: [PATCH 0931/1043] MIPS: Loongson64: Define PCI_IOBASE PCI_IOBASE is used to create VM maps for PCI I/O ports, it is required by generic PCI drivers to make memory mapped I/O range work. To deal with legacy drivers that have fixed I/O ports range we reserved 0x10000 in PCI_IOBASE, should be enough for i8259 i8042 stuff. Signed-off-by: Jiaxun Yang Signed-off-by: Thomas Bogendoerfer --- .../mips/include/asm/mach-loongson64/spaces.h | 8 ++++ arch/mips/loongson64/init.c | 42 ++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/arch/mips/include/asm/mach-loongson64/spaces.h b/arch/mips/include/asm/mach-loongson64/spaces.h index e85bc1d9c4f2..3de0ac9d8829 100644 --- a/arch/mips/include/asm/mach-loongson64/spaces.h +++ b/arch/mips/include/asm/mach-loongson64/spaces.h @@ -6,5 +6,13 @@ #define CAC_BASE _AC(0x9800000000000000, UL) #endif /* CONFIG_64BIT */ +/* Skip 128k to trap NULL pointer dereferences */ +#define PCI_IOBASE _AC(0xc000000000000000 + SZ_128K, UL) +#define PCI_IOSIZE SZ_16M +#define MAP_BASE (PCI_IOBASE + PCI_IOSIZE) + +/* Reserved at the start of PCI_IOBASE for legacy drivers */ +#define MMIO_LOWER_RESERVED 0x10000 + #include #endif diff --git a/arch/mips/loongson64/init.c b/arch/mips/loongson64/init.c index 23eeb85b1abf..59ddadace83f 100644 --- a/arch/mips/loongson64/init.c +++ b/arch/mips/loongson64/init.c @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -45,8 +46,7 @@ void __init prom_init(void) prom_init_env(); /* init base address of io space */ - set_io_port_base((unsigned long) - ioremap(LOONGSON_PCIIO_BASE, LOONGSON_PCIIO_SIZE)); + set_io_port_base(PCI_IOBASE); loongson_sysconf.early_config(); @@ -63,7 +63,45 @@ void __init prom_free_prom_memory(void) { } +static __init void reserve_pio_range(void) +{ + struct logic_pio_hwaddr *range; + + range = kzalloc(sizeof(*range), GFP_ATOMIC); + if (!range) + return; + + range->fwnode = &of_root->fwnode; + range->size = MMIO_LOWER_RESERVED; + range->hw_start = LOONGSON_PCIIO_BASE; + range->flags = LOGIC_PIO_CPU_MMIO; + + if (logic_pio_register_range(range)) { + pr_err("Failed to reserve PIO range for legacy ISA\n"); + goto free_range; + } + + if (WARN(range->io_start != 0, + "Reserved PIO range does not start from 0\n")) + goto unregister; + + /* + * i8259 would access I/O space, so mapping must be done here. + * Please remove it when all drivers can be managed by logic_pio. + */ + ioremap_page_range(PCI_IOBASE, PCI_IOBASE + MMIO_LOWER_RESERVED, + LOONGSON_PCIIO_BASE, + pgprot_device(PAGE_KERNEL)); + + return; +unregister: + logic_pio_unregister_range(range); +free_range: + kfree(range); +} + void __init arch_init_irq(void) { + reserve_pio_range(); irqchip_init(); } From 19a1f5ec699954d21be10f74ff71c2a7079e99ad Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 26 May 2020 18:10:58 +0200 Subject: [PATCH 0932/1043] sched: Fix smp_call_function_single_async() usage for ILB The recent commit: 90b5363acd47 ("sched: Clean up scheduler_ipi()") got smp_call_function_single_async() subtly wrong. Even though it will return -EBUSY when trying to re-use a csd, that condition is not atomic and still requires external serialization. The change in kick_ilb() got this wrong. While on first reading kick_ilb() has an atomic test-and-set that appears to serialize the use, the matching 'release' is not in the right place to actually guarantee this serialization. Rework the nohz_idle_balance() trigger so that the release is in the IPI callback and thus guarantees the required serialization for the CSD. Fixes: 90b5363acd47 ("sched: Clean up scheduler_ipi()") Reported-by: Qian Cai Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Frederic Weisbecker Cc: mgorman@techsingularity.net Link: https://lore.kernel.org/r/20200526161907.778543557@infradead.org --- kernel/sched/core.c | 38 +++++++++++--------------------------- kernel/sched/fair.c | 18 ++++++++---------- kernel/sched/sched.h | 1 + 3 files changed, 20 insertions(+), 37 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 95e457d4ed1c..2cacc1e44a84 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -637,41 +637,25 @@ void wake_up_nohz_cpu(int cpu) wake_up_idle_cpu(cpu); } -static inline bool got_nohz_idle_kick(void) -{ - int cpu = smp_processor_id(); - - if (!(atomic_read(nohz_flags(cpu)) & NOHZ_KICK_MASK)) - return false; - - if (idle_cpu(cpu) && !need_resched()) - return true; - - /* - * We can't run Idle Load Balance on this CPU for this time so we - * cancel it and clear NOHZ_BALANCE_KICK - */ - atomic_andnot(NOHZ_KICK_MASK, nohz_flags(cpu)); - return false; -} - static void nohz_csd_func(void *info) { struct rq *rq = info; + int cpu = cpu_of(rq); + unsigned int flags; - if (got_nohz_idle_kick()) { - rq->idle_balance = 1; + /* + * Release the rq::nohz_csd. + */ + flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(cpu)); + WARN_ON(!(flags & NOHZ_KICK_MASK)); + + rq->idle_balance = idle_cpu(cpu); + if (rq->idle_balance && !need_resched()) { + rq->nohz_idle_balance = flags; raise_softirq_irqoff(SCHED_SOFTIRQ); } } -#else /* CONFIG_NO_HZ_COMMON */ - -static inline bool got_nohz_idle_kick(void) -{ - return false; -} - #endif /* CONFIG_NO_HZ_COMMON */ #ifdef CONFIG_NO_HZ_FULL diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index dda9b194d225..2890bd54a088 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10024,6 +10024,10 @@ static void kick_ilb(unsigned int flags) if (ilb_cpu >= nr_cpu_ids) return; + /* + * Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets + * the first flag owns it; cleared by nohz_csd_func(). + */ flags = atomic_fetch_or(flags, nohz_flags(ilb_cpu)); if (flags & NOHZ_KICK_MASK) return; @@ -10371,20 +10375,14 @@ abort: */ static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle) { - int this_cpu = this_rq->cpu; - unsigned int flags; + unsigned int flags = this_rq->nohz_idle_balance; - if (!(atomic_read(nohz_flags(this_cpu)) & NOHZ_KICK_MASK)) + if (!flags) return false; - if (idle != CPU_IDLE) { - atomic_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu)); - return false; - } + this_rq->nohz_idle_balance = 0; - /* could be _relaxed() */ - flags = atomic_fetch_andnot(NOHZ_KICK_MASK, nohz_flags(this_cpu)); - if (!(flags & NOHZ_KICK_MASK)) + if (idle != CPU_IDLE) return false; _nohz_idle_balance(this_rq, flags, idle); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 4b32cff0dcbe..3c163cb5493f 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -951,6 +951,7 @@ struct rq { struct callback_head *balance_callback; + unsigned char nohz_idle_balance; unsigned char idle_balance; unsigned long misfit_task_load; From 52103be07d8b08311955f8c30e535c2dda290cf4 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 26 May 2020 18:10:59 +0200 Subject: [PATCH 0933/1043] smp: Optimize flush_smp_call_function_queue() The call_single_queue can contain (two) different callbacks, synchronous and asynchronous. The current interrupt handler runs them in-order, which means that remote CPUs that are waiting for their synchronous call can be delayed by running asynchronous callbacks. Rework the interrupt handler to first run the synchonous callbacks. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20200526161907.836818381@infradead.org --- kernel/smp.c | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/kernel/smp.c b/kernel/smp.c index 786092aabdcd..db2f73808db5 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -209,9 +209,9 @@ void generic_smp_call_function_single_interrupt(void) */ static void flush_smp_call_function_queue(bool warn_cpu_offline) { - struct llist_head *head; - struct llist_node *entry; call_single_data_t *csd, *csd_next; + struct llist_node *entry, *prev; + struct llist_head *head; static bool warned; lockdep_assert_irqs_disabled(); @@ -235,20 +235,39 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) csd->func); } + /* + * First; run all SYNC callbacks, people are waiting for us. + */ + prev = NULL; llist_for_each_entry_safe(csd, csd_next, entry, llist) { smp_call_func_t func = csd->func; void *info = csd->info; /* Do we wait until *after* callback? */ if (csd->flags & CSD_FLAG_SYNCHRONOUS) { + if (prev) { + prev->next = &csd_next->llist; + } else { + entry = &csd_next->llist; + } func(info); csd_unlock(csd); } else { - csd_unlock(csd); - func(info); + prev = &csd->llist; } } + /* + * Second; run all !SYNC callbacks. + */ + llist_for_each_entry_safe(csd, csd_next, entry, llist) { + smp_call_func_t func = csd->func; + void *info = csd->info; + + csd_unlock(csd); + func(info); + } + /* * Handle irq works queued remotely by irq_work_queue_on(). * Smp functions above are typically synchronous so they From afaa653c564da38c0b34c4baba31e88c46a8764c Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 26 May 2020 18:11:00 +0200 Subject: [PATCH 0934/1043] smp: Move irq_work_run() out of flush_smp_call_function_queue() This ensures flush_smp_call_function_queue() is strictly about call_single_queue. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20200526161907.895109676@infradead.org --- kernel/smp.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/kernel/smp.c b/kernel/smp.c index db2f73808db5..f720e38e880d 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -84,6 +84,7 @@ int smpcfd_dying_cpu(unsigned int cpu) * still pending. */ flush_smp_call_function_queue(false); + irq_work_run(); return 0; } @@ -191,6 +192,14 @@ static int generic_exec_single(int cpu, call_single_data_t *csd, void generic_smp_call_function_single_interrupt(void) { flush_smp_call_function_queue(true); + + /* + * Handle irq works queued remotely by irq_work_queue_on(). + * Smp functions above are typically synchronous so they + * better run first since some other CPUs may be busy waiting + * for them. + */ + irq_work_run(); } /** @@ -267,14 +276,6 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) csd_unlock(csd); func(info); } - - /* - * Handle irq works queued remotely by irq_work_queue_on(). - * Smp functions above are typically synchronous so they - * better run first since some other CPUs may be busy waiting - * for them. - */ - irq_work_run(); } /* From b2a02fc43a1f40ef4eb2fb2b06357382608d4d84 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 26 May 2020 18:11:01 +0200 Subject: [PATCH 0935/1043] smp: Optimize send_call_function_single_ipi() Just like the ttwu_queue_remote() IPI, make use of _TIF_POLLING_NRFLAG to avoid sending IPIs to idle CPUs. [ mingo: Fix UP build bug. ] Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20200526161907.953304789@infradead.org --- kernel/sched/core.c | 10 ++++++++++ kernel/sched/idle.c | 5 +++++ kernel/sched/sched.h | 7 ++++--- kernel/smp.c | 16 +++++++++++++++- 4 files changed, 34 insertions(+), 4 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 2cacc1e44a84..fa0d4990618e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2296,6 +2296,16 @@ static void wake_csd_func(void *info) sched_ttwu_pending(); } +void send_call_function_single_ipi(int cpu) +{ + struct rq *rq = cpu_rq(cpu); + + if (!set_nr_if_polling(rq->idle)) + arch_send_call_function_single_ipi(cpu); + else + trace_sched_wake_idle_without_ipi(cpu); +} + /* * Queue a task on the target CPUs wake_list and wake the CPU via IPI if * necessary. The wakee CPU on receipt of the IPI will queue the task diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index b743bf38f08f..387fd75ee6f7 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -289,6 +289,11 @@ static void do_idle(void) */ smp_mb__after_atomic(); + /* + * RCU relies on this call to be done outside of an RCU read-side + * critical section. + */ + flush_smp_call_function_from_idle(); sched_ttwu_pending(); schedule_idle(); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 3c163cb5493f..75b062999c43 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1506,11 +1506,12 @@ static inline void unregister_sched_domain_sysctl(void) } #endif -#else +extern void flush_smp_call_function_from_idle(void); +#else /* !CONFIG_SMP: */ +static inline void flush_smp_call_function_from_idle(void) { } static inline void sched_ttwu_pending(void) { } - -#endif /* CONFIG_SMP */ +#endif #include "stats.h" #include "autogroup.h" diff --git a/kernel/smp.c b/kernel/smp.c index f720e38e880d..9f1181375141 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -135,6 +135,8 @@ static __always_inline void csd_unlock(call_single_data_t *csd) static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data); +extern void send_call_function_single_ipi(int cpu); + /* * Insert a previously allocated call_single_data_t element * for execution on the given CPU. data must already have @@ -178,7 +180,7 @@ static int generic_exec_single(int cpu, call_single_data_t *csd, * equipped to do the right thing... */ if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu))) - arch_send_call_function_single_ipi(cpu); + send_call_function_single_ipi(cpu); return 0; } @@ -278,6 +280,18 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) } } +void flush_smp_call_function_from_idle(void) +{ + unsigned long flags; + + if (llist_empty(this_cpu_ptr(&call_single_queue))) + return; + + local_irq_save(flags); + flush_smp_call_function_queue(true); + local_irq_restore(flags); +} + /* * smp_call_function_single - Run a function on a specific CPU * @func: The function to run. This must be fast and non-blocking. From 4b44a21dd640b692d4e9b12d3e37c24825f90baa Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 26 May 2020 18:11:02 +0200 Subject: [PATCH 0936/1043] irq_work, smp: Allow irq_work on call_single_queue Currently irq_work_queue_on() will issue an unconditional arch_send_call_function_single_ipi() and has the handler do irq_work_run(). This is unfortunate in that it makes the IPI handler look at a second cacheline and it misses the opportunity to avoid the IPI. Instead note that struct irq_work and struct __call_single_data are very similar in layout, so use a few bits in the flags word to encode a type and stick the irq_work on the call_single_queue list. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20200526161908.011635912@infradead.org --- include/linux/irq_work.h | 7 ++- include/linux/smp.h | 23 +++++++- kernel/irq_work.c | 53 +++++++++-------- kernel/smp.c | 119 ++++++++++++++++++++++++--------------- 4 files changed, 130 insertions(+), 72 deletions(-) diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h index 3b752e80c017..f23a359c8f46 100644 --- a/include/linux/irq_work.h +++ b/include/linux/irq_work.h @@ -13,6 +13,8 @@ * busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed */ +/* flags share CSD_FLAG_ space */ + #define IRQ_WORK_PENDING BIT(0) #define IRQ_WORK_BUSY BIT(1) @@ -23,9 +25,12 @@ #define IRQ_WORK_CLAIMED (IRQ_WORK_PENDING | IRQ_WORK_BUSY) +/* + * structure shares layout with single_call_data_t. + */ struct irq_work { - atomic_t flags; struct llist_node llnode; + atomic_t flags; void (*func)(struct irq_work *); }; diff --git a/include/linux/smp.h b/include/linux/smp.h index cbc9162689d0..45ad6e30f398 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -16,17 +16,38 @@ typedef void (*smp_call_func_t)(void *info); typedef bool (*smp_cond_func_t)(int cpu, void *info); + +enum { + CSD_FLAG_LOCK = 0x01, + + /* IRQ_WORK_flags */ + + CSD_TYPE_ASYNC = 0x00, + CSD_TYPE_SYNC = 0x10, + CSD_TYPE_IRQ_WORK = 0x20, + CSD_FLAG_TYPE_MASK = 0xF0, +}; + +/* + * structure shares (partial) layout with struct irq_work + */ struct __call_single_data { struct llist_node llist; + unsigned int flags; smp_call_func_t func; void *info; - unsigned int flags; }; /* Use __aligned() to avoid to use 2 cache lines for 1 csd */ typedef struct __call_single_data call_single_data_t __aligned(sizeof(struct __call_single_data)); +/* + * Enqueue a llist_node on the call_single_queue; be very careful, read + * flush_smp_call_function_queue() in detail. + */ +extern void __smp_call_single_queue(int cpu, struct llist_node *node); + /* total number of cpus in this system (may exceed NR_CPUS) */ extern unsigned int total_cpus; diff --git a/kernel/irq_work.c b/kernel/irq_work.c index 48b5d1b6af4d..eca83965b631 100644 --- a/kernel/irq_work.c +++ b/kernel/irq_work.c @@ -31,7 +31,7 @@ static bool irq_work_claim(struct irq_work *work) { int oflags; - oflags = atomic_fetch_or(IRQ_WORK_CLAIMED, &work->flags); + oflags = atomic_fetch_or(IRQ_WORK_CLAIMED | CSD_TYPE_IRQ_WORK, &work->flags); /* * If the work is already pending, no need to raise the IPI. * The pairing atomic_fetch_andnot() in irq_work_run() makes sure @@ -102,8 +102,7 @@ bool irq_work_queue_on(struct irq_work *work, int cpu) if (cpu != smp_processor_id()) { /* Arch remote IPI send/receive backend aren't NMI safe */ WARN_ON_ONCE(in_nmi()); - if (llist_add(&work->llnode, &per_cpu(raised_list, cpu))) - arch_send_call_function_single_ipi(cpu); + __smp_call_single_queue(cpu, &work->llnode); } else { __irq_work_queue_local(work); } @@ -131,6 +130,31 @@ bool irq_work_needs_cpu(void) return true; } +void irq_work_single(void *arg) +{ + struct irq_work *work = arg; + int flags; + + /* + * Clear the PENDING bit, after this point the @work + * can be re-used. + * Make it immediately visible so that other CPUs trying + * to claim that work don't rely on us to handle their data + * while we are in the middle of the func. + */ + flags = atomic_fetch_andnot(IRQ_WORK_PENDING, &work->flags); + + lockdep_irq_work_enter(work); + work->func(work); + lockdep_irq_work_exit(work); + /* + * Clear the BUSY bit and return to the free state if + * no-one else claimed it meanwhile. + */ + flags &= ~IRQ_WORK_PENDING; + (void)atomic_cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY); +} + static void irq_work_run_list(struct llist_head *list) { struct irq_work *work, *tmp; @@ -142,27 +166,8 @@ static void irq_work_run_list(struct llist_head *list) return; llnode = llist_del_all(list); - llist_for_each_entry_safe(work, tmp, llnode, llnode) { - int flags; - /* - * Clear the PENDING bit, after this point the @work - * can be re-used. - * Make it immediately visible so that other CPUs trying - * to claim that work don't rely on us to handle their data - * while we are in the middle of the func. - */ - flags = atomic_fetch_andnot(IRQ_WORK_PENDING, &work->flags); - - lockdep_irq_work_enter(work); - work->func(work); - lockdep_irq_work_exit(work); - /* - * Clear the BUSY bit and return to the free state if - * no-one else claimed it meanwhile. - */ - flags &= ~IRQ_WORK_PENDING; - (void)atomic_cmpxchg(&work->flags, flags, flags & ~IRQ_WORK_BUSY); - } + llist_for_each_entry_safe(work, tmp, llnode, llnode) + irq_work_single(work); } /* diff --git a/kernel/smp.c b/kernel/smp.c index 9f1181375141..856562b80794 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -23,10 +23,8 @@ #include "smpboot.h" -enum { - CSD_FLAG_LOCK = 0x01, - CSD_FLAG_SYNCHRONOUS = 0x02, -}; + +#define CSD_TYPE(_csd) ((_csd)->flags & CSD_FLAG_TYPE_MASK) struct call_function_data { call_single_data_t __percpu *csd; @@ -137,15 +135,33 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data); extern void send_call_function_single_ipi(int cpu); +void __smp_call_single_queue(int cpu, struct llist_node *node) +{ + /* + * The list addition should be visible before sending the IPI + * handler locks the list to pull the entry off it because of + * normal cache coherency rules implied by spinlocks. + * + * If IPIs can go out of order to the cache coherency protocol + * in an architecture, sufficient synchronisation should be added + * to arch code to make it appear to obey cache coherency WRT + * locking and barrier primitives. Generic code isn't really + * equipped to do the right thing... + */ + if (llist_add(node, &per_cpu(call_single_queue, cpu))) + send_call_function_single_ipi(cpu); +} + /* * Insert a previously allocated call_single_data_t element * for execution on the given CPU. data must already have * ->func, ->info, and ->flags set. */ -static int generic_exec_single(int cpu, call_single_data_t *csd, - smp_call_func_t func, void *info) +static int generic_exec_single(int cpu, call_single_data_t *csd) { if (cpu == smp_processor_id()) { + smp_call_func_t func = csd->func; + void *info = csd->info; unsigned long flags; /* @@ -159,28 +175,12 @@ static int generic_exec_single(int cpu, call_single_data_t *csd, return 0; } - if ((unsigned)cpu >= nr_cpu_ids || !cpu_online(cpu)) { csd_unlock(csd); return -ENXIO; } - csd->func = func; - csd->info = info; - - /* - * The list addition should be visible before sending the IPI - * handler locks the list to pull the entry off it because of - * normal cache coherency rules implied by spinlocks. - * - * If IPIs can go out of order to the cache coherency protocol - * in an architecture, sufficient synchronisation should be added - * to arch code to make it appear to obey cache coherency WRT - * locking and barrier primitives. Generic code isn't really - * equipped to do the right thing... - */ - if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu))) - send_call_function_single_ipi(cpu); + __smp_call_single_queue(cpu, &csd->llist); return 0; } @@ -194,16 +194,10 @@ static int generic_exec_single(int cpu, call_single_data_t *csd, void generic_smp_call_function_single_interrupt(void) { flush_smp_call_function_queue(true); - - /* - * Handle irq works queued remotely by irq_work_queue_on(). - * Smp functions above are typically synchronous so they - * better run first since some other CPUs may be busy waiting - * for them. - */ - irq_work_run(); } +extern void irq_work_single(void *); + /** * flush_smp_call_function_queue - Flush pending smp-call-function callbacks * @@ -241,9 +235,21 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) * We don't have to use the _safe() variant here * because we are not invoking the IPI handlers yet. */ - llist_for_each_entry(csd, entry, llist) - pr_warn("IPI callback %pS sent to offline CPU\n", - csd->func); + llist_for_each_entry(csd, entry, llist) { + switch (CSD_TYPE(csd)) { + case CSD_TYPE_ASYNC: + case CSD_TYPE_SYNC: + case CSD_TYPE_IRQ_WORK: + pr_warn("IPI callback %pS sent to offline CPU\n", + csd->func); + break; + + default: + pr_warn("IPI callback, unknown type %d, sent to offline CPU\n", + CSD_TYPE(csd)); + break; + } + } } /* @@ -251,16 +257,17 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) */ prev = NULL; llist_for_each_entry_safe(csd, csd_next, entry, llist) { - smp_call_func_t func = csd->func; - void *info = csd->info; - /* Do we wait until *after* callback? */ - if (csd->flags & CSD_FLAG_SYNCHRONOUS) { + if (CSD_TYPE(csd) == CSD_TYPE_SYNC) { + smp_call_func_t func = csd->func; + void *info = csd->info; + if (prev) { prev->next = &csd_next->llist; } else { entry = &csd_next->llist; } + func(info); csd_unlock(csd); } else { @@ -272,11 +279,17 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) * Second; run all !SYNC callbacks. */ llist_for_each_entry_safe(csd, csd_next, entry, llist) { - smp_call_func_t func = csd->func; - void *info = csd->info; + int type = CSD_TYPE(csd); - csd_unlock(csd); - func(info); + if (type == CSD_TYPE_ASYNC) { + smp_call_func_t func = csd->func; + void *info = csd->info; + + csd_unlock(csd); + func(info); + } else if (type == CSD_TYPE_IRQ_WORK) { + irq_work_single(csd); + } } } @@ -305,7 +318,7 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info, { call_single_data_t *csd; call_single_data_t csd_stack = { - .flags = CSD_FLAG_LOCK | CSD_FLAG_SYNCHRONOUS, + .flags = CSD_FLAG_LOCK | CSD_TYPE_SYNC, }; int this_cpu; int err; @@ -339,7 +352,10 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info, csd_lock(csd); } - err = generic_exec_single(cpu, csd, func, info); + csd->func = func; + csd->info = info; + + err = generic_exec_single(cpu, csd); if (wait) csd_lock_wait(csd); @@ -385,7 +401,7 @@ int smp_call_function_single_async(int cpu, call_single_data_t *csd) csd->flags = CSD_FLAG_LOCK; smp_wmb(); - err = generic_exec_single(cpu, csd, csd->func, csd->info); + err = generic_exec_single(cpu, csd); out: preempt_enable(); @@ -500,7 +516,7 @@ static void smp_call_function_many_cond(const struct cpumask *mask, csd_lock(csd); if (wait) - csd->flags |= CSD_FLAG_SYNCHRONOUS; + csd->flags |= CSD_TYPE_SYNC; csd->func = func; csd->info = info; if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu))) @@ -632,6 +648,17 @@ void __init smp_init(void) { int num_nodes, num_cpus; + /* + * Ensure struct irq_work layout matches so that + * flush_smp_call_function_queue() can do horrible things. + */ + BUILD_BUG_ON(offsetof(struct irq_work, llnode) != + offsetof(struct __call_single_data, llist)); + BUILD_BUG_ON(offsetof(struct irq_work, func) != + offsetof(struct __call_single_data, func)); + BUILD_BUG_ON(offsetof(struct irq_work, flags) != + offsetof(struct __call_single_data, flags)); + idle_threads_init(); cpuhp_threads_init(); From 126c2092e5c8b28623cb890cd2930aa292410676 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 26 May 2020 18:11:03 +0200 Subject: [PATCH 0937/1043] sched: Add rq::ttwu_pending In preparation of removing rq->wake_list, replace the !list_empty(rq->wake_list) with rq->ttwu_pending. This is not fully equivalent as this new variable is racy. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20200526161908.070399698@infradead.org --- kernel/sched/core.c | 13 +++++++++++-- kernel/sched/debug.c | 1 - kernel/sched/fair.c | 2 +- kernel/sched/sched.h | 4 +++- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index fa0d4990618e..b71ed5ee97fd 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2275,13 +2275,21 @@ static int ttwu_remote(struct task_struct *p, int wake_flags) void sched_ttwu_pending(void) { struct rq *rq = this_rq(); - struct llist_node *llist = llist_del_all(&rq->wake_list); + struct llist_node *llist; struct task_struct *p, *t; struct rq_flags rf; + llist = llist_del_all(&rq->wake_list); if (!llist) return; + /* + * rq::ttwu_pending racy indication of out-standing wakeups. + * Races such that false-negatives are possible, since they + * are shorter lived that false-positives would be. + */ + WRITE_ONCE(rq->ttwu_pending, 0); + rq_lock_irqsave(rq, &rf); update_rq_clock(rq); @@ -2318,6 +2326,7 @@ static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); + WRITE_ONCE(rq->ttwu_pending, 1); if (llist_add(&p->wake_entry, &rq->wake_list)) { if (!set_nr_if_polling(rq->idle)) smp_call_function_single_async(cpu, &rq->wake_csd); @@ -4705,7 +4714,7 @@ int idle_cpu(int cpu) return 0; #ifdef CONFIG_SMP - if (!llist_empty(&rq->wake_list)) + if (rq->ttwu_pending) return 0; #endif diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 1c24a6bbdae2..36c54265bb2b 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -638,7 +638,6 @@ do { \ P(nr_running); P(nr_switches); - P(nr_load_updates); P(nr_uninterruptible); PN(next_balance); SEQ_printf(m, " .%-30s: %ld\n", "curr->pid", (long)(task_pid_nr(rq->curr))); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 2890bd54a088..0ed04d2a8959 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -8590,7 +8590,7 @@ static int idle_cpu_without(int cpu, struct task_struct *p) */ #ifdef CONFIG_SMP - if (!llist_empty(&rq->wake_list)) + if (rq->ttwu_pending) return 0; #endif diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 75b062999c43..c86fc94c54e5 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -895,7 +895,9 @@ struct rq { atomic_t nohz_flags; #endif /* CONFIG_NO_HZ_COMMON */ - unsigned long nr_load_updates; +#ifdef CONFIG_SMP + unsigned int ttwu_pending; +#endif u64 nr_switches; #ifdef CONFIG_UCLAMP_TASK From a148866489fbe243c936fe43e4525d8dbfa0318f Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 26 May 2020 18:11:04 +0200 Subject: [PATCH 0938/1043] sched: Replace rq::wake_list The recent commit: 90b5363acd47 ("sched: Clean up scheduler_ipi()") got smp_call_function_single_async() subtly wrong. Even though it will return -EBUSY when trying to re-use a csd, that condition is not atomic and still requires external serialization. The change in ttwu_queue_remote() got this wrong. While on first reading ttwu_queue_remote() has an atomic test-and-set that appears to serialize the use, the matching 'release' is not in the right place to actually guarantee this serialization. The actual race is vs the sched_ttwu_pending() call in the idle loop; that can run the wakeup-list without consuming the CSD. Instead of trying to chain the lists, merge them. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20200526161908.129371594@infradead.org --- include/linux/sched.h | 1 + include/linux/smp.h | 1 + kernel/sched/core.c | 25 +++++++---------------- kernel/sched/idle.c | 1 - kernel/sched/sched.h | 8 -------- kernel/smp.c | 47 ++++++++++++++++++++++++++++++++++++------- 6 files changed, 49 insertions(+), 34 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index ebc68706152a..e0f5f41cf2ee 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -654,6 +654,7 @@ struct task_struct { #ifdef CONFIG_SMP struct llist_node wake_entry; + unsigned int wake_entry_type; int on_cpu; #ifdef CONFIG_THREAD_INFO_IN_TASK /* Current CPU: */ diff --git a/include/linux/smp.h b/include/linux/smp.h index 45ad6e30f398..84f90e24ed6f 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -25,6 +25,7 @@ enum { CSD_TYPE_ASYNC = 0x00, CSD_TYPE_SYNC = 0x10, CSD_TYPE_IRQ_WORK = 0x20, + CSD_TYPE_TTWU = 0x30, CSD_FLAG_TYPE_MASK = 0xF0, }; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b71ed5ee97fd..b3c64c6b4d2e 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1538,7 +1538,7 @@ static int migration_cpu_stop(void *data) * __migrate_task() such that we will not miss enforcing cpus_ptr * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. */ - sched_ttwu_pending(); + flush_smp_call_function_from_idle(); raw_spin_lock(&p->pi_lock); rq_lock(rq, &rf); @@ -2272,14 +2272,13 @@ static int ttwu_remote(struct task_struct *p, int wake_flags) } #ifdef CONFIG_SMP -void sched_ttwu_pending(void) +void sched_ttwu_pending(void *arg) { + struct llist_node *llist = arg; struct rq *rq = this_rq(); - struct llist_node *llist; struct task_struct *p, *t; struct rq_flags rf; - llist = llist_del_all(&rq->wake_list); if (!llist) return; @@ -2299,11 +2298,6 @@ void sched_ttwu_pending(void) rq_unlock_irqrestore(rq, &rf); } -static void wake_csd_func(void *info) -{ - sched_ttwu_pending(); -} - void send_call_function_single_ipi(int cpu) { struct rq *rq = cpu_rq(cpu); @@ -2327,12 +2321,7 @@ static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED); WRITE_ONCE(rq->ttwu_pending, 1); - if (llist_add(&p->wake_entry, &rq->wake_list)) { - if (!set_nr_if_polling(rq->idle)) - smp_call_function_single_async(cpu, &rq->wake_csd); - else - trace_sched_wake_idle_without_ipi(cpu); - } + __smp_call_single_queue(cpu, &p->wake_entry); } void wake_up_if_idle(int cpu) @@ -2772,6 +2761,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) p->capture_control = NULL; #endif init_numa_balancing(clone_flags, p); +#ifdef CONFIG_SMP + p->wake_entry_type = CSD_TYPE_TTWU; +#endif } DEFINE_STATIC_KEY_FALSE(sched_numa_balancing); @@ -6564,7 +6556,6 @@ int sched_cpu_dying(unsigned int cpu) struct rq_flags rf; /* Handle pending wakeups and then migrate everything off */ - sched_ttwu_pending(); sched_tick_stop(cpu); rq_lock_irqsave(rq, &rf); @@ -6763,8 +6754,6 @@ void __init sched_init(void) rq->avg_idle = 2*sysctl_sched_migration_cost; rq->max_idle_balance_cost = sysctl_sched_migration_cost; - rq_csd_init(rq, &rq->wake_csd, wake_csd_func); - INIT_LIST_HEAD(&rq->cfs_tasks); rq_attach_root(rq, &def_root_domain); diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 387fd75ee6f7..05deb81bb3e3 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -294,7 +294,6 @@ static void do_idle(void) * critical section. */ flush_smp_call_function_from_idle(); - sched_ttwu_pending(); schedule_idle(); if (unlikely(klp_patch_pending(current))) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index c86fc94c54e5..1d4e94c1e5fe 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1023,11 +1023,6 @@ struct rq { unsigned int ttwu_local; #endif -#ifdef CONFIG_SMP - call_single_data_t wake_csd; - struct llist_head wake_list; -#endif - #ifdef CONFIG_CPU_IDLE /* Must be inspected within a rcu lock section */ struct cpuidle_state *idle_state; @@ -1371,8 +1366,6 @@ queue_balance_callback(struct rq *rq, rq->balance_callback = head; } -extern void sched_ttwu_pending(void); - #define rcu_dereference_check_sched_domain(p) \ rcu_dereference_check((p), \ lockdep_is_held(&sched_domains_mutex)) @@ -1512,7 +1505,6 @@ extern void flush_smp_call_function_from_idle(void); #else /* !CONFIG_SMP: */ static inline void flush_smp_call_function_from_idle(void) { } -static inline void sched_ttwu_pending(void) { } #endif #include "stats.h" diff --git a/kernel/smp.c b/kernel/smp.c index 856562b80794..0d61dc060b01 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -196,6 +196,7 @@ void generic_smp_call_function_single_interrupt(void) flush_smp_call_function_queue(true); } +extern void sched_ttwu_pending(void *); extern void irq_work_single(void *); /** @@ -244,6 +245,10 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) csd->func); break; + case CSD_TYPE_TTWU: + pr_warn("IPI task-wakeup sent to offline CPU\n"); + break; + default: pr_warn("IPI callback, unknown type %d, sent to offline CPU\n", CSD_TYPE(csd)); @@ -275,22 +280,43 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline) } } + if (!entry) + return; + /* * Second; run all !SYNC callbacks. */ + prev = NULL; llist_for_each_entry_safe(csd, csd_next, entry, llist) { int type = CSD_TYPE(csd); - if (type == CSD_TYPE_ASYNC) { - smp_call_func_t func = csd->func; - void *info = csd->info; + if (type != CSD_TYPE_TTWU) { + if (prev) { + prev->next = &csd_next->llist; + } else { + entry = &csd_next->llist; + } - csd_unlock(csd); - func(info); - } else if (type == CSD_TYPE_IRQ_WORK) { - irq_work_single(csd); + if (type == CSD_TYPE_ASYNC) { + smp_call_func_t func = csd->func; + void *info = csd->info; + + csd_unlock(csd); + func(info); + } else if (type == CSD_TYPE_IRQ_WORK) { + irq_work_single(csd); + } + + } else { + prev = &csd->llist; } } + + /* + * Third; only CSD_TYPE_TTWU is left, issue those. + */ + if (entry) + sched_ttwu_pending(entry); } void flush_smp_call_function_from_idle(void) @@ -659,6 +685,13 @@ void __init smp_init(void) BUILD_BUG_ON(offsetof(struct irq_work, flags) != offsetof(struct __call_single_data, flags)); + /* + * Assert the CSD_TYPE_TTWU layout is similar enough + * for task_struct to be on the @call_single_queue. + */ + BUILD_BUG_ON(offsetof(struct task_struct, wake_entry_type) - offsetof(struct task_struct, wake_entry) != + offsetof(struct __call_single_data, flags) - offsetof(struct __call_single_data, llist)); + idle_threads_init(); cpuhp_threads_init(); From 1f8db4150536431b031585ecc2a6793f69245de2 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Thu, 28 May 2020 11:01:34 +0200 Subject: [PATCH 0939/1043] sched/headers: Split out open-coded prototypes into kernel/sched/smp.h Move the prototypes for sched_ttwu_pending() and send_call_function_single_ipi() into the newly created kernel/sched/smp.h header, to make sure they are all the same, and to architectures happy that use -Wmissing-prototypes. Signed-off-by: Ingo Molnar --- kernel/sched/core.c | 1 + kernel/sched/smp.h | 9 +++++++++ kernel/smp.c | 5 +---- 3 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 kernel/sched/smp.h diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b3c64c6b4d2e..43ba2d4a8eca 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -20,6 +20,7 @@ #include "../smpboot.h" #include "pelt.h" +#include "smp.h" #define CREATE_TRACE_POINTS #include diff --git a/kernel/sched/smp.h b/kernel/sched/smp.h new file mode 100644 index 000000000000..9620e323162c --- /dev/null +++ b/kernel/sched/smp.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Scheduler internal SMP callback types and methods between the scheduler + * and other internal parts of the core kernel: + */ + +extern void sched_ttwu_pending(void *arg); + +extern void send_call_function_single_ipi(int cpu); diff --git a/kernel/smp.c b/kernel/smp.c index 0d61dc060b01..4dec04f7fdc5 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -22,7 +22,7 @@ #include #include "smpboot.h" - +#include "sched/smp.h" #define CSD_TYPE(_csd) ((_csd)->flags & CSD_FLAG_TYPE_MASK) @@ -133,8 +133,6 @@ static __always_inline void csd_unlock(call_single_data_t *csd) static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data); -extern void send_call_function_single_ipi(int cpu); - void __smp_call_single_queue(int cpu, struct llist_node *node) { /* @@ -196,7 +194,6 @@ void generic_smp_call_function_single_interrupt(void) flush_smp_call_function_queue(true); } -extern void sched_ttwu_pending(void *); extern void irq_work_single(void *); /** From fc5d1f1a42fba6266ab95dc3b84937933a9b5a66 Mon Sep 17 00:00:00 2001 From: Christoffer Dall Date: Sat, 1 Dec 2018 08:41:28 -0800 Subject: [PATCH 0940/1043] KVM: arm64: vgic-v3: Take cpu_if pointer directly instead of vcpu If we move the used_lrs field to the version-specific cpu interface structure, the following functions only operate on the struct vgic_v3_cpu_if and not the full vcpu: __vgic_v3_save_state __vgic_v3_restore_state __vgic_v3_activate_traps __vgic_v3_deactivate_traps __vgic_v3_save_aprs __vgic_v3_restore_aprs This is going to be very useful for nested virt, so move the used_lrs field and change the prototypes and implementations of these functions to take the cpu_if parameter directly. No functional change. Reviewed-by: James Morse Signed-off-by: Christoffer Dall Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_hyp.h | 12 ++++++------ arch/arm64/kvm/hyp/switch.c | 8 ++++---- arch/arm64/kvm/hyp/vgic-v3-sr.c | 33 ++++++++++---------------------- arch/arm64/kvm/vgic/vgic-v2.c | 10 +++++----- arch/arm64/kvm/vgic/vgic-v3.c | 14 ++++++++------ arch/arm64/kvm/vgic/vgic.c | 25 ++++++++++++++++-------- include/kvm/arm_vgic.h | 5 ++++- 7 files changed, 54 insertions(+), 53 deletions(-) diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h index fe57f60f06a8..4f67b0cdffe8 100644 --- a/arch/arm64/include/asm/kvm_hyp.h +++ b/arch/arm64/include/asm/kvm_hyp.h @@ -56,12 +56,12 @@ int __vgic_v2_perform_cpuif_access(struct kvm_vcpu *vcpu); -void __vgic_v3_save_state(struct kvm_vcpu *vcpu); -void __vgic_v3_restore_state(struct kvm_vcpu *vcpu); -void __vgic_v3_activate_traps(struct kvm_vcpu *vcpu); -void __vgic_v3_deactivate_traps(struct kvm_vcpu *vcpu); -void __vgic_v3_save_aprs(struct kvm_vcpu *vcpu); -void __vgic_v3_restore_aprs(struct kvm_vcpu *vcpu); +void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if); +void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if); +void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if); +void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if); +void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if); +void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if); int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu); void __timer_enable_traps(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c index 8a1e81a400e0..c07a45643cd4 100644 --- a/arch/arm64/kvm/hyp/switch.c +++ b/arch/arm64/kvm/hyp/switch.c @@ -270,8 +270,8 @@ static void __hyp_text __deactivate_vm(struct kvm_vcpu *vcpu) static void __hyp_text __hyp_vgic_save_state(struct kvm_vcpu *vcpu) { if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { - __vgic_v3_save_state(vcpu); - __vgic_v3_deactivate_traps(vcpu); + __vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3); + __vgic_v3_deactivate_traps(&vcpu->arch.vgic_cpu.vgic_v3); } } @@ -279,8 +279,8 @@ static void __hyp_text __hyp_vgic_save_state(struct kvm_vcpu *vcpu) static void __hyp_text __hyp_vgic_restore_state(struct kvm_vcpu *vcpu) { if (static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) { - __vgic_v3_activate_traps(vcpu); - __vgic_v3_restore_state(vcpu); + __vgic_v3_activate_traps(&vcpu->arch.vgic_cpu.vgic_v3); + __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3); } } diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c index 6b85773e15c4..10ed539835c1 100644 --- a/arch/arm64/kvm/hyp/vgic-v3-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c @@ -194,10 +194,9 @@ static u32 __hyp_text __vgic_v3_read_ap1rn(int n) return val; } -void __hyp_text __vgic_v3_save_state(struct kvm_vcpu *vcpu) +void __hyp_text __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if) { - struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; - u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs; + u64 used_lrs = cpu_if->used_lrs; /* * Make sure stores to the GIC via the memory mapped interface @@ -230,10 +229,9 @@ void __hyp_text __vgic_v3_save_state(struct kvm_vcpu *vcpu) } } -void __hyp_text __vgic_v3_restore_state(struct kvm_vcpu *vcpu) +void __hyp_text __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if) { - struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; - u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs; + u64 used_lrs = cpu_if->used_lrs; int i; if (used_lrs || cpu_if->its_vpe.its_vm) { @@ -257,10 +255,8 @@ void __hyp_text __vgic_v3_restore_state(struct kvm_vcpu *vcpu) } } -void __hyp_text __vgic_v3_activate_traps(struct kvm_vcpu *vcpu) +void __hyp_text __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if) { - struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; - /* * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a * Group0 interrupt (as generated in GICv2 mode) to be @@ -306,9 +302,8 @@ void __hyp_text __vgic_v3_activate_traps(struct kvm_vcpu *vcpu) write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2); } -void __hyp_text __vgic_v3_deactivate_traps(struct kvm_vcpu *vcpu) +void __hyp_text __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if) { - struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; u64 val; if (!cpu_if->vgic_sre) { @@ -333,15 +328,11 @@ void __hyp_text __vgic_v3_deactivate_traps(struct kvm_vcpu *vcpu) write_gicreg(0, ICH_HCR_EL2); } -void __hyp_text __vgic_v3_save_aprs(struct kvm_vcpu *vcpu) +void __hyp_text __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if) { - struct vgic_v3_cpu_if *cpu_if; u64 val; u32 nr_pre_bits; - vcpu = kern_hyp_va(vcpu); - cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; - val = read_gicreg(ICH_VTR_EL2); nr_pre_bits = vtr_to_nr_pre_bits(val); @@ -370,15 +361,11 @@ void __hyp_text __vgic_v3_save_aprs(struct kvm_vcpu *vcpu) } } -void __hyp_text __vgic_v3_restore_aprs(struct kvm_vcpu *vcpu) +void __hyp_text __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if) { - struct vgic_v3_cpu_if *cpu_if; u64 val; u32 nr_pre_bits; - vcpu = kern_hyp_va(vcpu); - cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; - val = read_gicreg(ICH_VTR_EL2); nr_pre_bits = vtr_to_nr_pre_bits(val); @@ -451,7 +438,7 @@ static int __hyp_text __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr, u64 *lr_val) { - unsigned int used_lrs = vcpu->arch.vgic_cpu.used_lrs; + unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs; u8 priority = GICv3_IDLE_PRIORITY; int i, lr = -1; @@ -490,7 +477,7 @@ static int __hyp_text __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, static int __hyp_text __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid, u64 *lr_val) { - unsigned int used_lrs = vcpu->arch.vgic_cpu.used_lrs; + unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs; int i; for (i = 0; i < used_lrs; i++) { diff --git a/arch/arm64/kvm/vgic/vgic-v2.c b/arch/arm64/kvm/vgic/vgic-v2.c index 621cc168fe3f..ebf53a4e1296 100644 --- a/arch/arm64/kvm/vgic/vgic-v2.c +++ b/arch/arm64/kvm/vgic/vgic-v2.c @@ -56,7 +56,7 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu) cpuif->vgic_hcr &= ~GICH_HCR_UIE; - for (lr = 0; lr < vgic_cpu->used_lrs; lr++) { + for (lr = 0; lr < vgic_cpu->vgic_v2.used_lrs; lr++) { u32 val = cpuif->vgic_lr[lr]; u32 cpuid, intid = val & GICH_LR_VIRTUALID; struct vgic_irq *irq; @@ -120,7 +120,7 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu) vgic_put_irq(vcpu->kvm, irq); } - vgic_cpu->used_lrs = 0; + cpuif->used_lrs = 0; } /* @@ -427,7 +427,7 @@ out: static void save_lrs(struct kvm_vcpu *vcpu, void __iomem *base) { struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; - u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs; + u64 used_lrs = cpu_if->used_lrs; u64 elrsr; int i; @@ -448,7 +448,7 @@ static void save_lrs(struct kvm_vcpu *vcpu, void __iomem *base) void vgic_v2_save_state(struct kvm_vcpu *vcpu) { void __iomem *base = kvm_vgic_global_state.vctrl_base; - u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs; + u64 used_lrs = vcpu->arch.vgic_cpu.vgic_v2.used_lrs; if (!base) return; @@ -463,7 +463,7 @@ void vgic_v2_restore_state(struct kvm_vcpu *vcpu) { struct vgic_v2_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v2; void __iomem *base = kvm_vgic_global_state.vctrl_base; - u64 used_lrs = vcpu->arch.vgic_cpu.used_lrs; + u64 used_lrs = cpu_if->used_lrs; int i; if (!base) diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 3ccd6d3cb4d3..76e2d85789ed 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -39,7 +39,7 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) cpuif->vgic_hcr &= ~ICH_HCR_UIE; - for (lr = 0; lr < vgic_cpu->used_lrs; lr++) { + for (lr = 0; lr < cpuif->used_lrs; lr++) { u64 val = cpuif->vgic_lr[lr]; u32 intid, cpuid; struct vgic_irq *irq; @@ -111,7 +111,7 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) vgic_put_irq(vcpu->kvm, irq); } - vgic_cpu->used_lrs = 0; + cpuif->used_lrs = 0; } /* Requires the irq to be locked already */ @@ -662,10 +662,10 @@ void vgic_v3_load(struct kvm_vcpu *vcpu) if (likely(cpu_if->vgic_sre)) kvm_call_hyp(__vgic_v3_write_vmcr, cpu_if->vgic_vmcr); - kvm_call_hyp(__vgic_v3_restore_aprs, vcpu); + kvm_call_hyp(__vgic_v3_restore_aprs, kern_hyp_va(cpu_if)); if (has_vhe()) - __vgic_v3_activate_traps(vcpu); + __vgic_v3_activate_traps(cpu_if); WARN_ON(vgic_v4_load(vcpu)); } @@ -680,12 +680,14 @@ void vgic_v3_vmcr_sync(struct kvm_vcpu *vcpu) void vgic_v3_put(struct kvm_vcpu *vcpu) { + struct vgic_v3_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v3; + WARN_ON(vgic_v4_put(vcpu, false)); vgic_v3_vmcr_sync(vcpu); - kvm_call_hyp(__vgic_v3_save_aprs, vcpu); + kvm_call_hyp(__vgic_v3_save_aprs, kern_hyp_va(cpu_if)); if (has_vhe()) - __vgic_v3_deactivate_traps(vcpu); + __vgic_v3_deactivate_traps(cpu_if); } diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 99b02ca730a8..c3643b7f101b 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -786,6 +786,7 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu) int count; bool multi_sgi; u8 prio = 0xff; + int i = 0; lockdep_assert_held(&vgic_cpu->ap_list_lock); @@ -827,11 +828,14 @@ static void vgic_flush_lr_state(struct kvm_vcpu *vcpu) } } - vcpu->arch.vgic_cpu.used_lrs = count; - /* Nuke remaining LRs */ - for ( ; count < kvm_vgic_global_state.nr_lr; count++) - vgic_clear_lr(vcpu, count); + for (i = count ; i < kvm_vgic_global_state.nr_lr; i++) + vgic_clear_lr(vcpu, i); + + if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) + vcpu->arch.vgic_cpu.vgic_v2.used_lrs = count; + else + vcpu->arch.vgic_cpu.vgic_v3.used_lrs = count; } static inline bool can_access_vgic_from_kernel(void) @@ -849,13 +853,13 @@ static inline void vgic_save_state(struct kvm_vcpu *vcpu) if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) vgic_v2_save_state(vcpu); else - __vgic_v3_save_state(vcpu); + __vgic_v3_save_state(&vcpu->arch.vgic_cpu.vgic_v3); } /* Sync back the hardware VGIC state into our emulation after a guest's run. */ void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu) { - struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu; + int used_lrs; /* An empty ap_list_head implies used_lrs == 0 */ if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head)) @@ -864,7 +868,12 @@ void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu) if (can_access_vgic_from_kernel()) vgic_save_state(vcpu); - if (vgic_cpu->used_lrs) + if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) + used_lrs = vcpu->arch.vgic_cpu.vgic_v2.used_lrs; + else + used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs; + + if (used_lrs) vgic_fold_lr_state(vcpu); vgic_prune_ap_list(vcpu); } @@ -874,7 +883,7 @@ static inline void vgic_restore_state(struct kvm_vcpu *vcpu) if (!static_branch_unlikely(&kvm_vgic_global_state.gicv3_cpuif)) vgic_v2_restore_state(vcpu); else - __vgic_v3_restore_state(vcpu); + __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3); } /* Flush our emulation state into the GIC hardware before entering the guest. */ diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index 69f4164d6477..a8d8fdcd3723 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -274,6 +274,8 @@ struct vgic_v2_cpu_if { u32 vgic_vmcr; u32 vgic_apr; u32 vgic_lr[VGIC_V2_MAX_LRS]; + + unsigned int used_lrs; }; struct vgic_v3_cpu_if { @@ -291,6 +293,8 @@ struct vgic_v3_cpu_if { * linking the Linux IRQ subsystem and the ITS together. */ struct its_vpe its_vpe; + + unsigned int used_lrs; }; struct vgic_cpu { @@ -300,7 +304,6 @@ struct vgic_cpu { struct vgic_v3_cpu_if vgic_v3; }; - unsigned int used_lrs; struct vgic_irq private_irqs[VGIC_NR_PRIVATE_IRQS]; raw_spinlock_t ap_list_lock; /* Protects the ap_list */ From 7ea90bdd70c9cf82dfbaa54e7d9f296928679224 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 20 Jun 2019 11:17:00 +0100 Subject: [PATCH 0941/1043] KVM: arm64: Refactor vcpu_{read,write}_sys_reg Extract the direct HW accessors for later reuse. Reviewed-by: James Morse Signed-off-by: Marc Zyngier --- arch/arm64/kvm/sys_regs.c | 128 +++++++++++++++++++++----------------- 1 file changed, 71 insertions(+), 57 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 620eaf11e672..50e328ca1419 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -64,11 +64,8 @@ static bool write_to_read_only(struct kvm_vcpu *vcpu, return false; } -u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg) +static bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val) { - if (!vcpu->arch.sysregs_loaded_on_cpu) - goto immediate_read; - /* * System registers listed in the switch are not saved on every * exit from the guest but are only saved on vcpu_put. @@ -79,40 +76,37 @@ u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg) * thread when emulating cross-VCPU communication. */ switch (reg) { - case CSSELR_EL1: return read_sysreg_s(SYS_CSSELR_EL1); - case SCTLR_EL1: return read_sysreg_s(SYS_SCTLR_EL12); - case ACTLR_EL1: return read_sysreg_s(SYS_ACTLR_EL1); - case CPACR_EL1: return read_sysreg_s(SYS_CPACR_EL12); - case TTBR0_EL1: return read_sysreg_s(SYS_TTBR0_EL12); - case TTBR1_EL1: return read_sysreg_s(SYS_TTBR1_EL12); - case TCR_EL1: return read_sysreg_s(SYS_TCR_EL12); - case ESR_EL1: return read_sysreg_s(SYS_ESR_EL12); - case AFSR0_EL1: return read_sysreg_s(SYS_AFSR0_EL12); - case AFSR1_EL1: return read_sysreg_s(SYS_AFSR1_EL12); - case FAR_EL1: return read_sysreg_s(SYS_FAR_EL12); - case MAIR_EL1: return read_sysreg_s(SYS_MAIR_EL12); - case VBAR_EL1: return read_sysreg_s(SYS_VBAR_EL12); - case CONTEXTIDR_EL1: return read_sysreg_s(SYS_CONTEXTIDR_EL12); - case TPIDR_EL0: return read_sysreg_s(SYS_TPIDR_EL0); - case TPIDRRO_EL0: return read_sysreg_s(SYS_TPIDRRO_EL0); - case TPIDR_EL1: return read_sysreg_s(SYS_TPIDR_EL1); - case AMAIR_EL1: return read_sysreg_s(SYS_AMAIR_EL12); - case CNTKCTL_EL1: return read_sysreg_s(SYS_CNTKCTL_EL12); - case PAR_EL1: return read_sysreg_s(SYS_PAR_EL1); - case DACR32_EL2: return read_sysreg_s(SYS_DACR32_EL2); - case IFSR32_EL2: return read_sysreg_s(SYS_IFSR32_EL2); - case DBGVCR32_EL2: return read_sysreg_s(SYS_DBGVCR32_EL2); + case CSSELR_EL1: *val = read_sysreg_s(SYS_CSSELR_EL1); break; + case SCTLR_EL1: *val = read_sysreg_s(SYS_SCTLR_EL12); break; + case ACTLR_EL1: *val = read_sysreg_s(SYS_ACTLR_EL1); break; + case CPACR_EL1: *val = read_sysreg_s(SYS_CPACR_EL12); break; + case TTBR0_EL1: *val = read_sysreg_s(SYS_TTBR0_EL12); break; + case TTBR1_EL1: *val = read_sysreg_s(SYS_TTBR1_EL12); break; + case TCR_EL1: *val = read_sysreg_s(SYS_TCR_EL12); break; + case ESR_EL1: *val = read_sysreg_s(SYS_ESR_EL12); break; + case AFSR0_EL1: *val = read_sysreg_s(SYS_AFSR0_EL12); break; + case AFSR1_EL1: *val = read_sysreg_s(SYS_AFSR1_EL12); break; + case FAR_EL1: *val = read_sysreg_s(SYS_FAR_EL12); break; + case MAIR_EL1: *val = read_sysreg_s(SYS_MAIR_EL12); break; + case VBAR_EL1: *val = read_sysreg_s(SYS_VBAR_EL12); break; + case CONTEXTIDR_EL1: *val = read_sysreg_s(SYS_CONTEXTIDR_EL12);break; + case TPIDR_EL0: *val = read_sysreg_s(SYS_TPIDR_EL0); break; + case TPIDRRO_EL0: *val = read_sysreg_s(SYS_TPIDRRO_EL0); break; + case TPIDR_EL1: *val = read_sysreg_s(SYS_TPIDR_EL1); break; + case AMAIR_EL1: *val = read_sysreg_s(SYS_AMAIR_EL12); break; + case CNTKCTL_EL1: *val = read_sysreg_s(SYS_CNTKCTL_EL12); break; + case PAR_EL1: *val = read_sysreg_s(SYS_PAR_EL1); break; + case DACR32_EL2: *val = read_sysreg_s(SYS_DACR32_EL2); break; + case IFSR32_EL2: *val = read_sysreg_s(SYS_IFSR32_EL2); break; + case DBGVCR32_EL2: *val = read_sysreg_s(SYS_DBGVCR32_EL2); break; + default: return false; } -immediate_read: - return __vcpu_sys_reg(vcpu, reg); + return true; } -void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg) +static bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg) { - if (!vcpu->arch.sysregs_loaded_on_cpu) - goto immediate_write; - /* * System registers listed in the switch are not restored on every * entry to the guest but are only restored on vcpu_load. @@ -122,32 +116,52 @@ void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg) * once, before running the VCPU, and never changed later. */ switch (reg) { - case CSSELR_EL1: write_sysreg_s(val, SYS_CSSELR_EL1); return; - case SCTLR_EL1: write_sysreg_s(val, SYS_SCTLR_EL12); return; - case ACTLR_EL1: write_sysreg_s(val, SYS_ACTLR_EL1); return; - case CPACR_EL1: write_sysreg_s(val, SYS_CPACR_EL12); return; - case TTBR0_EL1: write_sysreg_s(val, SYS_TTBR0_EL12); return; - case TTBR1_EL1: write_sysreg_s(val, SYS_TTBR1_EL12); return; - case TCR_EL1: write_sysreg_s(val, SYS_TCR_EL12); return; - case ESR_EL1: write_sysreg_s(val, SYS_ESR_EL12); return; - case AFSR0_EL1: write_sysreg_s(val, SYS_AFSR0_EL12); return; - case AFSR1_EL1: write_sysreg_s(val, SYS_AFSR1_EL12); return; - case FAR_EL1: write_sysreg_s(val, SYS_FAR_EL12); return; - case MAIR_EL1: write_sysreg_s(val, SYS_MAIR_EL12); return; - case VBAR_EL1: write_sysreg_s(val, SYS_VBAR_EL12); return; - case CONTEXTIDR_EL1: write_sysreg_s(val, SYS_CONTEXTIDR_EL12); return; - case TPIDR_EL0: write_sysreg_s(val, SYS_TPIDR_EL0); return; - case TPIDRRO_EL0: write_sysreg_s(val, SYS_TPIDRRO_EL0); return; - case TPIDR_EL1: write_sysreg_s(val, SYS_TPIDR_EL1); return; - case AMAIR_EL1: write_sysreg_s(val, SYS_AMAIR_EL12); return; - case CNTKCTL_EL1: write_sysreg_s(val, SYS_CNTKCTL_EL12); return; - case PAR_EL1: write_sysreg_s(val, SYS_PAR_EL1); return; - case DACR32_EL2: write_sysreg_s(val, SYS_DACR32_EL2); return; - case IFSR32_EL2: write_sysreg_s(val, SYS_IFSR32_EL2); return; - case DBGVCR32_EL2: write_sysreg_s(val, SYS_DBGVCR32_EL2); return; + case CSSELR_EL1: write_sysreg_s(val, SYS_CSSELR_EL1); break; + case SCTLR_EL1: write_sysreg_s(val, SYS_SCTLR_EL12); break; + case ACTLR_EL1: write_sysreg_s(val, SYS_ACTLR_EL1); break; + case CPACR_EL1: write_sysreg_s(val, SYS_CPACR_EL12); break; + case TTBR0_EL1: write_sysreg_s(val, SYS_TTBR0_EL12); break; + case TTBR1_EL1: write_sysreg_s(val, SYS_TTBR1_EL12); break; + case TCR_EL1: write_sysreg_s(val, SYS_TCR_EL12); break; + case ESR_EL1: write_sysreg_s(val, SYS_ESR_EL12); break; + case AFSR0_EL1: write_sysreg_s(val, SYS_AFSR0_EL12); break; + case AFSR1_EL1: write_sysreg_s(val, SYS_AFSR1_EL12); break; + case FAR_EL1: write_sysreg_s(val, SYS_FAR_EL12); break; + case MAIR_EL1: write_sysreg_s(val, SYS_MAIR_EL12); break; + case VBAR_EL1: write_sysreg_s(val, SYS_VBAR_EL12); break; + case CONTEXTIDR_EL1: write_sysreg_s(val, SYS_CONTEXTIDR_EL12);break; + case TPIDR_EL0: write_sysreg_s(val, SYS_TPIDR_EL0); break; + case TPIDRRO_EL0: write_sysreg_s(val, SYS_TPIDRRO_EL0); break; + case TPIDR_EL1: write_sysreg_s(val, SYS_TPIDR_EL1); break; + case AMAIR_EL1: write_sysreg_s(val, SYS_AMAIR_EL12); break; + case CNTKCTL_EL1: write_sysreg_s(val, SYS_CNTKCTL_EL12); break; + case PAR_EL1: write_sysreg_s(val, SYS_PAR_EL1); break; + case DACR32_EL2: write_sysreg_s(val, SYS_DACR32_EL2); break; + case IFSR32_EL2: write_sysreg_s(val, SYS_IFSR32_EL2); break; + case DBGVCR32_EL2: write_sysreg_s(val, SYS_DBGVCR32_EL2); break; + default: return false; } -immediate_write: + return true; +} + +u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg) +{ + u64 val = 0x8badf00d8badf00d; + + if (vcpu->arch.sysregs_loaded_on_cpu && + __vcpu_read_sys_reg_from_cpu(reg, &val)) + return val; + + return __vcpu_sys_reg(vcpu, reg); +} + +void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg) +{ + if (vcpu->arch.sysregs_loaded_on_cpu && + __vcpu_write_sys_reg_to_cpu(val, reg)) + return; + __vcpu_sys_reg(vcpu, reg) = val; } From 7ccadf23b8613c946f67e2b3c5e7f436858021aa Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 27 Jan 2020 11:54:42 +0000 Subject: [PATCH 0942/1043] KVM: arm64: Add missing reset handlers for PMU emulation As we're about to become a bit more harsh when it comes to the lack of reset callbacks, let's add the missing PMU reset handlers. Note that these only cover *CLR registers that were always covered by their *SET counterpart, so there is no semantic change here. Reviewed-by: James Morse Signed-off-by: Marc Zyngier --- arch/arm64/kvm/sys_regs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 50e328ca1419..9d28eabbdf97 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1546,7 +1546,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 }, { SYS_DESC(SYS_PMINTENSET_EL1), access_pminten, reset_unknown, PMINTENSET_EL1 }, - { SYS_DESC(SYS_PMINTENCLR_EL1), access_pminten, NULL, PMINTENSET_EL1 }, + { SYS_DESC(SYS_PMINTENCLR_EL1), access_pminten, reset_unknown, PMINTENSET_EL1 }, { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 }, { SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 }, @@ -1585,8 +1585,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_PMCR_EL0), access_pmcr, reset_pmcr, PMCR_EL0 }, { SYS_DESC(SYS_PMCNTENSET_EL0), access_pmcnten, reset_unknown, PMCNTENSET_EL0 }, - { SYS_DESC(SYS_PMCNTENCLR_EL0), access_pmcnten, NULL, PMCNTENSET_EL0 }, - { SYS_DESC(SYS_PMOVSCLR_EL0), access_pmovs, NULL, PMOVSSET_EL0 }, + { SYS_DESC(SYS_PMCNTENCLR_EL0), access_pmcnten, reset_unknown, PMCNTENSET_EL0 }, + { SYS_DESC(SYS_PMOVSCLR_EL0), access_pmovs, reset_unknown, PMOVSSET_EL0 }, { SYS_DESC(SYS_PMSWINC_EL0), access_pmswinc, reset_unknown, PMSWINC_EL0 }, { SYS_DESC(SYS_PMSELR_EL0), access_pmselr, reset_unknown, PMSELR_EL0 }, { SYS_DESC(SYS_PMCEID0_EL0), access_pmceid }, From bb44a8dbea259bc1dc2177b4bc90ca4e8fcbf659 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 27 Jan 2020 11:21:17 +0000 Subject: [PATCH 0943/1043] KVM: arm64: Move sysreg reset check to boot time Our sysreg reset check has become a bit silly, as it only checks whether a reset callback actually exists for a given sysreg entry, and apply the method if available. Doing the check at each vcpu reset is pretty dumb, as the tables never change. It is thus perfectly possible to do the same checks at boot time. This also allows us to introduce a sparse sys_regs[] array, something that will be required with ARMv8.4-NV. Signed-off-by: Marc Zyngier --- arch/arm64/kvm/sys_regs.c | 72 +++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 9d28eabbdf97..ad1d57501d6d 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -2087,12 +2087,37 @@ static const struct sys_reg_desc cp15_64_regs[] = { { SYS_DESC(SYS_AARCH32_CNTP_CVAL), access_arch_timer }, }; +static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n, + bool is_32) +{ + unsigned int i; + + for (i = 0; i < n; i++) { + if (!is_32 && table[i].reg && !table[i].reset) { + kvm_err("sys_reg table %p entry %d has lacks reset\n", + table, i); + return 1; + } + + if (i && cmp_sys_reg(&table[i-1], &table[i]) >= 0) { + kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1); + return 1; + } + } + + return 0; +} + /* Target specific emulation tables */ static struct kvm_sys_reg_target_table *target_tables[KVM_ARM_NUM_TARGETS]; void kvm_register_target_sys_reg_table(unsigned int target, struct kvm_sys_reg_target_table *table) { + if (check_sysreg_table(table->table64.table, table->table64.num, false) || + check_sysreg_table(table->table32.table, table->table32.num, true)) + return; + target_tables[target] = table; } @@ -2378,19 +2403,13 @@ static int emulate_sys_reg(struct kvm_vcpu *vcpu, } static void reset_sys_reg_descs(struct kvm_vcpu *vcpu, - const struct sys_reg_desc *table, size_t num, - unsigned long *bmap) + const struct sys_reg_desc *table, size_t num) { unsigned long i; for (i = 0; i < num; i++) - if (table[i].reset) { - int reg = table[i].reg; - + if (table[i].reset) table[i].reset(vcpu, &table[i]); - if (reg > 0 && reg < NR_SYS_REGS) - set_bit(reg, bmap); - } } /** @@ -2846,32 +2865,18 @@ int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices) return write_demux_regids(uindices); } -static int check_sysreg_table(const struct sys_reg_desc *table, unsigned int n) -{ - unsigned int i; - - for (i = 1; i < n; i++) { - if (cmp_sys_reg(&table[i-1], &table[i]) >= 0) { - kvm_err("sys_reg table %p out of order (%d)\n", table, i - 1); - return 1; - } - } - - return 0; -} - void kvm_sys_reg_table_init(void) { unsigned int i; struct sys_reg_desc clidr; /* Make sure tables are unique and in order. */ - BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs))); - BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs))); - BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs))); - BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs))); - BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs))); - BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs))); + BUG_ON(check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false)); + BUG_ON(check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true)); + BUG_ON(check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true)); + BUG_ON(check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true)); + BUG_ON(check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true)); + BUG_ON(check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs), false)); /* We abuse the reset function to overwrite the table itself. */ for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) @@ -2907,17 +2912,10 @@ void kvm_reset_sys_regs(struct kvm_vcpu *vcpu) { size_t num; const struct sys_reg_desc *table; - DECLARE_BITMAP(bmap, NR_SYS_REGS) = { 0, }; /* Generic chip reset first (so target could override). */ - reset_sys_reg_descs(vcpu, sys_reg_descs, ARRAY_SIZE(sys_reg_descs), bmap); + reset_sys_reg_descs(vcpu, sys_reg_descs, ARRAY_SIZE(sys_reg_descs)); table = get_target_table(vcpu->arch.target, true, &num); - reset_sys_reg_descs(vcpu, table, num, bmap); - - for (num = 1; num < NR_SYS_REGS; num++) { - if (WARN(!test_bit(num, bmap), - "Didn't reset __vcpu_sys_reg(%zi)\n", num)) - break; - } + reset_sys_reg_descs(vcpu, table, num); } From 349c330ced9764667678f4d2804fd4ebc16110c9 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sun, 12 Apr 2020 18:49:31 +0100 Subject: [PATCH 0944/1043] KVM: arm64: Don't use empty structures as CPU reset state Keeping empty structure as the vcpu state initializer is slightly wasteful: we only want to set pstate, and zero everything else. Just do that. Signed-off-by: Marc Zyngier --- arch/arm64/kvm/reset.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index 658f3a79617b..865c8aa670bc 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -36,15 +36,11 @@ static u32 kvm_ipa_limit; /* * ARMv8 Reset Values */ -static const struct kvm_regs default_regs_reset = { - .regs.pstate = (PSR_MODE_EL1h | PSR_A_BIT | PSR_I_BIT | - PSR_F_BIT | PSR_D_BIT), -}; +#define VCPU_RESET_PSTATE_EL1 (PSR_MODE_EL1h | PSR_A_BIT | PSR_I_BIT | \ + PSR_F_BIT | PSR_D_BIT) -static const struct kvm_regs default_regs_reset32 = { - .regs.pstate = (PSR_AA32_MODE_SVC | PSR_AA32_A_BIT | - PSR_AA32_I_BIT | PSR_AA32_F_BIT), -}; +#define VCPU_RESET_PSTATE_SVC (PSR_AA32_MODE_SVC | PSR_AA32_A_BIT | \ + PSR_AA32_I_BIT | PSR_AA32_F_BIT) static bool cpu_has_32bit_el1(void) { @@ -257,9 +253,9 @@ static int kvm_vcpu_enable_ptrauth(struct kvm_vcpu *vcpu) */ int kvm_reset_vcpu(struct kvm_vcpu *vcpu) { - const struct kvm_regs *cpu_reset; int ret = -EINVAL; bool loaded; + u32 pstate; /* Reset PMU outside of the non-preemptible section */ kvm_pmu_vcpu_reset(vcpu); @@ -290,16 +286,17 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu) if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features)) { if (!cpu_has_32bit_el1()) goto out; - cpu_reset = &default_regs_reset32; + pstate = VCPU_RESET_PSTATE_SVC; } else { - cpu_reset = &default_regs_reset; + pstate = VCPU_RESET_PSTATE_EL1; } break; } /* Reset core registers */ - memcpy(vcpu_gp_regs(vcpu), cpu_reset, sizeof(*cpu_reset)); + memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu))); + vcpu_gp_regs(vcpu)->regs.pstate = pstate; /* Reset system registers */ kvm_reset_sys_regs(vcpu); From c3b9c0043d25f66a73ebd8cbf34e7d6a9a2ccbb4 Mon Sep 17 00:00:00 2001 From: Tiezhu Yang Date: Thu, 28 May 2020 19:29:11 +0800 Subject: [PATCH 0945/1043] MIPS: Loongson64: Remove not used pci.c After commit 6423e59a64e7 ("MIPS: Loongson64: Switch to generic PCI driver"), arch/mips/loongson64/pci.c is not used any more, remove it. Signed-off-by: Tiezhu Yang Signed-off-by: Thomas Bogendoerfer --- arch/mips/loongson64/pci.c | 49 -------------------------------------- 1 file changed, 49 deletions(-) delete mode 100644 arch/mips/loongson64/pci.c diff --git a/arch/mips/loongson64/pci.c b/arch/mips/loongson64/pci.c deleted file mode 100644 index a440a2725a20..000000000000 --- a/arch/mips/loongson64/pci.c +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2007 Lemote, Inc. & Institute of Computing Technology - * Author: Fuxin Zhang, zhangfx@lemote.com - */ -#include - -#include -#include -#include - -static struct resource loongson_pci_mem_resource = { - .name = "pci memory space", - .start = LOONGSON_PCI_MEM_START, - .end = LOONGSON_PCI_MEM_END, - .flags = IORESOURCE_MEM, -}; - -static struct resource loongson_pci_io_resource = { - .name = "pci io space", - .start = LOONGSON_PCI_IO_START, - .end = IO_SPACE_LIMIT, - .flags = IORESOURCE_IO, -}; - -static struct pci_controller loongson_pci_controller = { - .pci_ops = &loongson_pci_ops, - .io_resource = &loongson_pci_io_resource, - .mem_resource = &loongson_pci_mem_resource, - .mem_offset = 0x00000000UL, - .io_offset = 0x00000000UL, -}; - - -extern int sbx00_acpi_init(void); - -static int __init pcibios_init(void) -{ - - loongson_pci_controller.io_map_base = mips_io_port_base; - loongson_pci_mem_resource.start = loongson_sysconf.pci_mem_start_addr; - loongson_pci_mem_resource.end = loongson_sysconf.pci_mem_end_addr; - - register_pci_controller(&loongson_pci_controller); - - return 0; -} - -arch_initcall(pcibios_init); From a43a67a2d715540c1368b9501a22b0373b5874c0 Mon Sep 17 00:00:00 2001 From: Goldwyn Rodrigues Date: Tue, 19 May 2020 09:14:18 -0500 Subject: [PATCH 0946/1043] btrfs: switch to iomap_dio_rw() for dio Switch from __blockdev_direct_IO() to iomap_dio_rw(). Rename btrfs_get_blocks_direct() to btrfs_dio_iomap_begin() and use it as iomap_begin() for iomap direct I/O functions. This function allocates and locks all the blocks required for the I/O. btrfs_submit_direct() is used as the submit_io() hook for direct I/O ops. Since we need direct I/O reads to go through iomap_dio_rw(), we change file_operations.read_iter() to a btrfs_file_read_iter() which calls btrfs_direct_IO() for direct reads and falls back to generic_file_buffered_read() for incomplete reads and buffered reads. We don't need address_space.direct_IO() anymore so set it to noop. Similarly, we don't need flags used in __blockdev_direct_IO(). iomap is capable of direct I/O reads from a hole, so we don't need to return -ENOENT. BTRFS direct I/O is now done under i_rwsem, shared in case of reads and exclusive in case of writes. This guards against simultaneous truncates. Use iomap->iomap_end() to check for failed or incomplete direct I/O: - for writes, call __endio_write_update_ordered() - for reads, unlock extents btrfs_dio_data is now hooked in iomap->private and not current->journal_info. It carries the reservation variable and the amount of data submitted, so we can calculate the amount of data to call __endio_write_update_ordered in case of an error. This patch removes last use of struct buffer_head from btrfs. Signed-off-by: Goldwyn Rodrigues Signed-off-by: David Sterba --- fs/btrfs/Kconfig | 1 + fs/btrfs/ctree.h | 1 + fs/btrfs/file.c | 21 +++- fs/btrfs/inode.c | 312 ++++++++++++++++++++++------------------------- 4 files changed, 166 insertions(+), 169 deletions(-) diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig index 575636f6491e..68b95ad82126 100644 --- a/fs/btrfs/Kconfig +++ b/fs/btrfs/Kconfig @@ -14,6 +14,7 @@ config BTRFS_FS select LZO_DECOMPRESS select ZSTD_COMPRESS select ZSTD_DECOMPRESS + select FS_IOMAP select RAID6_PQ select XOR_BLOCKS select SRCU diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 30ce7039bc27..47b8874eddff 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -2933,6 +2933,7 @@ int btrfs_writepage_cow_fixup(struct page *page, u64 start, u64 end); void btrfs_writepage_endio_finish_ordered(struct page *page, u64 start, u64 end, int uptodate); extern const struct dentry_operations btrfs_dentry_operations; +ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter); /* ioctl.c */ long btrfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg); diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 2c14312b05e8..dff89d04fd16 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -1819,7 +1819,7 @@ static ssize_t __btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) loff_t endbyte; int err; - written = generic_file_direct_write(iocb, from); + written = btrfs_direct_IO(iocb, from); if (written < 0 || !iov_iter_count(from)) return written; @@ -3476,9 +3476,26 @@ static int btrfs_file_open(struct inode *inode, struct file *filp) return generic_file_open(inode, filp); } +static ssize_t btrfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + ssize_t ret = 0; + + if (iocb->ki_flags & IOCB_DIRECT) { + struct inode *inode = file_inode(iocb->ki_filp); + + inode_lock_shared(inode); + ret = btrfs_direct_IO(iocb, to); + inode_unlock_shared(inode); + if (ret < 0) + return ret; + } + + return generic_file_buffered_read(iocb, to, ret); +} + const struct file_operations btrfs_file_operations = { .llseek = btrfs_file_llseek, - .read_iter = generic_file_read_iter, + .read_iter = btrfs_file_read_iter, .splice_read = generic_file_splice_read, .write_iter = btrfs_file_write_iter, .mmap = btrfs_file_mmap, diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 4ec7f34e6cd9..501842f8c648 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -5,7 +5,6 @@ #include #include -#include #include #include #include @@ -30,6 +29,7 @@ #include #include #include +#include #include #include "misc.h" #include "ctree.h" @@ -57,9 +57,9 @@ struct btrfs_iget_args { struct btrfs_dio_data { u64 reserve; - u64 unsubmitted_oe_range_start; - u64 unsubmitted_oe_range_end; - int overwrite; + loff_t length; + ssize_t submitted; + struct extent_changeset *data_reserved; }; static const struct inode_operations btrfs_dir_inode_operations; @@ -6984,7 +6984,7 @@ out: } static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend, - struct extent_state **cached_state, int writing) + struct extent_state **cached_state, bool writing) { struct btrfs_ordered_extent *ordered; int ret = 0; @@ -7122,30 +7122,7 @@ static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len, } -static int btrfs_get_blocks_direct_read(struct extent_map *em, - struct buffer_head *bh_result, - struct inode *inode, - u64 start, u64 len) -{ - struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - - if (em->block_start == EXTENT_MAP_HOLE || - test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) - return -ENOENT; - - len = min(len, em->len - (start - em->start)); - - bh_result->b_blocknr = (em->block_start + (start - em->start)) >> - inode->i_blkbits; - bh_result->b_size = len; - bh_result->b_bdev = fs_info->fs_devices->latest_bdev; - set_buffer_mapped(bh_result); - - return 0; -} - static int btrfs_get_blocks_direct_write(struct extent_map **map, - struct buffer_head *bh_result, struct inode *inode, struct btrfs_dio_data *dio_data, u64 start, u64 len) @@ -7207,7 +7184,6 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, } /* this will cow the extent */ - len = bh_result->b_size; free_extent_map(em); *map = em = btrfs_new_extent_direct(inode, start, len); if (IS_ERR(em)) { @@ -7218,64 +7194,73 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, len = min(len, em->len - (start - em->start)); skip_cow: - bh_result->b_blocknr = (em->block_start + (start - em->start)) >> - inode->i_blkbits; - bh_result->b_size = len; - bh_result->b_bdev = fs_info->fs_devices->latest_bdev; - set_buffer_mapped(bh_result); - - if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) - set_buffer_new(bh_result); - /* * Need to update the i_size under the extent lock so buffered * readers will get the updated i_size when we unlock. */ - if (!dio_data->overwrite && start + len > i_size_read(inode)) + if (start + len > i_size_read(inode)) i_size_write(inode, start + len); - WARN_ON(dio_data->reserve < len); dio_data->reserve -= len; - dio_data->unsubmitted_oe_range_end = start + len; - current->journal_info = dio_data; out: return ret; } -static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock, - struct buffer_head *bh_result, int create) +static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start, + loff_t length, unsigned flags, struct iomap *iomap, + struct iomap *srcmap) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct extent_map *em; struct extent_state *cached_state = NULL; struct btrfs_dio_data *dio_data = NULL; - u64 start = iblock << inode->i_blkbits; u64 lockstart, lockend; - u64 len = bh_result->b_size; + const bool write = !!(flags & IOMAP_WRITE); int ret = 0; + u64 len = length; + bool unlock_extents = false; - if (!create) + if (!write) len = min_t(u64, len, fs_info->sectorsize); lockstart = start; lockend = start + len - 1; - if (current->journal_info) { - /* - * Need to pull our outstanding extents and set journal_info to NULL so - * that anything that needs to check if there's a transaction doesn't get - * confused. - */ - dio_data = current->journal_info; - current->journal_info = NULL; + /* + * The generic stuff only does filemap_write_and_wait_range, which + * isn't enough if we've written compressed pages to this area, so we + * need to flush the dirty pages again to make absolutely sure that any + * outstanding dirty pages are on disk. + */ + if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, + &BTRFS_I(inode)->runtime_flags)) + ret = filemap_fdatawrite_range(inode->i_mapping, start, + start + length - 1); + + dio_data = kzalloc(sizeof(*dio_data), GFP_NOFS); + if (!dio_data) + return -ENOMEM; + + dio_data->length = length; + if (write) { + dio_data->reserve = round_up(length, fs_info->sectorsize); + ret = btrfs_delalloc_reserve_space(inode, + &dio_data->data_reserved, + start, dio_data->reserve); + if (ret) { + extent_changeset_free(dio_data->data_reserved); + kfree(dio_data); + return ret; + } } + iomap->private = dio_data; + /* * If this errors out it's because we couldn't invalidate pagecache for * this range and we need to fallback to buffered. */ - if (lock_extent_direct(inode, lockstart, lockend, &cached_state, - create)) { + if (lock_extent_direct(inode, lockstart, lockend, &cached_state, write)) { ret = -ENOTBLK; goto err; } @@ -7307,36 +7292,48 @@ static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock, goto unlock_err; } - if (create) { - ret = btrfs_get_blocks_direct_write(&em, bh_result, inode, - dio_data, start, len); + len = min(len, em->len - (start - em->start)); + if (write) { + ret = btrfs_get_blocks_direct_write(&em, inode, dio_data, + start, len); if (ret < 0) goto unlock_err; - - unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, - lockend, &cached_state); + unlock_extents = true; + /* Recalc len in case the new em is smaller than requested */ + len = min(len, em->len - (start - em->start)); } else { - ret = btrfs_get_blocks_direct_read(em, bh_result, inode, - start, len); - /* Can be negative only if we read from a hole */ - if (ret < 0) { - ret = 0; - free_extent_map(em); - goto unlock_err; - } /* * We need to unlock only the end area that we aren't using. * The rest is going to be unlocked by the endio routine. */ - lockstart = start + bh_result->b_size; - if (lockstart < lockend) { - unlock_extent_cached(&BTRFS_I(inode)->io_tree, - lockstart, lockend, &cached_state); - } else { - free_extent_state(cached_state); - } + lockstart = start + len; + if (lockstart < lockend) + unlock_extents = true; } + if (unlock_extents) + unlock_extent_cached(&BTRFS_I(inode)->io_tree, + lockstart, lockend, &cached_state); + else + free_extent_state(cached_state); + + /* + * Translate extent map information to iomap. + * We trim the extents (and move the addr) even though iomap code does + * that, since we have locked only the parts we are performing I/O in. + */ + if ((em->block_start == EXTENT_MAP_HOLE) || + (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) && !write)) { + iomap->addr = IOMAP_NULL_ADDR; + iomap->type = IOMAP_HOLE; + } else { + iomap->addr = em->block_start + (start - em->start); + iomap->type = IOMAP_MAPPED; + } + iomap->offset = start; + iomap->bdev = fs_info->fs_devices->latest_bdev; + iomap->length = len; + free_extent_map(em); return 0; @@ -7345,8 +7342,53 @@ unlock_err: unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend, &cached_state); err: - if (dio_data) - current->journal_info = dio_data; + if (dio_data) { + btrfs_delalloc_release_space(inode, dio_data->data_reserved, + start, dio_data->reserve, true); + btrfs_delalloc_release_extents(BTRFS_I(inode), dio_data->reserve); + extent_changeset_free(dio_data->data_reserved); + kfree(dio_data); + } + return ret; +} + +static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned flags, struct iomap *iomap) +{ + int ret = 0; + struct btrfs_dio_data *dio_data = iomap->private; + size_t submitted = dio_data->submitted; + const bool write = !!(flags & IOMAP_WRITE); + + if (!write && (iomap->type == IOMAP_HOLE)) { + /* If reading from a hole, unlock and return */ + unlock_extent(&BTRFS_I(inode)->io_tree, pos, pos + length - 1); + goto out; + } + + if (submitted < length) { + pos += submitted; + length -= submitted; + if (write) + __endio_write_update_ordered(inode, pos, length, false); + else + unlock_extent(&BTRFS_I(inode)->io_tree, pos, + pos + length - 1); + ret = -ENOTBLK; + } + + if (write) { + if (dio_data->reserve) + btrfs_delalloc_release_space(inode, + dio_data->data_reserved, pos, + dio_data->reserve, true); + btrfs_delalloc_release_extents(BTRFS_I(inode), dio_data->length); + extent_changeset_free(dio_data->data_reserved); + } +out: + kfree(dio_data); + iomap->private = NULL; + return ret; } @@ -7369,7 +7411,7 @@ static void btrfs_dio_private_put(struct btrfs_dio_private *dip) dip->logical_offset + dip->bytes - 1); } - dio_end_io(dip->dio_bio); + bio_endio(dip->dio_bio); kfree(dip); } @@ -7605,24 +7647,11 @@ static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio, dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9; dip->dio_bio = dio_bio; refcount_set(&dip->refs, 1); - - if (write) { - struct btrfs_dio_data *dio_data = current->journal_info; - - /* - * Setting range start and end to the same value means that - * no cleanup will happen in btrfs_direct_IO - */ - dio_data->unsubmitted_oe_range_end = dip->logical_offset + - dip->bytes; - dio_data->unsubmitted_oe_range_start = - dio_data->unsubmitted_oe_range_end; - } return dip; } -static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, - loff_t file_offset) +static blk_qc_t btrfs_submit_direct(struct inode *inode, struct iomap *iomap, + struct bio *dio_bio, loff_t file_offset) { const bool write = (bio_op(dio_bio) == REQ_OP_WRITE); const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM); @@ -7639,6 +7668,7 @@ static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, int ret; blk_status_t status; struct btrfs_io_geometry geom; + struct btrfs_dio_data *dio_data = iomap->private; dip = btrfs_create_dio_private(dio_bio, inode, file_offset); if (!dip) { @@ -7647,8 +7677,8 @@ static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, file_offset + dio_bio->bi_iter.bi_size - 1); } dio_bio->bi_status = BLK_STS_RESOURCE; - dio_end_io(dio_bio); - return; + bio_endio(dio_bio); + return BLK_QC_T_NONE; } if (!write && csum) { @@ -7719,15 +7749,17 @@ static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode, goto out_err; } + dio_data->submitted += clone_len; clone_offset += clone_len; start_sector += clone_len >> 9; file_offset += clone_len; } while (submit_len > 0); - return; + return BLK_QC_T_NONE; out_err: dip->dio_bio->bi_status = status; btrfs_dio_private_put(dip); + return BLK_QC_T_NONE; } static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info, @@ -7763,37 +7795,30 @@ out: return retval; } -static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +static const struct iomap_ops btrfs_dio_iomap_ops = { + .iomap_begin = btrfs_dio_iomap_begin, + .iomap_end = btrfs_dio_iomap_end, +}; + +static const struct iomap_dio_ops btrfs_dops = { + .submit_io = btrfs_submit_direct, +}; + +ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) { struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - struct btrfs_dio_data dio_data = { 0 }; struct extent_changeset *data_reserved = NULL; loff_t offset = iocb->ki_pos; size_t count = 0; - int flags = 0; - bool wakeup = true; bool relock = false; ssize_t ret; if (check_direct_IO(fs_info, iter, offset)) return 0; - inode_dio_begin(inode); - - /* - * The generic stuff only does filemap_write_and_wait_range, which - * isn't enough if we've written compressed pages to this area, so - * we need to flush the dirty pages again to make absolutely sure - * that any outstanding dirty pages are on disk. - */ count = iov_iter_count(iter); - if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, - &BTRFS_I(inode)->runtime_flags)) - filemap_fdatawrite_range(inode->i_mapping, offset, - offset + count - 1); - if (iov_iter_rw(iter) == WRITE) { /* * If the write DIO is beyond the EOF, we need update @@ -7801,71 +7826,24 @@ static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) * not unlock the i_mutex at this case. */ if (offset + count <= inode->i_size) { - dio_data.overwrite = 1; inode_unlock(inode); relock = true; } else if (iocb->ki_flags & IOCB_NOWAIT) { ret = -EAGAIN; goto out; } - ret = btrfs_delalloc_reserve_space(inode, &data_reserved, - offset, count); - if (ret) - goto out; - - /* - * We need to know how many extents we reserved so that we can - * do the accounting properly if we go over the number we - * originally calculated. Abuse current->journal_info for this. - */ - dio_data.reserve = round_up(count, - fs_info->sectorsize); - dio_data.unsubmitted_oe_range_start = (u64)offset; - dio_data.unsubmitted_oe_range_end = (u64)offset; - current->journal_info = &dio_data; down_read(&BTRFS_I(inode)->dio_sem); - } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK, - &BTRFS_I(inode)->runtime_flags)) { - inode_dio_end(inode); - flags = DIO_LOCKING | DIO_SKIP_HOLES; - wakeup = false; } - ret = __blockdev_direct_IO(iocb, inode, - fs_info->fs_devices->latest_bdev, - iter, btrfs_get_blocks_direct, NULL, - btrfs_submit_direct, flags); + ret = iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dops, + is_sync_kiocb(iocb)); + if (iov_iter_rw(iter) == WRITE) { up_read(&BTRFS_I(inode)->dio_sem); - current->journal_info = NULL; - if (ret < 0 && ret != -EIOCBQUEUED) { - if (dio_data.reserve) - btrfs_delalloc_release_space(inode, data_reserved, - offset, dio_data.reserve, true); - /* - * On error we might have left some ordered extents - * without submitting corresponding bios for them, so - * cleanup them up to avoid other tasks getting them - * and waiting for them to complete forever. - */ - if (dio_data.unsubmitted_oe_range_start < - dio_data.unsubmitted_oe_range_end) - __endio_write_update_ordered(inode, - dio_data.unsubmitted_oe_range_start, - dio_data.unsubmitted_oe_range_end - - dio_data.unsubmitted_oe_range_start, - false); - } else if (ret >= 0 && (size_t)ret < count) - btrfs_delalloc_release_space(inode, data_reserved, - offset, count - (size_t)ret, true); - btrfs_delalloc_release_extents(BTRFS_I(inode), count); } out: - if (wakeup) - inode_dio_end(inode); if (relock) inode_lock(inode); - extent_changeset_free(data_reserved); return ret; } @@ -10181,7 +10159,7 @@ static const struct address_space_operations btrfs_aops = { .writepage = btrfs_writepage, .writepages = btrfs_writepages, .readpages = btrfs_readpages, - .direct_IO = btrfs_direct_IO, + .direct_IO = noop_direct_IO, .invalidatepage = btrfs_invalidatepage, .releasepage = btrfs_releasepage, #ifdef CONFIG_MIGRATION From b75b7ca7c27dfd61dba368f390b7d4dc20b3a8cb Mon Sep 17 00:00:00 2001 From: Goldwyn Rodrigues Date: Mon, 25 Nov 2019 13:24:03 -0600 Subject: [PATCH 0947/1043] fs: remove dio_end_io() Since we removed the last user of dio_end_io(), remove the helper function dio_end_io(). Reviewed-by: Nikolay Borisov Reviewed-by: Johannes Thumshirn Reviewed-by: Christoph Hellwig Signed-off-by: Goldwyn Rodrigues Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/direct-io.c | 19 ------------------- include/linux/fs.h | 2 -- 2 files changed, 21 deletions(-) diff --git a/fs/direct-io.c b/fs/direct-io.c index 00b4d15bb811..c44d60f375bc 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -386,25 +386,6 @@ static void dio_bio_end_io(struct bio *bio) spin_unlock_irqrestore(&dio->bio_lock, flags); } -/** - * dio_end_io - handle the end io action for the given bio - * @bio: The direct io bio thats being completed - * - * This is meant to be called by any filesystem that uses their own dio_submit_t - * so that the DIO specific endio actions are dealt with after the filesystem - * has done it's completion work. - */ -void dio_end_io(struct bio *bio) -{ - struct dio *dio = bio->bi_private; - - if (dio->is_async) - dio_bio_end_aio(bio); - else - dio_bio_end_io(bio); -} -EXPORT_SYMBOL_GPL(dio_end_io); - static inline void dio_bio_alloc(struct dio *dio, struct dio_submit *sdio, struct block_device *bdev, diff --git a/include/linux/fs.h b/include/linux/fs.h index 366c533d30cd..e84623d5e173 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3187,8 +3187,6 @@ enum { DIO_SKIP_HOLES = 0x02, }; -void dio_end_io(struct bio *bio); - ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode, struct block_device *bdev, struct iov_iter *iter, get_block_t get_block, From 5f008163a559d566a0ee1190a0a24f3eec6f1ea7 Mon Sep 17 00:00:00 2001 From: Goldwyn Rodrigues Date: Wed, 4 Dec 2019 05:58:53 -0600 Subject: [PATCH 0948/1043] btrfs: remove BTRFS_INODE_READDIO_NEED_LOCK Since we now perform direct reads using i_rwsem, we can remove this inode flag used to co-ordinate unlocked reads. The truncate call takes i_rwsem. This means it is correctly synchronized with concurrent direct reads. Reviewed-by: Nikolay Borisov Reviewed-by: Johannes Thumshirn Signed-off-by: Goldwyn Rodrigues Signed-off-by: David Sterba --- fs/btrfs/btrfs_inode.h | 18 ------------------ fs/btrfs/inode.c | 3 --- 2 files changed, 21 deletions(-) diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h index e7d709505cb1..aeff56a0e105 100644 --- a/fs/btrfs/btrfs_inode.h +++ b/fs/btrfs/btrfs_inode.h @@ -28,7 +28,6 @@ enum { BTRFS_INODE_NEEDS_FULL_SYNC, BTRFS_INODE_COPY_EVERYTHING, BTRFS_INODE_IN_DELALLOC_LIST, - BTRFS_INODE_READDIO_NEED_LOCK, BTRFS_INODE_HAS_PROPS, BTRFS_INODE_SNAPSHOT_FLUSH, }; @@ -313,23 +312,6 @@ struct btrfs_dio_private { u8 csums[]; }; -/* - * Disable DIO read nolock optimization, so new dio readers will be forced - * to grab i_mutex. It is used to avoid the endless truncate due to - * nonlocked dio read. - */ -static inline void btrfs_inode_block_unlocked_dio(struct btrfs_inode *inode) -{ - set_bit(BTRFS_INODE_READDIO_NEED_LOCK, &inode->runtime_flags); - smp_mb(); -} - -static inline void btrfs_inode_resume_unlocked_dio(struct btrfs_inode *inode) -{ - smp_mb__before_atomic(); - clear_bit(BTRFS_INODE_READDIO_NEED_LOCK, &inode->runtime_flags); -} - /* Array of bytes with variable length, hexadecimal format 0x1234 */ #define CSUM_FMT "0x%*phN" #define CSUM_FMT_VALUE(size, bytes) size, bytes diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 501842f8c648..0c47f79da2e1 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -4750,10 +4750,7 @@ static int btrfs_setsize(struct inode *inode, struct iattr *attr) truncate_setsize(inode, newsize); - /* Disable nonlocked read DIO to avoid the endless truncate */ - btrfs_inode_block_unlocked_dio(BTRFS_I(inode)); inode_dio_wait(inode); - btrfs_inode_resume_unlocked_dio(BTRFS_I(inode)); ret = btrfs_truncate(inode, newsize == oldsize); if (ret && inode->i_nlink) { From d8f3e73587ce574f7a9bc165e0db69b0b148f6f8 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 19 May 2020 09:46:29 -0500 Subject: [PATCH 0949/1043] btrfs: split btrfs_direct_IO to read and write part The read and write versions don't have anything in common except for the call to iomap_dio_rw. So split this function, and merge each half into its only caller. Signed-off-by: Christoph Hellwig Signed-off-by: Goldwyn Rodrigues Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 3 ++ fs/btrfs/file.c | 88 ++++++++++++++++++++++++++++++++++++++++++------ fs/btrfs/inode.c | 82 ++------------------------------------------ 3 files changed, 83 insertions(+), 90 deletions(-) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 47b8874eddff..161533040978 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -28,6 +28,7 @@ #include #include #include +#include #include "extent-io-tree.h" #include "extent_io.h" #include "extent_map.h" @@ -2934,6 +2935,8 @@ void btrfs_writepage_endio_finish_ordered(struct page *page, u64 start, u64 end, int uptodate); extern const struct dentry_operations btrfs_dentry_operations; ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter); +extern const struct iomap_ops btrfs_dio_iomap_ops; +extern const struct iomap_dio_ops btrfs_dops; /* ioctl.c */ long btrfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg); diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index dff89d04fd16..fde125616687 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -1809,21 +1809,61 @@ again: return num_written ? num_written : ret; } -static ssize_t __btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) +static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info, + const struct iov_iter *iter, loff_t offset) +{ + const unsigned int blocksize_mask = fs_info->sectorsize - 1; + + if (offset & blocksize_mask) + return -EINVAL; + + if (iov_iter_alignment(iter) & blocksize_mask) + return -EINVAL; + + return 0; +} + +static ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) { struct file *file = iocb->ki_filp; struct inode *inode = file_inode(file); - loff_t pos; - ssize_t written; + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + loff_t pos = iocb->ki_pos; + ssize_t written = 0; ssize_t written_buffered; loff_t endbyte; int err; + size_t count = 0; + bool relock = false; - written = btrfs_direct_IO(iocb, from); + if (check_direct_IO(fs_info, from, pos)) + goto buffered; + + count = iov_iter_count(from); + /* + * If the write DIO is beyond the EOF, we need update the isize, but it + * is protected by i_mutex. So we can not unlock the i_mutex at this + * case. + */ + if (pos + count <= inode->i_size) { + inode_unlock(inode); + relock = true; + } else if (iocb->ki_flags & IOCB_NOWAIT) { + return -EAGAIN; + } + + down_read(&BTRFS_I(inode)->dio_sem); + written = iomap_dio_rw(iocb, from, &btrfs_dio_iomap_ops, &btrfs_dops, + is_sync_kiocb(iocb)); + up_read(&BTRFS_I(inode)->dio_sem); + + if (relock) + inode_lock(inode); if (written < 0 || !iov_iter_count(from)) return written; +buffered: pos = iocb->ki_pos; written_buffered = btrfs_buffered_write(iocb, from); if (written_buffered < 0) { @@ -1962,7 +2002,7 @@ static ssize_t btrfs_file_write_iter(struct kiocb *iocb, atomic_inc(&BTRFS_I(inode)->sync_writers); if (iocb->ki_flags & IOCB_DIRECT) { - num_written = __btrfs_direct_write(iocb, from); + num_written = btrfs_direct_write(iocb, from); } else { num_written = btrfs_buffered_write(iocb, from); if (num_written > 0) @@ -3476,16 +3516,44 @@ static int btrfs_file_open(struct inode *inode, struct file *filp) return generic_file_open(inode, filp); } +static int check_direct_read(struct btrfs_fs_info *fs_info, + const struct iov_iter *iter, loff_t offset) +{ + int ret; + int i, seg; + + ret = check_direct_IO(fs_info, iter, offset); + if (ret < 0) + return ret; + + for (seg = 0; seg < iter->nr_segs; seg++) + for (i = seg + 1; i < iter->nr_segs; i++) + if (iter->iov[seg].iov_base == iter->iov[i].iov_base) + return -EINVAL; + return 0; +} + +static ssize_t btrfs_direct_read(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + if (check_direct_read(btrfs_sb(inode->i_sb), to, iocb->ki_pos)) + return 0; + + inode_lock_shared(inode); + ret = iomap_dio_rw(iocb, to, &btrfs_dio_iomap_ops, &btrfs_dops, + is_sync_kiocb(iocb)); + inode_unlock_shared(inode); + return ret; +} + static ssize_t btrfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) { ssize_t ret = 0; if (iocb->ki_flags & IOCB_DIRECT) { - struct inode *inode = file_inode(iocb->ki_filp); - - inode_lock_shared(inode); - ret = btrfs_direct_IO(iocb, to); - inode_unlock_shared(inode); + ret = btrfs_direct_read(iocb, to); if (ret < 0) return ret; } diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 0c47f79da2e1..4583f0763571 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -29,7 +29,6 @@ #include #include #include -#include #include #include "misc.h" #include "ctree.h" @@ -7759,92 +7758,15 @@ out_err: return BLK_QC_T_NONE; } -static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info, - const struct iov_iter *iter, loff_t offset) -{ - int seg; - int i; - unsigned int blocksize_mask = fs_info->sectorsize - 1; - ssize_t retval = -EINVAL; - - if (offset & blocksize_mask) - goto out; - - if (iov_iter_alignment(iter) & blocksize_mask) - goto out; - - /* If this is a write we don't need to check anymore */ - if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter)) - return 0; - /* - * Check to make sure we don't have duplicate iov_base's in this - * iovec, if so return EINVAL, otherwise we'll get csum errors - * when reading back. - */ - for (seg = 0; seg < iter->nr_segs; seg++) { - for (i = seg + 1; i < iter->nr_segs; i++) { - if (iter->iov[seg].iov_base == iter->iov[i].iov_base) - goto out; - } - } - retval = 0; -out: - return retval; -} - -static const struct iomap_ops btrfs_dio_iomap_ops = { +const struct iomap_ops btrfs_dio_iomap_ops = { .iomap_begin = btrfs_dio_iomap_begin, .iomap_end = btrfs_dio_iomap_end, }; -static const struct iomap_dio_ops btrfs_dops = { +const struct iomap_dio_ops btrfs_dops = { .submit_io = btrfs_submit_direct, }; -ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) -{ - struct file *file = iocb->ki_filp; - struct inode *inode = file->f_mapping->host; - struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - struct extent_changeset *data_reserved = NULL; - loff_t offset = iocb->ki_pos; - size_t count = 0; - bool relock = false; - ssize_t ret; - - if (check_direct_IO(fs_info, iter, offset)) - return 0; - - count = iov_iter_count(iter); - if (iov_iter_rw(iter) == WRITE) { - /* - * If the write DIO is beyond the EOF, we need update - * the isize, but it is protected by i_mutex. So we can - * not unlock the i_mutex at this case. - */ - if (offset + count <= inode->i_size) { - inode_unlock(inode); - relock = true; - } else if (iocb->ki_flags & IOCB_NOWAIT) { - ret = -EAGAIN; - goto out; - } - down_read(&BTRFS_I(inode)->dio_sem); - } - - ret = iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dops, - is_sync_kiocb(iocb)); - - if (iov_iter_rw(iter) == WRITE) { - up_read(&BTRFS_I(inode)->dio_sem); - } -out: - if (relock) - inode_lock(inode); - extent_changeset_free(data_reserved); - return ret; -} - #define BTRFS_FIEMAP_FLAGS (FIEMAP_FLAG_SYNC) static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, From 995e9a166b6909c9bb4af8f51b9502f8b8c18291 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Wed, 27 May 2020 13:10:53 +0300 Subject: [PATCH 0950/1043] btrfs: open code key_search This function wraps the optimisation implemented by d7396f07358a ("Btrfs: optimize key searches in btrfs_search_slot") however this optimisation is really used in only one place - btrfs_search_slot. Just open code the optimisation and also add a comment explaining how it works since it's not clear just by looking at the code - the key point here is it depends on an internal invariant that BTRFS' btree provides, namely intermediate pointers always contain the key at slot0 at the child node. So in the case of exact match we can safely assume that the given key will always be in slot 0 on lower levels. Furthermore this results in a reduction of btrfs_search_slot's size: ./scripts/bloat-o-meter ctree.orig fs/btrfs/ctree.o add/remove: 0/0 grow/shrink: 0/1 up/down: 0/-75 (-75) Function old new delta btrfs_search_slot 2783 2708 -75 Total: Before=50423, After=50348, chg -0.15% Reviewed-by: Johannes Thumshirn Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 92775554d1cc..5f94414f3336 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -2488,19 +2488,6 @@ done: return ret; } -static int key_search(struct extent_buffer *b, const struct btrfs_key *key, - int *prev_cmp, int *slot) -{ - if (*prev_cmp != 0) { - *prev_cmp = btrfs_bin_search(b, key, slot); - return *prev_cmp; - } - - *slot = 0; - - return 0; -} - int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path, u64 iobjectid, u64 ioff, u8 key_type, struct btrfs_key *found_key) @@ -2770,9 +2757,23 @@ cow_done: } } - ret = key_search(b, key, &prev_cmp, &slot); - if (ret < 0) - goto done; + /* + * If btrfs_bin_search returns an exact match (prev_cmp == 0) + * we can safely assume the target key will always be in slot 0 + * on lower levels due to the invariants BTRFS' btree provides, + * namely that a btrfs_key_ptr entry always points to the + * lowest key in the child node, thus we can skip searching + * lower levels + */ + if (prev_cmp == 0) { + slot = 0; + ret = 0; + } else { + ret = btrfs_bin_search(b, key, &slot); + prev_cmp = ret; + if (ret < 0) + goto done; + } if (level == 0) { p->slots[level] = slot; @@ -2896,7 +2897,6 @@ int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, int level; int lowest_unlock = 1; u8 lowest_level = 0; - int prev_cmp = -1; lowest_level = p->lowest_level; WARN_ON(p->nodes[0] != NULL); @@ -2929,12 +2929,7 @@ again: */ btrfs_unlock_up_safe(p, level + 1); - /* - * Since we can unwind ebs we want to do a real search every - * time. - */ - prev_cmp = -1; - ret = key_search(b, key, &prev_cmp, &slot); + ret = btrfs_bin_search(b, key, &slot); if (ret < 0) goto done; From 213ff4b72a9c7509dd85979db64c66774f4f26c1 Mon Sep 17 00:00:00 2001 From: Nikolay Borisov Date: Wed, 27 May 2020 13:10:59 +0300 Subject: [PATCH 0951/1043] btrfs: remove redundant local variable in read_block_for_search The local 'b' variable is only used to directly read values from passed extent buffer. So eliminate it and directly use the input parameter. Furthermore this shrinks the size of the following functions: ./scripts/bloat-o-meter ctree.orig fs/btrfs/ctree.o add/remove: 0/0 grow/shrink: 0/2 up/down: 0/-73 (-73) Function old new delta read_block_for_search.isra 876 871 -5 push_node_left 1112 1044 -68 Total: Before=50348, After=50275, chg -0.14% Reviewed-by: Johannes Thumshirn Signed-off-by: Nikolay Borisov Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 5f94414f3336..3a7648bff42c 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -2335,16 +2335,15 @@ read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, struct btrfs_fs_info *fs_info = root->fs_info; u64 blocknr; u64 gen; - struct extent_buffer *b = *eb_ret; struct extent_buffer *tmp; struct btrfs_key first_key; int ret; int parent_level; - blocknr = btrfs_node_blockptr(b, slot); - gen = btrfs_node_ptr_generation(b, slot); - parent_level = btrfs_header_level(b); - btrfs_node_key_to_cpu(b, &first_key, slot); + blocknr = btrfs_node_blockptr(*eb_ret, slot); + gen = btrfs_node_ptr_generation(*eb_ret, slot); + parent_level = btrfs_header_level(*eb_ret); + btrfs_node_key_to_cpu(*eb_ret, &first_key, slot); tmp = find_extent_buffer(fs_info, blocknr); if (tmp) { From e2c8e92d1140754073ad3799eb6620c76bab2078 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 27 May 2020 11:15:53 +0100 Subject: [PATCH 0952/1043] btrfs: fix wrong file range cleanup after an error filling dealloc range If an error happens while running dellaloc in COW mode for a range, we can end up calling extent_clear_unlock_delalloc() for a range that goes beyond our range's end offset by 1 byte, which affects 1 extra page. This results in clearing bits and doing page operations (such as a page unlock) outside our target range. Fix that by calling extent_clear_unlock_delalloc() with an inclusive end offset, instead of an exclusive end offset, at cow_file_range(). Fixes: a315e68f6e8b30 ("Btrfs: fix invalid attempt to free reserved space on failure to cow range") CC: stable@vger.kernel.org # 4.14+ Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 4583f0763571..1b6cd937f214 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -1141,7 +1141,7 @@ out_unlock: */ if (extent_reserved) { extent_clear_unlock_delalloc(inode, start, - start + cur_alloc_size, + start + cur_alloc_size - 1, locked_page, clear_bits, page_ops); From 467dc47ea99c56e966e99d09dae54869850abeeb Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 27 May 2020 11:16:07 +0100 Subject: [PATCH 0953/1043] btrfs: fix space_info bytes_may_use underflow after nocow buffered write When doing a buffered write we always try to reserve data space for it, even when the file has the NOCOW bit set or the write falls into a file range covered by a prealloc extent. This is done both because it is expensive to check if we can do a nocow write (checking if an extent is shared through reflinks or if there's a hole in the range for example), and because when writeback starts we might actually need to fallback to COW mode (for example the block group containing the target extents was turned into RO mode due to a scrub or balance). When we are unable to reserve data space we check if we can do a nocow write, and if we can, we proceed with dirtying the pages and setting up the range for delalloc. In this case the bytes_may_use counter of the data space_info object is not incremented, unlike in the case where we are able to reserve data space (done through btrfs_check_data_free_space() which calls btrfs_alloc_data_chunk_ondemand()). Later when running delalloc we attempt to start writeback in nocow mode but we might revert back to cow mode, for example because in the meanwhile a block group was turned into RO mode by a scrub or relocation. The cow path after successfully allocating an extent ends up calling btrfs_add_reserved_bytes(), which expects the bytes_may_use counter of the data space_info object to have been incremented before - but we did not do it when the buffered write started, since there was not enough available data space. So btrfs_add_reserved_bytes() ends up decrementing the bytes_may_use counter anyway, and when the counter's current value is smaller then the size of the allocated extent we get a stack trace like the following: ------------[ cut here ]------------ WARNING: CPU: 0 PID: 20138 at fs/btrfs/space-info.h:115 btrfs_add_reserved_bytes+0x3d6/0x4e0 [btrfs] Modules linked in: btrfs blake2b_generic xor raid6_pq libcrc32c (...) CPU: 0 PID: 20138 Comm: kworker/u8:15 Not tainted 5.6.0-rc7-btrfs-next-58 #5 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.12.0-59-gc9ba5276e321-prebuilt.qemu.org 04/01/2014 Workqueue: writeback wb_workfn (flush-btrfs-1754) RIP: 0010:btrfs_add_reserved_bytes+0x3d6/0x4e0 [btrfs] Code: ff ff 48 (...) RSP: 0018:ffffbda18a4b3568 EFLAGS: 00010287 RAX: 0000000000000000 RBX: ffff9ca076f5d800 RCX: 0000000000000000 RDX: 0000000000000002 RSI: 0000000000000000 RDI: ffff9ca068470410 RBP: fffffffffffff000 R08: 0000000000000001 R09: 0000000000000000 R10: ffff9ca079d58040 R11: 0000000000000000 R12: ffff9ca068470400 R13: ffff9ca0408b2000 R14: 0000000000001000 R15: ffff9ca076f5d800 FS: 0000000000000000(0000) GS:ffff9ca07a600000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00005605dbfe7048 CR3: 0000000138570006 CR4: 00000000003606f0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: find_free_extent+0x4a0/0x16c0 [btrfs] btrfs_reserve_extent+0x91/0x180 [btrfs] cow_file_range+0x12d/0x490 [btrfs] run_delalloc_nocow+0x341/0xa40 [btrfs] btrfs_run_delalloc_range+0x1ea/0x6d0 [btrfs] ? find_lock_delalloc_range+0x221/0x250 [btrfs] writepage_delalloc+0xe8/0x150 [btrfs] __extent_writepage+0xe8/0x4c0 [btrfs] extent_write_cache_pages+0x237/0x530 [btrfs] ? btrfs_wq_submit_bio+0x9f/0xc0 [btrfs] extent_writepages+0x44/0xa0 [btrfs] do_writepages+0x23/0x80 __writeback_single_inode+0x59/0x700 writeback_sb_inodes+0x267/0x5f0 __writeback_inodes_wb+0x87/0xe0 wb_writeback+0x382/0x590 ? wb_workfn+0x4a2/0x6c0 wb_workfn+0x4a2/0x6c0 process_one_work+0x26d/0x6a0 worker_thread+0x4f/0x3e0 ? process_one_work+0x6a0/0x6a0 kthread+0x103/0x140 ? kthread_create_worker_on_cpu+0x70/0x70 ret_from_fork+0x3a/0x50 irq event stamp: 0 hardirqs last enabled at (0): [<0000000000000000>] 0x0 hardirqs last disabled at (0): [] copy_process+0x74f/0x2020 softirqs last enabled at (0): [] copy_process+0x74f/0x2020 softirqs last disabled at (0): [<0000000000000000>] 0x0 ---[ end trace f9f6ef8ec4cd8ec9 ]--- So to fix this, when falling back into cow mode check if space was not reserved, by testing for the bit EXTENT_NORESERVE in the respective file range, and if not, increment the bytes_may_use counter for the data space_info object. Also clear the EXTENT_NORESERVE bit from the range, so that if the cow path fails it decrements the bytes_may_use counter when clearing the delalloc range (through the btrfs_clear_delalloc_extent() callback). Fixes: 7ee9e4405f264e ("Btrfs: check if we can nocow if we don't have data space") CC: stable@vger.kernel.org # 4.4+ Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/inode.c | 61 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 1b6cd937f214..486b1da2fc5c 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -48,6 +48,7 @@ #include "qgroup.h" #include "delalloc-space.h" #include "block-group.h" +#include "space-info.h" struct btrfs_iget_args { u64 ino; @@ -1354,6 +1355,56 @@ static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info, return 1; } +static int fallback_to_cow(struct inode *inode, struct page *locked_page, + const u64 start, const u64 end, + int *page_started, unsigned long *nr_written) +{ + struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; + u64 range_start = start; + u64 count; + + /* + * If EXTENT_NORESERVE is set it means that when the buffered write was + * made we had not enough available data space and therefore we did not + * reserve data space for it, since we though we could do NOCOW for the + * respective file range (either there is prealloc extent or the inode + * has the NOCOW bit set). + * + * However when we need to fallback to COW mode (because for example the + * block group for the corresponding extent was turned to RO mode by a + * scrub or relocation) we need to do the following: + * + * 1) We increment the bytes_may_use counter of the data space info. + * If COW succeeds, it allocates a new data extent and after doing + * that it decrements the space info's bytes_may_use counter and + * increments its bytes_reserved counter by the same amount (we do + * this at btrfs_add_reserved_bytes()). So we need to increment the + * bytes_may_use counter to compensate (when space is reserved at + * buffered write time, the bytes_may_use counter is incremented); + * + * 2) We clear the EXTENT_NORESERVE bit from the range. We do this so + * that if the COW path fails for any reason, it decrements (through + * extent_clear_unlock_delalloc()) the bytes_may_use counter of the + * data space info, which we incremented in the step above. + */ + count = count_range_bits(io_tree, &range_start, end, end + 1 - start, + EXTENT_NORESERVE, 0); + if (count > 0) { + struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; + struct btrfs_space_info *sinfo = fs_info->data_sinfo; + + spin_lock(&sinfo->lock); + btrfs_space_info_update_bytes_may_use(fs_info, sinfo, count); + spin_unlock(&sinfo->lock); + + clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE, 0, 0, + NULL); + } + + return cow_file_range(inode, locked_page, start, end, page_started, + nr_written, 1); +} + /* * when nowcow writeback call back. This checks for snapshots or COW copies * of the extents that exist in the file, and COWs the file as required. @@ -1601,9 +1652,9 @@ out_check: * NOCOW, following one which needs to be COW'ed */ if (cow_start != (u64)-1) { - ret = cow_file_range(inode, locked_page, - cow_start, found_key.offset - 1, - page_started, nr_written, 1); + ret = fallback_to_cow(inode, locked_page, cow_start, + found_key.offset - 1, + page_started, nr_written); if (ret) { if (nocow) btrfs_dec_nocow_writers(fs_info, @@ -1692,8 +1743,8 @@ out_check: if (cow_start != (u64)-1) { cur_offset = end; - ret = cow_file_range(inode, locked_page, cow_start, end, - page_started, nr_written, 1); + ret = fallback_to_cow(inode, locked_page, cow_start, end, + page_started, nr_written); if (ret) goto error; } From 2166e5edce9ac1edf3b113d6091ef72fcac2d6c4 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 27 May 2020 11:16:19 +0100 Subject: [PATCH 0954/1043] btrfs: fix space_info bytes_may_use underflow during space cache writeout We always preallocate a data extent for writing a free space cache, which causes writeback to always try the nocow path first, since the free space inode has the prealloc bit set in its flags. However if the block group that contains the data extent for the space cache has been turned to RO mode due to a running scrub or balance for example, we have to fallback to the cow path. In that case once a new data extent is allocated we end up calling btrfs_add_reserved_bytes(), which decrements the counter named bytes_may_use from the data space_info object with the expection that this counter was previously incremented with the same amount (the size of the data extent). However when we started writeout of the space cache at cache_save_setup(), we incremented the value of the bytes_may_use counter through a call to btrfs_check_data_free_space() and then decremented it through a call to btrfs_prealloc_file_range_trans() immediately after. So when starting the writeback if we fallback to cow mode we have to increment the counter bytes_may_use of the data space_info again to compensate for the extent allocation done by the cow path. When this issue happens we are incorrectly decrementing the bytes_may_use counter and when its current value is smaller then the amount we try to subtract we end up with the following warning: ------------[ cut here ]------------ WARNING: CPU: 3 PID: 657 at fs/btrfs/space-info.h:115 btrfs_add_reserved_bytes+0x3d6/0x4e0 [btrfs] Modules linked in: btrfs blake2b_generic xor raid6_pq libcrc32c (...) CPU: 3 PID: 657 Comm: kworker/u8:7 Tainted: G W 5.6.0-rc7-btrfs-next-58 #5 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.12.0-59-gc9ba5276e321-prebuilt.qemu.org 04/01/2014 Workqueue: writeback wb_workfn (flush-btrfs-1591) RIP: 0010:btrfs_add_reserved_bytes+0x3d6/0x4e0 [btrfs] Code: ff ff 48 (...) RSP: 0000:ffffa41608f13660 EFLAGS: 00010287 RAX: 0000000000001000 RBX: ffff9615b93ae400 RCX: 0000000000000000 RDX: 0000000000000002 RSI: 0000000000000000 RDI: ffff9615b96ab410 RBP: fffffffffffee000 R08: 0000000000000001 R09: 0000000000000000 R10: ffff961585e62a40 R11: 0000000000000000 R12: ffff9615b96ab400 R13: ffff9615a1a2a000 R14: 0000000000012000 R15: ffff9615b93ae400 FS: 0000000000000000(0000) GS:ffff9615bb200000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 000055cbbc2ae178 CR3: 0000000115794006 CR4: 00000000003606e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: find_free_extent+0x4a0/0x16c0 [btrfs] btrfs_reserve_extent+0x91/0x180 [btrfs] cow_file_range+0x12d/0x490 [btrfs] btrfs_run_delalloc_range+0x9f/0x6d0 [btrfs] ? find_lock_delalloc_range+0x221/0x250 [btrfs] writepage_delalloc+0xe8/0x150 [btrfs] __extent_writepage+0xe8/0x4c0 [btrfs] extent_write_cache_pages+0x237/0x530 [btrfs] extent_writepages+0x44/0xa0 [btrfs] do_writepages+0x23/0x80 __writeback_single_inode+0x59/0x700 writeback_sb_inodes+0x267/0x5f0 __writeback_inodes_wb+0x87/0xe0 wb_writeback+0x382/0x590 ? wb_workfn+0x4a2/0x6c0 wb_workfn+0x4a2/0x6c0 process_one_work+0x26d/0x6a0 worker_thread+0x4f/0x3e0 ? process_one_work+0x6a0/0x6a0 kthread+0x103/0x140 ? kthread_create_worker_on_cpu+0x70/0x70 ret_from_fork+0x3a/0x50 irq event stamp: 0 hardirqs last enabled at (0): [<0000000000000000>] 0x0 hardirqs last disabled at (0): [] copy_process+0x74f/0x2020 softirqs last enabled at (0): [] copy_process+0x74f/0x2020 softirqs last disabled at (0): [<0000000000000000>] 0x0 ---[ end trace bd7c03622e0b0a52 ]--- ------------[ cut here ]------------ So fix this by incrementing the bytes_may_use counter of the data space_info when we fallback to the cow path. If the cow path is successful the counter is decremented after extent allocation (by btrfs_add_reserved_bytes()), if it fails it ends up being decremented as well when clearing the delalloc range (extent_clear_unlock_delalloc()). This could be triggered sporadically by the test case btrfs/061 from fstests. Fixes: 82d5902d9c681b ("Btrfs: Support reading/writing on disk free ino cache") CC: stable@vger.kernel.org # 4.4+ Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/inode.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 486b1da2fc5c..1242d0aa108d 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -1359,6 +1359,8 @@ static int fallback_to_cow(struct inode *inode, struct page *locked_page, const u64 start, const u64 end, int *page_started, unsigned long *nr_written) { + const bool is_space_ino = btrfs_is_free_space_inode(BTRFS_I(inode)); + const u64 range_bytes = end + 1 - start; struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; u64 range_start = start; u64 count; @@ -1386,19 +1388,27 @@ static int fallback_to_cow(struct inode *inode, struct page *locked_page, * that if the COW path fails for any reason, it decrements (through * extent_clear_unlock_delalloc()) the bytes_may_use counter of the * data space info, which we incremented in the step above. + * + * If we need to fallback to cow and the inode corresponds to a free + * space cache inode, we must also increment bytes_may_use of the data + * space_info for the same reason. Space caches always get a prealloc + * extent for them, however scrub or balance may have set the block + * group that contains that extent to RO mode. */ - count = count_range_bits(io_tree, &range_start, end, end + 1 - start, + count = count_range_bits(io_tree, &range_start, end, range_bytes, EXTENT_NORESERVE, 0); - if (count > 0) { + if (count > 0 || is_space_ino) { + const u64 bytes = is_space_ino ? range_bytes : count; struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; struct btrfs_space_info *sinfo = fs_info->data_sinfo; spin_lock(&sinfo->lock); - btrfs_space_info_update_bytes_may_use(fs_info, sinfo, count); + btrfs_space_info_update_bytes_may_use(fs_info, sinfo, bytes); spin_unlock(&sinfo->lock); - clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE, 0, 0, - NULL); + if (count > 0) + clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE, + 0, 0, NULL); } return cow_file_range(inode, locked_page, start, end, page_started, From d9d7d84d9906e1bc886c5e0fc66aaad26008264b Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Apr 2020 18:32:02 +0100 Subject: [PATCH 0955/1043] KVM: arm64: Parametrize exception entry with a target EL We currently assume that an exception is delivered to EL1, always. Once we emulate EL2, this no longer will be the case. To prepare for this, add a target_mode parameter. While we're at it, merge the computing of the target PC and PSTATE in a single function that updates both PC and CPSR after saving their previous values in the corresponding ELR/SPSR. This ensures that they are updated in the correct order (a pretty common source of bugs...). Reviewed-by: Mark Rutland Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/ptrace.h | 1 + arch/arm64/kvm/inject_fault.c | 75 +++++++++++++++++---------------- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index bf57308fcd63..953b6a1ce549 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -35,6 +35,7 @@ #define GIC_PRIO_PSR_I_SET (1 << 4) /* Additional SPSR bits not exposed in the UABI */ +#define PSR_MODE_THREAD_BIT (1 << 0) #define PSR_IL_BIT (1 << 20) /* AArch32-specific ptrace requests */ diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c index 6aafc2825c1c..e21fdd93027a 100644 --- a/arch/arm64/kvm/inject_fault.c +++ b/arch/arm64/kvm/inject_fault.c @@ -26,28 +26,12 @@ enum exception_type { except_type_serror = 0x180, }; -static u64 get_except_vector(struct kvm_vcpu *vcpu, enum exception_type type) -{ - u64 exc_offset; - - switch (*vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT)) { - case PSR_MODE_EL1t: - exc_offset = CURRENT_EL_SP_EL0_VECTOR; - break; - case PSR_MODE_EL1h: - exc_offset = CURRENT_EL_SP_ELx_VECTOR; - break; - case PSR_MODE_EL0t: - exc_offset = LOWER_EL_AArch64_VECTOR; - break; - default: - exc_offset = LOWER_EL_AArch32_VECTOR; - } - - return vcpu_read_sys_reg(vcpu, VBAR_EL1) + exc_offset + type; -} - /* + * This performs the exception entry at a given EL (@target_mode), stashing PC + * and PSTATE into ELR and SPSR respectively, and compute the new PC/PSTATE. + * The EL passed to this function *must* be a non-secure, privileged mode with + * bit 0 being set (PSTATE.SP == 1). + * * When an exception is taken, most PSTATE fields are left unchanged in the * handler. However, some are explicitly overridden (e.g. M[4:0]). Luckily all * of the inherited bits have the same position in the AArch64/AArch32 SPSR_ELx @@ -59,10 +43,35 @@ static u64 get_except_vector(struct kvm_vcpu *vcpu, enum exception_type type) * Here we manipulate the fields in order of the AArch64 SPSR_ELx layout, from * MSB to LSB. */ -static unsigned long get_except64_pstate(struct kvm_vcpu *vcpu) +static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, + enum exception_type type) { - unsigned long sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); - unsigned long old, new; + unsigned long sctlr, vbar, old, new, mode; + u64 exc_offset; + + mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT); + + if (mode == target_mode) + exc_offset = CURRENT_EL_SP_ELx_VECTOR; + else if ((mode | PSR_MODE_THREAD_BIT) == target_mode) + exc_offset = CURRENT_EL_SP_EL0_VECTOR; + else if (!(mode & PSR_MODE32_BIT)) + exc_offset = LOWER_EL_AArch64_VECTOR; + else + exc_offset = LOWER_EL_AArch32_VECTOR; + + switch (target_mode) { + case PSR_MODE_EL1h: + vbar = vcpu_read_sys_reg(vcpu, VBAR_EL1); + sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); + vcpu_write_elr_el1(vcpu, *vcpu_pc(vcpu)); + break; + default: + /* Don't do that */ + BUG(); + } + + *vcpu_pc(vcpu) = vbar + exc_offset + type; old = *vcpu_cpsr(vcpu); new = 0; @@ -105,9 +114,10 @@ static unsigned long get_except64_pstate(struct kvm_vcpu *vcpu) new |= PSR_I_BIT; new |= PSR_F_BIT; - new |= PSR_MODE_EL1h; + new |= target_mode; - return new; + *vcpu_cpsr(vcpu) = new; + vcpu_write_spsr(vcpu, old); } static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr) @@ -116,11 +126,7 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr bool is_aarch32 = vcpu_mode_is_32bit(vcpu); u32 esr = 0; - vcpu_write_elr_el1(vcpu, *vcpu_pc(vcpu)); - *vcpu_pc(vcpu) = get_except_vector(vcpu, except_type_sync); - - *vcpu_cpsr(vcpu) = get_except64_pstate(vcpu); - vcpu_write_spsr(vcpu, cpsr); + enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync); vcpu_write_sys_reg(vcpu, addr, FAR_EL1); @@ -148,14 +154,9 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr static void inject_undef64(struct kvm_vcpu *vcpu) { - unsigned long cpsr = *vcpu_cpsr(vcpu); u32 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT); - vcpu_write_elr_el1(vcpu, *vcpu_pc(vcpu)); - *vcpu_pc(vcpu) = get_except_vector(vcpu, except_type_sync); - - *vcpu_cpsr(vcpu) = get_except64_pstate(vcpu); - vcpu_write_spsr(vcpu, cpsr); + enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync); /* * Build an unknown exception, depending on the instruction From 8f7f4fe756bd5cfef73cf8234445081385bdbf7d Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 27 May 2020 11:38:57 +0100 Subject: [PATCH 0956/1043] KVM: arm64: Drop obsolete comment about sys_reg ordering The general comment about keeping the enum order in sync with the save/restore code has been obsolete for many years now. Just drop it. Note that there are other ordering requirements in the enum, such as the PtrAuth and PMU registers, which are still valid. Reported-by: James Morse Signed-off-by: Marc Zyngier --- arch/arm64/include/asm/kvm_host.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 69a338a390a6..59029e90b557 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -115,12 +115,8 @@ struct kvm_vcpu_fault_info { u64 disr_el1; /* Deferred [SError] Status Register */ }; -/* - * 0 is reserved as an invalid value. - * Order should be kept in sync with the save/restore code. - */ enum vcpu_sysreg { - __INVALID_SYSREG__, + __INVALID_SYSREG__, /* 0 is reserved as an invalid value */ MPIDR_EL1, /* MultiProcessor Affinity Register */ CSSELR_EL1, /* Cache Size Selection Register */ SCTLR_EL1, /* System Control Register */ From 1d0326f352bb094771df17f045bdbadff89a43e6 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Thu, 14 May 2020 02:25:55 +0200 Subject: [PATCH 0957/1043] genirq: Check irq_data_get_irq_chip() return value before use irq_data_get_irq_chip() can return NULL, however it is expected that this never happens. If a buggy driver leads to NULL being returned from irq_data_get_irq_chip(), warn about it instead of crashing the machine. Signed-off-by: Marek Vasut Signed-off-by: Thomas Gleixner To: linux-arm-kernel@lists.infradead.org --- kernel/irq/manage.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 453a8a0f4804..761911168438 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -2619,6 +2619,8 @@ int __irq_get_irqchip_state(struct irq_data *data, enum irqchip_irq_state which, do { chip = irq_data_get_irq_chip(data); + if (WARN_ON_ONCE(!chip)) + return -ENODEV; if (chip->irq_get_irqchip_state) break; #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY @@ -2696,6 +2698,8 @@ int irq_set_irqchip_state(unsigned int irq, enum irqchip_irq_state which, do { chip = irq_data_get_irq_chip(data); + if (WARN_ON_ONCE(!chip)) + return -ENODEV; if (chip->irq_set_irqchip_state) break; #ifdef CONFIG_IRQ_DOMAIN_HIERARCHY From c9d40913ac5a21eb2b976bb221a4677540e84eba Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 22 May 2020 11:21:49 -0400 Subject: [PATCH 0958/1043] KVM: x86: enable event window in inject_pending_event In case an interrupt arrives after nested.check_events but before the call to kvm_cpu_has_injectable_intr, we could end up enabling the interrupt window even if the interrupt is actually going to be a vmexit. This is useless rather than harmful, but it really complicates reasoning about SVM's handling of the VINTR intercept. We'd like to never bother with the VINTR intercept if V_INTR_MASKING=1 && INTERCEPT_INTR=1, because in that case there is no interrupt window and we can just exit the nested guest whenever we want. This patch moves the opening of the interrupt window inside inject_pending_event. This consolidates the check for pending interrupt/NMI/SMI in one place, and makes KVM's usage of immediate exits more consistent, extending it beyond just nested virtualization. There are two functional changes here. They only affect corner cases, but overall they simplify the inject_pending_event. - re-injection of still-pending events will also use req_immediate_exit instead of using interrupt-window intercepts. This should have no impact on performance on Intel since it simply replaces an interrupt-window or NMI-window exit for a preemption-timer exit. On AMD, which has no equivalent of the preemption time, it may incur some overhead but an actual effect on performance should only be visible in pathological cases. - kvm_arch_interrupt_allowed and kvm_vcpu_has_events will return true if an interrupt, NMI or SMI is blocked by nested_run_pending. This makes sense because entering the VM will allow it to make progress and deliver the event. Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 8 +-- arch/x86/kvm/svm/svm.c | 24 +++---- arch/x86/kvm/vmx/vmx.c | 20 +++--- arch/x86/kvm/x86.c | 119 ++++++++++++++++++-------------- 4 files changed, 93 insertions(+), 78 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index db261da578f3..7707bd4b0593 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1136,8 +1136,8 @@ struct kvm_x86_ops { void (*set_nmi)(struct kvm_vcpu *vcpu); void (*queue_exception)(struct kvm_vcpu *vcpu); void (*cancel_injection)(struct kvm_vcpu *vcpu); - bool (*interrupt_allowed)(struct kvm_vcpu *vcpu, bool for_injection); - bool (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); + int (*interrupt_allowed)(struct kvm_vcpu *vcpu, bool for_injection); + int (*nmi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); bool (*get_nmi_mask)(struct kvm_vcpu *vcpu); void (*set_nmi_mask)(struct kvm_vcpu *vcpu, bool masked); void (*enable_nmi_window)(struct kvm_vcpu *vcpu); @@ -1234,10 +1234,10 @@ struct kvm_x86_ops { void (*setup_mce)(struct kvm_vcpu *vcpu); - bool (*smi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); + int (*smi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); int (*pre_enter_smm)(struct kvm_vcpu *vcpu, char *smstate); int (*pre_leave_smm)(struct kvm_vcpu *vcpu, const char *smstate); - int (*enable_smi_window)(struct kvm_vcpu *vcpu); + void (*enable_smi_window)(struct kvm_vcpu *vcpu); int (*mem_enc_op)(struct kvm *kvm, void __user *argp); int (*mem_enc_reg_region)(struct kvm *kvm, struct kvm_enc_region *argp); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index d877a0f70cac..5da494847a2f 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3053,15 +3053,15 @@ bool svm_nmi_blocked(struct kvm_vcpu *vcpu) return ret; } -static bool svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) +static int svm_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) { struct vcpu_svm *svm = to_svm(vcpu); if (svm->nested.nested_run_pending) - return false; + return -EBUSY; /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(svm)) - return false; + return -EBUSY; return !svm_nmi_blocked(vcpu); } @@ -3112,18 +3112,18 @@ bool svm_interrupt_blocked(struct kvm_vcpu *vcpu) return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK); } -static bool svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) +static int svm_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) { struct vcpu_svm *svm = to_svm(vcpu); if (svm->nested.nested_run_pending) - return false; + return -EBUSY; /* * An IRQ must not be injected into L2 if it's supposed to VM-Exit, * e.g. if the IRQ arrived asynchronously after checking nested events. */ if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(svm)) - return false; + return -EBUSY; return !svm_interrupt_blocked(vcpu); } @@ -3793,15 +3793,15 @@ bool svm_smi_blocked(struct kvm_vcpu *vcpu) return is_smm(vcpu); } -static bool svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) +static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) { struct vcpu_svm *svm = to_svm(vcpu); if (svm->nested.nested_run_pending) - return false; + return -EBUSY; /* An SMI must not be injected into L2 if it's supposed to VM-Exit. */ if (for_injection && is_guest_mode(vcpu) && nested_exit_on_smi(svm)) - return false; + return -EBUSY; return !svm_smi_blocked(vcpu); } @@ -3848,7 +3848,7 @@ static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) return 0; } -static int enable_smi_window(struct kvm_vcpu *vcpu) +static void enable_smi_window(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); @@ -3856,9 +3856,9 @@ static int enable_smi_window(struct kvm_vcpu *vcpu) if (vgif_enabled(svm)) set_intercept(svm, INTERCEPT_STGI); /* STGI will cause a vm exit */ - return 1; + } else { + /* We must be in SMM; RSM will cause a vmexit anyway. */ } - return 0; } static bool svm_need_emulation_on_page_fault(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 4e76e30b661c..b3a41645e157 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4551,14 +4551,14 @@ bool vmx_nmi_blocked(struct kvm_vcpu *vcpu) GUEST_INTR_STATE_NMI)); } -static bool vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) +static int vmx_nmi_allowed(struct kvm_vcpu *vcpu, bool for_injection) { if (to_vmx(vcpu)->nested.nested_run_pending) - return false; + return -EBUSY; /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ if (for_injection && is_guest_mode(vcpu) && nested_exit_on_nmi(vcpu)) - return false; + return -EBUSY; return !vmx_nmi_blocked(vcpu); } @@ -4573,17 +4573,17 @@ bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu) (GUEST_INTR_STATE_STI | GUEST_INTR_STATE_MOV_SS)); } -static bool vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) +static int vmx_interrupt_allowed(struct kvm_vcpu *vcpu, bool for_injection) { if (to_vmx(vcpu)->nested.nested_run_pending) - return false; + return -EBUSY; /* * An IRQ must not be injected into L2 if it's supposed to VM-Exit, * e.g. if the IRQ arrived asynchronously after checking nested events. */ if (for_injection && is_guest_mode(vcpu) && nested_exit_on_intr(vcpu)) - return false; + return -EBUSY; return !vmx_interrupt_blocked(vcpu); } @@ -7757,11 +7757,11 @@ static void vmx_setup_mce(struct kvm_vcpu *vcpu) ~FEAT_CTL_LMCE_ENABLED; } -static bool vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) +static int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) { /* we need a nested vmexit to enter SMM, postpone if run is pending */ if (to_vmx(vcpu)->nested.nested_run_pending) - return false; + return -EBUSY; return !is_smm(vcpu); } @@ -7799,9 +7799,9 @@ static int vmx_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) return 0; } -static int enable_smi_window(struct kvm_vcpu *vcpu) +static void enable_smi_window(struct kvm_vcpu *vcpu) { - return 0; + /* RSM will cause a vmexit anyway. */ } static bool vmx_need_emulation_on_page_fault(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 77b9b4e66673..0ee828f60d05 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7714,7 +7714,7 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu) kvm_x86_ops.update_cr8_intercept(vcpu, tpr, max_irr); } -static int inject_pending_event(struct kvm_vcpu *vcpu) +static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit) { int r; bool can_inject = true; @@ -7760,8 +7760,8 @@ static int inject_pending_event(struct kvm_vcpu *vcpu) */ if (is_guest_mode(vcpu)) { r = kvm_x86_ops.nested_ops->check_events(vcpu); - if (r != 0) - return r; + if (r < 0) + goto busy; } /* try to inject new event if pending */ @@ -7799,27 +7799,69 @@ static int inject_pending_event(struct kvm_vcpu *vcpu) can_inject = false; } - /* Finish re-injection before considering new events */ - if (!can_inject) - return 0; - - if (vcpu->arch.smi_pending && - kvm_x86_ops.smi_allowed(vcpu, true)) { - vcpu->arch.smi_pending = false; - ++vcpu->arch.smi_count; - enter_smm(vcpu); - } else if (vcpu->arch.nmi_pending && - kvm_x86_ops.nmi_allowed(vcpu, true)) { - --vcpu->arch.nmi_pending; - vcpu->arch.nmi_injected = true; - kvm_x86_ops.set_nmi(vcpu); - } else if (kvm_cpu_has_injectable_intr(vcpu) && - kvm_x86_ops.interrupt_allowed(vcpu, true)) { - kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false); - kvm_x86_ops.set_irq(vcpu); + /* + * Finally, inject interrupt events. If an event cannot be injected + * due to architectural conditions (e.g. IF=0) a window-open exit + * will re-request KVM_REQ_EVENT. Sometimes however an event is pending + * and can architecturally be injected, but we cannot do it right now: + * an interrupt could have arrived just now and we have to inject it + * as a vmexit, or there could already an event in the queue, which is + * indicated by can_inject. In that case we request an immediate exit + * in order to make progress and get back here for another iteration. + * The kvm_x86_ops hooks communicate this by returning -EBUSY. + */ + if (vcpu->arch.smi_pending) { + r = can_inject ? kvm_x86_ops.smi_allowed(vcpu, true) : -EBUSY; + if (r < 0) + goto busy; + if (r) { + vcpu->arch.smi_pending = false; + ++vcpu->arch.smi_count; + enter_smm(vcpu); + can_inject = false; + } else + kvm_x86_ops.enable_smi_window(vcpu); } - return 0; + if (vcpu->arch.nmi_pending) { + r = can_inject ? kvm_x86_ops.nmi_allowed(vcpu, true) : -EBUSY; + if (r < 0) + goto busy; + if (r) { + --vcpu->arch.nmi_pending; + vcpu->arch.nmi_injected = true; + kvm_x86_ops.set_nmi(vcpu); + can_inject = false; + WARN_ON(kvm_x86_ops.nmi_allowed(vcpu, true) < 0); + } + if (vcpu->arch.nmi_pending) + kvm_x86_ops.enable_nmi_window(vcpu); + } + + if (kvm_cpu_has_injectable_intr(vcpu)) { + r = can_inject ? kvm_x86_ops.interrupt_allowed(vcpu, true) : -EBUSY; + if (r < 0) + goto busy; + if (r) { + kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false); + kvm_x86_ops.set_irq(vcpu); + WARN_ON(kvm_x86_ops.interrupt_allowed(vcpu, true) < 0); + } + if (kvm_cpu_has_injectable_intr(vcpu)) + kvm_x86_ops.enable_irq_window(vcpu); + } + + if (is_guest_mode(vcpu) && + kvm_x86_ops.nested_ops->hv_timer_pending && + kvm_x86_ops.nested_ops->hv_timer_pending(vcpu)) + *req_immediate_exit = true; + + WARN_ON(vcpu->arch.exception.pending); + return; + +busy: + *req_immediate_exit = true; + return; } static void process_nmi(struct kvm_vcpu *vcpu) @@ -8357,36 +8399,9 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) goto out; } - if (inject_pending_event(vcpu) != 0) - req_immediate_exit = true; - else { - /* Enable SMI/NMI/IRQ window open exits if needed. - * - * SMIs have three cases: - * 1) They can be nested, and then there is nothing to - * do here because RSM will cause a vmexit anyway. - * 2) There is an ISA-specific reason why SMI cannot be - * injected, and the moment when this changes can be - * intercepted. - * 3) Or the SMI can be pending because - * inject_pending_event has completed the injection - * of an IRQ or NMI from the previous vmexit, and - * then we request an immediate exit to inject the - * SMI. - */ - if (vcpu->arch.smi_pending && !is_smm(vcpu)) - if (!kvm_x86_ops.enable_smi_window(vcpu)) - req_immediate_exit = true; - if (vcpu->arch.nmi_pending) - kvm_x86_ops.enable_nmi_window(vcpu); - if (kvm_cpu_has_injectable_intr(vcpu) || req_int_win) - kvm_x86_ops.enable_irq_window(vcpu); - if (is_guest_mode(vcpu) && - kvm_x86_ops.nested_ops->hv_timer_pending && - kvm_x86_ops.nested_ops->hv_timer_pending(vcpu)) - req_immediate_exit = true; - WARN_ON(vcpu->arch.exception.pending); - } + inject_pending_event(vcpu, &req_immediate_exit); + if (req_int_win) + kvm_x86_ops.enable_irq_window(vcpu); if (kvm_lapic_enabled(vcpu)) { update_cr8_intercept(vcpu); From 7c86663b68bab393633d8312a0d25a3d004de182 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Sat, 16 May 2020 08:42:28 -0400 Subject: [PATCH 0959/1043] KVM: nSVM: inject exceptions via svm_check_nested_events This allows exceptions injected by the emulator to be properly delivered as vmexits. The code also becomes simpler, because we can just let all L0-intercepted exceptions go through the usual path. In particular, our emulation of the VMX #DB exit qualification is very much simplified, because the vmexit injection path can use kvm_deliver_exception_payload to update DR6. Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 + arch/x86/kvm/svm/nested.c | 138 +++++++++++++------------------- arch/x86/kvm/svm/svm.c | 9 --- arch/x86/kvm/svm/svm.h | 1 + arch/x86/kvm/x86.c | 13 +-- 5 files changed, 59 insertions(+), 104 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 7707bd4b0593..e6f2e1a2dab6 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1495,6 +1495,8 @@ void kvm_pic_clear_all(struct kvm_pic *pic, int irq_source_id); void kvm_inject_nmi(struct kvm_vcpu *vcpu); +void kvm_update_dr7(struct kvm_vcpu *vcpu); + int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn); int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva); void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index f39e0d578b9b..1ae13fd17028 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -111,6 +111,8 @@ void recalc_intercepts(struct vcpu_svm *svm) h = &svm->nested.hsave->control; g = &svm->nested; + svm->nested.host_intercept_exceptions = h->intercept_exceptions; + c->intercept_cr = h->intercept_cr; c->intercept_dr = h->intercept_dr; c->intercept_exceptions = h->intercept_exceptions; @@ -616,50 +618,6 @@ static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; } -/* DB exceptions for our internal use must not cause vmexit */ -static int nested_svm_intercept_db(struct vcpu_svm *svm) -{ - unsigned long dr6 = svm->vmcb->save.dr6; - - /* Always catch it and pass it to userspace if debugging. */ - if (svm->vcpu.guest_debug & - (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) - return NESTED_EXIT_HOST; - - /* if we're not singlestepping, it's not ours */ - if (!svm->nmi_singlestep) - goto reflected_db; - - /* if it's not a singlestep exception, it's not ours */ - if (!(dr6 & DR6_BS)) - goto reflected_db; - - /* if the guest is singlestepping, it should get the vmexit */ - if (svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF) { - disable_nmi_singlestep(svm); - goto reflected_db; - } - - /* it's ours, the nested hypervisor must not see this one */ - return NESTED_EXIT_HOST; - -reflected_db: - /* - * Synchronize guest DR6 here just like in kvm_deliver_exception_payload; - * it will be moved into the nested VMCB by nested_svm_vmexit. Once - * exceptions will be moved to svm_check_nested_events, all this stuff - * will just go away and we could just return NESTED_EXIT_HOST - * unconditionally. db_interception will queue the exception, which - * will be processed by svm_check_nested_events if a nested vmexit is - * required, and we will just use kvm_deliver_exception_payload to copy - * the payload to DR6 before vmexit. - */ - WARN_ON(svm->vcpu.arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT); - svm->vcpu.arch.dr6 &= ~(DR_TRAP_BITS | DR6_RTM); - svm->vcpu.arch.dr6 |= dr6 & ~DR6_FIXED_1; - return NESTED_EXIT_DONE; -} - static int nested_svm_intercept_ioio(struct vcpu_svm *svm) { unsigned port, size, iopm_len; @@ -710,20 +668,12 @@ static int nested_svm_intercept(struct vcpu_svm *svm) break; } case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { - u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); - if (svm->nested.intercept_exceptions & excp_bits) { - if (exit_code == SVM_EXIT_EXCP_BASE + DB_VECTOR) - vmexit = nested_svm_intercept_db(svm); - else if (exit_code == SVM_EXIT_EXCP_BASE + BP_VECTOR && - svm->vcpu.guest_debug & KVM_GUESTDBG_USE_SW_BP) - vmexit = NESTED_EXIT_HOST; - else - vmexit = NESTED_EXIT_DONE; - } - /* async page fault always cause vmexit */ - else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) && - svm->vcpu.arch.exception.nested_apf != 0) - vmexit = NESTED_EXIT_DONE; + /* + * Host-intercepted exceptions have been checked already in + * nested_svm_exit_special. There is nothing to do here, + * the vmexit is injected by svm_check_nested_events. + */ + vmexit = NESTED_EXIT_DONE; break; } case SVM_EXIT_ERR: { @@ -768,35 +718,45 @@ int nested_svm_check_permissions(struct vcpu_svm *svm) return 0; } -int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, - bool has_error_code, u32 error_code) +static bool nested_exit_on_exception(struct vcpu_svm *svm) { - int vmexit; + unsigned int nr = svm->vcpu.arch.exception.nr; - if (!is_guest_mode(&svm->vcpu)) - return 0; + return (svm->nested.intercept_exceptions & (1 << nr)); +} - vmexit = nested_svm_intercept(svm); - if (vmexit != NESTED_EXIT_DONE) - return 0; +static void nested_svm_inject_exception_vmexit(struct vcpu_svm *svm) +{ + unsigned int nr = svm->vcpu.arch.exception.nr; svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; svm->vmcb->control.exit_code_hi = 0; - svm->vmcb->control.exit_info_1 = error_code; + + if (svm->vcpu.arch.exception.has_error_code) + svm->vmcb->control.exit_info_1 = svm->vcpu.arch.exception.error_code; /* * EXITINFO2 is undefined for all exception intercepts other * than #PF. */ - if (svm->vcpu.arch.exception.nested_apf) - svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token; - else if (svm->vcpu.arch.exception.has_payload) - svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload; - else - svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2; + if (nr == PF_VECTOR) { + if (svm->vcpu.arch.exception.nested_apf) + svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token; + else if (svm->vcpu.arch.exception.has_payload) + svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload; + else + svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2; + } else if (nr == DB_VECTOR) { + /* See inject_pending_event. */ + kvm_deliver_exception_payload(&svm->vcpu); + if (svm->vcpu.arch.dr7 & DR7_GD) { + svm->vcpu.arch.dr7 &= ~DR7_GD; + kvm_update_dr7(&svm->vcpu); + } + } else + WARN_ON(svm->vcpu.arch.exception.has_payload); - svm->nested.exit_required = true; - return vmexit; + nested_svm_vmexit(svm); } static void nested_svm_smi(struct vcpu_svm *svm) @@ -835,6 +795,15 @@ static int svm_check_nested_events(struct kvm_vcpu *vcpu) kvm_event_needs_reinjection(vcpu) || svm->nested.exit_required || svm->nested.nested_run_pending; + if (vcpu->arch.exception.pending) { + if (block_nested_events) + return -EBUSY; + if (!nested_exit_on_exception(svm)) + return 0; + nested_svm_inject_exception_vmexit(svm); + return 0; + } + if (vcpu->arch.smi_pending && !svm_smi_blocked(vcpu)) { if (block_nested_events) return -EBUSY; @@ -872,18 +841,19 @@ int nested_svm_exit_special(struct vcpu_svm *svm) switch (exit_code) { case SVM_EXIT_INTR: case SVM_EXIT_NMI: - case SVM_EXIT_EXCP_BASE + MC_VECTOR: - return NESTED_EXIT_HOST; case SVM_EXIT_NPF: - /* For now we are always handling NPFs when using them */ - if (npt_enabled) - return NESTED_EXIT_HOST; - break; - case SVM_EXIT_EXCP_BASE + PF_VECTOR: - /* Trap async PF even if not shadowing */ - if (!npt_enabled || svm->vcpu.arch.apf.host_apf_reason) + return NESTED_EXIT_HOST; + case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { + u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); + + if (get_host_vmcb(svm)->control.intercept_exceptions & excp_bits) + return NESTED_EXIT_HOST; + else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR && + svm->vcpu.arch.apf.host_apf_reason) + /* Trap async PF even if not shadowing */ return NESTED_EXIT_HOST; break; + } default: break; } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 5da494847a2f..89b6fb1e0abc 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -331,17 +331,8 @@ static void svm_queue_exception(struct kvm_vcpu *vcpu) struct vcpu_svm *svm = to_svm(vcpu); unsigned nr = vcpu->arch.exception.nr; bool has_error_code = vcpu->arch.exception.has_error_code; - bool reinject = vcpu->arch.exception.injected; u32 error_code = vcpu->arch.exception.error_code; - /* - * If we are within a nested VM we'd better #VMEXIT and let the guest - * handle the exception - */ - if (!reinject && - nested_svm_check_exception(svm, nr, has_error_code, error_code)) - return; - kvm_deliver_exception_payload(&svm->vcpu); if (nr == BP_VECTOR && !nrips) { diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 5cc559ab862d..8342032291fc 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -86,6 +86,7 @@ struct nested_state { u64 hsave_msr; u64 vm_cr_msr; u64 vmcb; + u32 host_intercept_exceptions; /* These are the merged vectors */ u32 *msrpm; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 0ee828f60d05..f0fa610bed91 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1072,7 +1072,7 @@ static void kvm_update_dr0123(struct kvm_vcpu *vcpu) } } -static void kvm_update_dr7(struct kvm_vcpu *vcpu) +void kvm_update_dr7(struct kvm_vcpu *vcpu) { unsigned long dr7; @@ -1085,6 +1085,7 @@ static void kvm_update_dr7(struct kvm_vcpu *vcpu) if (dr7 & DR7_BP_EN_MASK) vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED; } +EXPORT_SYMBOL_GPL(kvm_update_dr7); static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) { @@ -7778,16 +7779,6 @@ static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit X86_EFLAGS_RF); if (vcpu->arch.exception.nr == DB_VECTOR) { - /* - * This code assumes that nSVM doesn't use - * check_nested_events(). If it does, the - * DR6/DR7 changes should happen before L1 - * gets a #VMEXIT for an intercepted #DB in - * L2. (Under VMX, on the other hand, the - * DR6/DR7 changes should not happen in the - * event of a VM-exit to L1 for an intercepted - * #DB in L2.) - */ kvm_deliver_exception_payload(vcpu); if (vcpu->arch.dr7 & DR7_GD) { vcpu->arch.dr7 &= ~DR7_GD; From bd279629f73f27f0931d09cd3da904b923fb6f35 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Sat, 16 May 2020 08:46:00 -0400 Subject: [PATCH 0960/1043] KVM: nSVM: remove exit_required All events now inject vmexits before vmentry rather than after vmexit. Therefore, exit_required is not set anymore and we can remove it. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 3 +-- arch/x86/kvm/svm/svm.c | 14 -------------- arch/x86/kvm/svm/svm.h | 3 --- 3 files changed, 1 insertion(+), 19 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 1ae13fd17028..3d17c62a84a3 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -792,8 +792,7 @@ static int svm_check_nested_events(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); bool block_nested_events = - kvm_event_needs_reinjection(vcpu) || svm->nested.exit_required || - svm->nested.nested_run_pending; + kvm_event_needs_reinjection(vcpu) || svm->nested.nested_run_pending; if (vcpu->arch.exception.pending) { if (block_nested_events) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 89b6fb1e0abc..545f63ebc720 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2889,13 +2889,6 @@ static int handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) if (npt_enabled) vcpu->arch.cr3 = svm->vmcb->save.cr3; - if (unlikely(svm->nested.exit_required)) { - nested_svm_vmexit(svm); - svm->nested.exit_required = false; - - return 1; - } - if (is_guest_mode(vcpu)) { int vmexit; @@ -3327,13 +3320,6 @@ static fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; - /* - * A vmexit emulation is required before the vcpu can be executed - * again. - */ - if (unlikely(svm->nested.exit_required)) - return EXIT_FASTPATH_NONE; - /* * Disable singlestep if we're injecting an interrupt/exception. * We don't want our modified rflags to be pushed on the stack where diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 8342032291fc..89fab75dd4f5 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -95,9 +95,6 @@ struct nested_state { u64 vmcb_msrpm; u64 vmcb_iopm; - /* A VMEXIT is required but not yet emulated */ - bool exit_required; - /* A VMRUN has started but has not yet been performed, so * we cannot inject a nested vmexit yet. */ bool nested_run_pending; From 5b672408660e309ce7d54462eea050d5e5d0fc3d Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Sat, 16 May 2020 08:50:35 -0400 Subject: [PATCH 0961/1043] KVM: nSVM: correctly inject INIT vmexits The usual drill at this point, except there is no code to remove because this case was not handled at all. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 3d17c62a84a3..dcac4c3510ab 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -25,6 +25,7 @@ #include "trace.h" #include "mmu.h" #include "x86.h" +#include "lapic.h" #include "svm.h" static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, @@ -788,11 +789,37 @@ static void nested_svm_intr(struct vcpu_svm *svm) nested_svm_vmexit(svm); } +static inline bool nested_exit_on_init(struct vcpu_svm *svm) +{ + return (svm->nested.intercept & (1ULL << INTERCEPT_INIT)); +} + +static void nested_svm_init(struct vcpu_svm *svm) +{ + svm->vmcb->control.exit_code = SVM_EXIT_INIT; + svm->vmcb->control.exit_info_1 = 0; + svm->vmcb->control.exit_info_2 = 0; + + nested_svm_vmexit(svm); +} + + static int svm_check_nested_events(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); bool block_nested_events = kvm_event_needs_reinjection(vcpu) || svm->nested.nested_run_pending; + struct kvm_lapic *apic = vcpu->arch.apic; + + if (lapic_in_kernel(vcpu) && + test_bit(KVM_APIC_INIT, &apic->pending_events)) { + if (block_nested_events) + return -EBUSY; + if (!nested_exit_on_init(svm)) + return 0; + nested_svm_init(svm); + return 0; + } if (vcpu->arch.exception.pending) { if (block_nested_events) From 978ce5837c7ed50e4ea30cc0fa20f2f820edf8ea Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 20 May 2020 08:37:37 -0400 Subject: [PATCH 0962/1043] KVM: SVM: always update CR3 in VMCB svm_load_mmu_pgd is delaying the write of GUEST_CR3 to prepare_vmcs02 as an optimization, but this is only correct before the nested vmentry. If userspace is modifying CR3 with KVM_SET_SREGS after the VM has already been put in guest mode, the value of CR3 will not be updated. Remove the optimization, which almost never triggers anyway. This was was added in commit 689f3bf21628 ("KVM: x86: unify callbacks to load paging root", 2020-03-16) just to keep the two vendor-specific modules closer, but we'll fix VMX too. Fixes: 689f3bf21628 ("KVM: x86: unify callbacks to load paging root") Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 6 +----- arch/x86/kvm/svm/svm.c | 16 +++++----------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index dcac4c3510ab..8756c9f463fd 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -256,11 +256,7 @@ void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, svm_set_efer(&svm->vcpu, nested_vmcb->save.efer); svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0); svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4); - if (npt_enabled) { - svm->vmcb->save.cr3 = nested_vmcb->save.cr3; - svm->vcpu.arch.cr3 = nested_vmcb->save.cr3; - } else - (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3); + (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3); /* Guest paging mode is active - reset mmu */ kvm_mmu_reset_context(&svm->vcpu); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 545f63ebc720..feb96a410f2d 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3447,7 +3447,6 @@ static fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long root) { struct vcpu_svm *svm = to_svm(vcpu); - bool update_guest_cr3 = true; unsigned long cr3; cr3 = __sme_set(root); @@ -3456,18 +3455,13 @@ static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long root) mark_dirty(svm->vmcb, VMCB_NPT); /* Loading L2's CR3 is handled by enter_svm_guest_mode. */ - if (is_guest_mode(vcpu)) - update_guest_cr3 = false; - else if (test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail)) - cr3 = vcpu->arch.cr3; - else /* CR3 is already up-to-date. */ - update_guest_cr3 = false; + if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail)) + return; + cr3 = vcpu->arch.cr3; } - if (update_guest_cr3) { - svm->vmcb->save.cr3 = cr3; - mark_dirty(svm->vmcb, VMCB_CR); - } + svm->vmcb->save.cr3 = cr3; + mark_dirty(svm->vmcb, VMCB_CR); } static int is_disabled(void) From df7e0681dd8acfa8d07816eaef232ded816d8a8c Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 20 May 2020 08:37:37 -0400 Subject: [PATCH 0963/1043] KVM: nVMX: always update CR3 in VMCS vmx_load_mmu_pgd is delaying the write of GUEST_CR3 to prepare_vmcs02 as an optimization, but this is only correct before the nested vmentry. If userspace is modifying CR3 with KVM_SET_SREGS after the VM has already been put in guest mode, the value of CR3 will not be updated. Remove the optimization, which almost never triggers anyway. Fixes: 04f11ef45810 ("KVM: nVMX: Always write vmcs02.GUEST_CR3 during nested VM-Enter") Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index b3a41645e157..7b55dc6230a9 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -3085,10 +3085,7 @@ void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, unsigned long pgd) spin_unlock(&to_kvm_vmx(kvm)->ept_pointer_lock); } - /* Loading vmcs02.GUEST_CR3 is handled by nested VM-Enter. */ - if (is_guest_mode(vcpu)) - update_guest_cr3 = false; - else if (!enable_unrestricted_guest && !is_paging(vcpu)) + if (!enable_unrestricted_guest && !is_paging(vcpu)) guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr; else if (test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail)) guest_cr3 = vcpu->arch.cr3; From 818e915fbac518e8c78e1877a0048d92d4965e5a Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Thu, 28 May 2020 23:27:49 +0800 Subject: [PATCH 0964/1043] irqchip: Add Loongson HyperTransport Vector support This controller appears on Loongson-3 chips for receiving interrupt vectors from PCH's PIC and PCH's PCIe MSI interrupts. Signed-off-by: Jiaxun Yang Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200528152757.1028711-2-jiaxun.yang@flygoat.com --- drivers/irqchip/Kconfig | 8 + drivers/irqchip/Makefile | 1 + drivers/irqchip/irq-loongson-htvec.c | 214 +++++++++++++++++++++++++++ 3 files changed, 223 insertions(+) create mode 100644 drivers/irqchip/irq-loongson-htvec.c diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index a85aada04a64..de4564e2ea88 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -532,4 +532,12 @@ config LOONGSON_HTPIC help Support for the Loongson-3 HyperTransport PIC Controller. +config LOONGSON_HTVEC + bool "Loongson3 HyperTransport Interrupt Vector Controller" + depends on MACH_LOONGSON64 || COMPILE_TEST + default MACH_LOONGSON64 + select IRQ_DOMAIN_HIERARCHY + help + Support for the Loongson3 HyperTransport Interrupt Vector Controller. + endmenu diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 37bbe39bf909..74561879f5a7 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -107,3 +107,4 @@ obj-$(CONFIG_TI_SCI_INTR_IRQCHIP) += irq-ti-sci-intr.o obj-$(CONFIG_TI_SCI_INTA_IRQCHIP) += irq-ti-sci-inta.o obj-$(CONFIG_LOONGSON_LIOINTC) += irq-loongson-liointc.o obj-$(CONFIG_LOONGSON_HTPIC) += irq-loongson-htpic.o +obj-$(CONFIG_LOONGSON_HTVEC) += irq-loongson-htvec.o diff --git a/drivers/irqchip/irq-loongson-htvec.c b/drivers/irqchip/irq-loongson-htvec.c new file mode 100644 index 000000000000..1ece9337c78d --- /dev/null +++ b/drivers/irqchip/irq-loongson-htvec.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020, Jiaxun Yang + * Loongson HyperTransport Interrupt Vector support + */ + +#define pr_fmt(fmt) "htvec: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Registers */ +#define HTVEC_EN_OFF 0x20 +#define HTVEC_MAX_PARENT_IRQ 4 + +#define VEC_COUNT_PER_REG 32 +#define VEC_REG_COUNT 4 +#define VEC_COUNT (VEC_COUNT_PER_REG * VEC_REG_COUNT) +#define VEC_REG_IDX(irq_id) ((irq_id) / VEC_COUNT_PER_REG) +#define VEC_REG_BIT(irq_id) ((irq_id) % VEC_COUNT_PER_REG) + +struct htvec { + void __iomem *base; + struct irq_domain *htvec_domain; + raw_spinlock_t htvec_lock; +}; + +static void htvec_irq_dispatch(struct irq_desc *desc) +{ + int i; + u32 pending; + bool handled = false; + struct irq_chip *chip = irq_desc_get_chip(desc); + struct htvec *priv = irq_desc_get_handler_data(desc); + + chained_irq_enter(chip, desc); + + for (i = 0; i < VEC_REG_COUNT; i++) { + pending = readl(priv->base + 4 * i); + while (pending) { + int bit = __ffs(pending); + + generic_handle_irq(irq_linear_revmap(priv->htvec_domain, bit + + VEC_COUNT_PER_REG * i)); + pending &= ~BIT(bit); + handled = true; + } + } + + if (!handled) + spurious_interrupt(); + + chained_irq_exit(chip, desc); +} + +static void htvec_ack_irq(struct irq_data *d) +{ + struct htvec *priv = irq_data_get_irq_chip_data(d); + + writel(BIT(VEC_REG_BIT(d->hwirq)), + priv->base + VEC_REG_IDX(d->hwirq) * 4); +} + +static void htvec_mask_irq(struct irq_data *d) +{ + u32 reg; + void __iomem *addr; + struct htvec *priv = irq_data_get_irq_chip_data(d); + + raw_spin_lock(&priv->htvec_lock); + addr = priv->base + HTVEC_EN_OFF; + addr += VEC_REG_IDX(d->hwirq) * 4; + reg = readl(addr); + reg &= ~BIT(VEC_REG_BIT(d->hwirq)); + writel(reg, addr); + raw_spin_unlock(&priv->htvec_lock); +} + +static void htvec_unmask_irq(struct irq_data *d) +{ + u32 reg; + void __iomem *addr; + struct htvec *priv = irq_data_get_irq_chip_data(d); + + raw_spin_lock(&priv->htvec_lock); + addr = priv->base + HTVEC_EN_OFF; + addr += VEC_REG_IDX(d->hwirq) * 4; + reg = readl(addr); + reg |= BIT(VEC_REG_BIT(d->hwirq)); + writel(reg, addr); + raw_spin_unlock(&priv->htvec_lock); +} + +static struct irq_chip htvec_irq_chip = { + .name = "LOONGSON_HTVEC", + .irq_mask = htvec_mask_irq, + .irq_unmask = htvec_unmask_irq, + .irq_ack = htvec_ack_irq, +}; + +static int htvec_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + unsigned long hwirq; + unsigned int type, i; + struct htvec *priv = domain->host_data; + + irq_domain_translate_onecell(domain, arg, &hwirq, &type); + + for (i = 0; i < nr_irqs; i++) { + irq_domain_set_info(domain, virq + i, hwirq + i, &htvec_irq_chip, + priv, handle_edge_irq, NULL, NULL); + } + + return 0; +} + +static void htvec_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } +} + +static const struct irq_domain_ops htvec_domain_ops = { + .translate = irq_domain_translate_onecell, + .alloc = htvec_domain_alloc, + .free = htvec_domain_free, +}; + +static void htvec_reset(struct htvec *priv) +{ + u32 idx; + + /* Clear IRQ cause registers, mask all interrupts */ + for (idx = 0; idx < VEC_REG_COUNT; idx++) { + writel_relaxed(0x0, priv->base + HTVEC_EN_OFF + 4 * idx); + writel_relaxed(0xFFFFFFFF, priv->base); + } +} + +static int htvec_of_init(struct device_node *node, + struct device_node *parent) +{ + struct htvec *priv; + int err, parent_irq[4], num_parents = 0, i; + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + raw_spin_lock_init(&priv->htvec_lock); + priv->base = of_iomap(node, 0); + if (!priv->base) { + err = -ENOMEM; + goto free_priv; + } + + /* Interrupt may come from any of the 4 interrupt line */ + for (i = 0; i < HTVEC_MAX_PARENT_IRQ; i++) { + parent_irq[i] = irq_of_parse_and_map(node, i); + if (parent_irq[i] <= 0) + break; + + num_parents++; + } + + if (!num_parents) { + pr_err("Failed to get parent irqs\n"); + err = -ENODEV; + goto iounmap_base; + } + + priv->htvec_domain = irq_domain_create_linear(of_node_to_fwnode(node), + VEC_COUNT, + &htvec_domain_ops, + priv); + if (!priv->htvec_domain) { + pr_err("Failed to create IRQ domain\n"); + err = -ENOMEM; + goto iounmap_base; + } + + htvec_reset(priv); + + for (i = 0; i < num_parents; i++) + irq_set_chained_handler_and_data(parent_irq[i], + htvec_irq_dispatch, priv); + + return 0; + +iounmap_base: + iounmap(priv->base); +free_priv: + kfree(priv); + + return err; +} + +IRQCHIP_DECLARE(htvec, "loongson,htvec-1.0", htvec_of_init); From 6c2832c3c6edc38ab58bad29731b4951c0a90cf8 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Thu, 28 May 2020 23:27:50 +0800 Subject: [PATCH 0965/1043] dt-bindings: interrupt-controller: Add Loongson HTVEC Add binding for Loongson-3 HyperTransport Interrupt Vector Controller. Reviewed-by: Rob Herring Signed-off-by: Jiaxun Yang Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200528152757.1028711-3-jiaxun.yang@flygoat.com --- .../interrupt-controller/loongson,htvec.yaml | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 Documentation/devicetree/bindings/interrupt-controller/loongson,htvec.yaml diff --git a/Documentation/devicetree/bindings/interrupt-controller/loongson,htvec.yaml b/Documentation/devicetree/bindings/interrupt-controller/loongson,htvec.yaml new file mode 100644 index 000000000000..e865cd8f96a9 --- /dev/null +++ b/Documentation/devicetree/bindings/interrupt-controller/loongson,htvec.yaml @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/interrupt-controller/loongson,htvec.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: Loongson-3 HyperTransport Interrupt Vector Controller + +maintainers: + - Jiaxun Yang + +description: + This interrupt controller is found in the Loongson-3 family of chips for + receiving vectorized interrupts from PCH's interrupt controller. + +properties: + compatible: + const: loongson,htvec-1.0 + + reg: + maxItems: 1 + + interrupts: + minItems: 1 + maxItems: 4 + description: Four parent interrupts that receive chained interrupts. + + interrupt-controller: true + + '#interrupt-cells': + const: 1 + +required: + - compatible + - reg + - interrupts + - interrupt-controller + - '#interrupt-cells' + +additionalProperties: false + +examples: + - | + #include + htvec: interrupt-controller@fb000080 { + compatible = "loongson,htvec-1.0"; + reg = <0xfb000080 0x40>; + interrupt-controller; + #interrupt-cells = <1>; + + interrupt-parent = <&liointc>; + interrupts = <24 IRQ_TYPE_LEVEL_HIGH>, + <25 IRQ_TYPE_LEVEL_HIGH>, + <26 IRQ_TYPE_LEVEL_HIGH>, + <27 IRQ_TYPE_LEVEL_HIGH>; + }; +... From ef8c01eb64ca6719da449dab0aa9424e13c58bd0 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Thu, 28 May 2020 23:27:51 +0800 Subject: [PATCH 0966/1043] irqchip: Add Loongson PCH PIC controller This controller appears on Loongson LS7A family of PCH to transform interrupts from devices into HyperTransport vectorized interrrupts and send them to procrssor's HT vector controller. Signed-off-by: Jiaxun Yang Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200528152757.1028711-4-jiaxun.yang@flygoat.com --- drivers/irqchip/Kconfig | 9 + drivers/irqchip/Makefile | 1 + drivers/irqchip/irq-loongson-pch-pic.c | 243 +++++++++++++++++++++++++ 3 files changed, 253 insertions(+) create mode 100644 drivers/irqchip/irq-loongson-pch-pic.c diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index de4564e2ea88..5524a621638c 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -540,4 +540,13 @@ config LOONGSON_HTVEC help Support for the Loongson3 HyperTransport Interrupt Vector Controller. +config LOONGSON_PCH_PIC + bool "Loongson PCH PIC Controller" + depends on MACH_LOONGSON64 || COMPILE_TEST + default MACH_LOONGSON64 + select IRQ_DOMAIN_HIERARCHY + select IRQ_FASTEOI_HIERARCHY_HANDLERS + help + Support for the Loongson PCH PIC Controller. + endmenu diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 74561879f5a7..acc72331cec8 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -108,3 +108,4 @@ obj-$(CONFIG_TI_SCI_INTA_IRQCHIP) += irq-ti-sci-inta.o obj-$(CONFIG_LOONGSON_LIOINTC) += irq-loongson-liointc.o obj-$(CONFIG_LOONGSON_HTPIC) += irq-loongson-htpic.o obj-$(CONFIG_LOONGSON_HTVEC) += irq-loongson-htvec.o +obj-$(CONFIG_LOONGSON_PCH_PIC) += irq-loongson-pch-pic.o diff --git a/drivers/irqchip/irq-loongson-pch-pic.c b/drivers/irqchip/irq-loongson-pch-pic.c new file mode 100644 index 000000000000..2a05b9305012 --- /dev/null +++ b/drivers/irqchip/irq-loongson-pch-pic.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020, Jiaxun Yang + * Loongson PCH PIC support + */ + +#define pr_fmt(fmt) "pch-pic: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Registers */ +#define PCH_PIC_MASK 0x20 +#define PCH_PIC_HTMSI_EN 0x40 +#define PCH_PIC_EDGE 0x60 +#define PCH_PIC_CLR 0x80 +#define PCH_PIC_AUTO0 0xc0 +#define PCH_PIC_AUTO1 0xe0 +#define PCH_INT_ROUTE(irq) (0x100 + irq) +#define PCH_INT_HTVEC(irq) (0x200 + irq) +#define PCH_PIC_POL 0x3e0 + +#define PIC_COUNT_PER_REG 32 +#define PIC_REG_COUNT 2 +#define PIC_COUNT (PIC_COUNT_PER_REG * PIC_REG_COUNT) +#define PIC_REG_IDX(irq_id) ((irq_id) / PIC_COUNT_PER_REG) +#define PIC_REG_BIT(irq_id) ((irq_id) % PIC_COUNT_PER_REG) + +struct pch_pic { + void __iomem *base; + struct irq_domain *pic_domain; + u32 ht_vec_base; + raw_spinlock_t pic_lock; +}; + +static void pch_pic_bitset(struct pch_pic *priv, int offset, int bit) +{ + u32 reg; + void __iomem *addr = priv->base + offset + PIC_REG_IDX(bit) * 4; + + raw_spin_lock(&priv->pic_lock); + reg = readl(addr); + reg |= BIT(PIC_REG_BIT(bit)); + writel(reg, addr); + raw_spin_unlock(&priv->pic_lock); +} + +static void pch_pic_bitclr(struct pch_pic *priv, int offset, int bit) +{ + u32 reg; + void __iomem *addr = priv->base + offset + PIC_REG_IDX(bit) * 4; + + raw_spin_lock(&priv->pic_lock); + reg = readl(addr); + reg &= ~BIT(PIC_REG_BIT(bit)); + writel(reg, addr); + raw_spin_unlock(&priv->pic_lock); +} + +static void pch_pic_eoi_irq(struct irq_data *d) +{ + u32 idx = PIC_REG_IDX(d->hwirq); + struct pch_pic *priv = irq_data_get_irq_chip_data(d); + + writel(BIT(PIC_REG_BIT(d->hwirq)), + priv->base + PCH_PIC_CLR + idx * 4); +} + +static void pch_pic_mask_irq(struct irq_data *d) +{ + struct pch_pic *priv = irq_data_get_irq_chip_data(d); + + pch_pic_bitset(priv, PCH_PIC_MASK, d->hwirq); + irq_chip_mask_parent(d); +} + +static void pch_pic_unmask_irq(struct irq_data *d) +{ + struct pch_pic *priv = irq_data_get_irq_chip_data(d); + + irq_chip_unmask_parent(d); + pch_pic_bitclr(priv, PCH_PIC_MASK, d->hwirq); +} + +static int pch_pic_set_type(struct irq_data *d, unsigned int type) +{ + struct pch_pic *priv = irq_data_get_irq_chip_data(d); + int ret = 0; + + switch (type) { + case IRQ_TYPE_EDGE_RISING: + pch_pic_bitset(priv, PCH_PIC_EDGE, d->hwirq); + pch_pic_bitclr(priv, PCH_PIC_POL, d->hwirq); + break; + case IRQ_TYPE_EDGE_FALLING: + pch_pic_bitset(priv, PCH_PIC_EDGE, d->hwirq); + pch_pic_bitset(priv, PCH_PIC_POL, d->hwirq); + break; + case IRQ_TYPE_LEVEL_HIGH: + pch_pic_bitclr(priv, PCH_PIC_EDGE, d->hwirq); + pch_pic_bitclr(priv, PCH_PIC_POL, d->hwirq); + break; + case IRQ_TYPE_LEVEL_LOW: + pch_pic_bitclr(priv, PCH_PIC_EDGE, d->hwirq); + pch_pic_bitset(priv, PCH_PIC_POL, d->hwirq); + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static struct irq_chip pch_pic_irq_chip = { + .name = "PCH PIC", + .irq_mask = pch_pic_mask_irq, + .irq_unmask = pch_pic_unmask_irq, + .irq_ack = irq_chip_ack_parent, + .irq_eoi = pch_pic_eoi_irq, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_set_type = pch_pic_set_type, +}; + +static int pch_pic_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + int err; + unsigned int type; + unsigned long hwirq; + struct irq_fwspec fwspec; + struct pch_pic *priv = domain->host_data; + + irq_domain_translate_twocell(domain, arg, &hwirq, &type); + + fwspec.fwnode = domain->parent->fwnode; + fwspec.param_count = 1; + fwspec.param[0] = hwirq + priv->ht_vec_base; + + err = irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec); + if (err) + return err; + + irq_domain_set_info(domain, virq, hwirq, + &pch_pic_irq_chip, priv, + handle_fasteoi_ack_irq, NULL, NULL); + irq_set_probe(virq); + + return 0; +} + +static const struct irq_domain_ops pch_pic_domain_ops = { + .translate = irq_domain_translate_twocell, + .alloc = pch_pic_alloc, + .free = irq_domain_free_irqs_parent, +}; + +static void pch_pic_reset(struct pch_pic *priv) +{ + int i; + + for (i = 0; i < PIC_COUNT; i++) { + /* Write vectore ID */ + writeb(priv->ht_vec_base + i, priv->base + PCH_INT_HTVEC(i)); + /* Hardcode route to HT0 Lo */ + writeb(1, priv->base + PCH_INT_ROUTE(i)); + } + + for (i = 0; i < PIC_REG_COUNT; i++) { + /* Clear IRQ cause registers, mask all interrupts */ + writel_relaxed(0xFFFFFFFF, priv->base + PCH_PIC_MASK + 4 * i); + writel_relaxed(0xFFFFFFFF, priv->base + PCH_PIC_CLR + 4 * i); + /* Clear auto bounce, we don't need that */ + writel_relaxed(0, priv->base + PCH_PIC_AUTO0 + 4 * i); + writel_relaxed(0, priv->base + PCH_PIC_AUTO1 + 4 * i); + /* Enable HTMSI transformer */ + writel_relaxed(0xFFFFFFFF, priv->base + PCH_PIC_HTMSI_EN + 4 * i); + } +} + +static int pch_pic_of_init(struct device_node *node, + struct device_node *parent) +{ + struct pch_pic *priv; + struct irq_domain *parent_domain; + int err; + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + raw_spin_lock_init(&priv->pic_lock); + priv->base = of_iomap(node, 0); + if (!priv->base) { + err = -ENOMEM; + goto free_priv; + } + + parent_domain = irq_find_host(parent); + if (!parent_domain) { + pr_err("Failed to find the parent domain\n"); + err = -ENXIO; + goto iounmap_base; + } + + if (of_property_read_u32(node, "loongson,pic-base-vec", + &priv->ht_vec_base)) { + pr_err("Failed to determine pic-base-vec\n"); + err = -EINVAL; + goto iounmap_base; + } + + priv->pic_domain = irq_domain_create_hierarchy(parent_domain, 0, + PIC_COUNT, + of_node_to_fwnode(node), + &pch_pic_domain_ops, + priv); + if (!priv->pic_domain) { + pr_err("Failed to create IRQ domain\n"); + err = -ENOMEM; + goto iounmap_base; + } + + pch_pic_reset(priv); + + return 0; + +iounmap_base: + iounmap(priv->base); +free_priv: + kfree(priv); + + return err; +} + +IRQCHIP_DECLARE(pch_pic, "loongson,pch-pic-1.0", pch_pic_of_init); From b6e4bc125fc517969f97d901b1845ebf47bbea26 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Thu, 28 May 2020 23:27:52 +0800 Subject: [PATCH 0967/1043] dt-bindings: interrupt-controller: Add Loongson PCH PIC Add binding for Loongson PCH PIC Controller. Reviewed-by: Rob Herring Signed-off-by: Jiaxun Yang Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200528152757.1028711-5-jiaxun.yang@flygoat.com --- .../loongson,pch-pic.yaml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 Documentation/devicetree/bindings/interrupt-controller/loongson,pch-pic.yaml diff --git a/Documentation/devicetree/bindings/interrupt-controller/loongson,pch-pic.yaml b/Documentation/devicetree/bindings/interrupt-controller/loongson,pch-pic.yaml new file mode 100644 index 000000000000..274adea13f33 --- /dev/null +++ b/Documentation/devicetree/bindings/interrupt-controller/loongson,pch-pic.yaml @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/interrupt-controller/loongson,pch-pic.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: Loongson PCH PIC Controller + +maintainers: + - Jiaxun Yang + +description: + This interrupt controller is found in the Loongson LS7A family of PCH for + transforming interrupts from on-chip devices into HyperTransport vectorized + interrupts. + +properties: + compatible: + const: loongson,pch-pic-1.0 + + reg: + maxItems: 1 + + loongson,pic-base-vec: + description: + u32 value of the base of parent HyperTransport vector allocated + to PCH PIC. + allOf: + - $ref: "/schemas/types.yaml#/definitions/uint32" + - minimum: 0 + maximum: 192 + + interrupt-controller: true + + '#interrupt-cells': + const: 2 + +required: + - compatible + - reg + - loongson,pic-base-vec + - interrupt-controller + - '#interrupt-cells' + +examples: + - | + #include + pic: interrupt-controller@10000000 { + compatible = "loongson,pch-pic-1.0"; + reg = <0x10000000 0x400>; + interrupt-controller; + #interrupt-cells = <2>; + loongson,pic-base-vec = <64>; + interrupt-parent = <&htvec>; + }; +... From 632dcc2c75ef6de3272aa4ddd8f19da1f1ace323 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Thu, 28 May 2020 23:27:53 +0800 Subject: [PATCH 0968/1043] irqchip: Add Loongson PCH MSI controller This controller appears on Loongson LS7A family of PCH to transform interrupts from PCI MSI into HyperTransport vectorized interrrupts and send them to procrssor's HT vector controller. Signed-off-by: Jiaxun Yang Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200528152757.1028711-6-jiaxun.yang@flygoat.com --- drivers/irqchip/Kconfig | 10 + drivers/irqchip/Makefile | 1 + drivers/irqchip/irq-loongson-pch-msi.c | 255 +++++++++++++++++++++++++ 3 files changed, 266 insertions(+) create mode 100644 drivers/irqchip/irq-loongson-pch-msi.c diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 5524a621638c..0b6b826dd843 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -549,4 +549,14 @@ config LOONGSON_PCH_PIC help Support for the Loongson PCH PIC Controller. +config LOONGSON_PCH_MSI + bool "Loongson PCH PIC Controller" + depends on MACH_LOONGSON64 || COMPILE_TEST + depends on PCI + default MACH_LOONGSON64 + select IRQ_DOMAIN_HIERARCHY + select PCI_MSI + help + Support for the Loongson PCH MSI Controller. + endmenu diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index acc72331cec8..3a4ce283189a 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -109,3 +109,4 @@ obj-$(CONFIG_LOONGSON_LIOINTC) += irq-loongson-liointc.o obj-$(CONFIG_LOONGSON_HTPIC) += irq-loongson-htpic.o obj-$(CONFIG_LOONGSON_HTVEC) += irq-loongson-htvec.o obj-$(CONFIG_LOONGSON_PCH_PIC) += irq-loongson-pch-pic.o +obj-$(CONFIG_LOONGSON_PCH_MSI) += irq-loongson-pch-msi.o diff --git a/drivers/irqchip/irq-loongson-pch-msi.c b/drivers/irqchip/irq-loongson-pch-msi.c new file mode 100644 index 000000000000..50becd21008c --- /dev/null +++ b/drivers/irqchip/irq-loongson-pch-msi.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020, Jiaxun Yang + * Loongson PCH MSI support + */ + +#define pr_fmt(fmt) "pch-msi: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +struct pch_msi_data { + struct mutex msi_map_lock; + phys_addr_t doorbell; + u32 irq_first; /* The vector number that MSIs starts */ + u32 num_irqs; /* The number of vectors for MSIs */ + unsigned long *msi_map; +}; + +static void pch_msi_mask_msi_irq(struct irq_data *d) +{ + pci_msi_mask_irq(d); + irq_chip_mask_parent(d); +} + +static void pch_msi_unmask_msi_irq(struct irq_data *d) +{ + irq_chip_unmask_parent(d); + pci_msi_unmask_irq(d); +} + +static struct irq_chip pch_msi_irq_chip = { + .name = "PCH PCI MSI", + .irq_mask = pch_msi_mask_msi_irq, + .irq_unmask = pch_msi_unmask_msi_irq, + .irq_ack = irq_chip_ack_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, +}; + +static int pch_msi_allocate_hwirq(struct pch_msi_data *priv, int num_req) +{ + int first; + + mutex_lock(&priv->msi_map_lock); + + first = bitmap_find_free_region(priv->msi_map, priv->num_irqs, + get_count_order(num_req)); + if (first < 0) { + mutex_unlock(&priv->msi_map_lock); + return -ENOSPC; + } + + mutex_unlock(&priv->msi_map_lock); + + return priv->irq_first + first; +} + +static void pch_msi_free_hwirq(struct pch_msi_data *priv, + int hwirq, int num_req) +{ + int first = hwirq - priv->irq_first; + + mutex_lock(&priv->msi_map_lock); + bitmap_release_region(priv->msi_map, first, get_count_order(num_req)); + mutex_unlock(&priv->msi_map_lock); +} + +static void pch_msi_compose_msi_msg(struct irq_data *data, + struct msi_msg *msg) +{ + struct pch_msi_data *priv = irq_data_get_irq_chip_data(data); + + msg->address_hi = upper_32_bits(priv->doorbell); + msg->address_lo = lower_32_bits(priv->doorbell); + msg->data = data->hwirq; +} + +static struct msi_domain_info pch_msi_domain_info = { + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX, + .chip = &pch_msi_irq_chip, +}; + +static struct irq_chip middle_irq_chip = { + .name = "PCH MSI", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_ack = irq_chip_ack_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_compose_msi_msg = pch_msi_compose_msi_msg, +}; + +static int pch_msi_parent_domain_alloc(struct irq_domain *domain, + unsigned int virq, int hwirq) +{ + struct irq_fwspec fwspec; + int ret; + + fwspec.fwnode = domain->parent->fwnode; + fwspec.param_count = 1; + fwspec.param[0] = hwirq; + + ret = irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec); + if (ret) + return ret; + + return 0; +} + +static int pch_msi_middle_domain_alloc(struct irq_domain *domain, + unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct pch_msi_data *priv = domain->host_data; + int hwirq, err, i; + + hwirq = pch_msi_allocate_hwirq(priv, nr_irqs); + if (hwirq < 0) + return hwirq; + + for (i = 0; i < nr_irqs; i++) { + err = pch_msi_parent_domain_alloc(domain, virq + i, hwirq + i); + if (err) + goto err_hwirq; + + irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, + &middle_irq_chip, priv); + } + + return 0; + +err_hwirq: + pch_msi_free_hwirq(priv, hwirq, nr_irqs); + irq_domain_free_irqs_parent(domain, virq, i - 1); + + return err; +} + +static void pch_msi_middle_domain_free(struct irq_domain *domain, + unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct pch_msi_data *priv = irq_data_get_irq_chip_data(d); + + irq_domain_free_irqs_parent(domain, virq, nr_irqs); + pch_msi_free_hwirq(priv, d->hwirq, nr_irqs); +} + +static const struct irq_domain_ops pch_msi_middle_domain_ops = { + .alloc = pch_msi_middle_domain_alloc, + .free = pch_msi_middle_domain_free, +}; + +static int pch_msi_init_domains(struct pch_msi_data *priv, + struct device_node *node, + struct irq_domain *parent) +{ + struct irq_domain *middle_domain, *msi_domain; + + middle_domain = irq_domain_create_linear(of_node_to_fwnode(node), + priv->num_irqs, + &pch_msi_middle_domain_ops, + priv); + if (!middle_domain) { + pr_err("Failed to create the MSI middle domain\n"); + return -ENOMEM; + } + + middle_domain->parent = parent; + irq_domain_update_bus_token(middle_domain, DOMAIN_BUS_NEXUS); + + msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(node), + &pch_msi_domain_info, + middle_domain); + if (!msi_domain) { + pr_err("Failed to create PCI MSI domain\n"); + irq_domain_remove(middle_domain); + return -ENOMEM; + } + + return 0; +} + +static int pch_msi_init(struct device_node *node, + struct device_node *parent) +{ + struct pch_msi_data *priv; + struct irq_domain *parent_domain; + struct resource res; + int ret; + + parent_domain = irq_find_host(parent); + if (!parent_domain) { + pr_err("Failed to find the parent domain\n"); + return -ENXIO; + } + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + mutex_init(&priv->msi_map_lock); + + ret = of_address_to_resource(node, 0, &res); + if (ret) { + pr_err("Failed to allocate resource\n"); + goto err_priv; + } + + priv->doorbell = res.start; + + if (of_property_read_u32(node, "loongson,msi-base-vec", + &priv->irq_first)) { + pr_err("Unable to parse MSI vec base\n"); + ret = -EINVAL; + goto err_priv; + } + + if (of_property_read_u32(node, "loongson,msi-num-vecs", + &priv->num_irqs)) { + pr_err("Unable to parse MSI vec number\n"); + ret = -EINVAL; + goto err_priv; + } + + priv->msi_map = bitmap_alloc(priv->num_irqs, GFP_KERNEL); + if (!priv->msi_map) { + ret = -ENOMEM; + goto err_priv; + } + + pr_debug("Registering %d MSIs, starting at %d\n", + priv->num_irqs, priv->irq_first); + + ret = pch_msi_init_domains(priv, node, parent_domain); + if (ret) + goto err_map; + + return 0; + +err_map: + kfree(priv->msi_map); +err_priv: + kfree(priv); + return ret; +} + +IRQCHIP_DECLARE(pch_msi, "loongson,pch-msi-1.0", pch_msi_init); From da10a4b626657387845f32d37141fc7d48ebbdb3 Mon Sep 17 00:00:00 2001 From: Jiaxun Yang Date: Thu, 28 May 2020 23:27:54 +0800 Subject: [PATCH 0969/1043] dt-bindings: interrupt-controller: Add Loongson PCH MSI Add binding for Loongson PCH MSI controller. Signed-off-by: Jiaxun Yang Reviewed-by: Rob Herring Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20200528152757.1028711-7-jiaxun.yang@flygoat.com --- .../loongson,pch-msi.yaml | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 Documentation/devicetree/bindings/interrupt-controller/loongson,pch-msi.yaml diff --git a/Documentation/devicetree/bindings/interrupt-controller/loongson,pch-msi.yaml b/Documentation/devicetree/bindings/interrupt-controller/loongson,pch-msi.yaml new file mode 100644 index 000000000000..1a5ebbdd219a --- /dev/null +++ b/Documentation/devicetree/bindings/interrupt-controller/loongson,pch-msi.yaml @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/interrupt-controller/loongson,pch-msi.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: Loongson PCH MSI Controller + +maintainers: + - Jiaxun Yang + +description: + This interrupt controller is found in the Loongson LS7A family of PCH for + transforming interrupts from PCIe MSI into HyperTransport vectorized + interrupts. + +properties: + compatible: + const: loongson,pch-msi-1.0 + + reg: + maxItems: 1 + + loongson,msi-base-vec: + description: + u32 value of the base of parent HyperTransport vector allocated + to PCH MSI. + allOf: + - $ref: "/schemas/types.yaml#/definitions/uint32" + - minimum: 0 + maximum: 255 + + loongson,msi-num-vecs: + description: + u32 value of the number of parent HyperTransport vectors allocated + to PCH MSI. + allOf: + - $ref: "/schemas/types.yaml#/definitions/uint32" + - minimum: 1 + maximum: 256 + + msi-controller: true + +required: + - compatible + - reg + - msi-controller + - loongson,msi-base-vec + - loongson,msi-num-vecs + +examples: + - | + #include + msi: msi-controller@2ff00000 { + compatible = "loongson,pch-msi-1.0"; + reg = <0x2ff00000 0x4>; + msi-controller; + loongson,msi-base-vec = <64>; + loongson,msi-num-vecs = <64>; + interrupt-parent = <&htvec>; + }; +... From f57a3fe44995a3820192e0cf7c3ebdecedd9586e Mon Sep 17 00:00:00 2001 From: Chao Yu Date: Fri, 29 May 2020 18:48:36 +0800 Subject: [PATCH 0970/1043] erofs: convert to use the new mount fs_context api Convert the erofs to use new internal mount API as the old one will be obsoleted and removed. This allows greater flexibility in communication of mount parameters between userspace, the VFS and the filesystem. See Documentation/filesystems/mount_api.txt for more information. Cc: Al Viro Cc: David Howells Signed-off-by: Chao Yu Link: https://lore.kernel.org/r/20200529104836.17843-1-hsiangkao@redhat.com Signed-off-by: Gao Xiang --- fs/erofs/internal.h | 27 +++-- fs/erofs/super.c | 253 ++++++++++++++++++++------------------------ fs/erofs/xattr.c | 4 +- fs/erofs/zdata.c | 4 +- 4 files changed, 132 insertions(+), 156 deletions(-) diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index 5eead7fdc7a6..1c077b7bb43d 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -46,6 +46,17 @@ typedef u64 erofs_off_t; /* data type for filesystem-wide blocks number */ typedef u32 erofs_blk_t; +struct erofs_fs_context { +#ifdef CONFIG_EROFS_FS_ZIP + /* current strategy of how to use managed cache */ + unsigned char cache_strategy; + + /* threshold for decompression synchronously */ + unsigned int max_sync_decompress_pages; +#endif + unsigned int mount_opt; +}; + struct erofs_sb_info { #ifdef CONFIG_EROFS_FS_ZIP /* list for all registered superblocks, mainly for shrinker */ @@ -55,14 +66,8 @@ struct erofs_sb_info { /* managed XArray arranged in physical block number */ struct xarray managed_pslots; - /* threshold for decompression synchronously */ - unsigned int max_sync_decompress_pages; - unsigned int shrinker_run_no; - /* current strategy of how to use managed cache */ - unsigned char cache_strategy; - /* pseudo inode to manage cached pages */ struct inode *managed_cache; #endif /* CONFIG_EROFS_FS_ZIP */ @@ -88,7 +93,7 @@ struct erofs_sb_info { u32 feature_compat; u32 feature_incompat; - unsigned int mount_opt; + struct erofs_fs_context ctx; /* options */ }; #define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info) @@ -98,17 +103,17 @@ struct erofs_sb_info { #define EROFS_MOUNT_XATTR_USER 0x00000010 #define EROFS_MOUNT_POSIX_ACL 0x00000020 -#define clear_opt(sbi, option) ((sbi)->mount_opt &= ~EROFS_MOUNT_##option) -#define set_opt(sbi, option) ((sbi)->mount_opt |= EROFS_MOUNT_##option) -#define test_opt(sbi, option) ((sbi)->mount_opt & EROFS_MOUNT_##option) +#define clear_opt(ctx, option) ((ctx)->mount_opt &= ~EROFS_MOUNT_##option) +#define set_opt(ctx, option) ((ctx)->mount_opt |= EROFS_MOUNT_##option) +#define test_opt(ctx, option) ((ctx)->mount_opt & EROFS_MOUNT_##option) -#ifdef CONFIG_EROFS_FS_ZIP enum { EROFS_ZIP_CACHE_DISABLED, EROFS_ZIP_CACHE_READAHEAD, EROFS_ZIP_CACHE_READAROUND }; +#ifdef CONFIG_EROFS_FS_ZIP #define EROFS_LOCKED_MAGIC (INT_MIN | 0xE0F510CCL) /* basic unit of the workstation of a super_block */ diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 8e46d204a0c2..7a13ffb07c23 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include "xattr.h" #define CREATE_TRACE_POINTS @@ -192,53 +194,18 @@ out: return ret; } -#ifdef CONFIG_EROFS_FS_ZIP -static int erofs_build_cache_strategy(struct super_block *sb, - substring_t *args) -{ - struct erofs_sb_info *sbi = EROFS_SB(sb); - const char *cs = match_strdup(args); - int err = 0; - - if (!cs) { - erofs_err(sb, "Not enough memory to store cache strategy"); - return -ENOMEM; - } - - if (!strcmp(cs, "disabled")) { - sbi->cache_strategy = EROFS_ZIP_CACHE_DISABLED; - } else if (!strcmp(cs, "readahead")) { - sbi->cache_strategy = EROFS_ZIP_CACHE_READAHEAD; - } else if (!strcmp(cs, "readaround")) { - sbi->cache_strategy = EROFS_ZIP_CACHE_READAROUND; - } else { - erofs_err(sb, "Unrecognized cache strategy \"%s\"", cs); - err = -EINVAL; - } - kfree(cs); - return err; -} -#else -static int erofs_build_cache_strategy(struct super_block *sb, - substring_t *args) -{ - erofs_info(sb, "EROFS compression is disabled, so cache strategy is ignored"); - return 0; -} -#endif - /* set up default EROFS parameters */ -static void erofs_default_options(struct erofs_sb_info *sbi) +static void erofs_default_options(struct erofs_fs_context *ctx) { #ifdef CONFIG_EROFS_FS_ZIP - sbi->cache_strategy = EROFS_ZIP_CACHE_READAROUND; - sbi->max_sync_decompress_pages = 3; + ctx->cache_strategy = EROFS_ZIP_CACHE_READAROUND; + ctx->max_sync_decompress_pages = 3; #endif #ifdef CONFIG_EROFS_FS_XATTR - set_opt(sbi, XATTR_USER); + set_opt(ctx, XATTR_USER); #endif #ifdef CONFIG_EROFS_FS_POSIX_ACL - set_opt(sbi, POSIX_ACL); + set_opt(ctx, POSIX_ACL); #endif } @@ -251,73 +218,62 @@ enum { Opt_err }; -static match_table_t erofs_tokens = { - {Opt_user_xattr, "user_xattr"}, - {Opt_nouser_xattr, "nouser_xattr"}, - {Opt_acl, "acl"}, - {Opt_noacl, "noacl"}, - {Opt_cache_strategy, "cache_strategy=%s"}, - {Opt_err, NULL} +static const struct constant_table erofs_param_cache_strategy[] = { + {"disabled", EROFS_ZIP_CACHE_DISABLED}, + {"readahead", EROFS_ZIP_CACHE_READAHEAD}, + {"readaround", EROFS_ZIP_CACHE_READAROUND}, + {} }; -static int erofs_parse_options(struct super_block *sb, char *options) +static const struct fs_parameter_spec erofs_fs_parameters[] = { + fsparam_flag_no("user_xattr", Opt_user_xattr), + fsparam_flag_no("acl", Opt_acl), + fsparam_enum("cache_strategy", Opt_cache_strategy, + erofs_param_cache_strategy), + {} +}; + +static int erofs_fc_parse_param(struct fs_context *fc, + struct fs_parameter *param) { - substring_t args[MAX_OPT_ARGS]; - char *p; - int err; + struct erofs_fs_context *ctx __maybe_unused = fc->fs_private; + struct fs_parse_result result; + int opt; - if (!options) - return 0; + opt = fs_parse(fc, erofs_fs_parameters, param, &result); + if (opt < 0) + return opt; - while ((p = strsep(&options, ","))) { - int token; - - if (!*p) - continue; - - args[0].to = args[0].from = NULL; - token = match_token(p, erofs_tokens, args); - - switch (token) { + switch (opt) { + case Opt_user_xattr: #ifdef CONFIG_EROFS_FS_XATTR - case Opt_user_xattr: - set_opt(EROFS_SB(sb), XATTR_USER); - break; - case Opt_nouser_xattr: - clear_opt(EROFS_SB(sb), XATTR_USER); - break; + if (result.boolean) + set_opt(ctx, XATTR_USER); + else + clear_opt(ctx, XATTR_USER); #else - case Opt_user_xattr: - erofs_info(sb, "user_xattr options not supported"); - break; - case Opt_nouser_xattr: - erofs_info(sb, "nouser_xattr options not supported"); - break; + errorfc(fc, "{,no}user_xattr options not supported"); #endif + break; + case Opt_acl: #ifdef CONFIG_EROFS_FS_POSIX_ACL - case Opt_acl: - set_opt(EROFS_SB(sb), POSIX_ACL); - break; - case Opt_noacl: - clear_opt(EROFS_SB(sb), POSIX_ACL); - break; + if (result.boolean) + set_opt(ctx, POSIX_ACL); + else + clear_opt(ctx, POSIX_ACL); #else - case Opt_acl: - erofs_info(sb, "acl options not supported"); - break; - case Opt_noacl: - erofs_info(sb, "noacl options not supported"); - break; + errorfc(fc, "{,no}acl options not supported"); #endif - case Opt_cache_strategy: - err = erofs_build_cache_strategy(sb, args); - if (err) - return err; - break; - default: - erofs_err(sb, "Unrecognized mount option \"%s\" or missing value", p); - return -EINVAL; - } + break; + case Opt_cache_strategy: +#ifdef CONFIG_EROFS_FS_ZIP + ctx->cache_strategy = result.uint_32; +#else + errorfc(fc, "compression not supported, cache_strategy ignored"); +#endif + break; + default: + return -ENOPARAM; } return 0; } @@ -381,10 +337,11 @@ static int erofs_init_managed_cache(struct super_block *sb) static int erofs_init_managed_cache(struct super_block *sb) { return 0; } #endif -static int erofs_fill_super(struct super_block *sb, void *data, int silent) +static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) { struct inode *inode; struct erofs_sb_info *sbi; + struct erofs_fs_context *ctx = fc->fs_private; int err; sb->s_magic = EROFS_SUPER_MAGIC; @@ -410,18 +367,13 @@ static int erofs_fill_super(struct super_block *sb, void *data, int silent) sb->s_op = &erofs_sops; sb->s_xattr = erofs_xattr_handlers; - /* set erofs default mount options */ - erofs_default_options(sbi); - - err = erofs_parse_options(sb, data); - if (err) - return err; - - if (test_opt(sbi, POSIX_ACL)) + if (test_opt(ctx, POSIX_ACL)) sb->s_flags |= SB_POSIXACL; else sb->s_flags &= ~SB_POSIXACL; + sbi->ctx = *ctx; + #ifdef CONFIG_EROFS_FS_ZIP xa_init(&sbi->managed_pslots); #endif @@ -448,15 +400,58 @@ static int erofs_fill_super(struct super_block *sb, void *data, int silent) if (err) return err; - erofs_info(sb, "mounted with opts: %s, root inode @ nid %llu.", - (char *)data, ROOT_NID(sbi)); + erofs_info(sb, "mounted with root inode @ nid %llu.", ROOT_NID(sbi)); return 0; } -static struct dentry *erofs_mount(struct file_system_type *fs_type, int flags, - const char *dev_name, void *data) +static int erofs_fc_get_tree(struct fs_context *fc) { - return mount_bdev(fs_type, flags, dev_name, data, erofs_fill_super); + return get_tree_bdev(fc, erofs_fc_fill_super); +} + +static int erofs_fc_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct erofs_sb_info *sbi = EROFS_SB(sb); + struct erofs_fs_context *ctx = fc->fs_private; + + DBG_BUGON(!sb_rdonly(sb)); + + if (test_opt(ctx, POSIX_ACL)) + fc->sb_flags |= SB_POSIXACL; + else + fc->sb_flags &= ~SB_POSIXACL; + + sbi->ctx = *ctx; + + fc->sb_flags |= SB_RDONLY; + return 0; +} + +static void erofs_fc_free(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static const struct fs_context_operations erofs_context_ops = { + .parse_param = erofs_fc_parse_param, + .get_tree = erofs_fc_get_tree, + .reconfigure = erofs_fc_reconfigure, + .free = erofs_fc_free, +}; + +static int erofs_init_fs_context(struct fs_context *fc) +{ + fc->fs_private = kzalloc(sizeof(struct erofs_fs_context), GFP_KERNEL); + if (!fc->fs_private) + return -ENOMEM; + + /* set default mount options */ + erofs_default_options(fc->fs_private); + + fc->ops = &erofs_context_ops; + + return 0; } /* @@ -495,7 +490,7 @@ static void erofs_put_super(struct super_block *sb) static struct file_system_type erofs_fs_type = { .owner = THIS_MODULE, .name = "erofs", - .mount = erofs_mount, + .init_fs_context = erofs_init_fs_context, .kill_sb = erofs_kill_sb, .fs_flags = FS_REQUIRES_DEV, }; @@ -576,61 +571,37 @@ static int erofs_statfs(struct dentry *dentry, struct kstatfs *buf) static int erofs_show_options(struct seq_file *seq, struct dentry *root) { struct erofs_sb_info *sbi __maybe_unused = EROFS_SB(root->d_sb); + struct erofs_fs_context *ctx __maybe_unused = &sbi->ctx; #ifdef CONFIG_EROFS_FS_XATTR - if (test_opt(sbi, XATTR_USER)) + if (test_opt(ctx, XATTR_USER)) seq_puts(seq, ",user_xattr"); else seq_puts(seq, ",nouser_xattr"); #endif #ifdef CONFIG_EROFS_FS_POSIX_ACL - if (test_opt(sbi, POSIX_ACL)) + if (test_opt(ctx, POSIX_ACL)) seq_puts(seq, ",acl"); else seq_puts(seq, ",noacl"); #endif #ifdef CONFIG_EROFS_FS_ZIP - if (sbi->cache_strategy == EROFS_ZIP_CACHE_DISABLED) { + if (ctx->cache_strategy == EROFS_ZIP_CACHE_DISABLED) seq_puts(seq, ",cache_strategy=disabled"); - } else if (sbi->cache_strategy == EROFS_ZIP_CACHE_READAHEAD) { + else if (ctx->cache_strategy == EROFS_ZIP_CACHE_READAHEAD) seq_puts(seq, ",cache_strategy=readahead"); - } else if (sbi->cache_strategy == EROFS_ZIP_CACHE_READAROUND) { + else if (ctx->cache_strategy == EROFS_ZIP_CACHE_READAROUND) seq_puts(seq, ",cache_strategy=readaround"); - } #endif return 0; } -static int erofs_remount(struct super_block *sb, int *flags, char *data) -{ - struct erofs_sb_info *sbi = EROFS_SB(sb); - unsigned int org_mnt_opt = sbi->mount_opt; - int err; - - DBG_BUGON(!sb_rdonly(sb)); - err = erofs_parse_options(sb, data); - if (err) - goto out; - - if (test_opt(sbi, POSIX_ACL)) - sb->s_flags |= SB_POSIXACL; - else - sb->s_flags &= ~SB_POSIXACL; - - *flags |= SB_RDONLY; - return 0; -out: - sbi->mount_opt = org_mnt_opt; - return err; -} - const struct super_operations erofs_sops = { .put_super = erofs_put_super, .alloc_inode = erofs_alloc_inode, .free_inode = erofs_free_inode, .statfs = erofs_statfs, .show_options = erofs_show_options, - .remount_fs = erofs_remount, }; module_init(erofs_module_init); diff --git a/fs/erofs/xattr.c b/fs/erofs/xattr.c index b766c3ee5fa8..87e437e7b34f 100644 --- a/fs/erofs/xattr.c +++ b/fs/erofs/xattr.c @@ -422,7 +422,7 @@ static int shared_getxattr(struct inode *inode, struct getxattr_iter *it) static bool erofs_xattr_user_list(struct dentry *dentry) { - return test_opt(EROFS_SB(dentry->d_sb), XATTR_USER); + return test_opt(&EROFS_SB(dentry->d_sb)->ctx, XATTR_USER); } static bool erofs_xattr_trusted_list(struct dentry *dentry) @@ -469,7 +469,7 @@ static int erofs_xattr_generic_get(const struct xattr_handler *handler, switch (handler->flags) { case EROFS_XATTR_INDEX_USER: - if (!test_opt(sbi, XATTR_USER)) + if (!test_opt(&sbi->ctx, XATTR_USER)) return -EOPNOTSUPP; break; case EROFS_XATTR_INDEX_TRUSTED: diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index c4b6c9aa87ec..5086b1218aac 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -615,7 +615,7 @@ restart_now: goto err_out; /* preload all compressed pages (maybe downgrade role if necessary) */ - if (should_alloc_managed_pages(fe, sbi->cache_strategy, map->m_la)) + if (should_alloc_managed_pages(fe, sbi->ctx.cache_strategy, map->m_la)) cache_strategy = DELAYEDALLOC; else cache_strategy = DONTALLOC; @@ -1302,7 +1302,7 @@ static int z_erofs_readpage(struct file *file, struct page *page) static bool should_decompress_synchronously(struct erofs_sb_info *sbi, unsigned int nr) { - return nr <= sbi->max_sync_decompress_pages; + return nr <= sbi->ctx.max_sync_decompress_pages; } static int z_erofs_readpages(struct file *filp, struct address_space *mapping, From 34f853b849eb6a509eb8f40f2f5946ebb1f62739 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Thu, 28 May 2020 04:48:44 -0400 Subject: [PATCH 0971/1043] erofs: suppress false positive last_block warning As Andrew mentioned, some rare specific gcc versions could report last_block uninitialized warning. Actually last_block doesn't need to be uninitialized first from its implementation due to bio == NULL condition. After a bio is allocated, last_block will be assigned then. The detailed analysis is in this thread [1]. So let's silence those confusing gccs simply. [1] https://lore.kernel.org/r/20200421072839.GA13867@hsiangkao-HP-ZHAN-66-Pro-G1 Cc: Andrew Morton Reviewed-by: Chao Yu Link: https://lore.kernel.org/r/20200528084844.23359-1-hsiangkao@redhat.com Signed-off-by: Gao Xiang --- fs/erofs/data.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/erofs/data.c b/fs/erofs/data.c index fc3a8d8064f8..2812645b361e 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -265,7 +265,7 @@ submit_bio_out: */ static int erofs_raw_access_readpage(struct file *file, struct page *page) { - erofs_off_t last_block; + erofs_off_t uninitialized_var(last_block); struct bio *bio; trace_erofs_readpage(page, true); @@ -285,7 +285,7 @@ static int erofs_raw_access_readpages(struct file *filp, struct list_head *pages, unsigned int nr_pages) { - erofs_off_t last_block; + erofs_off_t uninitialized_var(last_block); struct bio *bio = NULL; gfp_t gfp = readahead_gfp_mask(mapping); struct page *page = list_last_entry(pages, struct page, lru); From b3878a6aac1bfc52cf531d2568ae054348b2334a Mon Sep 17 00:00:00 2001 From: Thomas Bogendoerfer Date: Sat, 30 May 2020 10:55:25 +0200 Subject: [PATCH 0972/1043] MIPS: Fix build warning about "PTR_STR" redefinition PTR_STR is redefined when CONFIG_TEST_PRINTF is set. This causes the following build warning: CC lib/test_printf.o lib/test_printf.c:214:0: warning: "PTR_STR" redefined #define PTR_STR "ffff0123456789ab" ^ In file included from ./arch/mips/include/asm/dsemul.h:11:0, from ./arch/mips/include/asm/processor.h:22, from ./arch/mips/include/asm/thread_info.h:16, from ./include/linux/thread_info.h:38, from ./include/asm-generic/preempt.h:5, from ./arch/mips/include/generated/asm/preempt.h:1, from ./include/linux/preempt.h:78, from ./include/linux/spinlock.h:51, from ./include/linux/seqlock.h:36, from ./include/linux/time.h:6, from ./include/linux/stat.h:19, from ./include/linux/module.h:13, from lib/test_printf.c:10: ./arch/mips/include/asm/inst.h:20:0: note: this is the location of the previous definition #define PTR_STR ".dword" ^ Instead of renaming PTR_STR we move the unaligned macros to a new file, which is only included inside MIPS code. This way we can safely include asm.h and can use STR(PTR) again. Fixes: e701656ec4db ("MIPS: inst.h: Stop including asm.h to avoid various build failures") Cc: Maciej W. Rozycki" Reported-by: Tiezhu Yang Co-developed-by: Huacai Chen Signed-off-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/asm/inst.h | 779 ------------------------- arch/mips/include/asm/unaligned-emul.h | 779 +++++++++++++++++++++++++ arch/mips/kernel/unaligned.c | 1 + arch/mips/loongson64/cop2-ex.c | 1 + 4 files changed, 781 insertions(+), 779 deletions(-) create mode 100644 arch/mips/include/asm/unaligned-emul.h diff --git a/arch/mips/include/asm/inst.h b/arch/mips/include/asm/inst.h index 6d74ba33b923..22912f78401c 100644 --- a/arch/mips/include/asm/inst.h +++ b/arch/mips/include/asm/inst.h @@ -13,13 +13,6 @@ #include -#if (_MIPS_SZPTR == 32) -#define PTR_STR ".word" -#endif -#if (_MIPS_SZPTR == 64) -#define PTR_STR ".dword" -#endif - /* HACHACHAHCAHC ... */ /* In case some other massaging is needed, keep MIPSInst as wrapper */ @@ -92,776 +85,4 @@ struct mm_decoded_insn { /* Recode table from 16-bit register notation to 32-bit GPR. Do NOT export!!! */ extern const int reg16to32[]; -#ifdef __BIG_ENDIAN -#define _LoadHW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ (".set\tnoat\n" \ - "1:\t"type##_lb("%0", "0(%2)")"\n" \ - "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - "3:\t.set\tat\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#ifndef CONFIG_CPU_NO_LOAD_STORE_LR -#define _LoadW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - "1:\t"type##_lwl("%0", "(%2)")"\n" \ - "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ -/* For CPUs without lwl instruction */ -#define _LoadW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n" \ - ".set\tnoat\n\t" \ - "1:"type##_lb("%0", "0(%2)")"\n\t" \ - "2:"type##_lbu("$1", "1(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "3:"type##_lbu("$1", "2(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "4:"type##_lbu("$1", "3(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - ".set\tpop\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%1, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 11b\n\t" \ - PTR_STR"\t2b, 11b\n\t" \ - PTR_STR"\t3b, 11b\n\t" \ - PTR_STR"\t4b, 11b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ - -#define _LoadHWU(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tnoat\n" \ - "1:\t"type##_lbu("%0", "0(%2)")"\n" \ - "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".set\tat\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#ifndef CONFIG_CPU_NO_LOAD_STORE_LR -#define _LoadWU(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - "1:\t"type##_lwl("%0", "(%2)")"\n" \ - "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\ - "dsll\t%0, %0, 32\n\t" \ - "dsrl\t%0, %0, 32\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - "\t.section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#define _LoadDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - "1:\tldl\t%0, (%2)\n" \ - "2:\tldr\t%0, 7(%2)\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - "\t.section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ -/* For CPUs without lwl and ldl instructions */ -#define _LoadWU(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:"type##_lbu("%0", "0(%2)")"\n\t" \ - "2:"type##_lbu("$1", "1(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "3:"type##_lbu("$1", "2(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "4:"type##_lbu("$1", "3(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - ".set\tpop\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%1, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 11b\n\t" \ - PTR_STR"\t2b, 11b\n\t" \ - PTR_STR"\t3b, 11b\n\t" \ - PTR_STR"\t4b, 11b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#define _LoadDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:lb\t%0, 0(%2)\n\t" \ - "2:lbu\t $1, 1(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "3:lbu\t$1, 2(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "4:lbu\t$1, 3(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "5:lbu\t$1, 4(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "6:lbu\t$1, 5(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "7:lbu\t$1, 6(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "8:lbu\t$1, 7(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - ".set\tpop\n\t" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%1, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 11b\n\t" \ - PTR_STR"\t2b, 11b\n\t" \ - PTR_STR"\t3b, 11b\n\t" \ - PTR_STR"\t4b, 11b\n\t" \ - PTR_STR"\t5b, 11b\n\t" \ - PTR_STR"\t6b, 11b\n\t" \ - PTR_STR"\t7b, 11b\n\t" \ - PTR_STR"\t8b, 11b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ - - -#define _StoreHW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tnoat\n" \ - "1:\t"type##_sb("%1", "1(%2)")"\n" \ - "srl\t$1, %1, 0x8\n" \ - "2:\t"type##_sb("$1", "0(%2)")"\n" \ - ".set\tat\n\t" \ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT));\ -} while (0) - -#ifndef CONFIG_CPU_NO_LOAD_STORE_LR -#define _StoreW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - "1:\t"type##_swl("%1", "(%2)")"\n" \ - "2:\t"type##_swr("%1", "3(%2)")"\n\t"\ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#define _StoreDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - "1:\tsdl\t%1,(%2)\n" \ - "2:\tsdr\t%1, 7(%2)\n\t" \ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ -#define _StoreW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:"type##_sb("%1", "3(%2)")"\n\t" \ - "srl\t$1, %1, 0x8\n\t" \ - "2:"type##_sb("$1", "2(%2)")"\n\t" \ - "srl\t$1, $1, 0x8\n\t" \ - "3:"type##_sb("$1", "1(%2)")"\n\t" \ - "srl\t$1, $1, 0x8\n\t" \ - "4:"type##_sb("$1", "0(%2)")"\n\t" \ - ".set\tpop\n\t" \ - "li\t%0, 0\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%0, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 11b\n\t" \ - PTR_STR"\t2b, 11b\n\t" \ - PTR_STR"\t3b, 11b\n\t" \ - PTR_STR"\t4b, 11b\n\t" \ - ".previous" \ - : "=&r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT) \ - : "memory"); \ -} while (0) - -#define _StoreDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:sb\t%1, 7(%2)\n\t" \ - "dsrl\t$1, %1, 0x8\n\t" \ - "2:sb\t$1, 6(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "3:sb\t$1, 5(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "4:sb\t$1, 4(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "5:sb\t$1, 3(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "6:sb\t$1, 2(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "7:sb\t$1, 1(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "8:sb\t$1, 0(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - ".set\tpop\n\t" \ - "li\t%0, 0\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%0, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 11b\n\t" \ - PTR_STR"\t2b, 11b\n\t" \ - PTR_STR"\t3b, 11b\n\t" \ - PTR_STR"\t4b, 11b\n\t" \ - PTR_STR"\t5b, 11b\n\t" \ - PTR_STR"\t6b, 11b\n\t" \ - PTR_STR"\t7b, 11b\n\t" \ - PTR_STR"\t8b, 11b\n\t" \ - ".previous" \ - : "=&r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT) \ - : "memory"); \ -} while (0) - -#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ - -#else /* __BIG_ENDIAN */ - -#define _LoadHW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ (".set\tnoat\n" \ - "1:\t"type##_lb("%0", "1(%2)")"\n" \ - "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - "3:\t.set\tat\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#ifndef CONFIG_CPU_NO_LOAD_STORE_LR -#define _LoadW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - "1:\t"type##_lwl("%0", "3(%2)")"\n" \ - "2:\t"type##_lwr("%0", "(%2)")"\n\t"\ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ -/* For CPUs without lwl instruction */ -#define _LoadW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n" \ - ".set\tnoat\n\t" \ - "1:"type##_lb("%0", "3(%2)")"\n\t" \ - "2:"type##_lbu("$1", "2(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "3:"type##_lbu("$1", "1(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "4:"type##_lbu("$1", "0(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - ".set\tpop\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%1, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 11b\n\t" \ - PTR_STR"\t2b, 11b\n\t" \ - PTR_STR"\t3b, 11b\n\t" \ - PTR_STR"\t4b, 11b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ - - -#define _LoadHWU(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tnoat\n" \ - "1:\t"type##_lbu("%0", "1(%2)")"\n" \ - "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".set\tat\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#ifndef CONFIG_CPU_NO_LOAD_STORE_LR -#define _LoadWU(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - "1:\t"type##_lwl("%0", "3(%2)")"\n" \ - "2:\t"type##_lwr("%0", "(%2)")"\n\t"\ - "dsll\t%0, %0, 32\n\t" \ - "dsrl\t%0, %0, 32\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - "\t.section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#define _LoadDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - "1:\tldl\t%0, 7(%2)\n" \ - "2:\tldr\t%0, (%2)\n\t" \ - "li\t%1, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - "\t.section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%1, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ -/* For CPUs without lwl and ldl instructions */ -#define _LoadWU(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:"type##_lbu("%0", "3(%2)")"\n\t" \ - "2:"type##_lbu("$1", "2(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "3:"type##_lbu("$1", "1(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "4:"type##_lbu("$1", "0(%2)")"\n\t" \ - "sll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - ".set\tpop\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%1, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 11b\n\t" \ - PTR_STR"\t2b, 11b\n\t" \ - PTR_STR"\t3b, 11b\n\t" \ - PTR_STR"\t4b, 11b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#define _LoadDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:lb\t%0, 7(%2)\n\t" \ - "2:lbu\t$1, 6(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "3:lbu\t$1, 5(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "4:lbu\t$1, 4(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "5:lbu\t$1, 3(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "6:lbu\t$1, 2(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "7:lbu\t$1, 1(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "8:lbu\t$1, 0(%2)\n\t" \ - "dsll\t%0, 0x8\n\t" \ - "or\t%0, $1\n\t" \ - "li\t%1, 0\n" \ - ".set\tpop\n\t" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%1, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 11b\n\t" \ - PTR_STR"\t2b, 11b\n\t" \ - PTR_STR"\t3b, 11b\n\t" \ - PTR_STR"\t4b, 11b\n\t" \ - PTR_STR"\t5b, 11b\n\t" \ - PTR_STR"\t6b, 11b\n\t" \ - PTR_STR"\t7b, 11b\n\t" \ - PTR_STR"\t8b, 11b\n\t" \ - ".previous" \ - : "=&r" (value), "=r" (res) \ - : "r" (addr), "i" (-EFAULT)); \ -} while (0) -#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ - -#define _StoreHW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tnoat\n" \ - "1:\t"type##_sb("%1", "0(%2)")"\n" \ - "srl\t$1,%1, 0x8\n" \ - "2:\t"type##_sb("$1", "1(%2)")"\n" \ - ".set\tat\n\t" \ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT));\ -} while (0) - -#ifndef CONFIG_CPU_NO_LOAD_STORE_LR -#define _StoreW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - "1:\t"type##_swl("%1", "3(%2)")"\n" \ - "2:\t"type##_swr("%1", "(%2)")"\n\t"\ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#define _StoreDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - "1:\tsdl\t%1, 7(%2)\n" \ - "2:\tsdr\t%1, (%2)\n\t" \ - "li\t%0, 0\n" \ - "3:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "4:\tli\t%0, %3\n\t" \ - "j\t3b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 4b\n\t" \ - PTR_STR"\t2b, 4b\n\t" \ - ".previous" \ - : "=r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT)); \ -} while (0) - -#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ -/* For CPUs without swl and sdl instructions */ -#define _StoreW(addr, value, res, type) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:"type##_sb("%1", "0(%2)")"\n\t" \ - "srl\t$1, %1, 0x8\n\t" \ - "2:"type##_sb("$1", "1(%2)")"\n\t" \ - "srl\t$1, $1, 0x8\n\t" \ - "3:"type##_sb("$1", "2(%2)")"\n\t" \ - "srl\t$1, $1, 0x8\n\t" \ - "4:"type##_sb("$1", "3(%2)")"\n\t" \ - ".set\tpop\n\t" \ - "li\t%0, 0\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%0, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 11b\n\t" \ - PTR_STR"\t2b, 11b\n\t" \ - PTR_STR"\t3b, 11b\n\t" \ - PTR_STR"\t4b, 11b\n\t" \ - ".previous" \ - : "=&r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT) \ - : "memory"); \ -} while (0) - -#define _StoreDW(addr, value, res) \ -do { \ - __asm__ __volatile__ ( \ - ".set\tpush\n\t" \ - ".set\tnoat\n\t" \ - "1:sb\t%1, 0(%2)\n\t" \ - "dsrl\t$1, %1, 0x8\n\t" \ - "2:sb\t$1, 1(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "3:sb\t$1, 2(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "4:sb\t$1, 3(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "5:sb\t$1, 4(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "6:sb\t$1, 5(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "7:sb\t$1, 6(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - "8:sb\t$1, 7(%2)\n\t" \ - "dsrl\t$1, $1, 0x8\n\t" \ - ".set\tpop\n\t" \ - "li\t%0, 0\n" \ - "10:\n\t" \ - ".insn\n\t" \ - ".section\t.fixup,\"ax\"\n\t" \ - "11:\tli\t%0, %3\n\t" \ - "j\t10b\n\t" \ - ".previous\n\t" \ - ".section\t__ex_table,\"a\"\n\t" \ - PTR_STR"\t1b, 11b\n\t" \ - PTR_STR"\t2b, 11b\n\t" \ - PTR_STR"\t3b, 11b\n\t" \ - PTR_STR"\t4b, 11b\n\t" \ - PTR_STR"\t5b, 11b\n\t" \ - PTR_STR"\t6b, 11b\n\t" \ - PTR_STR"\t7b, 11b\n\t" \ - PTR_STR"\t8b, 11b\n\t" \ - ".previous" \ - : "=&r" (res) \ - : "r" (value), "r" (addr), "i" (-EFAULT) \ - : "memory"); \ -} while (0) - -#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ -#endif - -#define LoadHWU(addr, value, res) _LoadHWU(addr, value, res, kernel) -#define LoadHWUE(addr, value, res) _LoadHWU(addr, value, res, user) -#define LoadWU(addr, value, res) _LoadWU(addr, value, res, kernel) -#define LoadWUE(addr, value, res) _LoadWU(addr, value, res, user) -#define LoadHW(addr, value, res) _LoadHW(addr, value, res, kernel) -#define LoadHWE(addr, value, res) _LoadHW(addr, value, res, user) -#define LoadW(addr, value, res) _LoadW(addr, value, res, kernel) -#define LoadWE(addr, value, res) _LoadW(addr, value, res, user) -#define LoadDW(addr, value, res) _LoadDW(addr, value, res) - -#define StoreHW(addr, value, res) _StoreHW(addr, value, res, kernel) -#define StoreHWE(addr, value, res) _StoreHW(addr, value, res, user) -#define StoreW(addr, value, res) _StoreW(addr, value, res, kernel) -#define StoreWE(addr, value, res) _StoreW(addr, value, res, user) -#define StoreDW(addr, value, res) _StoreDW(addr, value, res) - #endif /* _ASM_INST_H */ diff --git a/arch/mips/include/asm/unaligned-emul.h b/arch/mips/include/asm/unaligned-emul.h new file mode 100644 index 000000000000..2022b18944b9 --- /dev/null +++ b/arch/mips/include/asm/unaligned-emul.h @@ -0,0 +1,779 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_MIPS_UNALIGNED_EMUL_H +#define _ASM_MIPS_UNALIGNED_EMUL_H + +#include + +#ifdef __BIG_ENDIAN +#define _LoadHW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ (".set\tnoat\n" \ + "1:\t"type##_lb("%0", "0(%2)")"\n" \ + "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + "3:\t.set\tat\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR +#define _LoadW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + "1:\t"type##_lwl("%0", "(%2)")"\n" \ + "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ +/* For CPUs without lwl instruction */ +#define _LoadW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n" \ + ".set\tnoat\n\t" \ + "1:"type##_lb("%0", "0(%2)")"\n\t" \ + "2:"type##_lbu("$1", "1(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:"type##_lbu("$1", "2(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:"type##_lbu("$1", "3(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ + +#define _LoadHWU(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tnoat\n" \ + "1:\t"type##_lbu("%0", "0(%2)")"\n" \ + "2:\t"type##_lbu("$1", "1(%2)")"\n\t"\ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".set\tat\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR +#define _LoadWU(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + "1:\t"type##_lwl("%0", "(%2)")"\n" \ + "2:\t"type##_lwr("%0", "3(%2)")"\n\t"\ + "dsll\t%0, %0, 32\n\t" \ + "dsrl\t%0, %0, 32\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + "\t.section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#define _LoadDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + "1:\tldl\t%0, (%2)\n" \ + "2:\tldr\t%0, 7(%2)\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + "\t.section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ +/* For CPUs without lwl and ldl instructions */ +#define _LoadWU(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:"type##_lbu("%0", "0(%2)")"\n\t" \ + "2:"type##_lbu("$1", "1(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:"type##_lbu("$1", "2(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:"type##_lbu("$1", "3(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#define _LoadDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:lb\t%0, 0(%2)\n\t" \ + "2:lbu\t $1, 1(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:lbu\t$1, 2(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:lbu\t$1, 3(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "5:lbu\t$1, 4(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "6:lbu\t$1, 5(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "7:lbu\t$1, 6(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "8:lbu\t$1, 7(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n\t" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + STR(PTR)"\t5b, 11b\n\t" \ + STR(PTR)"\t6b, 11b\n\t" \ + STR(PTR)"\t7b, 11b\n\t" \ + STR(PTR)"\t8b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ + + +#define _StoreHW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tnoat\n" \ + "1:\t"type##_sb("%1", "1(%2)")"\n" \ + "srl\t$1, %1, 0x8\n" \ + "2:\t"type##_sb("$1", "0(%2)")"\n" \ + ".set\tat\n\t" \ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT));\ +} while (0) + +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR +#define _StoreW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + "1:\t"type##_swl("%1", "(%2)")"\n" \ + "2:\t"type##_swr("%1", "3(%2)")"\n\t"\ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#define _StoreDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + "1:\tsdl\t%1,(%2)\n" \ + "2:\tsdr\t%1, 7(%2)\n\t" \ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ +#define _StoreW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:"type##_sb("%1", "3(%2)")"\n\t" \ + "srl\t$1, %1, 0x8\n\t" \ + "2:"type##_sb("$1", "2(%2)")"\n\t" \ + "srl\t$1, $1, 0x8\n\t" \ + "3:"type##_sb("$1", "1(%2)")"\n\t" \ + "srl\t$1, $1, 0x8\n\t" \ + "4:"type##_sb("$1", "0(%2)")"\n\t" \ + ".set\tpop\n\t" \ + "li\t%0, 0\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%0, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT) \ + : "memory"); \ +} while (0) + +#define _StoreDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:sb\t%1, 7(%2)\n\t" \ + "dsrl\t$1, %1, 0x8\n\t" \ + "2:sb\t$1, 6(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "3:sb\t$1, 5(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "4:sb\t$1, 4(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "5:sb\t$1, 3(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "6:sb\t$1, 2(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "7:sb\t$1, 1(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "8:sb\t$1, 0(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + ".set\tpop\n\t" \ + "li\t%0, 0\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%0, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + STR(PTR)"\t5b, 11b\n\t" \ + STR(PTR)"\t6b, 11b\n\t" \ + STR(PTR)"\t7b, 11b\n\t" \ + STR(PTR)"\t8b, 11b\n\t" \ + ".previous" \ + : "=&r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT) \ + : "memory"); \ +} while (0) + +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ + +#else /* __BIG_ENDIAN */ + +#define _LoadHW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ (".set\tnoat\n" \ + "1:\t"type##_lb("%0", "1(%2)")"\n" \ + "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + "3:\t.set\tat\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR +#define _LoadW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + "1:\t"type##_lwl("%0", "3(%2)")"\n" \ + "2:\t"type##_lwr("%0", "(%2)")"\n\t"\ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ +/* For CPUs without lwl instruction */ +#define _LoadW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n" \ + ".set\tnoat\n\t" \ + "1:"type##_lb("%0", "3(%2)")"\n\t" \ + "2:"type##_lbu("$1", "2(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:"type##_lbu("$1", "1(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:"type##_lbu("$1", "0(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ + + +#define _LoadHWU(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tnoat\n" \ + "1:\t"type##_lbu("%0", "1(%2)")"\n" \ + "2:\t"type##_lbu("$1", "0(%2)")"\n\t"\ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".set\tat\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR +#define _LoadWU(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + "1:\t"type##_lwl("%0", "3(%2)")"\n" \ + "2:\t"type##_lwr("%0", "(%2)")"\n\t"\ + "dsll\t%0, %0, 32\n\t" \ + "dsrl\t%0, %0, 32\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + "\t.section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#define _LoadDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + "1:\tldl\t%0, 7(%2)\n" \ + "2:\tldr\t%0, (%2)\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + "\t.section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ +/* For CPUs without lwl and ldl instructions */ +#define _LoadWU(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:"type##_lbu("%0", "3(%2)")"\n\t" \ + "2:"type##_lbu("$1", "2(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:"type##_lbu("$1", "1(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:"type##_lbu("$1", "0(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#define _LoadDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:lb\t%0, 7(%2)\n\t" \ + "2:lbu\t$1, 6(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "3:lbu\t$1, 5(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "4:lbu\t$1, 4(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "5:lbu\t$1, 3(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "6:lbu\t$1, 2(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "7:lbu\t$1, 1(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "8:lbu\t$1, 0(%2)\n\t" \ + "dsll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + ".set\tpop\n\t" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%1, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + STR(PTR)"\t5b, 11b\n\t" \ + STR(PTR)"\t6b, 11b\n\t" \ + STR(PTR)"\t7b, 11b\n\t" \ + STR(PTR)"\t8b, 11b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); \ +} while (0) +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ + +#define _StoreHW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tnoat\n" \ + "1:\t"type##_sb("%1", "0(%2)")"\n" \ + "srl\t$1,%1, 0x8\n" \ + "2:\t"type##_sb("$1", "1(%2)")"\n" \ + ".set\tat\n\t" \ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT));\ +} while (0) + +#ifndef CONFIG_CPU_NO_LOAD_STORE_LR +#define _StoreW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + "1:\t"type##_swl("%1", "3(%2)")"\n" \ + "2:\t"type##_swr("%1", "(%2)")"\n\t"\ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#define _StoreDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + "1:\tsdl\t%1, 7(%2)\n" \ + "2:\tsdr\t%1, (%2)\n\t" \ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT)); \ +} while (0) + +#else /* CONFIG_CPU_NO_LOAD_STORE_LR */ +/* For CPUs without swl and sdl instructions */ +#define _StoreW(addr, value, res, type) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:"type##_sb("%1", "0(%2)")"\n\t" \ + "srl\t$1, %1, 0x8\n\t" \ + "2:"type##_sb("$1", "1(%2)")"\n\t" \ + "srl\t$1, $1, 0x8\n\t" \ + "3:"type##_sb("$1", "2(%2)")"\n\t" \ + "srl\t$1, $1, 0x8\n\t" \ + "4:"type##_sb("$1", "3(%2)")"\n\t" \ + ".set\tpop\n\t" \ + "li\t%0, 0\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%0, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + ".previous" \ + : "=&r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT) \ + : "memory"); \ +} while (0) + +#define _StoreDW(addr, value, res) \ +do { \ + __asm__ __volatile__ ( \ + ".set\tpush\n\t" \ + ".set\tnoat\n\t" \ + "1:sb\t%1, 0(%2)\n\t" \ + "dsrl\t$1, %1, 0x8\n\t" \ + "2:sb\t$1, 1(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "3:sb\t$1, 2(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "4:sb\t$1, 3(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "5:sb\t$1, 4(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "6:sb\t$1, 5(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "7:sb\t$1, 6(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + "8:sb\t$1, 7(%2)\n\t" \ + "dsrl\t$1, $1, 0x8\n\t" \ + ".set\tpop\n\t" \ + "li\t%0, 0\n" \ + "10:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "11:\tli\t%0, %3\n\t" \ + "j\t10b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 11b\n\t" \ + STR(PTR)"\t2b, 11b\n\t" \ + STR(PTR)"\t3b, 11b\n\t" \ + STR(PTR)"\t4b, 11b\n\t" \ + STR(PTR)"\t5b, 11b\n\t" \ + STR(PTR)"\t6b, 11b\n\t" \ + STR(PTR)"\t7b, 11b\n\t" \ + STR(PTR)"\t8b, 11b\n\t" \ + ".previous" \ + : "=&r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT) \ + : "memory"); \ +} while (0) + +#endif /* CONFIG_CPU_NO_LOAD_STORE_LR */ +#endif + +#define LoadHWU(addr, value, res) _LoadHWU(addr, value, res, kernel) +#define LoadHWUE(addr, value, res) _LoadHWU(addr, value, res, user) +#define LoadWU(addr, value, res) _LoadWU(addr, value, res, kernel) +#define LoadWUE(addr, value, res) _LoadWU(addr, value, res, user) +#define LoadHW(addr, value, res) _LoadHW(addr, value, res, kernel) +#define LoadHWE(addr, value, res) _LoadHW(addr, value, res, user) +#define LoadW(addr, value, res) _LoadW(addr, value, res, kernel) +#define LoadWE(addr, value, res) _LoadW(addr, value, res, user) +#define LoadDW(addr, value, res) _LoadDW(addr, value, res) + +#define StoreHW(addr, value, res) _StoreHW(addr, value, res, kernel) +#define StoreHWE(addr, value, res) _StoreHW(addr, value, res, user) +#define StoreW(addr, value, res) _StoreW(addr, value, res, kernel) +#define StoreWE(addr, value, res) _StoreW(addr, value, res, user) +#define StoreDW(addr, value, res) _StoreDW(addr, value, res) + +#endif /* _ASM_MIPS_UNALIGNED_EMUL_H */ diff --git a/arch/mips/kernel/unaligned.c b/arch/mips/kernel/unaligned.c index 19b906a31c81..0adce604fa44 100644 --- a/arch/mips/kernel/unaligned.c +++ b/arch/mips/kernel/unaligned.c @@ -89,6 +89,7 @@ #include #include #include +#include #include #include diff --git a/arch/mips/loongson64/cop2-ex.c b/arch/mips/loongson64/cop2-ex.c index af0600dfe83c..f130f62129b8 100644 --- a/arch/mips/loongson64/cop2-ex.c +++ b/arch/mips/loongson64/cop2-ex.c @@ -23,6 +23,7 @@ #include #include #include +#include static int loongson_cu2_call(struct notifier_block *nfb, unsigned long action, void *data) From 360428f8c0cd857006a8a3f515946285370489ac Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 30 May 2020 14:54:17 +0300 Subject: [PATCH 0973/1043] io_uring: move timeouts flushing to a helper Separate flushing offset timeouts io_commit_cqring() by moving it into a helper. Just a preparation, makes following patches clearer. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index de6547e68626..ffd0ec7a5a7b 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -988,23 +988,6 @@ static inline bool req_need_defer(struct io_kiocb *req) return false; } -static struct io_kiocb *io_get_timeout_req(struct io_ring_ctx *ctx) -{ - struct io_kiocb *req; - - req = list_first_entry_or_null(&ctx->timeout_list, struct io_kiocb, list); - if (req) { - if (req->flags & REQ_F_TIMEOUT_NOSEQ) - return NULL; - if (!__req_need_defer(req)) { - list_del_init(&req->list); - return req; - } - } - - return NULL; -} - static void __io_commit_cqring(struct io_ring_ctx *ctx) { struct io_rings *rings = ctx->rings; @@ -1133,13 +1116,24 @@ static void __io_queue_deferred(struct io_ring_ctx *ctx) } while (!list_empty(&ctx->defer_list)); } +static void io_flush_timeouts(struct io_ring_ctx *ctx) +{ + while (!list_empty(&ctx->timeout_list)) { + struct io_kiocb *req = list_first_entry(&ctx->timeout_list, + struct io_kiocb, list); + + if (req->flags & REQ_F_TIMEOUT_NOSEQ) + break; + if (__req_need_defer(req)) + break; + list_del_init(&req->list); + io_kill_timeout(req); + } +} + static void io_commit_cqring(struct io_ring_ctx *ctx) { - struct io_kiocb *req; - - while ((req = io_get_timeout_req(ctx)) != NULL) - io_kill_timeout(req); - + io_flush_timeouts(ctx); __io_commit_cqring(ctx); if (unlikely(!list_empty(&ctx->defer_list))) From bfe68a221905de37e65394a6d58c1e5f3e545d2f Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 30 May 2020 14:54:18 +0300 Subject: [PATCH 0974/1043] io_uring: off timeouts based only on completions Offset timeouts wait not for sqe->off non-timeout CQEs, but rather sqe->off + number of prior inflight requests. Wait exactly for sqe->off non-timeout completions Reported-by: Jens Axboe Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 65 +++++++++++---------------------------------------- 1 file changed, 14 insertions(+), 51 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index ffd0ec7a5a7b..9f11feb8a3ac 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -394,7 +394,8 @@ struct io_timeout { struct file *file; u64 addr; int flags; - u32 count; + u32 off; + u32 target_seq; }; struct io_rw { @@ -1124,8 +1125,10 @@ static void io_flush_timeouts(struct io_ring_ctx *ctx) if (req->flags & REQ_F_TIMEOUT_NOSEQ) break; - if (__req_need_defer(req)) + if (req->timeout.target_seq != ctx->cached_cq_tail + - atomic_read(&ctx->cq_timeouts)) break; + list_del_init(&req->list); io_kill_timeout(req); } @@ -4660,20 +4663,8 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) * We could be racing with timeout deletion. If the list is empty, * then timeout lookup already found it and will be handling it. */ - if (!list_empty(&req->list)) { - struct io_kiocb *prev; - - /* - * Adjust the reqs sequence before the current one because it - * will consume a slot in the cq_ring and the cq_tail - * pointer will be increased, otherwise other timeout reqs may - * return in advance without waiting for enough wait_nr. - */ - prev = req; - list_for_each_entry_continue_reverse(prev, &ctx->timeout_list, list) - prev->sequence++; + if (!list_empty(&req->list)) list_del_init(&req->list); - } io_cqring_fill_event(req, -ETIME); io_commit_cqring(ctx); @@ -4765,7 +4756,7 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, if (flags & ~IORING_TIMEOUT_ABS) return -EINVAL; - req->timeout.count = off; + req->timeout.off = off; if (!req->io && io_alloc_async_ctx(req)) return -ENOMEM; @@ -4789,13 +4780,10 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, static int io_timeout(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; - struct io_timeout_data *data; + struct io_timeout_data *data = &req->io->timeout; struct list_head *entry; - unsigned span = 0; - u32 count = req->timeout.count; - u32 seq = req->sequence; + u32 tail, off = req->timeout.off; - data = &req->io->timeout; spin_lock_irq(&ctx->completion_lock); /* @@ -4803,13 +4791,14 @@ static int io_timeout(struct io_kiocb *req) * timeout event to be satisfied. If it isn't set, then this is * a pure timeout request, sequence isn't used. */ - if (!count) { + if (!off) { req->flags |= REQ_F_TIMEOUT_NOSEQ; entry = ctx->timeout_list.prev; goto add; } - req->sequence = seq + count; + tail = ctx->cached_cq_tail - atomic_read(&ctx->cq_timeouts); + req->timeout.target_seq = tail + off; /* * Insertion sort, ensuring the first entry in the list is always @@ -4817,39 +4806,13 @@ static int io_timeout(struct io_kiocb *req) */ list_for_each_prev(entry, &ctx->timeout_list) { struct io_kiocb *nxt = list_entry(entry, struct io_kiocb, list); - unsigned nxt_seq; - long long tmp, tmp_nxt; - u32 nxt_offset = nxt->timeout.count; if (nxt->flags & REQ_F_TIMEOUT_NOSEQ) continue; - - /* - * Since seq + count can overflow, use type long - * long to store it. - */ - tmp = (long long)seq + count; - nxt_seq = nxt->sequence - nxt_offset; - tmp_nxt = (long long)nxt_seq + nxt_offset; - - /* - * cached_sq_head may overflow, and it will never overflow twice - * once there is some timeout req still be valid. - */ - if (seq < nxt_seq) - tmp += UINT_MAX; - - if (tmp > tmp_nxt) + /* nxt.seq is behind @tail, otherwise would've been completed */ + if (off >= nxt->timeout.target_seq - tail) break; - - /* - * Sequence of reqs after the insert one and itself should - * be adjusted because each timeout req consumes a slot. - */ - span++; - nxt->sequence++; } - req->sequence -= span; add: list_add(&req->list, entry); data->timer.function = io_timeout_fn; From 7b53d59859bc932b37895d2d37388e7fa29af7a5 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 30 May 2020 14:19:15 +0300 Subject: [PATCH 0975/1043] io_uring: fix overflowed reqs cancellation Overflowed requests in io_uring_cancel_files() should be shed only of inflight and overflowed refs. All other left references are owned by someone else. If refcount_sub_and_test() fails, it will go further and put put extra ref, don't do that. Also, don't need to do io_wq_cancel_work() for overflowed reqs, they will be let go shortly anyway. Signed-off-by: Pavel Begunkov Signed-off-by: Jens Axboe --- fs/io_uring.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/io_uring.c b/fs/io_uring.c index 9f11feb8a3ac..732ec73ec3c0 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -7449,10 +7449,11 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx, finish_wait(&ctx->inflight_wait, &wait); continue; } + } else { + io_wq_cancel_work(ctx->io_wq, &cancel_req->work); + io_put_req(cancel_req); } - io_wq_cancel_work(ctx->io_wq, &cancel_req->work); - io_put_req(cancel_req); schedule(); finish_wait(&ctx->inflight_wait, &wait); } From 70768ebaa5872e11f68d71761bb9fa1546cb451e Mon Sep 17 00:00:00 2001 From: WANG Xuerui Date: Sat, 30 May 2020 15:32:41 +0800 Subject: [PATCH 0976/1043] MIPS: Loongson64: Guard against future cores without CPUCFG Previously it was thought that all future Loongson cores would come with native CPUCFG. From new information shared by Huacai this is definitely not true (maybe some future 2K cores, for example), so collisions at PRID_REV level are inevitable. The CPU model matching needs to take PRID_IMP into consideration. The emulation logic needs to be disabled for those future cores as well, as we cannot possibly encode their non-discoverable features right now. Reported-by: Huacai Chen Cc: Jiaxun Yang Signed-off-by: WANG Xuerui Reviewed-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- .../include/asm/mach-loongson64/cpucfg-emul.h | 11 ++++++ arch/mips/kernel/traps.c | 4 ++ arch/mips/loongson64/cpucfg-emul.c | 37 ++++++++++--------- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/arch/mips/include/asm/mach-loongson64/cpucfg-emul.h b/arch/mips/include/asm/mach-loongson64/cpucfg-emul.h index 01dc308df7b2..d64af19c210d 100644 --- a/arch/mips/include/asm/mach-loongson64/cpucfg-emul.h +++ b/arch/mips/include/asm/mach-loongson64/cpucfg-emul.h @@ -12,6 +12,12 @@ void loongson3_cpucfg_synthesize_data(struct cpuinfo_mips *c); +static inline bool loongson3_cpucfg_emulation_enabled(struct cpuinfo_mips *c) +{ + /* All supported cores have non-zero LOONGSON_CFG1 data. */ + return c->loongson3_cpucfg_data[0] != 0; +} + static inline u32 loongson3_cpucfg_read_synthesized(struct cpuinfo_mips *c, __u64 sel) { @@ -53,6 +59,11 @@ static inline void loongson3_cpucfg_synthesize_data(struct cpuinfo_mips *c) { } +static inline bool loongson3_cpucfg_emulation_enabled(struct cpuinfo_mips *c) +{ + return false; +} + static inline u32 loongson3_cpucfg_read_synthesized(struct cpuinfo_mips *c, __u64 sel) { diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 2d5b16daf331..22f805a73921 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -722,6 +722,10 @@ static int simulate_loongson3_cpucfg(struct pt_regs *regs, perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); + /* Do not emulate on unsupported core models. */ + if (!loongson3_cpucfg_emulation_enabled(¤t_cpu_data)) + return -1; + regs->regs[rd] = loongson3_cpucfg_read_synthesized( ¤t_cpu_data, sel); diff --git a/arch/mips/loongson64/cpucfg-emul.c b/arch/mips/loongson64/cpucfg-emul.c index fdd52b21c1df..c16023a13379 100644 --- a/arch/mips/loongson64/cpucfg-emul.c +++ b/arch/mips/loongson64/cpucfg-emul.c @@ -134,13 +134,9 @@ void loongson3_cpucfg_synthesize_data(struct cpuinfo_mips *c) c->loongson3_cpucfg_data[1] = 0; c->loongson3_cpucfg_data[2] = 0; - /* Add CPUCFG features non-discoverable otherwise. - * - * All Loongson processors covered by CPUCFG emulation have distinct - * PRID_REV, so take advantage of this. - */ - switch (c->processor_id & PRID_REV_MASK) { - case PRID_REV_LOONGSON3A_R1: + /* Add CPUCFG features non-discoverable otherwise. */ + switch (c->processor_id & (PRID_IMP_MASK | PRID_REV_MASK)) { + case PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R1: c->loongson3_cpucfg_data[0] |= (LOONGSON_CFG1_LSLDR0 | LOONGSON_CFG1_LSSYNCI | LOONGSON_CFG1_LSUCA | LOONGSON_CFG1_LLSYNC | LOONGSON_CFG1_TGTSYNC); @@ -153,8 +149,8 @@ void loongson3_cpucfg_synthesize_data(struct cpuinfo_mips *c) LOONGSON_CFG3_LCAMVW_REV1); break; - case PRID_REV_LOONGSON3B_R1: - case PRID_REV_LOONGSON3B_R2: + case PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3B_R1: + case PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3B_R2: c->loongson3_cpucfg_data[0] |= (LOONGSON_CFG1_LSLDR0 | LOONGSON_CFG1_LSSYNCI | LOONGSON_CFG1_LSUCA | LOONGSON_CFG1_LLSYNC | LOONGSON_CFG1_TGTSYNC); @@ -167,10 +163,10 @@ void loongson3_cpucfg_synthesize_data(struct cpuinfo_mips *c) LOONGSON_CFG3_LCAMVW_REV1); break; - case PRID_REV_LOONGSON2K_R1_0: - case PRID_REV_LOONGSON2K_R1_1: - case PRID_REV_LOONGSON2K_R1_2: - case PRID_REV_LOONGSON2K_R1_3: + case PRID_IMP_LOONGSON_64R | PRID_REV_LOONGSON2K_R1_0: + case PRID_IMP_LOONGSON_64R | PRID_REV_LOONGSON2K_R1_1: + case PRID_IMP_LOONGSON_64R | PRID_REV_LOONGSON2K_R1_2: + case PRID_IMP_LOONGSON_64R | PRID_REV_LOONGSON2K_R1_3: decode_loongson_config6(c); probe_uca(c); @@ -183,10 +179,10 @@ void loongson3_cpucfg_synthesize_data(struct cpuinfo_mips *c) c->loongson3_cpucfg_data[2] = 0; break; - case PRID_REV_LOONGSON3A_R2_0: - case PRID_REV_LOONGSON3A_R2_1: - case PRID_REV_LOONGSON3A_R3_0: - case PRID_REV_LOONGSON3A_R3_1: + case PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_0: + case PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_1: + case PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R3_0: + case PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R3_1: decode_loongson_config6(c); probe_uca(c); @@ -203,6 +199,13 @@ void loongson3_cpucfg_synthesize_data(struct cpuinfo_mips *c) LOONGSON_CFG3_LCAMKW_REV1 | LOONGSON_CFG3_LCAMVW_REV1); break; + + default: + /* It is possible that some future Loongson cores still do + * not have CPUCFG, so do not emulate anything for these + * cores. + */ + return; } /* This feature is set by firmware, but all known Loongson-64 systems From f06da27eb82e358ca389ccb9d13de61e94e77a47 Mon Sep 17 00:00:00 2001 From: WANG Xuerui Date: Sat, 30 May 2020 15:32:42 +0800 Subject: [PATCH 0977/1043] MIPS: Expose Loongson CPUCFG availability via HWCAP The point is to allow userspace to probe for CPUCFG without possibly triggering invalid instructions. In addition to that, future Loongson feature bits could all be stuffed into CPUCFG bit fields (or "leaves" in x86-speak) if Loongson does not make mistakes, so ELF HWCAP bits are conserved. Userspace can determine native CPUCFG availability by checking the LCSRP (Loongson CSR Present) bit in CPUCFG output after seeing CPUCFG bit in HWCAP. Native CPUCFG always sets the LCSRP bit, as CPUCFG is part of the Loongson CSR ASE, while the emulation intentionally leaves this bit clear. The other existing Loongson-specific HWCAP bits are, to my best knowledge, unused, as (1) they are fairly recent additions, (2) Loongson never back-ported the patch into their kernel fork, and (3) Loongson's existing installed base rarely upgrade, if ever; However, they are still considered userspace ABI, hence unfortunately unremovable. But hopefully at least we could stop adding new Loongson HWCAP bits in the future. Cc: Paul Burton Cc: Jiaxun Yang Cc: Huacai Chen Signed-off-by: WANG Xuerui Reviewed-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/include/uapi/asm/hwcap.h | 1 + arch/mips/loongson64/cpucfg-emul.c | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/arch/mips/include/uapi/asm/hwcap.h b/arch/mips/include/uapi/asm/hwcap.h index 1ade1daa4921..b7e02bdc1985 100644 --- a/arch/mips/include/uapi/asm/hwcap.h +++ b/arch/mips/include/uapi/asm/hwcap.h @@ -17,5 +17,6 @@ #define HWCAP_LOONGSON_MMI (1 << 11) #define HWCAP_LOONGSON_EXT (1 << 12) #define HWCAP_LOONGSON_EXT2 (1 << 13) +#define HWCAP_LOONGSON_CPUCFG (1 << 14) #endif /* _UAPI_ASM_HWCAP_H */ diff --git a/arch/mips/loongson64/cpucfg-emul.c b/arch/mips/loongson64/cpucfg-emul.c index c16023a13379..ca75f07252df 100644 --- a/arch/mips/loongson64/cpucfg-emul.c +++ b/arch/mips/loongson64/cpucfg-emul.c @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -128,7 +129,7 @@ void loongson3_cpucfg_synthesize_data(struct cpuinfo_mips *c) /* CPUs with CPUCFG support don't need to synthesize anything. */ if (cpu_has_cfg()) - return; + goto have_cpucfg_now; c->loongson3_cpucfg_data[0] = 0; c->loongson3_cpucfg_data[1] = 0; @@ -217,4 +218,10 @@ void loongson3_cpucfg_synthesize_data(struct cpuinfo_mips *c) patch_cpucfg_sel1(c); patch_cpucfg_sel2(c); patch_cpucfg_sel3(c); + +have_cpucfg_now: + /* We have usable CPUCFG now, emulated or not. + * Announce CPUCFG availability to userspace via hwcap. + */ + elf_hwcap |= HWCAP_LOONGSON_CPUCFG; } From dd25ed7361fb543d32f0da66e1e492563e64938e Mon Sep 17 00:00:00 2001 From: WANG Xuerui Date: Sat, 30 May 2020 15:32:43 +0800 Subject: [PATCH 0978/1043] MIPS: Loongson64: Reorder CPUCFG model match arms Originally the match arms are ordered by model release date, however the LOONGSON_64R cores are even more reduced capability-wise. So put them at top of the switch block. Suggested-by: Huacai Chen Signed-off-by: WANG Xuerui Reviewed-by: Huacai Chen Signed-off-by: Thomas Bogendoerfer --- arch/mips/loongson64/cpucfg-emul.c | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/arch/mips/loongson64/cpucfg-emul.c b/arch/mips/loongson64/cpucfg-emul.c index ca75f07252df..cd619b47ba1f 100644 --- a/arch/mips/loongson64/cpucfg-emul.c +++ b/arch/mips/loongson64/cpucfg-emul.c @@ -137,6 +137,22 @@ void loongson3_cpucfg_synthesize_data(struct cpuinfo_mips *c) /* Add CPUCFG features non-discoverable otherwise. */ switch (c->processor_id & (PRID_IMP_MASK | PRID_REV_MASK)) { + case PRID_IMP_LOONGSON_64R | PRID_REV_LOONGSON2K_R1_0: + case PRID_IMP_LOONGSON_64R | PRID_REV_LOONGSON2K_R1_1: + case PRID_IMP_LOONGSON_64R | PRID_REV_LOONGSON2K_R1_2: + case PRID_IMP_LOONGSON_64R | PRID_REV_LOONGSON2K_R1_3: + decode_loongson_config6(c); + probe_uca(c); + + c->loongson3_cpucfg_data[0] |= (LOONGSON_CFG1_LSLDR0 | + LOONGSON_CFG1_LSSYNCI | LOONGSON_CFG1_LLSYNC | + LOONGSON_CFG1_TGTSYNC); + c->loongson3_cpucfg_data[1] |= (LOONGSON_CFG2_LBT1 | + LOONGSON_CFG2_LBT2 | LOONGSON_CFG2_LPMP | + LOONGSON_CFG2_LPM_REV2); + c->loongson3_cpucfg_data[2] = 0; + break; + case PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R1: c->loongson3_cpucfg_data[0] |= (LOONGSON_CFG1_LSLDR0 | LOONGSON_CFG1_LSSYNCI | LOONGSON_CFG1_LSUCA | @@ -164,22 +180,6 @@ void loongson3_cpucfg_synthesize_data(struct cpuinfo_mips *c) LOONGSON_CFG3_LCAMVW_REV1); break; - case PRID_IMP_LOONGSON_64R | PRID_REV_LOONGSON2K_R1_0: - case PRID_IMP_LOONGSON_64R | PRID_REV_LOONGSON2K_R1_1: - case PRID_IMP_LOONGSON_64R | PRID_REV_LOONGSON2K_R1_2: - case PRID_IMP_LOONGSON_64R | PRID_REV_LOONGSON2K_R1_3: - decode_loongson_config6(c); - probe_uca(c); - - c->loongson3_cpucfg_data[0] |= (LOONGSON_CFG1_LSLDR0 | - LOONGSON_CFG1_LSSYNCI | LOONGSON_CFG1_LLSYNC | - LOONGSON_CFG1_TGTSYNC); - c->loongson3_cpucfg_data[1] |= (LOONGSON_CFG2_LBT1 | - LOONGSON_CFG2_LBT2 | LOONGSON_CFG2_LPMP | - LOONGSON_CFG2_LPM_REV2); - c->loongson3_cpucfg_data[2] = 0; - break; - case PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_0: case PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R2_1: case PRID_IMP_LOONGSON_64C | PRID_REV_LOONGSON3A_R3_0: From 389500696810f5aa8fc1fe6f375b9dabd111e1d6 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sun, 31 May 2020 12:06:03 +0200 Subject: [PATCH 0979/1043] MIPS: ralink: bootrom: mark a function as __init to save some memory 'bootrom_setup()' is only called via 'postcore_initcall'. It can be marked as __init to save a few bytes of memory. Signed-off-by: Christophe JAILLET Signed-off-by: Thomas Bogendoerfer --- arch/mips/ralink/bootrom.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/mips/ralink/bootrom.c b/arch/mips/ralink/bootrom.c index 88bcce59beeb..94ca8379b83c 100644 --- a/arch/mips/ralink/bootrom.c +++ b/arch/mips/ralink/bootrom.c @@ -31,7 +31,7 @@ static const struct file_operations bootrom_file_ops = { .release = single_release, }; -static int bootrom_setup(void) +static int __init bootrom_setup(void) { debugfs_create_file("bootrom", 0444, NULL, NULL, &bootrom_file_ops); return 0; From 9bd0bd264578fe191bf5d2ff23f9887b91862536 Mon Sep 17 00:00:00 2001 From: Chuanhong Guo Date: Sat, 28 Mar 2020 12:14:57 +0800 Subject: [PATCH 0980/1043] MIPS: ralink: drop ralink_clk_init for mt7621 ralink_clk_init is only called in arch/mips/ralink/clk.c which isn't compiled for mt7621. And it doesn't export a proper cpu clock. Drop this unused function. Signed-off-by: Chuanhong Guo Signed-off-by: Thomas Bogendoerfer --- arch/mips/ralink/mt7621.c | 43 --------------------------------------- 1 file changed, 43 deletions(-) diff --git a/arch/mips/ralink/mt7621.c b/arch/mips/ralink/mt7621.c index 0accb80db709..ca0ac607b0f3 100644 --- a/arch/mips/ralink/mt7621.c +++ b/arch/mips/ralink/mt7621.c @@ -20,11 +20,6 @@ #include "common.h" -#define SYSC_REG_SYSCFG 0x10 -#define SYSC_REG_CPLL_CLKCFG0 0x2c -#define SYSC_REG_CUR_CLK_STS 0x44 -#define CPU_CLK_SEL (BIT(30) | BIT(31)) - #define MT7621_GPIO_MODE_UART1 1 #define MT7621_GPIO_MODE_I2C 2 #define MT7621_GPIO_MODE_UART3_MASK 0x3 @@ -115,44 +110,6 @@ phys_addr_t mips_cpc_default_phys_base(void) panic("Cannot detect cpc address"); } -void __init ralink_clk_init(void) -{ - int cpu_fdiv = 0; - int cpu_ffrac = 0; - int fbdiv = 0; - u32 clk_sts, syscfg; - u8 clk_sel = 0, xtal_mode; - u32 cpu_clk; - - if ((rt_sysc_r32(SYSC_REG_CPLL_CLKCFG0) & CPU_CLK_SEL) != 0) - clk_sel = 1; - - switch (clk_sel) { - case 0: - clk_sts = rt_sysc_r32(SYSC_REG_CUR_CLK_STS); - cpu_fdiv = ((clk_sts >> 8) & 0x1F); - cpu_ffrac = (clk_sts & 0x1F); - cpu_clk = (500 * cpu_ffrac / cpu_fdiv) * 1000 * 1000; - break; - - case 1: - fbdiv = ((rt_sysc_r32(0x648) >> 4) & 0x7F) + 1; - syscfg = rt_sysc_r32(SYSC_REG_SYSCFG); - xtal_mode = (syscfg >> 6) & 0x7; - if (xtal_mode >= 6) { - /* 25Mhz Xtal */ - cpu_clk = 25 * fbdiv * 1000 * 1000; - } else if (xtal_mode >= 3) { - /* 40Mhz Xtal */ - cpu_clk = 40 * fbdiv * 1000 * 1000; - } else { - /* 20Mhz Xtal */ - cpu_clk = 20 * fbdiv * 1000 * 1000; - } - break; - } -} - void __init ralink_of_remap(void) { rt_sysc_membase = plat_of_remap_node("mtk,mt7621-sysc"); From d77aeb5d403d379ff458e04fc07b5b86700270f2 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Mon, 1 Jun 2020 09:45:27 +0200 Subject: [PATCH 0981/1043] irqchip: Fix "Loongson HyperTransport Vector support" driver build on all non-MIPS platforms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit: 818e915fbac5: ("irqchip: Add Loongson HyperTransport Vector support") Added a MIPS-only driver, but turned on compilation on all other architectures as well: config LOONGSON_HTVEC bool "Loongson3 HyperTransport Interrupt Vector Controller" depends on MACH_LOONGSON64 || COMPILE_TEST But this driver was never build tested on any other architecture than MIPS: drivers/irqchip/irq-loongson-htvec.c: In function ‘htvec_irq_dispatch’: drivers/irqchip/irq-loongson-htvec.c:59:3: error: implicit declaration of function ‘spurious_interrupt’; did you mean ‘smp_reboot_interrupt’? [-Werror=implicit-function-declaration] Because spurious_interrupt() only exists on MIPS. So make it MIPS-only. Signed-off-by: Ingo Molnar Cc: Jiaxun Yang Cc: Marc Zyngier Cc: Thomas Gleixner Signed-off-by: Ingo Molnar --- drivers/irqchip/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 0b6b826dd843..66b9a68f5e9f 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -534,7 +534,7 @@ config LOONGSON_HTPIC config LOONGSON_HTVEC bool "Loongson3 HyperTransport Interrupt Vector Controller" - depends on MACH_LOONGSON64 || COMPILE_TEST + depends on MACH_LOONGSON64 default MACH_LOONGSON64 select IRQ_DOMAIN_HIERARCHY help From 45f08f4cd6679adebca9fb17a46df40e251cd2ff Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 1 Jun 2020 04:17:45 -0400 Subject: [PATCH 0982/1043] KVM: check userspace_addr for all memslots The userspace_addr alignment and range checks are not performed for private memory slots that are prepared by KVM itself. This is unnecessary and makes it questionable to use __*_user functions to access memory later on. We also rely on the userspace address being aligned since we have an entire family of functions to map gfn to pfn. Fortunately skipping the check is completely unnecessary. Only x86 uses private memslots and their userspace_addr is obtained from vm_mmap, therefore it must be below PAGE_OFFSET. In fact, any attempt to pass an address above PAGE_OFFSET would have failed because such an address would return true for kvm_is_error_hva. Reported-by: Linus Torvalds Signed-off-by: Paolo Bonzini --- virt/kvm/kvm_main.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 731c1e517716..08184f571669 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1223,10 +1223,9 @@ int __kvm_set_memory_region(struct kvm *kvm, if (mem->guest_phys_addr & (PAGE_SIZE - 1)) return -EINVAL; /* We can read the guest memory with __xxx_user() later on. */ - if ((id < KVM_USER_MEM_SLOTS) && - ((mem->userspace_addr & (PAGE_SIZE - 1)) || + if ((mem->userspace_addr & (PAGE_SIZE - 1)) || !access_ok((void __user *)(unsigned long)mem->userspace_addr, - mem->memory_size))) + mem->memory_size)) return -EINVAL; if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) return -EINVAL; From 69c9dfa24bb7bac5c9e2bd4d3f631e35b91e3733 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 13 May 2020 12:57:26 -0400 Subject: [PATCH 0983/1043] KVM: nSVM: move map argument out of enter_svm_guest_mode Unmapping the nested VMCB in enter_svm_guest_mode is a bit of a wart, since the map argument is not used elsewhere in the function. There are just two callers, and those are also the place where kvm_vcpu_map is called, so it is cleaner to unmap there. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 14 ++++++-------- arch/x86/kvm/svm/svm.c | 3 ++- arch/x86/kvm/svm/svm.h | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 8756c9f463fd..8e98d5e420a5 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -229,7 +229,7 @@ static bool nested_vmcb_checks(struct vmcb *vmcb) } void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, - struct vmcb *nested_vmcb, struct kvm_host_map *map) + struct vmcb *nested_vmcb) { bool evaluate_pending_interrupts = is_intercept(svm, INTERCEPT_VINTR) || @@ -304,8 +304,6 @@ void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, svm->vmcb->control.pause_filter_thresh = nested_vmcb->control.pause_filter_thresh; - kvm_vcpu_unmap(&svm->vcpu, map, true); - /* Enter Guest-Mode */ enter_guest_mode(&svm->vcpu); @@ -368,10 +366,7 @@ int nested_svm_vmrun(struct vcpu_svm *svm) nested_vmcb->control.exit_code_hi = 0; nested_vmcb->control.exit_info_1 = 0; nested_vmcb->control.exit_info_2 = 0; - - kvm_vcpu_unmap(&svm->vcpu, &map, true); - - return ret; + goto out; } trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa, @@ -414,7 +409,7 @@ int nested_svm_vmrun(struct vcpu_svm *svm) copy_vmcb_control_area(hsave, vmcb); svm->nested.nested_run_pending = 1; - enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, &map); + enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb); if (!nested_svm_vmrun_msrpm(svm)) { svm->vmcb->control.exit_code = SVM_EXIT_ERR; @@ -425,6 +420,9 @@ int nested_svm_vmrun(struct vcpu_svm *svm) nested_svm_vmexit(svm); } +out: + kvm_vcpu_unmap(&svm->vcpu, &map, true); + return ret; } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index feb96a410f2d..76b3f553815e 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3814,7 +3814,8 @@ static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) if (kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb), &map) == -EINVAL) return 1; nested_vmcb = map.hva; - enter_svm_guest_mode(svm, vmcb, nested_vmcb, &map); + enter_svm_guest_mode(svm, vmcb, nested_vmcb); + kvm_vcpu_unmap(&svm->vcpu, &map, true); } return 0; } diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 89fab75dd4f5..33e3f09d7a8e 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -395,7 +395,7 @@ static inline bool nested_exit_on_nmi(struct vcpu_svm *svm) } void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, - struct vmcb *nested_vmcb, struct kvm_host_map *map); + struct vmcb *nested_vmcb); int nested_svm_vmrun(struct vcpu_svm *svm); void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb); int nested_svm_vmexit(struct vcpu_svm *svm); From 3e06f0163f7f75364290bf338f4e279f6e3e404f Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 13 May 2020 13:07:26 -0400 Subject: [PATCH 0984/1043] KVM: nSVM: extract load_nested_vmcb_control When restoring SVM nested state, the control state cache in svm->nested will have to be filled, but the save state will not have to be moved into svm->vmcb. Therefore, pull the code that handles the control area into a separate function. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 8e98d5e420a5..fc0c6d1678eb 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -228,6 +228,23 @@ static bool nested_vmcb_checks(struct vmcb *vmcb) return true; } +static void load_nested_vmcb_control(struct vcpu_svm *svm, + struct vmcb_control_area *control) +{ + svm->nested.nested_cr3 = control->nested_cr3; + + svm->nested.vmcb_msrpm = control->msrpm_base_pa & ~0x0fffULL; + svm->nested.vmcb_iopm = control->iopm_base_pa & ~0x0fffULL; + + /* cache intercepts */ + svm->nested.intercept_cr = control->intercept_cr; + svm->nested.intercept_dr = control->intercept_dr; + svm->nested.intercept_exceptions = control->intercept_exceptions; + svm->nested.intercept = control->intercept; + + svm->vcpu.arch.tsc_offset += control->tsc_offset; +} + void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, struct vmcb *nested_vmcb) { @@ -235,15 +252,16 @@ void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, is_intercept(svm, INTERCEPT_VINTR) || is_intercept(svm, INTERCEPT_IRET); + svm->nested.vmcb = vmcb_gpa; if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF) svm->vcpu.arch.hflags |= HF_HIF_MASK; else svm->vcpu.arch.hflags &= ~HF_HIF_MASK; - if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) { - svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3; + load_nested_vmcb_control(svm, &nested_vmcb->control); + + if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) nested_svm_init_mmu_context(&svm->vcpu); - } /* Load the nested guest state */ svm->vmcb->save.es = nested_vmcb->save.es; @@ -274,25 +292,15 @@ void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, svm->vcpu.arch.dr6 = nested_vmcb->save.dr6; svm->vmcb->save.cpl = nested_vmcb->save.cpl; - svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL; - svm->nested.vmcb_iopm = nested_vmcb->control.iopm_base_pa & ~0x0fffULL; - - /* cache intercepts */ - svm->nested.intercept_cr = nested_vmcb->control.intercept_cr; - svm->nested.intercept_dr = nested_vmcb->control.intercept_dr; - svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions; - svm->nested.intercept = nested_vmcb->control.intercept; - svm_flush_tlb(&svm->vcpu); - svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK; if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK) svm->vcpu.arch.hflags |= HF_VINTR_MASK; else svm->vcpu.arch.hflags &= ~HF_VINTR_MASK; - svm->vcpu.arch.tsc_offset += nested_vmcb->control.tsc_offset; svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset; + svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK; svm->vmcb->control.virt_ext = nested_vmcb->control.virt_ext; svm->vmcb->control.int_vector = nested_vmcb->control.int_vector; svm->vmcb->control.int_state = nested_vmcb->control.int_state; @@ -313,8 +321,6 @@ void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, */ recalc_intercepts(svm); - svm->nested.vmcb = vmcb_gpa; - /* * If L1 had a pending IRQ/NMI before executing VMRUN, * which wasn't delivered because it was disallowed (e.g. From f241d711b2d17f79ae5fad78c5644c674fce42ba Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 18 May 2020 10:56:43 -0400 Subject: [PATCH 0985/1043] KVM: nSVM: extract preparation of VMCB for nested run Split out filling svm->vmcb.save and svm->vmcb.control before VMRUN. Only the latter will be useful when restoring nested SVM state. This patch introduces no semantic change, so the MMU setup is still done in nested_prepare_vmcb_save. The next patch will clean up things. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 40 +++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index fc0c6d1678eb..73be7af79453 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -245,21 +245,8 @@ static void load_nested_vmcb_control(struct vcpu_svm *svm, svm->vcpu.arch.tsc_offset += control->tsc_offset; } -void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, - struct vmcb *nested_vmcb) +static void nested_prepare_vmcb_save(struct vcpu_svm *svm, struct vmcb *nested_vmcb) { - bool evaluate_pending_interrupts = - is_intercept(svm, INTERCEPT_VINTR) || - is_intercept(svm, INTERCEPT_IRET); - - svm->nested.vmcb = vmcb_gpa; - if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF) - svm->vcpu.arch.hflags |= HF_HIF_MASK; - else - svm->vcpu.arch.hflags &= ~HF_HIF_MASK; - - load_nested_vmcb_control(svm, &nested_vmcb->control); - if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) nested_svm_init_mmu_context(&svm->vcpu); @@ -291,7 +278,10 @@ void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, svm->vmcb->save.dr7 = nested_vmcb->save.dr7; svm->vcpu.arch.dr6 = nested_vmcb->save.dr6; svm->vmcb->save.cpl = nested_vmcb->save.cpl; +} +static void nested_prepare_vmcb_control(struct vcpu_svm *svm, struct vmcb *nested_vmcb) +{ svm_flush_tlb(&svm->vcpu); if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK) svm->vcpu.arch.hflags |= HF_VINTR_MASK; @@ -321,6 +311,26 @@ void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, */ recalc_intercepts(svm); + mark_all_dirty(svm->vmcb); +} + +void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, + struct vmcb *nested_vmcb) +{ + bool evaluate_pending_interrupts = + is_intercept(svm, INTERCEPT_VINTR) || + is_intercept(svm, INTERCEPT_IRET); + + svm->nested.vmcb = vmcb_gpa; + if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF) + svm->vcpu.arch.hflags |= HF_HIF_MASK; + else + svm->vcpu.arch.hflags &= ~HF_HIF_MASK; + + load_nested_vmcb_control(svm, &nested_vmcb->control); + nested_prepare_vmcb_save(svm, nested_vmcb); + nested_prepare_vmcb_control(svm, nested_vmcb); + /* * If L1 had a pending IRQ/NMI before executing VMRUN, * which wasn't delivered because it was disallowed (e.g. @@ -336,8 +346,6 @@ void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, enable_gif(svm); if (unlikely(evaluate_pending_interrupts)) kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); - - mark_all_dirty(svm->vmcb); } int nested_svm_vmrun(struct vcpu_svm *svm) From 69cb877487de3662e08081cf036f8fe0e4a3b806 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 22 May 2020 05:27:46 -0400 Subject: [PATCH 0986/1043] KVM: nSVM: move MMU setup to nested_prepare_vmcb_control Everything that is needed during nested state restore is now part of nested_prepare_vmcb_control. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 73be7af79453..a85cc7376a82 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -247,9 +247,6 @@ static void load_nested_vmcb_control(struct vcpu_svm *svm, static void nested_prepare_vmcb_save(struct vcpu_svm *svm, struct vmcb *nested_vmcb) { - if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) - nested_svm_init_mmu_context(&svm->vcpu); - /* Load the nested guest state */ svm->vmcb->save.es = nested_vmcb->save.es; svm->vmcb->save.cs = nested_vmcb->save.cs; @@ -263,9 +260,6 @@ static void nested_prepare_vmcb_save(struct vcpu_svm *svm, struct vmcb *nested_v svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4); (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3); - /* Guest paging mode is active - reset mmu */ - kvm_mmu_reset_context(&svm->vcpu); - svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2; kvm_rax_write(&svm->vcpu, nested_vmcb->save.rax); kvm_rsp_write(&svm->vcpu, nested_vmcb->save.rsp); @@ -282,6 +276,12 @@ static void nested_prepare_vmcb_save(struct vcpu_svm *svm, struct vmcb *nested_v static void nested_prepare_vmcb_control(struct vcpu_svm *svm, struct vmcb *nested_vmcb) { + if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) + nested_svm_init_mmu_context(&svm->vcpu); + + /* Guest paging mode is active - reset mmu */ + kvm_mmu_reset_context(&svm->vcpu); + svm_flush_tlb(&svm->vcpu); if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK) svm->vcpu.arch.hflags |= HF_VINTR_MASK; From 18fc6c55d1f449ff57778a4003c838a79d62b5a8 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 18 May 2020 11:07:08 -0400 Subject: [PATCH 0987/1043] KVM: nSVM: clean up tsc_offset update Use l1_tsc_offset to compute svm->vcpu.arch.tsc_offset and svm->vmcb->control.tsc_offset, instead of relying on hsave. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index a85cc7376a82..5ca403a69148 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -241,8 +241,6 @@ static void load_nested_vmcb_control(struct vcpu_svm *svm, svm->nested.intercept_dr = control->intercept_dr; svm->nested.intercept_exceptions = control->intercept_exceptions; svm->nested.intercept = control->intercept; - - svm->vcpu.arch.tsc_offset += control->tsc_offset; } static void nested_prepare_vmcb_save(struct vcpu_svm *svm, struct vmcb *nested_vmcb) @@ -288,7 +286,8 @@ static void nested_prepare_vmcb_control(struct vcpu_svm *svm, struct vmcb *neste else svm->vcpu.arch.hflags &= ~HF_VINTR_MASK; - svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset; + svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset = + svm->vcpu.arch.l1_tsc_offset + nested_vmcb->control.tsc_offset; svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK; svm->vmcb->control.virt_ext = nested_vmcb->control.virt_ext; @@ -553,7 +552,9 @@ int nested_svm_vmexit(struct vcpu_svm *svm) /* Restore the original control entries */ copy_vmcb_control_area(vmcb, hsave); - svm->vcpu.arch.tsc_offset = svm->vmcb->control.tsc_offset; + svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset = + svm->vcpu.arch.l1_tsc_offset; + kvm_clear_exception_queue(&svm->vcpu); kvm_clear_interrupt_queue(&svm->vcpu); From 2f675917efc8a4d469f8be95e6ea55b2310bfd0d Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 18 May 2020 15:21:22 -0400 Subject: [PATCH 0988/1043] KVM: nSVM: pass vmcb_control_area to copy_vmcb_control_area This will come in handy when we put a struct vmcb_control_area in svm->nested. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 5ca403a69148..fd9742c1a860 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -141,11 +141,9 @@ void recalc_intercepts(struct vcpu_svm *svm) c->intercept |= g->intercept; } -static void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb) +static void copy_vmcb_control_area(struct vmcb_control_area *dst, + struct vmcb_control_area *from) { - struct vmcb_control_area *dst = &dst_vmcb->control; - struct vmcb_control_area *from = &from_vmcb->control; - dst->intercept_cr = from->intercept_cr; dst->intercept_dr = from->intercept_dr; dst->intercept_exceptions = from->intercept_exceptions; @@ -419,7 +417,7 @@ int nested_svm_vmrun(struct vcpu_svm *svm) else hsave->save.cr3 = kvm_read_cr3(&svm->vcpu); - copy_vmcb_control_area(hsave, vmcb); + copy_vmcb_control_area(&hsave->control, &vmcb->control); svm->nested.nested_run_pending = 1; enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb); @@ -550,7 +548,7 @@ int nested_svm_vmexit(struct vcpu_svm *svm) nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK; /* Restore the original control entries */ - copy_vmcb_control_area(vmcb, hsave); + copy_vmcb_control_area(&vmcb->control, &hsave->control); svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset; From 7923ef4f6ec4a25a902bd827446eac860b01fd1c Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 18 May 2020 15:24:46 -0400 Subject: [PATCH 0989/1043] KVM: nSVM: remove trailing padding for struct vmcb_control_area Allow placing the VMCB structs on the stack or in other structs without wasting too much space. Add BUILD_BUG_ON as a quick safeguard against typos. Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/svm.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index 6ece8561ba66..8a1f5382a4ea 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -96,7 +96,6 @@ struct __attribute__ ((__packed__)) vmcb_control_area { u8 reserved_6[8]; /* Offset 0xe8 */ u64 avic_logical_id; /* Offset 0xf0 */ u64 avic_physical_id; /* Offset 0xf8 */ - u8 reserved_7[768]; }; @@ -203,8 +202,16 @@ struct __attribute__ ((__packed__)) vmcb_save_area { u64 last_excp_to; }; + +static inline void __unused_size_checks(void) +{ + BUILD_BUG_ON(sizeof(struct vmcb_save_area) != 0x298); + BUILD_BUG_ON(sizeof(struct vmcb_control_area) != 256); +} + struct __attribute__ ((__packed__)) vmcb { struct vmcb_control_area control; + u8 reserved_control[1024 - sizeof(struct vmcb_control_area)]; struct vmcb_save_area save; }; From e670bf68f4b701506d51f007917ab633894294d0 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 13 May 2020 13:16:12 -0400 Subject: [PATCH 0990/1043] KVM: nSVM: save all control fields in svm->nested In preparation for nested SVM save/restore, store all data that matters from the VMCB control area into svm->nested. It will then become part of the nested SVM state that is saved by KVM_SET_NESTED_STATE and restored by KVM_GET_NESTED_STATE, just like the cached vmcs12 for nVMX. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 73 +++++++++++++++++---------------------- arch/x86/kvm/svm/svm.c | 4 +-- arch/x86/kvm/svm/svm.h | 20 +++-------- 3 files changed, 39 insertions(+), 58 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index fd9742c1a860..1e5f460b5540 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -60,7 +60,7 @@ static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu, static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index) { struct vcpu_svm *svm = to_svm(vcpu); - u64 cr3 = svm->nested.nested_cr3; + u64 cr3 = svm->nested.ctl.nested_cr3; u64 pdpte; int ret; @@ -75,7 +75,7 @@ static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - return svm->nested.nested_cr3; + return svm->nested.ctl.nested_cr3; } static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) @@ -100,8 +100,7 @@ static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu) void recalc_intercepts(struct vcpu_svm *svm) { - struct vmcb_control_area *c, *h; - struct nested_state *g; + struct vmcb_control_area *c, *h, *g; mark_dirty(svm->vmcb, VMCB_INTERCEPTS); @@ -110,7 +109,7 @@ void recalc_intercepts(struct vcpu_svm *svm) c = &svm->vmcb->control; h = &svm->nested.hsave->control; - g = &svm->nested; + g = &svm->nested.ctl; svm->nested.host_intercept_exceptions = h->intercept_exceptions; @@ -180,7 +179,7 @@ static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) */ int i; - if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT))) + if (!(svm->nested.ctl.intercept & (1ULL << INTERCEPT_MSR_PROT))) return true; for (i = 0; i < MSRPM_OFFSETS; i++) { @@ -191,7 +190,7 @@ static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) break; p = msrpm_offsets[i]; - offset = svm->nested.vmcb_msrpm + (p * 4); + offset = svm->nested.ctl.msrpm_base_pa + (p * 4); if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4)) return false; @@ -229,16 +228,10 @@ static bool nested_vmcb_checks(struct vmcb *vmcb) static void load_nested_vmcb_control(struct vcpu_svm *svm, struct vmcb_control_area *control) { - svm->nested.nested_cr3 = control->nested_cr3; + copy_vmcb_control_area(&svm->nested.ctl, control); - svm->nested.vmcb_msrpm = control->msrpm_base_pa & ~0x0fffULL; - svm->nested.vmcb_iopm = control->iopm_base_pa & ~0x0fffULL; - - /* cache intercepts */ - svm->nested.intercept_cr = control->intercept_cr; - svm->nested.intercept_dr = control->intercept_dr; - svm->nested.intercept_exceptions = control->intercept_exceptions; - svm->nested.intercept = control->intercept; + svm->nested.ctl.msrpm_base_pa &= ~0x0fffULL; + svm->nested.ctl.iopm_base_pa &= ~0x0fffULL; } static void nested_prepare_vmcb_save(struct vcpu_svm *svm, struct vmcb *nested_vmcb) @@ -270,34 +263,32 @@ static void nested_prepare_vmcb_save(struct vcpu_svm *svm, struct vmcb *nested_v svm->vmcb->save.cpl = nested_vmcb->save.cpl; } -static void nested_prepare_vmcb_control(struct vcpu_svm *svm, struct vmcb *nested_vmcb) +static void nested_prepare_vmcb_control(struct vcpu_svm *svm) { - if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) + if (svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) nested_svm_init_mmu_context(&svm->vcpu); /* Guest paging mode is active - reset mmu */ kvm_mmu_reset_context(&svm->vcpu); svm_flush_tlb(&svm->vcpu); - if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK) + if (svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) svm->vcpu.arch.hflags |= HF_VINTR_MASK; else svm->vcpu.arch.hflags &= ~HF_VINTR_MASK; svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset = - svm->vcpu.arch.l1_tsc_offset + nested_vmcb->control.tsc_offset; + svm->vcpu.arch.l1_tsc_offset + svm->nested.ctl.tsc_offset; - svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK; - svm->vmcb->control.virt_ext = nested_vmcb->control.virt_ext; - svm->vmcb->control.int_vector = nested_vmcb->control.int_vector; - svm->vmcb->control.int_state = nested_vmcb->control.int_state; - svm->vmcb->control.event_inj = nested_vmcb->control.event_inj; - svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err; + svm->vmcb->control.int_ctl = svm->nested.ctl.int_ctl | V_INTR_MASKING_MASK; + svm->vmcb->control.virt_ext = svm->nested.ctl.virt_ext; + svm->vmcb->control.int_vector = svm->nested.ctl.int_vector; + svm->vmcb->control.int_state = svm->nested.ctl.int_state; + svm->vmcb->control.event_inj = svm->nested.ctl.event_inj; + svm->vmcb->control.event_inj_err = svm->nested.ctl.event_inj_err; - svm->vmcb->control.pause_filter_count = - nested_vmcb->control.pause_filter_count; - svm->vmcb->control.pause_filter_thresh = - nested_vmcb->control.pause_filter_thresh; + svm->vmcb->control.pause_filter_count = svm->nested.ctl.pause_filter_count; + svm->vmcb->control.pause_filter_thresh = svm->nested.ctl.pause_filter_thresh; /* Enter Guest-Mode */ enter_guest_mode(&svm->vcpu); @@ -326,7 +317,7 @@ void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, load_nested_vmcb_control(svm, &nested_vmcb->control); nested_prepare_vmcb_save(svm, nested_vmcb); - nested_prepare_vmcb_control(svm, nested_vmcb); + nested_prepare_vmcb_control(svm); /* * If L1 had a pending IRQ/NMI before executing VMRUN, @@ -556,7 +547,7 @@ int nested_svm_vmexit(struct vcpu_svm *svm) kvm_clear_exception_queue(&svm->vcpu); kvm_clear_interrupt_queue(&svm->vcpu); - svm->nested.nested_cr3 = 0; + svm->nested.ctl.nested_cr3 = 0; /* Restore selected save entries */ svm->vmcb->save.es = hsave->save.es; @@ -606,7 +597,7 @@ static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) u32 offset, msr, value; int write, mask; - if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT))) + if (!(svm->nested.ctl.intercept & (1ULL << INTERCEPT_MSR_PROT))) return NESTED_EXIT_HOST; msr = svm->vcpu.arch.regs[VCPU_REGS_RCX]; @@ -620,7 +611,7 @@ static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) /* Offset is in 32 bit units but need in 8 bit units */ offset *= 4; - if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4)) + if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.ctl.msrpm_base_pa + offset, &value, 4)) return NESTED_EXIT_DONE; return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST; @@ -633,13 +624,13 @@ static int nested_svm_intercept_ioio(struct vcpu_svm *svm) u8 start_bit; u64 gpa; - if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT))) + if (!(svm->nested.ctl.intercept & (1ULL << INTERCEPT_IOIO_PROT))) return NESTED_EXIT_HOST; port = svm->vmcb->control.exit_info_1 >> 16; size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; - gpa = svm->nested.vmcb_iopm + (port / 8); + gpa = svm->nested.ctl.iopm_base_pa + (port / 8); start_bit = port % 8; iopm_len = (start_bit + size > 8) ? 2 : 1; mask = (0xf >> (4 - size)) << start_bit; @@ -665,13 +656,13 @@ static int nested_svm_intercept(struct vcpu_svm *svm) break; case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: { u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0); - if (svm->nested.intercept_cr & bit) + if (svm->nested.ctl.intercept_cr & bit) vmexit = NESTED_EXIT_DONE; break; } case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: { u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0); - if (svm->nested.intercept_dr & bit) + if (svm->nested.ctl.intercept_dr & bit) vmexit = NESTED_EXIT_DONE; break; } @@ -690,7 +681,7 @@ static int nested_svm_intercept(struct vcpu_svm *svm) } default: { u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR); - if (svm->nested.intercept & exit_bits) + if (svm->nested.ctl.intercept & exit_bits) vmexit = NESTED_EXIT_DONE; } } @@ -730,7 +721,7 @@ static bool nested_exit_on_exception(struct vcpu_svm *svm) { unsigned int nr = svm->vcpu.arch.exception.nr; - return (svm->nested.intercept_exceptions & (1 << nr)); + return (svm->nested.ctl.intercept_exceptions & (1 << nr)); } static void nested_svm_inject_exception_vmexit(struct vcpu_svm *svm) @@ -798,7 +789,7 @@ static void nested_svm_intr(struct vcpu_svm *svm) static inline bool nested_exit_on_init(struct vcpu_svm *svm) { - return (svm->nested.intercept & (1ULL << INTERCEPT_INIT)); + return (svm->nested.ctl.intercept & (1ULL << INTERCEPT_INIT)); } static void nested_svm_init(struct vcpu_svm *svm) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 76b3f553815e..ec98c5979656 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2173,7 +2173,7 @@ static bool check_selective_cr0_intercepted(struct vcpu_svm *svm, bool ret = false; u64 intercept; - intercept = svm->nested.intercept; + intercept = svm->nested.ctl.intercept; if (!is_guest_mode(&svm->vcpu) || (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))) @@ -3649,7 +3649,7 @@ static int svm_check_intercept(struct kvm_vcpu *vcpu, info->intercept == x86_intercept_clts) break; - intercept = svm->nested.intercept; + intercept = svm->nested.ctl.intercept; if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))) break; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 33e3f09d7a8e..dd5418f20256 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -91,22 +91,12 @@ struct nested_state { /* These are the merged vectors */ u32 *msrpm; - /* gpa pointers to the real vectors */ - u64 vmcb_msrpm; - u64 vmcb_iopm; - /* A VMRUN has started but has not yet been performed, so * we cannot inject a nested vmexit yet. */ bool nested_run_pending; - /* cache for intercepts of the guest */ - u32 intercept_cr; - u32 intercept_dr; - u32 intercept_exceptions; - u64 intercept; - - /* Nested Paging related state */ - u64 nested_cr3; + /* cache for control fields of the guest */ + struct vmcb_control_area ctl; }; struct vcpu_svm { @@ -381,17 +371,17 @@ static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu) static inline bool nested_exit_on_smi(struct vcpu_svm *svm) { - return (svm->nested.intercept & (1ULL << INTERCEPT_SMI)); + return (svm->nested.ctl.intercept & (1ULL << INTERCEPT_SMI)); } static inline bool nested_exit_on_intr(struct vcpu_svm *svm) { - return (svm->nested.intercept & (1ULL << INTERCEPT_INTR)); + return (svm->nested.ctl.intercept & (1ULL << INTERCEPT_INTR)); } static inline bool nested_exit_on_nmi(struct vcpu_svm *svm) { - return (svm->nested.intercept & (1ULL << INTERCEPT_NMI)); + return (svm->nested.ctl.intercept & (1ULL << INTERCEPT_NMI)); } void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, From d8e4e58f4bd4bb55d2640a841c3606333930f0e1 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 22 May 2020 07:38:20 -0400 Subject: [PATCH 0991/1043] KVM: nSVM: restore clobbered INT_CTL fields after clearing VINTR Restore the INT_CTL value from the guest's VMCB once we've stopped using it, so that virtual interrupts can be injected as requested by L1. V_TPR is up-to-date however, and it can change if the guest writes to CR8, so keep it. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index ec98c5979656..4122ba86bac2 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1365,9 +1365,17 @@ static void svm_set_vintr(struct vcpu_svm *svm) static void svm_clear_vintr(struct vcpu_svm *svm) { + const u32 mask = V_TPR_MASK | V_GIF_ENABLE_MASK | V_GIF_MASK | V_INTR_MASKING_MASK; clr_intercept(svm, INTERCEPT_VINTR); - svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; + /* Drop int_ctl fields related to VINTR injection. */ + svm->vmcb->control.int_ctl &= mask; + if (is_guest_mode(&svm->vcpu)) { + WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) != + (svm->nested.ctl.int_ctl & V_TPR_MASK)); + svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl & ~mask; + } + mark_dirty(svm->vmcb, VMCB_INTR); } From 2d8a42be0e2b15a4e0b20349f27bb8288db5ebe6 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 22 May 2020 03:50:14 -0400 Subject: [PATCH 0992/1043] KVM: nSVM: synchronize VMCB controls updated by the processor on every vmexit The control state changes on every L2->L0 vmexit, and we will have to serialize it in the nested state. So keep it up to date in svm->nested.ctl and just copy them back to the nested VMCB in nested_svm_vmexit. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 57 ++++++++++++++++++++++----------------- arch/x86/kvm/svm/svm.c | 5 +++- arch/x86/kvm/svm/svm.h | 1 + 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 1e5f460b5540..921466eba556 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -234,6 +234,34 @@ static void load_nested_vmcb_control(struct vcpu_svm *svm, svm->nested.ctl.iopm_base_pa &= ~0x0fffULL; } +/* + * Synchronize fields that are written by the processor, so that + * they can be copied back into the nested_vmcb. + */ +void sync_nested_vmcb_control(struct vcpu_svm *svm) +{ + u32 mask; + svm->nested.ctl.event_inj = svm->vmcb->control.event_inj; + svm->nested.ctl.event_inj_err = svm->vmcb->control.event_inj_err; + + /* Only a few fields of int_ctl are written by the processor. */ + mask = V_IRQ_MASK | V_TPR_MASK; + if (!(svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) && + is_intercept(svm, SVM_EXIT_VINTR)) { + /* + * In order to request an interrupt window, L0 is usurping + * svm->vmcb->control.int_ctl and possibly setting V_IRQ + * even if it was clear in L1's VMCB. Restoring it would be + * wrong. However, in this case V_IRQ will remain true until + * interrupt_window_interception calls svm_clear_vintr and + * restores int_ctl. We can just leave it aside. + */ + mask &= ~V_IRQ_MASK; + } + svm->nested.ctl.int_ctl &= ~mask; + svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask; +} + static void nested_prepare_vmcb_save(struct vcpu_svm *svm, struct vmcb *nested_vmcb) { /* Load the nested guest state */ @@ -471,6 +499,7 @@ int nested_svm_vmexit(struct vcpu_svm *svm) /* Exit Guest-Mode */ leave_guest_mode(&svm->vcpu); svm->nested.vmcb = 0; + WARN_ON_ONCE(svm->nested.nested_run_pending); /* in case we halted in L2 */ svm->vcpu.arch.mp_state = KVM_MP_STATE_RUNNABLE; @@ -497,8 +526,6 @@ int nested_svm_vmexit(struct vcpu_svm *svm) nested_vmcb->save.dr6 = svm->vcpu.arch.dr6; nested_vmcb->save.cpl = vmcb->save.cpl; - nested_vmcb->control.int_ctl = vmcb->control.int_ctl; - nested_vmcb->control.int_vector = vmcb->control.int_vector; nested_vmcb->control.int_state = vmcb->control.int_state; nested_vmcb->control.exit_code = vmcb->control.exit_code; nested_vmcb->control.exit_code_hi = vmcb->control.exit_code_hi; @@ -510,34 +537,16 @@ int nested_svm_vmexit(struct vcpu_svm *svm) if (svm->nrips_enabled) nested_vmcb->control.next_rip = vmcb->control.next_rip; - /* - * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have - * to make sure that we do not lose injected events. So check event_inj - * here and copy it to exit_int_info if it is valid. - * Exit_int_info and event_inj can't be both valid because the case - * below only happens on a VMRUN instruction intercept which has - * no valid exit_int_info set. - */ - if (vmcb->control.event_inj & SVM_EVTINJ_VALID) { - struct vmcb_control_area *nc = &nested_vmcb->control; - - nc->exit_int_info = vmcb->control.event_inj; - nc->exit_int_info_err = vmcb->control.event_inj_err; - } - - nested_vmcb->control.tlb_ctl = 0; - nested_vmcb->control.event_inj = 0; - nested_vmcb->control.event_inj_err = 0; + nested_vmcb->control.int_ctl = svm->nested.ctl.int_ctl; + nested_vmcb->control.tlb_ctl = svm->nested.ctl.tlb_ctl; + nested_vmcb->control.event_inj = svm->nested.ctl.event_inj; + nested_vmcb->control.event_inj_err = svm->nested.ctl.event_inj_err; nested_vmcb->control.pause_filter_count = svm->vmcb->control.pause_filter_count; nested_vmcb->control.pause_filter_thresh = svm->vmcb->control.pause_filter_thresh; - /* We always set V_INTR_MASKING and remember the old value in hflags */ - if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK)) - nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK; - /* Restore the original control entries */ copy_vmcb_control_area(&vmcb->control, &hsave->control); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 4122ba86bac2..b710e62ace16 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3427,7 +3427,10 @@ static fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) sync_cr8_to_lapic(vcpu); svm->next_rip = 0; - svm->nested.nested_run_pending = 0; + if (is_guest_mode(&svm->vcpu)) { + sync_nested_vmcb_control(svm); + svm->nested.nested_run_pending = 0; + } svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index dd5418f20256..7e79f0af1204 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -394,6 +394,7 @@ int nested_svm_check_permissions(struct vcpu_svm *svm); int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, bool has_error_code, u32 error_code); int nested_svm_exit_special(struct vcpu_svm *svm); +void sync_nested_vmcb_control(struct vcpu_svm *svm); extern struct kvm_x86_nested_ops svm_nested_ops; From 31031098feb9e1233176896d31eaf766c13429df Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 22 May 2020 12:33:52 -0400 Subject: [PATCH 0993/1043] KVM: nSVM: remove unnecessary if kvm_vcpu_apicv_active must be false when nested virtualization is enabled, so there is no need to check it in clgi_interception. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index b710e62ace16..7383f821eb3b 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2015,8 +2015,7 @@ static int clgi_interception(struct vcpu_svm *svm) disable_gif(svm); /* After a CLGI no interrupts should come */ - if (!kvm_vcpu_apicv_active(&svm->vcpu)) - svm_clear_vintr(svm); + svm_clear_vintr(svm); return ret; } From ffdf7f9e80ac1b6fea3bc6a65ea1f264bc226eab Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 22 May 2020 12:18:27 -0400 Subject: [PATCH 0994/1043] KVM: nSVM: extract svm_set_gif Extract the code that is needed to implement CLGI and STGI, so that we can run it from VMRUN and vmexit (and in the future, KVM_SET_NESTED_STATE). Skip the request for KVM_REQ_EVENT unless needed, subsuming the evaluate_pending_interrupts optimization that is found in enter_svm_guest_mode. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/irq.c | 1 + arch/x86/kvm/svm/nested.c | 22 ++--------------- arch/x86/kvm/svm/svm.c | 51 ++++++++++++++++++++++++++------------- arch/x86/kvm/svm/svm.h | 1 + 4 files changed, 38 insertions(+), 37 deletions(-) diff --git a/arch/x86/kvm/irq.c b/arch/x86/kvm/irq.c index 54f7ea68083b..99d118ffc67d 100644 --- a/arch/x86/kvm/irq.c +++ b/arch/x86/kvm/irq.c @@ -83,6 +83,7 @@ int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v) return kvm_apic_has_interrupt(v) != -1; /* LAPIC */ } +EXPORT_SYMBOL_GPL(kvm_cpu_has_injectable_intr); /* * check if there is pending interrupt without diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 921466eba556..7e4a506828c9 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -333,10 +333,6 @@ static void nested_prepare_vmcb_control(struct vcpu_svm *svm) void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, struct vmcb *nested_vmcb) { - bool evaluate_pending_interrupts = - is_intercept(svm, INTERCEPT_VINTR) || - is_intercept(svm, INTERCEPT_IRET); - svm->nested.vmcb = vmcb_gpa; if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF) svm->vcpu.arch.hflags |= HF_HIF_MASK; @@ -347,21 +343,7 @@ void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, nested_prepare_vmcb_save(svm, nested_vmcb); nested_prepare_vmcb_control(svm); - /* - * If L1 had a pending IRQ/NMI before executing VMRUN, - * which wasn't delivered because it was disallowed (e.g. - * interrupts disabled), L0 needs to evaluate if this pending - * event should cause an exit from L2 to L1 or be delivered - * directly to L2. - * - * Usually this would be handled by the processor noticing an - * IRQ/NMI window request. However, VMRUN can unblock interrupts - * by implicitly setting GIF, so force L0 to perform pending event - * evaluation by requesting a KVM_REQ_EVENT. - */ - enable_gif(svm); - if (unlikely(evaluate_pending_interrupts)) - kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); + svm_set_gif(svm, true); } int nested_svm_vmrun(struct vcpu_svm *svm) @@ -505,7 +487,7 @@ int nested_svm_vmexit(struct vcpu_svm *svm) svm->vcpu.arch.mp_state = KVM_MP_STATE_RUNNABLE; /* Give the current vmcb to the guest */ - disable_gif(svm); + svm_set_gif(svm, false); nested_vmcb->save.es = vmcb->save.es; nested_vmcb->save.cs = vmcb->save.cs; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 7383f821eb3b..e48e4173bc60 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1981,6 +1981,38 @@ static int vmrun_interception(struct vcpu_svm *svm) return nested_svm_vmrun(svm); } +void svm_set_gif(struct vcpu_svm *svm, bool value) +{ + if (value) { + /* + * If VGIF is enabled, the STGI intercept is only added to + * detect the opening of the SMI/NMI window; remove it now. + * Likewise, clear the VINTR intercept, we will set it + * again while processing KVM_REQ_EVENT if needed. + */ + if (vgif_enabled(svm)) + clr_intercept(svm, INTERCEPT_STGI); + if (is_intercept(svm, SVM_EXIT_VINTR)) + svm_clear_vintr(svm); + + enable_gif(svm); + if (svm->vcpu.arch.smi_pending || + svm->vcpu.arch.nmi_pending || + kvm_cpu_has_injectable_intr(&svm->vcpu)) + kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); + } else { + disable_gif(svm); + + /* + * After a CLGI no interrupts should come. But if vGIF is + * in use, we still rely on the VINTR intercept (rather than + * STGI) to detect an open interrupt window. + */ + if (!vgif_enabled(svm)) + svm_clear_vintr(svm); + } +} + static int stgi_interception(struct vcpu_svm *svm) { int ret; @@ -1988,18 +2020,8 @@ static int stgi_interception(struct vcpu_svm *svm) if (nested_svm_check_permissions(svm)) return 1; - /* - * If VGIF is enabled, the STGI intercept is only added to - * detect the opening of the SMI/NMI window; remove it now. - */ - if (vgif_enabled(svm)) - clr_intercept(svm, INTERCEPT_STGI); - ret = kvm_skip_emulated_instruction(&svm->vcpu); - kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); - - enable_gif(svm); - + svm_set_gif(svm, true); return ret; } @@ -2011,12 +2033,7 @@ static int clgi_interception(struct vcpu_svm *svm) return 1; ret = kvm_skip_emulated_instruction(&svm->vcpu); - - disable_gif(svm); - - /* After a CLGI no interrupts should come */ - svm_clear_vintr(svm); - + svm_set_gif(svm, false); return ret; } diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 7e79f0af1204..10b7b55720a0 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -357,6 +357,7 @@ void disable_nmi_singlestep(struct vcpu_svm *svm); bool svm_smi_blocked(struct kvm_vcpu *vcpu); bool svm_nmi_blocked(struct kvm_vcpu *vcpu); bool svm_interrupt_blocked(struct kvm_vcpu *vcpu); +void svm_set_gif(struct vcpu_svm *svm, bool value); /* nested.c */ From 91b7130cb6606d8c6b3b77e54426b3f3a83f48b1 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 22 May 2020 12:28:52 -0400 Subject: [PATCH 0995/1043] KVM: SVM: preserve VGIF across VMCB switch There is only one GIF flag for the whole processor, so make sure it is not clobbered when switching to L2 (in which case we also have to include the V_GIF_ENABLE_MASK, lest we confuse enable_gif/disable_gif/gif_set). When going back, L1 could in theory have entered L2 without issuing a CLGI so make sure the svm_set_gif is done last, after svm->vmcb->control.int_ctl has been copied back from hsave. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 7e4a506828c9..6c7f0bffdf01 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -293,6 +293,7 @@ static void nested_prepare_vmcb_save(struct vcpu_svm *svm, struct vmcb *nested_v static void nested_prepare_vmcb_control(struct vcpu_svm *svm) { + const u32 mask = V_INTR_MASKING_MASK | V_GIF_ENABLE_MASK | V_GIF_MASK; if (svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) nested_svm_init_mmu_context(&svm->vcpu); @@ -308,7 +309,10 @@ static void nested_prepare_vmcb_control(struct vcpu_svm *svm) svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset + svm->nested.ctl.tsc_offset; - svm->vmcb->control.int_ctl = svm->nested.ctl.int_ctl | V_INTR_MASKING_MASK; + svm->vmcb->control.int_ctl = + (svm->nested.ctl.int_ctl & ~mask) | + (svm->nested.hsave->control.int_ctl & mask); + svm->vmcb->control.virt_ext = svm->nested.ctl.virt_ext; svm->vmcb->control.int_vector = svm->nested.ctl.int_vector; svm->vmcb->control.int_state = svm->nested.ctl.int_state; From 36e2e98363e6c13288de6824b51866292dbc151d Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 22 May 2020 06:04:57 -0400 Subject: [PATCH 0996/1043] KVM: nSVM: synthesize correct EXITINTINFO on vmexit This bit was added to nested VMX right when nested_run_pending was introduced, but it is not yet there in nSVM. Since we can have pending events that L0 injected directly into L2 on vmentry, we have to transfer them into L1's queue. For this to work, one important change is required: svm_complete_interrupts (which clears the "injected" fields from the previous VMRUN, and updates them from svm->vmcb's EXITINTINFO) must be placed before we inject the vmexit. This is not too scary though; VMX even does it in vmx_vcpu_run. While at it, the nested_vmexit_inject tracepoint is moved towards the end of nested_svm_vmexit. This ensures that the synthesized EXITINTINFO is visible in the trace. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 59 +++++++++++++++++++++++++++++++-------- arch/x86/kvm/svm/svm.c | 4 +-- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 6c7f0bffdf01..c3c04fef11df 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -262,6 +262,43 @@ void sync_nested_vmcb_control(struct vcpu_svm *svm) svm->nested.ctl.int_ctl |= svm->vmcb->control.int_ctl & mask; } +/* + * Transfer any event that L0 or L1 wanted to inject into L2 to + * EXIT_INT_INFO. + */ +static void nested_vmcb_save_pending_event(struct vcpu_svm *svm, + struct vmcb *nested_vmcb) +{ + struct kvm_vcpu *vcpu = &svm->vcpu; + u32 exit_int_info = 0; + unsigned int nr; + + if (vcpu->arch.exception.injected) { + nr = vcpu->arch.exception.nr; + exit_int_info = nr | SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_EXEPT; + + if (vcpu->arch.exception.has_error_code) { + exit_int_info |= SVM_EVTINJ_VALID_ERR; + nested_vmcb->control.exit_int_info_err = + vcpu->arch.exception.error_code; + } + + } else if (vcpu->arch.nmi_injected) { + exit_int_info = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; + + } else if (vcpu->arch.interrupt.injected) { + nr = vcpu->arch.interrupt.nr; + exit_int_info = nr | SVM_EVTINJ_VALID; + + if (vcpu->arch.interrupt.soft) + exit_int_info |= SVM_EVTINJ_TYPE_SOFT; + else + exit_int_info |= SVM_EVTINJ_TYPE_INTR; + } + + nested_vmcb->control.exit_int_info = exit_int_info; +} + static void nested_prepare_vmcb_save(struct vcpu_svm *svm, struct vmcb *nested_vmcb) { /* Load the nested guest state */ @@ -466,13 +503,6 @@ int nested_svm_vmexit(struct vcpu_svm *svm) struct vmcb *vmcb = svm->vmcb; struct kvm_host_map map; - trace_kvm_nested_vmexit_inject(vmcb->control.exit_code, - vmcb->control.exit_info_1, - vmcb->control.exit_info_2, - vmcb->control.exit_int_info, - vmcb->control.exit_int_info_err, - KVM_ISA_SVM); - rc = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->nested.vmcb), &map); if (rc) { if (rc == -EINVAL) @@ -517,8 +547,9 @@ int nested_svm_vmexit(struct vcpu_svm *svm) nested_vmcb->control.exit_code_hi = vmcb->control.exit_code_hi; nested_vmcb->control.exit_info_1 = vmcb->control.exit_info_1; nested_vmcb->control.exit_info_2 = vmcb->control.exit_info_2; - nested_vmcb->control.exit_int_info = vmcb->control.exit_int_info; - nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err; + + if (nested_vmcb->control.exit_code != SVM_EXIT_ERR) + nested_vmcb_save_pending_event(svm, nested_vmcb); if (svm->nrips_enabled) nested_vmcb->control.next_rip = vmcb->control.next_rip; @@ -539,9 +570,6 @@ int nested_svm_vmexit(struct vcpu_svm *svm) svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset; - kvm_clear_exception_queue(&svm->vcpu); - kvm_clear_interrupt_queue(&svm->vcpu); - svm->nested.ctl.nested_cr3 = 0; /* Restore selected save entries */ @@ -570,6 +598,13 @@ int nested_svm_vmexit(struct vcpu_svm *svm) mark_all_dirty(svm->vmcb); + trace_kvm_nested_vmexit_inject(nested_vmcb->control.exit_code, + nested_vmcb->control.exit_info_1, + nested_vmcb->control.exit_info_2, + nested_vmcb->control.exit_int_info, + nested_vmcb->control.exit_int_info_err, + KVM_ISA_SVM); + kvm_vcpu_unmap(&svm->vcpu, &map, true); nested_svm_uninit_mmu_context(&svm->vcpu); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index e48e4173bc60..422b1cc62a20 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2913,6 +2913,8 @@ static int handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) if (npt_enabled) vcpu->arch.cr3 = svm->vmcb->save.cr3; + svm_complete_interrupts(svm); + if (is_guest_mode(vcpu)) { int vmexit; @@ -2932,8 +2934,6 @@ static int handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) return 1; } - svm_complete_interrupts(svm); - if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) { kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; kvm_run->fail_entry.hardware_entry_failure_reason From e9fd761a46b8655aa5ff84b4adc0c8cf43952145 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 13 May 2020 13:28:23 -0400 Subject: [PATCH 0997/1043] KVM: nSVM: remove HF_VINTR_MASK Now that the int_ctl field is stored in svm->nested.ctl.int_ctl, we can use it instead of vcpu->arch.hflags to check whether L2 is running in V_INTR_MASKING mode. Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 - arch/x86/kvm/svm/nested.c | 6 +----- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/svm/svm.h | 4 +++- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index e6f2e1a2dab6..0dfc522f96cc 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1596,7 +1596,6 @@ enum { #define HF_GIF_MASK (1 << 0) #define HF_HIF_MASK (1 << 1) -#define HF_VINTR_MASK (1 << 2) #define HF_NMI_MASK (1 << 3) #define HF_IRET_MASK (1 << 4) #define HF_GUEST_MASK (1 << 5) /* VCPU is in guest-mode */ diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index c3c04fef11df..6967fe884eaf 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -118,7 +118,7 @@ void recalc_intercepts(struct vcpu_svm *svm) c->intercept_exceptions = h->intercept_exceptions; c->intercept = h->intercept; - if (svm->vcpu.arch.hflags & HF_VINTR_MASK) { + if (g->int_ctl & V_INTR_MASKING_MASK) { /* We only want the cr8 intercept bits of L1 */ c->intercept_cr &= ~(1U << INTERCEPT_CR8_READ); c->intercept_cr &= ~(1U << INTERCEPT_CR8_WRITE); @@ -338,10 +338,6 @@ static void nested_prepare_vmcb_control(struct vcpu_svm *svm) kvm_mmu_reset_context(&svm->vcpu); svm_flush_tlb(&svm->vcpu); - if (svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) - svm->vcpu.arch.hflags |= HF_VINTR_MASK; - else - svm->vcpu.arch.hflags &= ~HF_VINTR_MASK; svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset = svm->vcpu.arch.l1_tsc_offset + svm->nested.ctl.tsc_offset; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 422b1cc62a20..6867dba4f736 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3104,7 +3104,7 @@ bool svm_interrupt_blocked(struct kvm_vcpu *vcpu) if (is_guest_mode(vcpu)) { /* As long as interrupts are being delivered... */ - if ((svm->vcpu.arch.hflags & HF_VINTR_MASK) + if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) ? !(svm->vcpu.arch.hflags & HF_HIF_MASK) : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF)) return true; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 10b7b55720a0..be8e830f83fa 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -367,7 +367,9 @@ void svm_set_gif(struct vcpu_svm *svm, bool value); static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu) { - return is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK); + struct vcpu_svm *svm = to_svm(vcpu); + + return is_guest_mode(vcpu) && (svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK); } static inline bool nested_exit_on_smi(struct vcpu_svm *svm) From 08245e6d2e589f2b6e9e275ddb343e2ec9ce94ec Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 19 May 2020 09:21:04 -0400 Subject: [PATCH 0998/1043] KVM: nSVM: remove HF_HIF_MASK The L1 flags can be found in the save area of svm->nested.hsave, fish it from there so that there is one fewer thing to migrate. Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 - arch/x86/kvm/svm/nested.c | 5 ----- arch/x86/kvm/svm/svm.c | 2 +- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 0dfc522f96cc..3485f8454088 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1595,7 +1595,6 @@ enum { }; #define HF_GIF_MASK (1 << 0) -#define HF_HIF_MASK (1 << 1) #define HF_NMI_MASK (1 << 3) #define HF_IRET_MASK (1 << 4) #define HF_GUEST_MASK (1 << 5) /* VCPU is in guest-mode */ diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 6967fe884eaf..65ecc8586f75 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -371,11 +371,6 @@ void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, struct vmcb *nested_vmcb) { svm->nested.vmcb = vmcb_gpa; - if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF) - svm->vcpu.arch.hflags |= HF_HIF_MASK; - else - svm->vcpu.arch.hflags &= ~HF_HIF_MASK; - load_nested_vmcb_control(svm, &nested_vmcb->control); nested_prepare_vmcb_save(svm, nested_vmcb); nested_prepare_vmcb_control(svm); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 6867dba4f736..bc08221f6743 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3105,7 +3105,7 @@ bool svm_interrupt_blocked(struct kvm_vcpu *vcpu) if (is_guest_mode(vcpu)) { /* As long as interrupts are being delivered... */ if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) - ? !(svm->vcpu.arch.hflags & HF_HIF_MASK) + ? !(svm->nested.hsave->save.rflags & X86_EFLAGS_IF) : !(kvm_get_rflags(vcpu) & X86_EFLAGS_IF)) return true; From ca46d739e3caf44dcd3db9eb8da30d0ff3aa9180 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 18 May 2020 13:02:15 -0400 Subject: [PATCH 0999/1043] KVM: nSVM: split nested_vmcb_check_controls The authoritative state does not come from the VMCB once in guest mode, but KVM_SET_NESTED_STATE can still perform checks on L1's provided SVM controls because we get them from userspace. Therefore, split out a function to do them. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 65ecc8586f75..bd3a89cd4070 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -203,6 +203,21 @@ static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm) return true; } +static bool nested_vmcb_check_controls(struct vmcb_control_area *control) +{ + if ((control->intercept & (1ULL << INTERCEPT_VMRUN)) == 0) + return false; + + if (control->asid == 0) + return false; + + if ((control->nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && + !npt_enabled) + return false; + + return true; +} + static bool nested_vmcb_checks(struct vmcb *vmcb) { if ((vmcb->save.efer & EFER_SVME) == 0) @@ -212,17 +227,7 @@ static bool nested_vmcb_checks(struct vmcb *vmcb) (vmcb->save.cr0 & X86_CR0_NW)) return false; - if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0) - return false; - - if (vmcb->control.asid == 0) - return false; - - if ((vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) && - !npt_enabled) - return false; - - return true; + return nested_vmcb_check_controls(&vmcb->control); } static void load_nested_vmcb_control(struct vcpu_svm *svm, From c513f484c5582a8efadf3d72298e2285b041536e Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 18 May 2020 13:08:37 -0400 Subject: [PATCH 1000/1043] KVM: nSVM: leave guest mode when clearing EFER.SVME According to the AMD manual, the effect of turning off EFER.SVME while a guest is running is undefined. We make it leave guest mode immediately, similar to the effect of clearing the VMX bit in MSR_IA32_FEAT_CTL. Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/nested.c | 16 ++++++++++++++++ arch/x86/kvm/svm/svm.c | 10 ++++++++-- arch/x86/kvm/svm/svm.h | 1 + 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index bd3a89cd4070..369eca73fe3e 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -618,6 +618,22 @@ int nested_svm_vmexit(struct vcpu_svm *svm) return 0; } +/* + * Forcibly leave nested mode in order to be able to reset the VCPU later on. + */ +void svm_leave_nested(struct vcpu_svm *svm) +{ + if (is_guest_mode(&svm->vcpu)) { + struct vmcb *hsave = svm->nested.hsave; + struct vmcb *vmcb = svm->vmcb; + + svm->nested.nested_run_pending = 0; + leave_guest_mode(&svm->vcpu); + copy_vmcb_control_area(&vmcb->control, &hsave->control); + nested_svm_uninit_mmu_context(&svm->vcpu); + } +} + static int nested_svm_exit_handled_msr(struct vcpu_svm *svm) { u32 offset, msr, value; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index bc08221f6743..b4db9a980469 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -265,6 +265,7 @@ static int get_npt_level(struct kvm_vcpu *vcpu) void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) { + struct vcpu_svm *svm = to_svm(vcpu); vcpu->arch.efer = efer; if (!npt_enabled) { @@ -275,8 +276,13 @@ void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) efer &= ~EFER_LME; } - to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME; - mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR); + if (!(efer & EFER_SVME)) { + svm_leave_nested(svm); + svm_set_gif(svm, true); + } + + svm->vmcb->save.efer = efer | EFER_SVME; + mark_dirty(svm->vmcb, VMCB_CR); } static int is_external_interrupt(u32 info) diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index be8e830f83fa..6ac4c00a5d82 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -389,6 +389,7 @@ static inline bool nested_exit_on_nmi(struct vcpu_svm *svm) void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa, struct vmcb *nested_vmcb); +void svm_leave_nested(struct vcpu_svm *svm); int nested_svm_vmrun(struct vcpu_svm *svm); void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb); int nested_svm_vmexit(struct vcpu_svm *svm); From 929d1cfaa6926ccee28d8d4220e0b4e2defd9cd1 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 19 May 2020 06:18:31 -0400 Subject: [PATCH 1001/1043] KVM: MMU: pass arbitrary CR0/CR4/EFER to kvm_init_shadow_mmu This allows fetching the registers from the hsave area when setting up the NPT shadow MMU, and is needed for KVM_SET_NESTED_STATE (which runs long after the CR0, CR4 and EFER values in vcpu have been switched to hold L2 guest state). Signed-off-by: Paolo Bonzini --- arch/x86/kvm/mmu.h | 2 +- arch/x86/kvm/mmu/mmu.c | 14 +++++++++----- arch/x86/kvm/svm/nested.c | 5 ++++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index 048e865ad485..0ad06bfe2c2c 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -57,7 +57,7 @@ void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context); void kvm_init_mmu(struct kvm_vcpu *vcpu, bool reset_roots); -void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu); +void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, u32 cr0, u32 cr4, u32 efer); void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, bool accessed_dirty, gpa_t new_eptp); bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index fd1c9145505c..2e62a03410c7 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4952,7 +4952,7 @@ kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only) return role; } -void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu) +void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, u32 cr0, u32 cr4, u32 efer) { struct kvm_mmu *context = vcpu->arch.mmu; union kvm_mmu_role new_role = @@ -4961,11 +4961,11 @@ void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu) if (new_role.as_u64 == context->mmu_role.as_u64) return; - if (!is_paging(vcpu)) + if (!(cr0 & X86_CR0_PG)) nonpaging_init_context(vcpu, context); - else if (is_long_mode(vcpu)) + else if (efer & EFER_LMA) paging64_init_context(vcpu, context); - else if (is_pae(vcpu)) + else if (cr4 & X86_CR4_PAE) paging32E_init_context(vcpu, context); else paging32_init_context(vcpu, context); @@ -5043,7 +5043,11 @@ static void init_kvm_softmmu(struct kvm_vcpu *vcpu) { struct kvm_mmu *context = vcpu->arch.mmu; - kvm_init_shadow_mmu(vcpu); + kvm_init_shadow_mmu(vcpu, + kvm_read_cr0_bits(vcpu, X86_CR0_PG), + kvm_read_cr4_bits(vcpu, X86_CR4_PAE), + vcpu->arch.efer); + context->get_guest_pgd = get_cr3; context->get_pdptr = kvm_pdptr_read; context->inject_page_fault = kvm_inject_page_fault; diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 369eca73fe3e..c712fe577029 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -80,10 +80,13 @@ static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu) static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) { + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb *hsave = svm->nested.hsave; + WARN_ON(mmu_is_nested(vcpu)); vcpu->arch.mmu = &vcpu->arch.guest_mmu; - kvm_init_shadow_mmu(vcpu); + kvm_init_shadow_mmu(vcpu, X86_CR0_PG, hsave->save.cr4, hsave->save.efer); vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; From ed881297338625f3799cce8774d198ef05a858e6 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Fri, 29 May 2020 15:04:06 +0200 Subject: [PATCH 1002/1043] selftests: kvm: introduce cpu_has_svm() check Many tests will want to check if the CPU is Intel or AMD in guest code, add cpu_has_svm() and put it as static inline to svm_util.h. Signed-off-by: Vitaly Kuznetsov Message-Id: <20200529130407.57176-1-vkuznets@redhat.com> Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/include/x86_64/svm_util.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/testing/selftests/kvm/include/x86_64/svm_util.h b/tools/testing/selftests/kvm/include/x86_64/svm_util.h index cd037917fece..674151d24fcf 100644 --- a/tools/testing/selftests/kvm/include/x86_64/svm_util.h +++ b/tools/testing/selftests/kvm/include/x86_64/svm_util.h @@ -35,4 +35,14 @@ void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_r void run_guest(struct vmcb *vmcb, uint64_t vmcb_gpa); void nested_svm_check_supported(void); +static inline bool cpu_has_svm(void) +{ + u32 eax = 0x80000001, ecx; + + asm("cpuid" : + "=a" (eax), "=c" (ecx) : "0" (eax) : "ebx", "edx"); + + return ecx & CPUID_SVM; +} + #endif /* SELFTEST_KVM_SVM_UTILS_H */ From 10b910cb7edebac478d329244ca479b7962fb46e Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 19 May 2020 13:23:05 -0400 Subject: [PATCH 1003/1043] selftests: kvm: add a SVM version of state-test The test is similar to the existing one for VMX, but simpler because we don't have to test shadow VMCS or vmptrld/vmptrst/vmclear. Signed-off-by: Paolo Bonzini --- .../testing/selftests/kvm/x86_64/state_test.c | 62 +++++++++++++++---- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/tools/testing/selftests/kvm/x86_64/state_test.c b/tools/testing/selftests/kvm/x86_64/state_test.c index 5b1a016edf55..d43b6f99b66c 100644 --- a/tools/testing/selftests/kvm/x86_64/state_test.c +++ b/tools/testing/selftests/kvm/x86_64/state_test.c @@ -18,14 +18,46 @@ #include "kvm_util.h" #include "processor.h" #include "vmx.h" +#include "svm_util.h" #define VCPU_ID 5 +#define L2_GUEST_STACK_SIZE 256 -void l2_guest_code(void) +void svm_l2_guest_code(void) +{ + GUEST_SYNC(4); + /* Exit to L1 */ + vmcall(); + GUEST_SYNC(6); + /* Done, exit to L1 and never come back. */ + vmcall(); +} + +static void svm_l1_guest_code(struct svm_test_data *svm) +{ + unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; + struct vmcb *vmcb = svm->vmcb; + + GUEST_ASSERT(svm->vmcb_gpa); + /* Prepare for L2 execution. */ + generic_svm_setup(svm, svm_l2_guest_code, + &l2_guest_stack[L2_GUEST_STACK_SIZE]); + + GUEST_SYNC(3); + run_guest(vmcb, svm->vmcb_gpa); + GUEST_ASSERT(vmcb->control.exit_code == SVM_EXIT_VMMCALL); + GUEST_SYNC(5); + vmcb->save.rip += 3; + run_guest(vmcb, svm->vmcb_gpa); + GUEST_ASSERT(vmcb->control.exit_code == SVM_EXIT_VMMCALL); + GUEST_SYNC(7); +} + +void vmx_l2_guest_code(void) { GUEST_SYNC(6); - /* Exit to L1 */ + /* Exit to L1 */ vmcall(); /* L1 has now set up a shadow VMCS for us. */ @@ -42,10 +74,9 @@ void l2_guest_code(void) vmcall(); } -void l1_guest_code(struct vmx_pages *vmx_pages) +static void vmx_l1_guest_code(struct vmx_pages *vmx_pages) { -#define L2_GUEST_STACK_SIZE 64 - unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; + unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; GUEST_ASSERT(vmx_pages->vmcs_gpa); GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); @@ -56,7 +87,7 @@ void l1_guest_code(struct vmx_pages *vmx_pages) GUEST_SYNC(4); GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); - prepare_vmcs(vmx_pages, l2_guest_code, + prepare_vmcs(vmx_pages, vmx_l2_guest_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]); GUEST_SYNC(5); @@ -106,20 +137,24 @@ void l1_guest_code(struct vmx_pages *vmx_pages) GUEST_ASSERT(vmresume()); } -void guest_code(struct vmx_pages *vmx_pages) +static void __attribute__((__flatten__)) guest_code(void *arg) { GUEST_SYNC(1); GUEST_SYNC(2); - if (vmx_pages) - l1_guest_code(vmx_pages); + if (arg) { + if (cpu_has_svm()) + svm_l1_guest_code(arg); + else + vmx_l1_guest_code(arg); + } GUEST_DONE(); } int main(int argc, char *argv[]) { - vm_vaddr_t vmx_pages_gva = 0; + vm_vaddr_t nested_gva = 0; struct kvm_regs regs1, regs2; struct kvm_vm *vm; @@ -136,8 +171,11 @@ int main(int argc, char *argv[]) vcpu_regs_get(vm, VCPU_ID, ®s1); if (kvm_check_cap(KVM_CAP_NESTED_STATE)) { - vcpu_alloc_vmx(vm, &vmx_pages_gva); - vcpu_args_set(vm, VCPU_ID, 1, vmx_pages_gva); + if (kvm_get_supported_cpuid_entry(0x80000001)->ecx & CPUID_SVM) + vcpu_alloc_svm(vm, &nested_gva); + else + vcpu_alloc_vmx(vm, &nested_gva); + vcpu_args_set(vm, VCPU_ID, 1, nested_gva); } else { pr_info("will skip nested state checks\n"); vcpu_args_set(vm, VCPU_ID, 1, 0); From 8ec107c89b19bc37a7ec364f3e1c92ae4d961b78 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Fri, 29 May 2020 15:04:07 +0200 Subject: [PATCH 1004/1043] selftests: kvm: fix smm test on SVM KVM_CAP_NESTED_STATE is now supported for AMD too but smm test acts like it is still Intel only. Signed-off-by: Vitaly Kuznetsov Message-Id: <20200529130407.57176-2-vkuznets@redhat.com> Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/x86_64/smm_test.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/kvm/x86_64/smm_test.c b/tools/testing/selftests/kvm/x86_64/smm_test.c index 8230b6bc6b8f..6f8f478b3ceb 100644 --- a/tools/testing/selftests/kvm/x86_64/smm_test.c +++ b/tools/testing/selftests/kvm/x86_64/smm_test.c @@ -17,6 +17,7 @@ #include "kvm_util.h" #include "vmx.h" +#include "svm_util.h" #define VCPU_ID 1 @@ -58,7 +59,7 @@ void self_smi(void) APIC_DEST_SELF | APIC_INT_ASSERT | APIC_DM_SMI); } -void guest_code(struct vmx_pages *vmx_pages) +void guest_code(void *arg) { uint64_t apicbase = rdmsr(MSR_IA32_APICBASE); @@ -72,8 +73,11 @@ void guest_code(struct vmx_pages *vmx_pages) sync_with_host(4); - if (vmx_pages) { - GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + if (arg) { + if (cpu_has_svm()) + generic_svm_setup(arg, NULL, NULL); + else + GUEST_ASSERT(prepare_for_vmx_operation(arg)); sync_with_host(5); @@ -87,7 +91,7 @@ void guest_code(struct vmx_pages *vmx_pages) int main(int argc, char *argv[]) { - vm_vaddr_t vmx_pages_gva = 0; + vm_vaddr_t nested_gva = 0; struct kvm_regs regs; struct kvm_vm *vm; @@ -114,8 +118,11 @@ int main(int argc, char *argv[]) vcpu_set_msr(vm, VCPU_ID, MSR_IA32_SMBASE, SMRAM_GPA); if (kvm_check_cap(KVM_CAP_NESTED_STATE)) { - vcpu_alloc_vmx(vm, &vmx_pages_gva); - vcpu_args_set(vm, VCPU_ID, 1, vmx_pages_gva); + if (kvm_get_supported_cpuid_entry(0x80000001)->ecx & CPUID_SVM) + vcpu_alloc_svm(vm, &nested_gva); + else + vcpu_alloc_vmx(vm, &nested_gva); + vcpu_args_set(vm, VCPU_ID, 1, nested_gva); } else { pr_info("will skip SMM test with VMX enabled\n"); vcpu_args_set(vm, VCPU_ID, 1, 0); From cc440cdad5b7a4c1de12dace725209eb3e0cf663 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 13 May 2020 13:36:32 -0400 Subject: [PATCH 1005/1043] KVM: nSVM: implement KVM_GET_NESTED_STATE and KVM_SET_NESTED_STATE Similar to VMX, the state that is captured through the currently available IOCTLs is a mix of L1 and L2 state, dependent on whether the L2 guest was running at the moment when the process was interrupted to save its state. In particular, the SVM-specific state for nested virtualization includes the L1 saved state (including the interrupt flag), the cached L2 controls, and the GIF. Signed-off-by: Paolo Bonzini --- arch/x86/include/uapi/asm/kvm.h | 17 +++- arch/x86/kvm/cpuid.h | 5 ++ arch/x86/kvm/svm/nested.c | 147 ++++++++++++++++++++++++++++++++ arch/x86/kvm/svm/svm.c | 1 + arch/x86/kvm/vmx/nested.c | 5 -- arch/x86/kvm/x86.c | 3 +- 6 files changed, 171 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h index 3f3f780c8c65..12075a9de1c1 100644 --- a/arch/x86/include/uapi/asm/kvm.h +++ b/arch/x86/include/uapi/asm/kvm.h @@ -385,18 +385,22 @@ struct kvm_sync_regs { #define KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT (1 << 4) #define KVM_STATE_NESTED_FORMAT_VMX 0 -#define KVM_STATE_NESTED_FORMAT_SVM 1 /* unused */ +#define KVM_STATE_NESTED_FORMAT_SVM 1 #define KVM_STATE_NESTED_GUEST_MODE 0x00000001 #define KVM_STATE_NESTED_RUN_PENDING 0x00000002 #define KVM_STATE_NESTED_EVMCS 0x00000004 #define KVM_STATE_NESTED_MTF_PENDING 0x00000008 +#define KVM_STATE_NESTED_GIF_SET 0x00000100 #define KVM_STATE_NESTED_SMM_GUEST_MODE 0x00000001 #define KVM_STATE_NESTED_SMM_VMXON 0x00000002 #define KVM_STATE_NESTED_VMX_VMCS_SIZE 0x1000 +#define KVM_STATE_NESTED_SVM_VMCB_SIZE 0x1000 + + struct kvm_vmx_nested_state_data { __u8 vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE]; __u8 shadow_vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE]; @@ -411,6 +415,15 @@ struct kvm_vmx_nested_state_hdr { } smm; }; +struct kvm_svm_nested_state_data { + /* Save area only used if KVM_STATE_NESTED_RUN_PENDING. */ + __u8 vmcb12[KVM_STATE_NESTED_SVM_VMCB_SIZE]; +}; + +struct kvm_svm_nested_state_hdr { + __u64 vmcb_pa; +}; + /* for KVM_CAP_NESTED_STATE */ struct kvm_nested_state { __u16 flags; @@ -419,6 +432,7 @@ struct kvm_nested_state { union { struct kvm_vmx_nested_state_hdr vmx; + struct kvm_svm_nested_state_hdr svm; /* Pad the header to 128 bytes. */ __u8 pad[120]; @@ -431,6 +445,7 @@ struct kvm_nested_state { */ union { struct kvm_vmx_nested_state_data vmx[0]; + struct kvm_svm_nested_state_data svm[0]; } data; }; diff --git a/arch/x86/kvm/cpuid.h b/arch/x86/kvm/cpuid.h index 63a70f6a3df3..05434cd9342f 100644 --- a/arch/x86/kvm/cpuid.h +++ b/arch/x86/kvm/cpuid.h @@ -303,4 +303,9 @@ static __always_inline void kvm_cpu_cap_check_and_set(unsigned int x86_feature) kvm_cpu_cap_set(x86_feature); } +static inline bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa) +{ + return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu)); +} + #endif diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index c712fe577029..6b1049148c1b 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -25,6 +25,7 @@ #include "trace.h" #include "mmu.h" #include "x86.h" +#include "cpuid.h" #include "lapic.h" #include "svm.h" @@ -238,6 +239,8 @@ static void load_nested_vmcb_control(struct vcpu_svm *svm, { copy_vmcb_control_area(&svm->nested.ctl, control); + /* Copy it here because nested_svm_check_controls will check it. */ + svm->nested.ctl.asid = control->asid; svm->nested.ctl.msrpm_base_pa &= ~0x0fffULL; svm->nested.ctl.iopm_base_pa &= ~0x0fffULL; } @@ -930,6 +933,150 @@ int nested_svm_exit_special(struct vcpu_svm *svm) return NESTED_EXIT_CONTINUE; } +static int svm_get_nested_state(struct kvm_vcpu *vcpu, + struct kvm_nested_state __user *user_kvm_nested_state, + u32 user_data_size) +{ + struct vcpu_svm *svm; + struct kvm_nested_state kvm_state = { + .flags = 0, + .format = KVM_STATE_NESTED_FORMAT_SVM, + .size = sizeof(kvm_state), + }; + struct vmcb __user *user_vmcb = (struct vmcb __user *) + &user_kvm_nested_state->data.svm[0]; + + if (!vcpu) + return kvm_state.size + KVM_STATE_NESTED_SVM_VMCB_SIZE; + + svm = to_svm(vcpu); + + if (user_data_size < kvm_state.size) + goto out; + + /* First fill in the header and copy it out. */ + if (is_guest_mode(vcpu)) { + kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb; + kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE; + kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE; + + if (svm->nested.nested_run_pending) + kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING; + } + + if (gif_set(svm)) + kvm_state.flags |= KVM_STATE_NESTED_GIF_SET; + + if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state))) + return -EFAULT; + + if (!is_guest_mode(vcpu)) + goto out; + + /* + * Copy over the full size of the VMCB rather than just the size + * of the structs. + */ + if (clear_user(user_vmcb, KVM_STATE_NESTED_SVM_VMCB_SIZE)) + return -EFAULT; + if (copy_to_user(&user_vmcb->control, &svm->nested.ctl, + sizeof(user_vmcb->control))) + return -EFAULT; + if (copy_to_user(&user_vmcb->save, &svm->nested.hsave->save, + sizeof(user_vmcb->save))) + return -EFAULT; + +out: + return kvm_state.size; +} + +static int svm_set_nested_state(struct kvm_vcpu *vcpu, + struct kvm_nested_state __user *user_kvm_nested_state, + struct kvm_nested_state *kvm_state) +{ + struct vcpu_svm *svm = to_svm(vcpu); + struct vmcb *hsave = svm->nested.hsave; + struct vmcb __user *user_vmcb = (struct vmcb __user *) + &user_kvm_nested_state->data.svm[0]; + struct vmcb_control_area ctl; + struct vmcb_save_area save; + u32 cr0; + + if (kvm_state->format != KVM_STATE_NESTED_FORMAT_SVM) + return -EINVAL; + + if (kvm_state->flags & ~(KVM_STATE_NESTED_GUEST_MODE | + KVM_STATE_NESTED_RUN_PENDING | + KVM_STATE_NESTED_GIF_SET)) + return -EINVAL; + + /* + * If in guest mode, vcpu->arch.efer actually refers to the L2 guest's + * EFER.SVME, but EFER.SVME still has to be 1 for VMRUN to succeed. + */ + if (!(vcpu->arch.efer & EFER_SVME)) { + /* GIF=1 and no guest mode are required if SVME=0. */ + if (kvm_state->flags != KVM_STATE_NESTED_GIF_SET) + return -EINVAL; + } + + /* SMM temporarily disables SVM, so we cannot be in guest mode. */ + if (is_smm(vcpu) && (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) + return -EINVAL; + + if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE)) { + svm_leave_nested(svm); + goto out_set_gif; + } + + if (!page_address_valid(vcpu, kvm_state->hdr.svm.vmcb_pa)) + return -EINVAL; + if (kvm_state->size < sizeof(*kvm_state) + KVM_STATE_NESTED_SVM_VMCB_SIZE) + return -EINVAL; + if (copy_from_user(&ctl, &user_vmcb->control, sizeof(ctl))) + return -EFAULT; + if (copy_from_user(&save, &user_vmcb->save, sizeof(save))) + return -EFAULT; + + if (!nested_vmcb_check_controls(&ctl)) + return -EINVAL; + + /* + * Processor state contains L2 state. Check that it is + * valid for guest mode (see nested_vmcb_checks). + */ + cr0 = kvm_read_cr0(vcpu); + if (((cr0 & X86_CR0_CD) == 0) && (cr0 & X86_CR0_NW)) + return -EINVAL; + + /* + * Validate host state saved from before VMRUN (see + * nested_svm_check_permissions). + * TODO: validate reserved bits for all saved state. + */ + if (!(save.cr0 & X86_CR0_PG)) + return -EINVAL; + + /* + * All checks done, we can enter guest mode. L1 control fields + * come from the nested save state. Guest state is already + * in the registers, the save area of the nested state instead + * contains saved L1 state. + */ + copy_vmcb_control_area(&hsave->control, &svm->vmcb->control); + hsave->save = save; + + svm->nested.vmcb = kvm_state->hdr.svm.vmcb_pa; + load_nested_vmcb_control(svm, &ctl); + nested_prepare_vmcb_control(svm); + +out_set_gif: + svm_set_gif(svm, !!(kvm_state->flags & KVM_STATE_NESTED_GIF_SET)); + return 0; +} + struct kvm_x86_nested_ops svm_nested_ops = { .check_events = svm_check_nested_events, + .get_state = svm_get_nested_state, + .set_state = svm_set_nested_state, }; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index b4db9a980469..3871bfb40594 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1193,6 +1193,7 @@ static int svm_create_vcpu(struct kvm_vcpu *vcpu) svm->avic_is_running = true; svm->nested.hsave = page_address(hsave_page); + clear_page(svm->nested.hsave); svm->msrpm = page_address(msrpm_pages); svm_vcpu_init_msrpm(svm->msrpm); diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 51ebb60e1533..106fc6fceb97 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -437,11 +437,6 @@ static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu, } } -static bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa) -{ - return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu)); -} - static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index f0fa610bed91..d4aa7dc662d5 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -4628,7 +4628,8 @@ long kvm_arch_vcpu_ioctl(struct file *filp, if (kvm_state.flags & ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE - | KVM_STATE_NESTED_EVMCS | KVM_STATE_NESTED_MTF_PENDING)) + | KVM_STATE_NESTED_EVMCS | KVM_STATE_NESTED_MTF_PENDING + | KVM_STATE_NESTED_GIF_SET)) break; /* nested_run_pending implies guest_mode. */ From a8387d0b471f7c8d4d936af0d6dc44dff5c41c6c Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 29 May 2020 05:42:55 -0400 Subject: [PATCH 1006/1043] Revert "KVM: No need to retry for hva_to_pfn_remapped()" This reverts commit 5b494aea13fe9ec67365510c0d75835428cbb303. If unlocked==true then the vma pointer could be invalidated, so the 2nd follow_pfn() is potentially racy: we do need to get out and redo find_vma_intersection(). Signed-off-by: Peter Xu Signed-off-by: Paolo Bonzini --- virt/kvm/kvm_main.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index da6da386b591..c9c6db5f77c2 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1831,6 +1831,8 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma, r = fixup_user_fault(current, current->mm, addr, (write_fault ? FAULT_FLAG_WRITE : 0), &unlocked); + if (unlocked) + return -EAGAIN; if (r) return r; @@ -1901,12 +1903,15 @@ static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async, goto exit; } +retry: vma = find_vma_intersection(current->mm, addr, addr + 1); if (vma == NULL) pfn = KVM_PFN_ERR_FAULT; else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) { r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable, &pfn); + if (r == -EAGAIN) + goto retry; if (r < 0) pfn = KVM_PFN_ERR_FAULT; } else { From f4a9fdd5f12ad48700ad5dcb7a0dc7cf9a3bcd5a Mon Sep 17 00:00:00 2001 From: "Gustavo A. R. Silva" Date: Thu, 7 May 2020 13:56:18 -0500 Subject: [PATCH 1007/1043] KVM: VMX: Replace zero-length array with flexible-array The current codebase makes use of the zero-length array language extension to the C90 standard, but the preferred mechanism to declare variable-length types such as these ones is a flexible array member[1][2], introduced in C99: struct foo { int stuff; struct boo array[]; }; By making use of the mechanism above, we will get a compiler warning in case the flexible array does not occur last in the structure, which will help us prevent some kind of undefined behavior bugs from being inadvertently introduced[3] to the codebase from now on. Also, notice that, dynamic memory allocations won't be affected by this change: "Flexible array members have incomplete type, and so the sizeof operator may not be applied. As a quirk of the original implementation of zero-length arrays, sizeof evaluates to zero."[1] sizeof(flexible-array-member) triggers a warning because flexible array members have incomplete type[1]. There are some instances of code in which the sizeof operator is being incorrectly/erroneously applied to zero-length arrays and the result is zero. Such instances may be hiding some bugs. So, this work (flexible-array member conversions) will also help to get completely rid of those sorts of issues. This issue was found with the help of Coccinelle. [1] https://gcc.gnu.org/onlinedocs/gcc/Zero-Length.html [2] https://github.com/KSPP/linux/issues/21 [3] commit 76497732932f ("cxgb3/l2t: Fix undefined behaviour") Signed-off-by: Gustavo A. R. Silva Message-Id: <20200507185618.GA14831@embeddedor> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmcs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/vmcs.h b/arch/x86/kvm/vmx/vmcs.h index 481ad879197b..5c0ff80b85c0 100644 --- a/arch/x86/kvm/vmx/vmcs.h +++ b/arch/x86/kvm/vmx/vmcs.h @@ -19,7 +19,7 @@ struct vmcs_hdr { struct vmcs { struct vmcs_hdr hdr; u32 abort; - char data[0]; + char data[]; }; DECLARE_PER_CPU(struct vmcs *, current_vmcs); From 84b09f33a5de528d05c007d9847403a364dfe35e Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Mon, 25 May 2020 16:41:16 +0200 Subject: [PATCH 1008/1043] Revert "KVM: async_pf: Fix #DF due to inject "Page not Present" and "Page Ready" exceptions simultaneously" Commit 9a6e7c39810e (""KVM: async_pf: Fix #DF due to inject "Page not Present" and "Page Ready" exceptions simultaneously") added a protection against 'page ready' notification coming before 'page not present' is delivered. This situation seems to be impossible since commit 2a266f23550b ("KVM MMU: check pending exception before injecting APF) which added 'vcpu->arch.exception.pending' check to kvm_can_do_async_pf. On x86, kvm_arch_async_page_present() has only one call site: kvm_check_async_pf_completion() loop and we only enter the loop when kvm_arch_can_inject_async_page_present(vcpu) which when async pf msr is enabled, translates into kvm_can_do_async_pf(). There is also one problem with the cancellation mechanism. We don't seem to check that the 'page not present' notification we're canceling matches the 'page ready' notification so in theory, we may erroneously drop two valid events. Revert the commit. Reviewed-by: Gavin Shan Signed-off-by: Vitaly Kuznetsov Message-Id: <20200525144125.143875-2-vkuznets@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index d4aa7dc662d5..b9a5ff7b922c 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10427,13 +10427,6 @@ static int apf_put_user(struct kvm_vcpu *vcpu, u32 val) sizeof(val)); } -static int apf_get_user(struct kvm_vcpu *vcpu, u32 *val) -{ - - return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, val, - sizeof(u32)); -} - static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu) { if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu)) @@ -10498,7 +10491,6 @@ void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) { struct x86_exception fault; - u32 val; if (work->wakeup_all) work->arch.token = ~0; /* broadcast wakeup */ @@ -10507,19 +10499,7 @@ void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa); if (vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED && - !apf_get_user(vcpu, &val)) { - if (val == KVM_PV_REASON_PAGE_NOT_PRESENT && - vcpu->arch.exception.pending && - vcpu->arch.exception.nr == PF_VECTOR && - !apf_put_user(vcpu, 0)) { - vcpu->arch.exception.injected = false; - vcpu->arch.exception.pending = false; - vcpu->arch.exception.nr = 0; - vcpu->arch.exception.has_error_code = false; - vcpu->arch.exception.error_code = 0; - vcpu->arch.exception.has_payload = false; - vcpu->arch.exception.payload = 0; - } else if (!apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) { + !apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) { fault.vector = PF_VECTOR; fault.error_code_valid = true; fault.error_code = 0; @@ -10527,7 +10507,6 @@ void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, fault.address = work->arch.token; fault.async_page_fault = true; kvm_inject_page_fault(vcpu, &fault); - } } vcpu->arch.apf.halted = false; vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; From 68fd66f100d196d35ab3008d4c69af3a0d7e7200 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Mon, 25 May 2020 16:41:17 +0200 Subject: [PATCH 1009/1043] KVM: x86: extend struct kvm_vcpu_pv_apf_data with token info Currently, APF mechanism relies on the #PF abuse where the token is being passed through CR2. If we switch to using interrupts to deliver page-ready notifications we need a different way to pass the data. Extent the existing 'struct kvm_vcpu_pv_apf_data' with token information for page-ready notifications. While on it, rename 'reason' to 'flags'. This doesn't change the semantics as we only have reasons '1' and '2' and these can be treated as bit flags but KVM_PV_REASON_PAGE_READY is going away with interrupt based delivery making 'reason' name misleading. The newly introduced apf_put_user_ready() temporary puts both flags and token information, this will be changed to put token only when we switch to interrupt based notifications. Signed-off-by: Vitaly Kuznetsov Message-Id: <20200525144125.143875-3-vkuznets@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/include/asm/kvm_para.h | 4 ++-- arch/x86/include/uapi/asm/kvm_para.h | 5 +++-- arch/x86/kernel/kvm.c | 16 ++++++++-------- arch/x86/kvm/mmu/mmu.c | 6 +++--- arch/x86/kvm/svm/nested.c | 2 +- arch/x86/kvm/svm/svm.c | 3 ++- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/vmx/vmx.c | 4 ++-- arch/x86/kvm/x86.c | 17 +++++++++++++---- 10 files changed, 36 insertions(+), 25 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 3485f8454088..033f6173a857 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -770,7 +770,7 @@ struct kvm_vcpu_arch { u64 msr_val; u32 id; bool send_user_only; - u32 host_apf_reason; + u32 host_apf_flags; unsigned long nested_apf_token; bool delivery_as_pf_vmexit; } apf; diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h index 118e5c2379f9..57fd1966c4ea 100644 --- a/arch/x86/include/asm/kvm_para.h +++ b/arch/x86/include/asm/kvm_para.h @@ -90,7 +90,7 @@ unsigned int kvm_arch_para_features(void); unsigned int kvm_arch_para_hints(void); void kvm_async_pf_task_wait_schedule(u32 token); void kvm_async_pf_task_wake(u32 token); -u32 kvm_read_and_reset_pf_reason(void); +u32 kvm_read_and_reset_apf_flags(void); void kvm_disable_steal_time(void); bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token); @@ -131,7 +131,7 @@ static inline unsigned int kvm_arch_para_hints(void) return 0; } -static inline u32 kvm_read_and_reset_pf_reason(void) +static inline u32 kvm_read_and_reset_apf_flags(void) { return 0; } diff --git a/arch/x86/include/uapi/asm/kvm_para.h b/arch/x86/include/uapi/asm/kvm_para.h index 2a8e0b6b9805..d1cd5c0f431a 100644 --- a/arch/x86/include/uapi/asm/kvm_para.h +++ b/arch/x86/include/uapi/asm/kvm_para.h @@ -112,8 +112,9 @@ struct kvm_mmu_op_release_pt { #define KVM_PV_REASON_PAGE_READY 2 struct kvm_vcpu_pv_apf_data { - __u32 reason; - __u8 pad[60]; + __u32 flags; + __u32 token; /* Used for page ready notification only */ + __u8 pad[56]; __u32 enabled; }; diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index b3d9b0d7a37d..d6f22a3a1f7d 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -218,23 +218,23 @@ again: } EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake); -u32 kvm_read_and_reset_pf_reason(void) +u32 kvm_read_and_reset_apf_flags(void) { - u32 reason = 0; + u32 flags = 0; if (__this_cpu_read(apf_reason.enabled)) { - reason = __this_cpu_read(apf_reason.reason); - __this_cpu_write(apf_reason.reason, 0); + flags = __this_cpu_read(apf_reason.flags); + __this_cpu_write(apf_reason.flags, 0); } - return reason; + return flags; } -EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason); -NOKPROBE_SYMBOL(kvm_read_and_reset_pf_reason); +EXPORT_SYMBOL_GPL(kvm_read_and_reset_apf_flags); +NOKPROBE_SYMBOL(kvm_read_and_reset_apf_flags); bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token) { - u32 reason = kvm_read_and_reset_pf_reason(); + u32 reason = kvm_read_and_reset_apf_flags(); switch (reason) { case KVM_PV_REASON_PAGE_NOT_PRESENT: diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 2e62a03410c7..5de1929cfc55 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -4164,7 +4164,7 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, #endif vcpu->arch.l1tf_flush_l1d = true; - switch (vcpu->arch.apf.host_apf_reason) { + switch (vcpu->arch.apf.host_apf_flags) { default: trace_kvm_page_fault(fault_address, error_code); @@ -4174,13 +4174,13 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code, insn_len); break; case KVM_PV_REASON_PAGE_NOT_PRESENT: - vcpu->arch.apf.host_apf_reason = 0; + vcpu->arch.apf.host_apf_flags = 0; local_irq_disable(); kvm_async_pf_task_wait_schedule(fault_address); local_irq_enable(); break; case KVM_PV_REASON_PAGE_READY: - vcpu->arch.apf.host_apf_reason = 0; + vcpu->arch.apf.host_apf_flags = 0; local_irq_disable(); kvm_async_pf_task_wake(fault_address); local_irq_enable(); diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 6b1049148c1b..8a6db11dcb43 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -921,7 +921,7 @@ int nested_svm_exit_special(struct vcpu_svm *svm) if (get_host_vmcb(svm)->control.intercept_exceptions & excp_bits) return NESTED_EXIT_HOST; else if (exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR && - svm->vcpu.arch.apf.host_apf_reason) + svm->vcpu.arch.apf.host_apf_flags) /* Trap async PF even if not shadowing */ return NESTED_EXIT_HOST; break; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 3871bfb40594..9e333b91ff78 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3459,7 +3459,8 @@ static fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) /* if exit due to PF check for async PF */ if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) - svm->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason(); + svm->vcpu.arch.apf.host_apf_flags = + kvm_read_and_reset_apf_flags(); if (npt_enabled) { vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR); diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 106fc6fceb97..119a2f7395d6 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5652,7 +5652,7 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) if (is_nmi(intr_info)) return true; else if (is_page_fault(intr_info)) - return vcpu->arch.apf.host_apf_reason || !enable_ept; + return vcpu->arch.apf.host_apf_flags || !enable_ept; else if (is_debug(intr_info) && vcpu->guest_debug & (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 7b55dc6230a9..5a43af0061ef 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -4765,7 +4765,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) if (is_page_fault(intr_info)) { cr2 = vmx_get_exit_qual(vcpu); /* EPT won't cause page fault directly */ - WARN_ON_ONCE(!vcpu->arch.apf.host_apf_reason && enable_ept); + WARN_ON_ONCE(!vcpu->arch.apf.host_apf_flags && enable_ept); return kvm_handle_page_fault(vcpu, error_code, cr2, NULL, 0); } @@ -6360,7 +6360,7 @@ static void handle_exception_nmi_irqoff(struct vcpu_vmx *vmx) /* if exit due to PF check for async PF */ if (is_page_fault(intr_info)) { - vmx->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason(); + vmx->vcpu.arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); /* Handle machine checks before interrupts are enabled */ } else if (is_machine_check(intr_info)) { kvm_machine_check(); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index b9a5ff7b922c..84aa3c1519ed 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2690,7 +2690,7 @@ static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) } if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa, - sizeof(u32))) + sizeof(u64))) return 1; vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS); @@ -10420,8 +10420,17 @@ static void kvm_del_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn) } } -static int apf_put_user(struct kvm_vcpu *vcpu, u32 val) +static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu) { + u32 reason = KVM_PV_REASON_PAGE_NOT_PRESENT; + + return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &reason, + sizeof(reason)); +} + +static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token) +{ + u64 val = (u64)token << 32 | KVM_PV_REASON_PAGE_READY; return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val, sizeof(val)); @@ -10466,7 +10475,7 @@ void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, kvm_add_async_pf_gfn(vcpu, work->arch.gfn); if (kvm_can_deliver_async_pf(vcpu) && - !apf_put_user(vcpu, KVM_PV_REASON_PAGE_NOT_PRESENT)) { + !apf_put_user_notpresent(vcpu)) { fault.vector = PF_VECTOR; fault.error_code_valid = true; fault.error_code = 0; @@ -10499,7 +10508,7 @@ void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa); if (vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED && - !apf_put_user(vcpu, KVM_PV_REASON_PAGE_READY)) { + !apf_put_user_ready(vcpu, work->arch.token)) { fault.vector = PF_VECTOR; fault.error_code_valid = true; fault.error_code = 0; From 7c0ade6c9023b2b90b757e2927b306bec1cc4ca6 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Mon, 25 May 2020 16:41:18 +0200 Subject: [PATCH 1010/1043] KVM: rename kvm_arch_can_inject_async_page_present() to kvm_arch_can_dequeue_async_page_present() An innocent reader of the following x86 KVM code: bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu) { if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED)) return true; ... may get very confused: if APF mechanism is not enabled, why do we report that we 'can inject async page present'? In reality, upon injection kvm_arch_async_page_present() will check the same condition again and, in case APF is disabled, will just drop the item. This is fine as the guest which deliberately disabled APF doesn't expect to get any APF notifications. Rename kvm_arch_can_inject_async_page_present() to kvm_arch_can_dequeue_async_page_present() to make it clear what we are checking: if the item can be dequeued (meaning either injected or just dropped). On s390 kvm_arch_can_inject_async_page_present() always returns 'true' so the rename doesn't matter much. Signed-off-by: Vitaly Kuznetsov Message-Id: <20200525144125.143875-4-vkuznets@redhat.com> Signed-off-by: Paolo Bonzini --- arch/s390/include/asm/kvm_host.h | 2 +- arch/s390/kvm/kvm-s390.c | 2 +- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/x86.c | 2 +- virt/kvm/async_pf.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 176f74cc2257..9d23bde1363b 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -973,7 +973,7 @@ struct kvm_arch_async_pf { unsigned long pfault_token; }; -bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu); +bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu); void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work); diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index a560a368f92c..06bde4bad205 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3943,7 +3943,7 @@ void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, /* s390 will always inject the page directly */ } -bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu) +bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu) { /* * s390 will always inject the page directly, diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 033f6173a857..f3897e417b69 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1660,7 +1660,7 @@ void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, struct kvm_async_pf *work); void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work); -bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu); +bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu); extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn); int kvm_skip_emulated_instruction(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 84aa3c1519ed..0e79b37b2b7e 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10521,7 +10521,7 @@ void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; } -bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu) +bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu) { if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED)) return true; diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c index 10b533f641a6..82e53f180a1a 100644 --- a/virt/kvm/async_pf.c +++ b/virt/kvm/async_pf.c @@ -134,7 +134,7 @@ void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu) struct kvm_async_pf *work; while (!list_empty_careful(&vcpu->async_pf.done) && - kvm_arch_can_inject_async_page_present(vcpu)) { + kvm_arch_can_dequeue_async_page_present(vcpu)) { spin_lock(&vcpu->async_pf.lock); work = list_first_entry(&vcpu->async_pf.done, typeof(*work), link); From 0958f0cefede403037653e44de0e3332d10b0e1a Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Mon, 25 May 2020 16:41:19 +0200 Subject: [PATCH 1011/1043] KVM: introduce kvm_read_guest_offset_cached() We already have kvm_write_guest_offset_cached(), introduce read analogue. Signed-off-by: Vitaly Kuznetsov Message-Id: <20200525144125.143875-5-vkuznets@redhat.com> Signed-off-by: Paolo Bonzini --- include/linux/kvm_host.h | 3 +++ virt/kvm/kvm_main.c | 19 ++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 161684696610..f43b59b1294c 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -734,6 +734,9 @@ int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset, int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len); int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, void *data, unsigned long len); +int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, + void *data, unsigned int offset, + unsigned long len); int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data, int offset, int len); int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data, diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index c9c6db5f77c2..e2af6ffce9c9 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2510,13 +2510,15 @@ int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, } EXPORT_SYMBOL_GPL(kvm_write_guest_cached); -int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, - void *data, unsigned long len) +int kvm_read_guest_offset_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, + void *data, unsigned int offset, + unsigned long len) { struct kvm_memslots *slots = kvm_memslots(kvm); int r; + gpa_t gpa = ghc->gpa + offset; - BUG_ON(len > ghc->len); + BUG_ON(len + offset > ghc->len); if (slots->generation != ghc->generation) { if (__kvm_gfn_to_hva_cache_init(slots, ghc, ghc->gpa, ghc->len)) @@ -2527,14 +2529,21 @@ int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, return -EFAULT; if (unlikely(!ghc->memslot)) - return kvm_read_guest(kvm, ghc->gpa, data, len); + return kvm_read_guest(kvm, gpa, data, len); - r = __copy_from_user(data, (void __user *)ghc->hva, len); + r = __copy_from_user(data, (void __user *)ghc->hva + offset, len); if (r) return -EFAULT; return 0; } +EXPORT_SYMBOL_GPL(kvm_read_guest_offset_cached); + +int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc, + void *data, unsigned long len) +{ + return kvm_read_guest_offset_cached(kvm, ghc, data, 0, len); +} EXPORT_SYMBOL_GPL(kvm_read_guest_cached); int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len) From 2635b5c4a0e407b84f68e188c719f28ba0e9ae1b Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Mon, 25 May 2020 16:41:20 +0200 Subject: [PATCH 1012/1043] KVM: x86: interrupt based APF 'page ready' event delivery Concerns were expressed around APF delivery via synthetic #PF exception as in some cases such delivery may collide with real page fault. For 'page ready' notifications we can easily switch to using an interrupt instead. Introduce new MSR_KVM_ASYNC_PF_INT mechanism and deprecate the legacy one. One notable difference between the two mechanisms is that interrupt may not get handled immediately so whenever we would like to deliver next event (regardless of its type) we must be sure the guest had read and cleared previous event in the slot. While on it, get rid on 'type 1/type 2' names for APF events in the documentation as they are causing confusion. Use 'page not present' and 'page ready' everywhere instead. Signed-off-by: Vitaly Kuznetsov Message-Id: <20200525144125.143875-6-vkuznets@redhat.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/msr.rst | 98 ++++++++++++++++++++-------- arch/x86/include/asm/kvm_host.h | 4 +- arch/x86/include/uapi/asm/kvm_para.h | 12 +++- arch/x86/kvm/x86.c | 93 ++++++++++++++++++-------- 4 files changed, 150 insertions(+), 57 deletions(-) diff --git a/Documentation/virt/kvm/msr.rst b/Documentation/virt/kvm/msr.rst index 33892036672d..be08df12f31a 100644 --- a/Documentation/virt/kvm/msr.rst +++ b/Documentation/virt/kvm/msr.rst @@ -190,41 +190,68 @@ MSR_KVM_ASYNC_PF_EN: 0x4b564d02 data: - Bits 63-6 hold 64-byte aligned physical address of a - 64 byte memory area which must be in guest RAM and must be - zeroed. Bits 5-3 are reserved and should be zero. Bit 0 is 1 - when asynchronous page faults are enabled on the vcpu 0 when - disabled. Bit 1 is 1 if asynchronous page faults can be injected - when vcpu is in cpl == 0. Bit 2 is 1 if asynchronous page faults - are delivered to L1 as #PF vmexits. Bit 2 can be set only if - KVM_FEATURE_ASYNC_PF_VMEXIT is present in CPUID. + Asynchronous page fault (APF) control MSR. - First 4 byte of 64 byte memory location will be written to by - the hypervisor at the time of asynchronous page fault (APF) - injection to indicate type of asynchronous page fault. Value - of 1 means that the page referred to by the page fault is not - present. Value 2 means that the page is now available. Disabling - interrupt inhibits APFs. Guest must not enable interrupt - before the reason is read, or it may be overwritten by another - APF. Since APF uses the same exception vector as regular page - fault guest must reset the reason to 0 before it does - something that can generate normal page fault. If during page - fault APF reason is 0 it means that this is regular page - fault. + Bits 63-6 hold 64-byte aligned physical address of a 64 byte memory area + which must be in guest RAM and must be zeroed. This memory is expected + to hold a copy of the following structure:: - During delivery of type 1 APF cr2 contains a token that will - be used to notify a guest when missing page becomes - available. When page becomes available type 2 APF is sent with - cr2 set to the token associated with the page. There is special - kind of token 0xffffffff which tells vcpu that it should wake - up all processes waiting for APFs and no individual type 2 APFs - will be sent. + struct kvm_vcpu_pv_apf_data { + /* Used for 'page not present' events delivered via #PF */ + __u32 flags; + + /* Used for 'page ready' events delivered via interrupt notification */ + __u32 token; + + __u8 pad[56]; + __u32 enabled; + }; + + Bits 5-4 of the MSR are reserved and should be zero. Bit 0 is set to 1 + when asynchronous page faults are enabled on the vcpu, 0 when disabled. + Bit 1 is 1 if asynchronous page faults can be injected when vcpu is in + cpl == 0. Bit 2 is 1 if asynchronous page faults are delivered to L1 as + #PF vmexits. Bit 2 can be set only if KVM_FEATURE_ASYNC_PF_VMEXIT is + present in CPUID. Bit 3 enables interrupt based delivery of 'page ready' + events. + + 'Page not present' events are currently always delivered as synthetic + #PF exception. During delivery of these events APF CR2 register contains + a token that will be used to notify the guest when missing page becomes + available. Also, to make it possible to distinguish between real #PF and + APF, first 4 bytes of 64 byte memory location ('flags') will be written + to by the hypervisor at the time of injection. Only first bit of 'flags' + is currently supported, when set, it indicates that the guest is dealing + with asynchronous 'page not present' event. If during a page fault APF + 'flags' is '0' it means that this is regular page fault. Guest is + supposed to clear 'flags' when it is done handling #PF exception so the + next event can be delivered. + + Note, since APF 'page not present' events use the same exception vector + as regular page fault, guest must reset 'flags' to '0' before it does + something that can generate normal page fault. + + Bytes 5-7 of 64 byte memory location ('token') will be written to by the + hypervisor at the time of APF 'page ready' event injection. The content + of these bytes is a token which was previously delivered as 'page not + present' event. The event indicates the page in now available. Guest is + supposed to write '0' to 'token' when it is done handling 'page ready' + event so the next one can be delivered. + + Note, MSR_KVM_ASYNC_PF_INT MSR specifying the interrupt vector for 'page + ready' APF delivery needs to be written to before enabling APF mechanism + in MSR_KVM_ASYNC_PF_EN or interrupt #0 can get injected. + + Note, previously, 'page ready' events were delivered via the same #PF + exception as 'page not present' events but this is now deprecated. If + bit 3 (interrupt based delivery) is not set APF events are not delivered. If APF is disabled while there are outstanding APFs, they will not be delivered. - Currently type 2 APF will be always delivered on the same vcpu as - type 1 was, but guest should not rely on that. + Currently 'page ready' APF events will be always delivered on the + same vcpu as 'page not present' event was, but guest should not rely on + that. MSR_KVM_STEAL_TIME: 0x4b564d03 @@ -319,3 +346,16 @@ data: KVM guests can request the host not to poll on HLT, for example if they are performing polling themselves. + +MSR_KVM_ASYNC_PF_INT: + 0x4b564d06 + +data: + Second asynchronous page fault (APF) control MSR. + + Bits 0-7: APIC vector for delivery of 'page ready' APF events. + Bits 8-63: Reserved + + Interrupt vector for asynchnonous 'page ready' notifications delivery. + The vector has to be set up before asynchronous page fault mechanism + is enabled in MSR_KVM_ASYNC_PF_EN. diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index f3897e417b69..2d39571451a0 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -767,7 +767,9 @@ struct kvm_vcpu_arch { bool halted; gfn_t gfns[ASYNC_PF_PER_VCPU]; struct gfn_to_hva_cache data; - u64 msr_val; + u64 msr_en_val; /* MSR_KVM_ASYNC_PF_EN */ + u64 msr_int_val; /* MSR_KVM_ASYNC_PF_INT */ + u16 vec; u32 id; bool send_user_only; u32 host_apf_flags; diff --git a/arch/x86/include/uapi/asm/kvm_para.h b/arch/x86/include/uapi/asm/kvm_para.h index d1cd5c0f431a..1d37d616b1fc 100644 --- a/arch/x86/include/uapi/asm/kvm_para.h +++ b/arch/x86/include/uapi/asm/kvm_para.h @@ -50,6 +50,7 @@ #define MSR_KVM_STEAL_TIME 0x4b564d03 #define MSR_KVM_PV_EOI_EN 0x4b564d04 #define MSR_KVM_POLL_CONTROL 0x4b564d05 +#define MSR_KVM_ASYNC_PF_INT 0x4b564d06 struct kvm_steal_time { __u64 steal; @@ -81,6 +82,11 @@ struct kvm_clock_pairing { #define KVM_ASYNC_PF_ENABLED (1 << 0) #define KVM_ASYNC_PF_SEND_ALWAYS (1 << 1) #define KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT (1 << 2) +#define KVM_ASYNC_PF_DELIVERY_AS_INT (1 << 3) + +/* MSR_KVM_ASYNC_PF_INT */ +#define KVM_ASYNC_PF_VEC_MASK GENMASK(7, 0) + /* Operations for KVM_HC_MMU_OP */ #define KVM_MMU_OP_WRITE_PTE 1 @@ -112,8 +118,12 @@ struct kvm_mmu_op_release_pt { #define KVM_PV_REASON_PAGE_READY 2 struct kvm_vcpu_pv_apf_data { + /* Used for 'page not present' events delivered via #PF */ __u32 flags; - __u32 token; /* Used for page ready notification only */ + + /* Used for 'page ready' events delivered via interrupt notification */ + __u32 token; + __u8 pad[56]; __u32 enabled; }; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 0e79b37b2b7e..e6f3ec5193b2 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1248,7 +1248,7 @@ static const u32 emulated_msrs_all[] = { HV_X64_MSR_TSC_EMULATION_STATUS, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME, - MSR_KVM_PV_EOI_EN, + MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_IA32_TSC_ADJUST, MSR_IA32_TSCDEADLINE, @@ -2673,17 +2673,24 @@ out: return r; } +static inline bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu) +{ + u64 mask = KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT; + + return (vcpu->arch.apf.msr_en_val & mask) == mask; +} + static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) { gpa_t gpa = data & ~0x3f; - /* Bits 3:5 are reserved, Should be zero */ - if (data & 0x38) + /* Bits 4:5 are reserved, Should be zero */ + if (data & 0x30) return 1; - vcpu->arch.apf.msr_val = data; + vcpu->arch.apf.msr_en_val = data; - if (!(data & KVM_ASYNC_PF_ENABLED)) { + if (!kvm_pv_async_pf_enabled(vcpu)) { kvm_clear_async_pf_completion_queue(vcpu); kvm_async_pf_hash_reset(vcpu); return 0; @@ -2695,7 +2702,25 @@ static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data) vcpu->arch.apf.send_user_only = !(data & KVM_ASYNC_PF_SEND_ALWAYS); vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT; + kvm_async_pf_wakeup_all(vcpu); + + return 0; +} + +static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data) +{ + /* Bits 8-63 are reserved */ + if (data >> 8) + return 1; + + if (!lapic_in_kernel(vcpu)) + return 1; + + vcpu->arch.apf.msr_int_val = data; + + vcpu->arch.apf.vec = data & KVM_ASYNC_PF_VEC_MASK; + return 0; } @@ -2917,6 +2942,10 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (kvm_pv_enable_async_pf(vcpu, data)) return 1; break; + case MSR_KVM_ASYNC_PF_INT: + if (kvm_pv_enable_async_pf_int(vcpu, data)) + return 1; + break; case MSR_KVM_STEAL_TIME: if (unlikely(!sched_info_on())) @@ -3191,7 +3220,10 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) msr_info->data = vcpu->arch.time; break; case MSR_KVM_ASYNC_PF_EN: - msr_info->data = vcpu->arch.apf.msr_val; + msr_info->data = vcpu->arch.apf.msr_en_val; + break; + case MSR_KVM_ASYNC_PF_INT: + msr_info->data = vcpu->arch.apf.msr_int_val; break; case MSR_KVM_STEAL_TIME: msr_info->data = vcpu->arch.st.msr_val; @@ -9553,7 +9585,8 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) vcpu->arch.cr2 = 0; kvm_make_request(KVM_REQ_EVENT, vcpu); - vcpu->arch.apf.msr_val = 0; + vcpu->arch.apf.msr_en_val = 0; + vcpu->arch.apf.msr_int_val = 0; vcpu->arch.st.msr_val = 0; kvmclock_reset(vcpu); @@ -10430,10 +10463,22 @@ static inline int apf_put_user_notpresent(struct kvm_vcpu *vcpu) static inline int apf_put_user_ready(struct kvm_vcpu *vcpu, u32 token) { - u64 val = (u64)token << 32 | KVM_PV_REASON_PAGE_READY; + unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token); - return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apf.data, &val, - sizeof(val)); + return kvm_write_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data, + &token, offset, sizeof(token)); +} + +static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu) +{ + unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token); + u32 val; + + if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data, + &val, offset, sizeof(val))) + return false; + + return !val; } static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu) @@ -10441,9 +10486,8 @@ static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu) if (!vcpu->arch.apf.delivery_as_pf_vmexit && is_guest_mode(vcpu)) return false; - if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED) || - (vcpu->arch.apf.send_user_only && - kvm_x86_ops.get_cpl(vcpu) == 0)) + if (!kvm_pv_async_pf_enabled(vcpu) || + (vcpu->arch.apf.send_user_only && kvm_x86_ops.get_cpl(vcpu) == 0)) return false; return true; @@ -10499,7 +10543,10 @@ void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) { - struct x86_exception fault; + struct kvm_lapic_irq irq = { + .delivery_mode = APIC_DM_FIXED, + .vector = vcpu->arch.apf.vec + }; if (work->wakeup_all) work->arch.token = ~0; /* broadcast wakeup */ @@ -10507,26 +10554,20 @@ void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, kvm_del_async_pf_gfn(vcpu, work->arch.gfn); trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa); - if (vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED && - !apf_put_user_ready(vcpu, work->arch.token)) { - fault.vector = PF_VECTOR; - fault.error_code_valid = true; - fault.error_code = 0; - fault.nested_page_fault = false; - fault.address = work->arch.token; - fault.async_page_fault = true; - kvm_inject_page_fault(vcpu, &fault); - } + if (kvm_pv_async_pf_enabled(vcpu) && + !apf_put_user_ready(vcpu, work->arch.token)) + kvm_apic_set_irq(vcpu, &irq, NULL); + vcpu->arch.apf.halted = false; vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; } bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu) { - if (!(vcpu->arch.apf.msr_val & KVM_ASYNC_PF_ENABLED)) + if (!kvm_pv_async_pf_enabled(vcpu)) return true; else - return kvm_can_do_async_pf(vcpu); + return apf_pageready_slot_free(vcpu); } void kvm_arch_start_assignment(struct kvm *kvm) From 557a961abbe06ed9dfd3b55ef7bd6e68295cda3d Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Mon, 25 May 2020 16:41:21 +0200 Subject: [PATCH 1013/1043] KVM: x86: acknowledgment mechanism for async pf page ready notifications If two page ready notifications happen back to back the second one is not delivered and the only mechanism we currently have is kvm_check_async_pf_completion() check in vcpu_run() loop. The check will only be performed with the next vmexit when it happens and in some cases it may take a while. With interrupt based page ready notification delivery the situation is even worse: unlike exceptions, interrupts are not handled immediately so we must check if the slot is empty. This is slow and unnecessary. Introduce dedicated MSR_KVM_ASYNC_PF_ACK MSR to communicate the fact that the slot is free and host should check its notification queue. Mandate using it for interrupt based 'page ready' APF event delivery. As kvm_check_async_pf_completion() is going away from vcpu_run() we need a way to communicate the fact that vcpu->async_pf.done queue has transitioned from empty to non-empty state. Introduce kvm_arch_async_page_present_queued() and KVM_REQ_APF_READY to do the job. Signed-off-by: Vitaly Kuznetsov Message-Id: <20200525144125.143875-7-vkuznets@redhat.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/msr.rst | 15 ++++++++++++++- arch/s390/include/asm/kvm_host.h | 2 ++ arch/x86/include/asm/kvm_host.h | 3 +++ arch/x86/include/uapi/asm/kvm_para.h | 1 + arch/x86/kvm/x86.c | 26 ++++++++++++++++++++++---- virt/kvm/async_pf.c | 10 ++++++++++ 6 files changed, 52 insertions(+), 5 deletions(-) diff --git a/Documentation/virt/kvm/msr.rst b/Documentation/virt/kvm/msr.rst index be08df12f31a..9b107889b033 100644 --- a/Documentation/virt/kvm/msr.rst +++ b/Documentation/virt/kvm/msr.rst @@ -236,7 +236,9 @@ data: of these bytes is a token which was previously delivered as 'page not present' event. The event indicates the page in now available. Guest is supposed to write '0' to 'token' when it is done handling 'page ready' - event so the next one can be delivered. + event and to write 1' to MSR_KVM_ASYNC_PF_ACK after clearing the location; + writing to the MSR forces KVM to re-scan its queue and deliver the next + pending notification. Note, MSR_KVM_ASYNC_PF_INT MSR specifying the interrupt vector for 'page ready' APF delivery needs to be written to before enabling APF mechanism @@ -359,3 +361,14 @@ data: Interrupt vector for asynchnonous 'page ready' notifications delivery. The vector has to be set up before asynchronous page fault mechanism is enabled in MSR_KVM_ASYNC_PF_EN. + +MSR_KVM_ASYNC_PF_ACK: + 0x4b564d07 + +data: + Asynchronous page fault (APF) acknowledgment. + + When the guest is done processing 'page ready' APF event and 'token' + field in 'struct kvm_vcpu_pv_apf_data' is cleared it is supposed to + write '1' to bit 0 of the MSR, this causes the host to re-scan its queue + and check if there are more notifications pending. diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 9d23bde1363b..3d554887794e 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -984,6 +984,8 @@ void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, struct kvm_async_pf *work); +static inline void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) {} + void kvm_arch_crypto_clear_masks(struct kvm *kvm); void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm, unsigned long *aqm, unsigned long *adm); diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 2d39571451a0..f345bcf38cfb 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -86,6 +86,7 @@ #define KVM_REQ_TLB_FLUSH_CURRENT KVM_ARCH_REQ(26) #define KVM_REQ_HV_TLB_FLUSH \ KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_NO_WAKEUP) +#define KVM_REQ_APF_READY KVM_ARCH_REQ(28) #define CR0_RESERVED_BITS \ (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ @@ -775,6 +776,7 @@ struct kvm_vcpu_arch { u32 host_apf_flags; unsigned long nested_apf_token; bool delivery_as_pf_vmexit; + bool pageready_pending; } apf; /* OSVW MSRs (AMD only) */ @@ -1662,6 +1664,7 @@ void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, struct kvm_async_pf *work); void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work); +void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu); bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu); extern bool kvm_find_async_pf_gfn(struct kvm_vcpu *vcpu, gfn_t gfn); diff --git a/arch/x86/include/uapi/asm/kvm_para.h b/arch/x86/include/uapi/asm/kvm_para.h index 1d37d616b1fc..7ac20df80ba8 100644 --- a/arch/x86/include/uapi/asm/kvm_para.h +++ b/arch/x86/include/uapi/asm/kvm_para.h @@ -51,6 +51,7 @@ #define MSR_KVM_PV_EOI_EN 0x4b564d04 #define MSR_KVM_POLL_CONTROL 0x4b564d05 #define MSR_KVM_ASYNC_PF_INT 0x4b564d06 +#define MSR_KVM_ASYNC_PF_ACK 0x4b564d07 struct kvm_steal_time { __u64 steal; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index e6f3ec5193b2..c9d709a672f3 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1248,7 +1248,7 @@ static const u32 emulated_msrs_all[] = { HV_X64_MSR_TSC_EMULATION_STATUS, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME, - MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, + MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK, MSR_IA32_TSC_ADJUST, MSR_IA32_TSCDEADLINE, @@ -2946,6 +2946,12 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (kvm_pv_enable_async_pf_int(vcpu, data)) return 1; break; + case MSR_KVM_ASYNC_PF_ACK: + if (data & 0x1) { + vcpu->arch.apf.pageready_pending = false; + kvm_check_async_pf_completion(vcpu); + } + break; case MSR_KVM_STEAL_TIME: if (unlikely(!sched_info_on())) @@ -3225,6 +3231,9 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case MSR_KVM_ASYNC_PF_INT: msr_info->data = vcpu->arch.apf.msr_int_val; break; + case MSR_KVM_ASYNC_PF_ACK: + msr_info->data = 0; + break; case MSR_KVM_STEAL_TIME: msr_info->data = vcpu->arch.st.msr_val; break; @@ -8413,6 +8422,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) kvm_hv_process_stimers(vcpu); if (kvm_check_request(KVM_REQ_APICV_UPDATE, vcpu)) kvm_vcpu_update_apicv(vcpu); + if (kvm_check_request(KVM_REQ_APF_READY, vcpu)) + kvm_check_async_pf_completion(vcpu); } if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win) { @@ -8664,8 +8675,6 @@ static int vcpu_run(struct kvm_vcpu *vcpu) break; } - kvm_check_async_pf_completion(vcpu); - if (signal_pending(current)) { r = -EINTR; vcpu->run->exit_reason = KVM_EXIT_INTR; @@ -10555,13 +10564,22 @@ void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa); if (kvm_pv_async_pf_enabled(vcpu) && - !apf_put_user_ready(vcpu, work->arch.token)) + !apf_put_user_ready(vcpu, work->arch.token)) { + vcpu->arch.apf.pageready_pending = true; kvm_apic_set_irq(vcpu, &irq, NULL); + } vcpu->arch.apf.halted = false; vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; } +void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) +{ + kvm_make_request(KVM_REQ_APF_READY, vcpu); + if (!vcpu->arch.apf.pageready_pending) + kvm_vcpu_kick(vcpu); +} + bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu) { if (!kvm_pv_async_pf_enabled(vcpu)) diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c index 82e53f180a1a..f1e07fae84e9 100644 --- a/virt/kvm/async_pf.c +++ b/virt/kvm/async_pf.c @@ -51,6 +51,7 @@ static void async_pf_execute(struct work_struct *work) unsigned long addr = apf->addr; gpa_t cr2_or_gpa = apf->cr2_or_gpa; int locked = 1; + bool first; might_sleep(); @@ -69,10 +70,14 @@ static void async_pf_execute(struct work_struct *work) kvm_arch_async_page_present(vcpu, apf); spin_lock(&vcpu->async_pf.lock); + first = list_empty(&vcpu->async_pf.done); list_add_tail(&apf->link, &vcpu->async_pf.done); apf->vcpu = NULL; spin_unlock(&vcpu->async_pf.lock); + if (!IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC) && first) + kvm_arch_async_page_present_queued(vcpu); + /* * apf may be freed by kvm_check_async_pf_completion() after * this point @@ -201,6 +206,7 @@ retry_sync: int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu) { struct kvm_async_pf *work; + bool first; if (!list_empty_careful(&vcpu->async_pf.done)) return 0; @@ -213,9 +219,13 @@ int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu) INIT_LIST_HEAD(&work->queue); /* for list_del to work */ spin_lock(&vcpu->async_pf.lock); + first = list_empty(&vcpu->async_pf.done); list_add_tail(&work->link, &vcpu->async_pf.done); spin_unlock(&vcpu->async_pf.lock); + if (!IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC) && first) + kvm_arch_async_page_present_queued(vcpu); + vcpu->async_pf.queued++; return 0; } From 72de5fa4c16195827252b961ba44028a39dfeaff Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Mon, 25 May 2020 16:41:22 +0200 Subject: [PATCH 1014/1043] KVM: x86: announce KVM_FEATURE_ASYNC_PF_INT Introduce new capability to indicate that KVM supports interrupt based delivery of 'page ready' APF events. This includes support for both MSR_KVM_ASYNC_PF_INT and MSR_KVM_ASYNC_PF_ACK. Signed-off-by: Vitaly Kuznetsov Message-Id: <20200525144125.143875-8-vkuznets@redhat.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/cpuid.rst | 6 ++++++ Documentation/virt/kvm/msr.rst | 12 ++++++++---- arch/x86/include/uapi/asm/kvm_para.h | 1 + arch/x86/kvm/cpuid.c | 3 ++- arch/x86/kvm/x86.c | 1 + include/uapi/linux/kvm.h | 1 + 6 files changed, 19 insertions(+), 5 deletions(-) diff --git a/Documentation/virt/kvm/cpuid.rst b/Documentation/virt/kvm/cpuid.rst index f721c89327ec..a7dff9186bed 100644 --- a/Documentation/virt/kvm/cpuid.rst +++ b/Documentation/virt/kvm/cpuid.rst @@ -86,6 +86,12 @@ KVM_FEATURE_PV_SCHED_YIELD 13 guest checks this feature bit before using paravirtualized sched yield. +KVM_FEATURE_ASYNC_PF_INT 14 guest checks this feature bit + before using the second async + pf control msr 0x4b564d06 and + async pf acknowledgment msr + 0x4b564d07. + KVM_FEATURE_CLOCSOURCE_STABLE_BIT 24 host will warn if no guest-side per-cpu warps are expeced in kvmclock diff --git a/Documentation/virt/kvm/msr.rst b/Documentation/virt/kvm/msr.rst index 9b107889b033..e37a14c323d2 100644 --- a/Documentation/virt/kvm/msr.rst +++ b/Documentation/virt/kvm/msr.rst @@ -213,7 +213,8 @@ data: cpl == 0. Bit 2 is 1 if asynchronous page faults are delivered to L1 as #PF vmexits. Bit 2 can be set only if KVM_FEATURE_ASYNC_PF_VMEXIT is present in CPUID. Bit 3 enables interrupt based delivery of 'page ready' - events. + events. Bit 3 can only be set if KVM_FEATURE_ASYNC_PF_INT is present in + CPUID. 'Page not present' events are currently always delivered as synthetic #PF exception. During delivery of these events APF CR2 register contains @@ -242,7 +243,8 @@ data: Note, MSR_KVM_ASYNC_PF_INT MSR specifying the interrupt vector for 'page ready' APF delivery needs to be written to before enabling APF mechanism - in MSR_KVM_ASYNC_PF_EN or interrupt #0 can get injected. + in MSR_KVM_ASYNC_PF_EN or interrupt #0 can get injected. The MSR is + available if KVM_FEATURE_ASYNC_PF_INT is present in CPUID. Note, previously, 'page ready' events were delivered via the same #PF exception as 'page not present' events but this is now deprecated. If @@ -360,7 +362,8 @@ data: Interrupt vector for asynchnonous 'page ready' notifications delivery. The vector has to be set up before asynchronous page fault mechanism - is enabled in MSR_KVM_ASYNC_PF_EN. + is enabled in MSR_KVM_ASYNC_PF_EN. The MSR is only available if + KVM_FEATURE_ASYNC_PF_INT is present in CPUID. MSR_KVM_ASYNC_PF_ACK: 0x4b564d07 @@ -371,4 +374,5 @@ data: When the guest is done processing 'page ready' APF event and 'token' field in 'struct kvm_vcpu_pv_apf_data' is cleared it is supposed to write '1' to bit 0 of the MSR, this causes the host to re-scan its queue - and check if there are more notifications pending. + and check if there are more notifications pending. The MSR is available + if KVM_FEATURE_ASYNC_PF_INT is present in CPUID. diff --git a/arch/x86/include/uapi/asm/kvm_para.h b/arch/x86/include/uapi/asm/kvm_para.h index 7ac20df80ba8..812e9b4c1114 100644 --- a/arch/x86/include/uapi/asm/kvm_para.h +++ b/arch/x86/include/uapi/asm/kvm_para.h @@ -31,6 +31,7 @@ #define KVM_FEATURE_PV_SEND_IPI 11 #define KVM_FEATURE_POLL_CONTROL 12 #define KVM_FEATURE_PV_SCHED_YIELD 13 +#define KVM_FEATURE_ASYNC_PF_INT 14 #define KVM_HINTS_REALTIME 0 diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index cd708b0b460a..a9f1905896fb 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -711,7 +711,8 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) (1 << KVM_FEATURE_ASYNC_PF_VMEXIT) | (1 << KVM_FEATURE_PV_SEND_IPI) | (1 << KVM_FEATURE_POLL_CONTROL) | - (1 << KVM_FEATURE_PV_SCHED_YIELD); + (1 << KVM_FEATURE_PV_SCHED_YIELD) | + (1 << KVM_FEATURE_ASYNC_PF_INT); if (sched_info_on()) entry->eax |= (1 << KVM_FEATURE_STEAL_TIME); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index c9d709a672f3..ae3a7f2fbda2 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3446,6 +3446,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_X86_ROBUST_SINGLESTEP: case KVM_CAP_XSAVE: case KVM_CAP_ASYNC_PF: + case KVM_CAP_ASYNC_PF_INT: case KVM_CAP_GET_TSC_KHZ: case KVM_CAP_KVMCLOCK_CTRL: case KVM_CAP_READONLY_MEM: diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index ac9eba0289d1..fc0075988b80 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -1018,6 +1018,7 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_S390_PROTECTED 180 #define KVM_CAP_PPC_SECURE_GUEST 181 #define KVM_CAP_HALT_POLL 182 +#define KVM_CAP_ASYNC_PF_INT 183 #ifdef KVM_CAP_IRQ_ROUTING From cbd717585b8038d93c309176bb563a5c6de60ac7 Mon Sep 17 00:00:00 2001 From: Wei Wang Date: Fri, 29 May 2020 15:43:44 +0800 Subject: [PATCH 1015/1043] KVM: x86/pmu: Tweak kvm_pmu_get_msr to pass 'struct msr_data' in Change kvm_pmu_get_msr() to get the msr_data struct, as the host_initiated field from the struct could be used by get_msr. This also makes this API consistent with kvm_pmu_set_msr. No functional changes. Signed-off-by: Wei Wang Message-Id: <20200529074347.124619-2-like.xu@linux.intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/pmu.c | 4 ++-- arch/x86/kvm/pmu.h | 4 ++-- arch/x86/kvm/svm/pmu.c | 7 ++++--- arch/x86/kvm/vmx/pmu_intel.c | 19 +++++++++++-------- arch/x86/kvm/x86.c | 4 ++-- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index a5078841bdac..b86346903f2e 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -397,9 +397,9 @@ static void kvm_pmu_mark_pmc_in_use(struct kvm_vcpu *vcpu, u32 msr) __set_bit(pmc->idx, pmu->pmc_in_use); } -int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *data) +int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { - return kvm_x86_ops.pmu_ops->get_msr(vcpu, msr, data); + return kvm_x86_ops.pmu_ops->get_msr(vcpu, msr_info); } int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h index a6c78a797cb1..ab85eed8a6cc 100644 --- a/arch/x86/kvm/pmu.h +++ b/arch/x86/kvm/pmu.h @@ -32,7 +32,7 @@ struct kvm_pmu_ops { struct kvm_pmc *(*msr_idx_to_pmc)(struct kvm_vcpu *vcpu, u32 msr); int (*is_valid_rdpmc_ecx)(struct kvm_vcpu *vcpu, unsigned int idx); bool (*is_valid_msr)(struct kvm_vcpu *vcpu, u32 msr); - int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr, u64 *data); + int (*get_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr_info); int (*set_msr)(struct kvm_vcpu *vcpu, struct msr_data *msr_info); void (*refresh)(struct kvm_vcpu *vcpu); void (*init)(struct kvm_vcpu *vcpu); @@ -147,7 +147,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu); int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned pmc, u64 *data); int kvm_pmu_is_valid_rdpmc_ecx(struct kvm_vcpu *vcpu, unsigned int idx); bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr); -int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *data); +int kvm_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info); int kvm_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info); void kvm_pmu_refresh(struct kvm_vcpu *vcpu); void kvm_pmu_reset(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index ce0b10fe5e2b..035da07500e8 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -215,21 +215,22 @@ static struct kvm_pmc *amd_msr_idx_to_pmc(struct kvm_vcpu *vcpu, u32 msr) return pmc; } -static int amd_pmu_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *data) +static int amd_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); struct kvm_pmc *pmc; + u32 msr = msr_info->index; /* MSR_PERFCTRn */ pmc = get_gp_pmc_amd(pmu, msr, PMU_TYPE_COUNTER); if (pmc) { - *data = pmc_read_counter(pmc); + msr_info->data = pmc_read_counter(pmc); return 0; } /* MSR_EVNTSELn */ pmc = get_gp_pmc_amd(pmu, msr, PMU_TYPE_EVNTSEL); if (pmc) { - *data = pmc->eventsel; + msr_info->data = pmc->eventsel; return 0; } diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 7c857737b438..e1a303fefc16 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -184,35 +184,38 @@ static struct kvm_pmc *intel_msr_idx_to_pmc(struct kvm_vcpu *vcpu, u32 msr) return pmc; } -static int intel_pmu_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *data) +static int intel_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); struct kvm_pmc *pmc; + u32 msr = msr_info->index; switch (msr) { case MSR_CORE_PERF_FIXED_CTR_CTRL: - *data = pmu->fixed_ctr_ctrl; + msr_info->data = pmu->fixed_ctr_ctrl; return 0; case MSR_CORE_PERF_GLOBAL_STATUS: - *data = pmu->global_status; + msr_info->data = pmu->global_status; return 0; case MSR_CORE_PERF_GLOBAL_CTRL: - *data = pmu->global_ctrl; + msr_info->data = pmu->global_ctrl; return 0; case MSR_CORE_PERF_GLOBAL_OVF_CTRL: - *data = pmu->global_ovf_ctrl; + msr_info->data = pmu->global_ovf_ctrl; return 0; default: if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0))) { u64 val = pmc_read_counter(pmc); - *data = val & pmu->counter_bitmask[KVM_PMC_GP]; + msr_info->data = + val & pmu->counter_bitmask[KVM_PMC_GP]; return 0; } else if ((pmc = get_fixed_pmc(pmu, msr))) { u64 val = pmc_read_counter(pmc); - *data = val & pmu->counter_bitmask[KVM_PMC_FIXED]; + msr_info->data = + val & pmu->counter_bitmask[KVM_PMC_FIXED]; return 0; } else if ((pmc = get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0))) { - *data = pmc->eventsel; + msr_info->data = pmc->eventsel; return 0; } } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index ae3a7f2fbda2..7e34ff784695 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3148,7 +3148,7 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR1: case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL1: if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) - return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data); + return kvm_pmu_get_msr(vcpu, msr_info); msr_info->data = 0; break; case MSR_IA32_UCODE_REV: @@ -3316,7 +3316,7 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; default: if (kvm_pmu_is_valid_msr(vcpu, msr_info->index)) - return kvm_pmu_get_msr(vcpu, msr_info->index, &msr_info->data); + return kvm_pmu_get_msr(vcpu, msr_info); if (!ignore_msrs) { vcpu_debug_ratelimited(vcpu, "unhandled rdmsr: 0x%x\n", msr_info->index); From 27461da31089ace6966e4f1695cba7eb87ffbe4f Mon Sep 17 00:00:00 2001 From: Like Xu Date: Fri, 29 May 2020 15:43:45 +0800 Subject: [PATCH 1016/1043] KVM: x86/pmu: Support full width counting Intel CPUs have a new alternative MSR range (starting from MSR_IA32_PMC0) for GP counters that allows writing the full counter width. Enable this range from a new capability bit (IA32_PERF_CAPABILITIES.FW_WRITE[bit 13]). The guest would query CPUID to get the counter width, and sign extends the counter values as needed. The traditional MSRs always limit to 32bit, even though the counter internally is larger (48 or 57 bits). When the new capability is set, use the alternative range which do not have these restrictions. This lowers the overhead of perf stat slightly because it has to do less interrupts to accumulate the counter value. Signed-off-by: Like Xu Message-Id: <20200529074347.124619-3-like.xu@linux.intel.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/cpuid.c | 2 +- arch/x86/kvm/vmx/capabilities.h | 11 +++++++ arch/x86/kvm/vmx/pmu_intel.c | 52 ++++++++++++++++++++++++++++++--- arch/x86/kvm/vmx/vmx.c | 3 ++ arch/x86/kvm/x86.c | 2 ++ 6 files changed, 66 insertions(+), 5 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index f345bcf38cfb..b878fcd164ce 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -601,6 +601,7 @@ struct kvm_vcpu_arch { u64 ia32_xss; u64 microcode_version; u64 arch_capabilities; + u64 perf_capabilities; /* * Paging state of the vcpu diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index a9f1905896fb..253b8e875ccd 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -296,7 +296,7 @@ void kvm_set_cpu_caps(void) F(XMM3) | F(PCLMULQDQ) | 0 /* DTES64, MONITOR */ | 0 /* DS-CPL, VMX, SMX, EST */ | 0 /* TM2 */ | F(SSSE3) | 0 /* CNXT-ID */ | 0 /* Reserved */ | - F(FMA) | F(CX16) | 0 /* xTPR Update, PDCM */ | + F(FMA) | F(CX16) | 0 /* xTPR Update */ | F(PDCM) | F(PCID) | 0 /* Reserved, DCA */ | F(XMM4_1) | F(XMM4_2) | F(X2APIC) | F(MOVBE) | F(POPCNT) | 0 /* Reserved*/ | F(AES) | F(XSAVE) | 0 /* OSXSAVE */ | F(AVX) | diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index 8903475f751e..4bbd8b448d22 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -18,6 +18,8 @@ extern int __read_mostly pt_mode; #define PT_MODE_SYSTEM 0 #define PT_MODE_HOST_GUEST 1 +#define PMU_CAP_FW_WRITES (1ULL << 13) + struct nested_vmx_msrs { /* * We only store the "true" versions of the VMX capability MSRs. We @@ -367,4 +369,13 @@ static inline bool vmx_pt_mode_is_host_guest(void) return pt_mode == PT_MODE_HOST_GUEST; } +static inline u64 vmx_get_perf_capabilities(void) +{ + /* + * Since counters are virtualized, KVM would support full + * width counting unconditionally, even if the host lacks it. + */ + return PMU_CAP_FW_WRITES; +} + #endif /* __KVM_X86_VMX_CAPS_H */ diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index e1a303fefc16..d33d890b605f 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -18,6 +18,8 @@ #include "nested.h" #include "pmu.h" +#define MSR_PMC_FULL_WIDTH_BIT (MSR_IA32_PMC0 - MSR_IA32_PERFCTR0) + static struct kvm_event_hw_type_mapping intel_arch_events[] = { /* Index must match CPUID 0x0A.EBX bit vector */ [0] = { 0x3c, 0x00, PERF_COUNT_HW_CPU_CYCLES }, @@ -150,6 +152,22 @@ static struct kvm_pmc *intel_rdpmc_ecx_to_pmc(struct kvm_vcpu *vcpu, return &counters[array_index_nospec(idx, num_counters)]; } +static inline bool fw_writes_is_enabled(struct kvm_vcpu *vcpu) +{ + if (!guest_cpuid_has(vcpu, X86_FEATURE_PDCM)) + return false; + + return vcpu->arch.perf_capabilities & PMU_CAP_FW_WRITES; +} + +static inline struct kvm_pmc *get_fw_gp_pmc(struct kvm_pmu *pmu, u32 msr) +{ + if (!fw_writes_is_enabled(pmu_to_vcpu(pmu))) + return NULL; + + return get_gp_pmc(pmu, msr, MSR_IA32_PMC0); +} + static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) { struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); @@ -162,10 +180,13 @@ static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) case MSR_CORE_PERF_GLOBAL_OVF_CTRL: ret = pmu->version > 1; break; + case MSR_IA32_PERF_CAPABILITIES: + ret = guest_cpuid_has(vcpu, X86_FEATURE_PDCM); + break; default: ret = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0) || get_gp_pmc(pmu, msr, MSR_P6_EVNTSEL0) || - get_fixed_pmc(pmu, msr); + get_fixed_pmc(pmu, msr) || get_fw_gp_pmc(pmu, msr); break; } @@ -203,8 +224,15 @@ static int intel_pmu_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) case MSR_CORE_PERF_GLOBAL_OVF_CTRL: msr_info->data = pmu->global_ovf_ctrl; return 0; + case MSR_IA32_PERF_CAPABILITIES: + if (!msr_info->host_initiated && + !guest_cpuid_has(vcpu, X86_FEATURE_PDCM)) + return 1; + msr_info->data = vcpu->arch.perf_capabilities; + return 0; default: - if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0))) { + if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) || + (pmc = get_gp_pmc(pmu, msr, MSR_IA32_PMC0))) { u64 val = pmc_read_counter(pmc); msr_info->data = val & pmu->counter_bitmask[KVM_PMC_GP]; @@ -261,9 +289,22 @@ static int intel_pmu_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return 0; } break; + case MSR_IA32_PERF_CAPABILITIES: + if (!msr_info->host_initiated) + return 1; + if (guest_cpuid_has(vcpu, X86_FEATURE_PDCM) ? + (data & ~vmx_get_perf_capabilities()) : data) + return 1; + vcpu->arch.perf_capabilities = data; + return 0; default: - if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0))) { - if (!msr_info->host_initiated) + if ((pmc = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0)) || + (pmc = get_gp_pmc(pmu, msr, MSR_IA32_PMC0))) { + if ((msr & MSR_PMC_FULL_WIDTH_BIT) && + (data & ~pmu->counter_bitmask[KVM_PMC_GP])) + return 1; + if (!msr_info->host_initiated && + !(msr & MSR_PMC_FULL_WIDTH_BIT)) data = (s64)(s32)data; pmc->counter += data - pmc_read_counter(pmc); if (pmc->perf_event) @@ -303,6 +344,7 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) pmu->counter_bitmask[KVM_PMC_FIXED] = 0; pmu->version = 0; pmu->reserved_bits = 0xffffffff00200000ull; + vcpu->arch.perf_capabilities = 0; entry = kvm_find_cpuid_entry(vcpu, 0xa, 0); if (!entry) @@ -315,6 +357,8 @@ static void intel_pmu_refresh(struct kvm_vcpu *vcpu) return; perf_get_x86_pmu_capability(&x86_pmu); + if (guest_cpuid_has(vcpu, X86_FEATURE_PDCM)) + vcpu->arch.perf_capabilities = vmx_get_perf_capabilities(); pmu->nr_arch_gp_counters = min_t(int, eax.split.num_counters, x86_pmu.num_counters_gp); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 5a43af0061ef..170cc76a581f 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1788,6 +1788,9 @@ static int vmx_get_msr_feature(struct kvm_msr_entry *msr) if (!nested) return 1; return vmx_get_vmx_msr(&vmcs_config.nested, msr->index, &msr->data); + case MSR_IA32_PERF_CAPABILITIES: + msr->data = vmx_get_perf_capabilities(); + return 0; default: return 1; } diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 7e34ff784695..ca8a57312291 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1253,6 +1253,7 @@ static const u32 emulated_msrs_all[] = { MSR_IA32_TSC_ADJUST, MSR_IA32_TSCDEADLINE, MSR_IA32_ARCH_CAPABILITIES, + MSR_IA32_PERF_CAPABILITIES, MSR_IA32_MISC_ENABLE, MSR_IA32_MCG_STATUS, MSR_IA32_MCG_CTL, @@ -1319,6 +1320,7 @@ static const u32 msr_based_features_all[] = { MSR_F10H_DECFG, MSR_IA32_UCODE_REV, MSR_IA32_ARCH_CAPABILITIES, + MSR_IA32_PERF_CAPABILITIES, }; static u32 msr_based_features[ARRAY_SIZE(msr_based_features_all)]; From f7d31e65368aeef973fab788aa22c4f1d5a6af66 Mon Sep 17 00:00:00 2001 From: Jon Doron Date: Fri, 24 Apr 2020 14:37:40 +0300 Subject: [PATCH 1017/1043] x86/kvm/hyper-v: Explicitly align hcall param for kvm_hyperv_exit The problem the patch is trying to address is the fact that 'struct kvm_hyperv_exit' has different layout on when compiling in 32 and 64 bit modes. In 64-bit mode the default alignment boundary is 64 bits thus forcing extra gaps after 'type' and 'msr' but in 32-bit mode the boundary is at 32 bits thus no extra gaps. This is an issue as even when the kernel is 64 bit, the userspace using the interface can be both 32 and 64 bit but the same 32 bit userspace has to work with 32 bit kernel. The issue is fixed by forcing the 64 bit layout, this leads to ABI change for 32 bit builds and while we are obviously breaking '32 bit userspace with 32 bit kernel' case, we're fixing the '32 bit userspace with 64 bit kernel' one. As the interface has no (known) users and 32 bit KVM is rather baroque nowadays, this seems like a reasonable decision. Reviewed-by: Vitaly Kuznetsov Signed-off-by: Jon Doron Message-Id: <20200424113746.3473563-2-arilou@gmail.com> Reviewed-by: Roman Kagan Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 2 ++ include/uapi/linux/kvm.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index d871dacb984e..d0569db2b1e2 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -5067,9 +5067,11 @@ EOI was received. #define KVM_EXIT_HYPERV_SYNIC 1 #define KVM_EXIT_HYPERV_HCALL 2 __u32 type; + __u32 pad1; union { struct { __u32 msr; + __u32 pad2; __u64 control; __u64 evt_page; __u64 msg_page; diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index fc0075988b80..cc04aceb8793 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -189,9 +189,11 @@ struct kvm_hyperv_exit { #define KVM_EXIT_HYPERV_SYNIC 1 #define KVM_EXIT_HYPERV_HCALL 2 __u32 type; + __u32 pad1; union { struct { __u32 msr; + __u32 pad2; __u64 control; __u64 evt_page; __u64 msg_page; From 850448f35aaf45215276468d63c91ab1e230cf06 Mon Sep 17 00:00:00 2001 From: Peter Shier Date: Tue, 26 May 2020 14:51:06 -0700 Subject: [PATCH 1018/1043] KVM: nVMX: Fix VMX preemption timer migration Add new field to hold preemption timer expiration deadline appended to struct kvm_vmx_nested_state_hdr. This is to prevent the first VM-Enter after migration from incorrectly restarting the timer with the full timer value instead of partially decayed timer value. KVM_SET_NESTED_STATE restarts timer using migrated state regardless of whether L1 sets VM_EXIT_SAVE_VMX_PREEMPTION_TIMER. Fixes: cf8b84f48a593 ("kvm: nVMX: Prepare for checkpointing L2 state") Signed-off-by: Peter Shier Signed-off-by: Makarand Sonare Message-Id: <20200526215107.205814-2-makarandsonare@google.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 4 +++ arch/x86/include/uapi/asm/kvm.h | 3 ++ arch/x86/kvm/vmx/nested.c | 55 ++++++++++++++++++++++++++++----- arch/x86/kvm/vmx/vmx.h | 2 ++ 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index d0569db2b1e2..db22e834ce2a 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -4334,9 +4334,13 @@ Errors: #define KVM_STATE_NESTED_VMX_SMM_GUEST_MODE 0x00000001 #define KVM_STATE_NESTED_VMX_SMM_VMXON 0x00000002 +#define KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE 0x00000001 + struct kvm_vmx_nested_state_hdr { + __u32 flags; __u64 vmxon_pa; __u64 vmcs12_pa; + __u64 preemption_timer_deadline; struct { __u16 flags; diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h index 12075a9de1c1..17c5a038f42d 100644 --- a/arch/x86/include/uapi/asm/kvm.h +++ b/arch/x86/include/uapi/asm/kvm.h @@ -400,6 +400,7 @@ struct kvm_sync_regs { #define KVM_STATE_NESTED_SVM_VMCB_SIZE 0x1000 +#define KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE 0x00000001 struct kvm_vmx_nested_state_data { __u8 vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE]; @@ -407,8 +408,10 @@ struct kvm_vmx_nested_state_data { }; struct kvm_vmx_nested_state_hdr { + __u32 flags; __u64 vmxon_pa; __u64 vmcs12_pa; + __u64 preemption_timer_deadline; struct { __u16 flags; diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 119a2f7395d6..da87bb8670bb 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -2087,9 +2087,29 @@ static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer) return HRTIMER_NORESTART; } -static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu) +static u64 vmx_calc_preemption_timer_value(struct kvm_vcpu *vcpu) +{ + struct vcpu_vmx *vmx = to_vmx(vcpu); + struct vmcs12 *vmcs12 = get_vmcs12(vcpu); + u64 timer_value = 0; + + u64 l1_scaled_tsc = kvm_read_l1_tsc(vcpu, rdtsc()) >> + VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; + + if (!vmx->nested.has_preemption_timer_deadline) { + timer_value = vmcs12->vmx_preemption_timer_value; + vmx->nested.preemption_timer_deadline = timer_value + + l1_scaled_tsc; + vmx->nested.has_preemption_timer_deadline = true; + } else if (l1_scaled_tsc < vmx->nested.preemption_timer_deadline) + timer_value = vmx->nested.preemption_timer_deadline - + l1_scaled_tsc; + return timer_value; +} + +static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu, + u64 preemption_timeout) { - u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value; struct vcpu_vmx *vmx = to_vmx(vcpu); /* @@ -3348,8 +3368,10 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, * the timer. */ vmx->nested.preemption_timer_expired = false; - if (nested_cpu_has_preemption_timer(vmcs12)) - vmx_start_preemption_timer(vcpu); + if (nested_cpu_has_preemption_timer(vmcs12)) { + u64 timer_value = vmx_calc_preemption_timer_value(vcpu); + vmx_start_preemption_timer(vcpu, timer_value); + } /* * Note no nested_vmx_succeed or nested_vmx_fail here. At this point @@ -3457,6 +3479,7 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) * the nested entry. */ vmx->nested.nested_run_pending = 1; + vmx->nested.has_preemption_timer_deadline = false; status = nested_vmx_enter_non_root_mode(vcpu, true); if (unlikely(status != NVMX_VMENTRY_SUCCESS)) goto vmentry_failed; @@ -3957,9 +3980,10 @@ static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE; if (nested_cpu_has_preemption_timer(vmcs12) && - vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER) - vmcs12->vmx_preemption_timer_value = - vmx_get_preemption_timer_value(vcpu); + vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER && + !vmx->nested.nested_run_pending) + vmcs12->vmx_preemption_timer_value = + vmx_get_preemption_timer_value(vcpu); /* * In some cases (usually, nested EPT), L2 is allowed to change its @@ -5891,8 +5915,10 @@ static int vmx_get_nested_state(struct kvm_vcpu *vcpu, .flags = 0, .format = KVM_STATE_NESTED_FORMAT_VMX, .size = sizeof(kvm_state), + .hdr.vmx.flags = 0, .hdr.vmx.vmxon_pa = -1ull, .hdr.vmx.vmcs12_pa = -1ull, + .hdr.vmx.preemption_timer_deadline = 0, }; struct kvm_vmx_nested_state_data __user *user_vmx_nested_state = &user_kvm_nested_state->data.vmx[0]; @@ -5934,6 +5960,14 @@ static int vmx_get_nested_state(struct kvm_vcpu *vcpu, if (vmx->nested.mtf_pending) kvm_state.flags |= KVM_STATE_NESTED_MTF_PENDING; + + if (nested_cpu_has_preemption_timer(vmcs12) && + vmx->nested.has_preemption_timer_deadline) { + kvm_state.hdr.vmx.flags |= + KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE; + kvm_state.hdr.vmx.preemption_timer_deadline = + vmx->nested.preemption_timer_deadline; + } } } @@ -5979,7 +6013,6 @@ static int vmx_get_nested_state(struct kvm_vcpu *vcpu, get_shadow_vmcs12(vcpu), VMCS12_SIZE)) return -EFAULT; } - out: return kvm_state.size; } @@ -6141,6 +6174,12 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu, goto error_guest_mode; } + if (kvm_state->hdr.vmx.flags & KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE) { + vmx->nested.has_preemption_timer_deadline = true; + vmx->nested.preemption_timer_deadline = + kvm_state->hdr.vmx.preemption_timer_deadline; + } + if (nested_vmx_check_controls(vcpu, vmcs12) || nested_vmx_check_host_state(vcpu, vmcs12) || nested_vmx_check_guest_state(vcpu, vmcs12, &ignored)) diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 298ddef79d00..672c28f17e49 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -169,6 +169,8 @@ struct nested_vmx { u16 posted_intr_nv; struct hrtimer preemption_timer; + u64 preemption_timer_deadline; + bool has_preemption_timer_deadline; bool preemption_timer_expired; /* to migrate it to L2 if VM_ENTRY_LOAD_DEBUG_CONTROLS is off */ From 8d7fbf01f9afc9ea1381de705013aa3ca453009f Mon Sep 17 00:00:00 2001 From: Makarand Sonare Date: Tue, 26 May 2020 14:51:07 -0700 Subject: [PATCH 1019/1043] KVM: selftests: VMX preemption timer migration test When a nested VM with a VMX-preemption timer is migrated, verify that the nested VM and its parent VM observe the VMX-preemption timer exit close to the original expiration deadline. Signed-off-by: Makarand Sonare Reviewed-by: Jim Mattson Message-Id: <20200526215107.205814-3-makarandsonare@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/nested.c | 12 +- tools/arch/x86/include/uapi/asm/kvm.h | 1 + tools/testing/selftests/kvm/.gitignore | 1 + tools/testing/selftests/kvm/Makefile | 1 + .../testing/selftests/kvm/include/kvm_util.h | 2 + .../selftests/kvm/include/x86_64/processor.h | 11 +- .../selftests/kvm/include/x86_64/vmx.h | 27 ++ .../kvm/x86_64/vmx_preemption_timer_test.c | 255 ++++++++++++++++++ 8 files changed, 298 insertions(+), 12 deletions(-) create mode 100644 tools/testing/selftests/kvm/x86_64/vmx_preemption_timer_test.c diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index da87bb8670bb..9c74a732b08d 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -2091,20 +2091,16 @@ static u64 vmx_calc_preemption_timer_value(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - u64 timer_value = 0; u64 l1_scaled_tsc = kvm_read_l1_tsc(vcpu, rdtsc()) >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE; if (!vmx->nested.has_preemption_timer_deadline) { - timer_value = vmcs12->vmx_preemption_timer_value; - vmx->nested.preemption_timer_deadline = timer_value + - l1_scaled_tsc; + vmx->nested.preemption_timer_deadline = + vmcs12->vmx_preemption_timer_value + l1_scaled_tsc; vmx->nested.has_preemption_timer_deadline = true; - } else if (l1_scaled_tsc < vmx->nested.preemption_timer_deadline) - timer_value = vmx->nested.preemption_timer_deadline - - l1_scaled_tsc; - return timer_value; + } + return vmx->nested.preemption_timer_deadline - l1_scaled_tsc; } static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu, diff --git a/tools/arch/x86/include/uapi/asm/kvm.h b/tools/arch/x86/include/uapi/asm/kvm.h index 3f3f780c8c65..43e24903812c 100644 --- a/tools/arch/x86/include/uapi/asm/kvm.h +++ b/tools/arch/x86/include/uapi/asm/kvm.h @@ -400,6 +400,7 @@ struct kvm_sync_regs { struct kvm_vmx_nested_state_data { __u8 vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE]; __u8 shadow_vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE]; + __u64 preemption_timer_deadline; }; struct kvm_vmx_nested_state_hdr { diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 222e50104296..f159718f90c0 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -10,6 +10,7 @@ /x86_64/set_sregs_test /x86_64/smm_test /x86_64/state_test +/x86_64/vmx_preemption_timer_test /x86_64/svm_vmcall_test /x86_64/sync_regs_test /x86_64/vmx_close_while_nested_test diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index 29200b606bcc..b4ff112e5c7e 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -46,6 +46,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/platform_info_test TEST_GEN_PROGS_x86_64 += x86_64/set_sregs_test TEST_GEN_PROGS_x86_64 += x86_64/smm_test TEST_GEN_PROGS_x86_64 += x86_64/state_test +TEST_GEN_PROGS_x86_64 += x86_64/vmx_preemption_timer_test TEST_GEN_PROGS_x86_64 += x86_64/svm_vmcall_test TEST_GEN_PROGS_x86_64 += x86_64/sync_regs_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_close_while_nested_test diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index e244c6ecfc1d..919e161dd289 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -314,6 +314,8 @@ void ucall_uninit(struct kvm_vm *vm); void ucall(uint64_t cmd, int nargs, ...); uint64_t get_ucall(struct kvm_vm *vm, uint32_t vcpu_id, struct ucall *uc); +#define GUEST_SYNC_ARGS(stage, arg1, arg2, arg3, arg4) \ + ucall(UCALL_SYNC, 6, "hello", stage, arg1, arg2, arg3, arg4) #define GUEST_SYNC(stage) ucall(UCALL_SYNC, 2, "hello", stage) #define GUEST_DONE() ucall(UCALL_DONE, 0) #define __GUEST_ASSERT(_condition, _nargs, _args...) do { \ diff --git a/tools/testing/selftests/kvm/include/x86_64/processor.h b/tools/testing/selftests/kvm/include/x86_64/processor.h index 7428513a4c68..7cb19eca6c72 100644 --- a/tools/testing/selftests/kvm/include/x86_64/processor.h +++ b/tools/testing/selftests/kvm/include/x86_64/processor.h @@ -79,13 +79,16 @@ static inline uint64_t get_desc64_base(const struct desc64 *desc) static inline uint64_t rdtsc(void) { uint32_t eax, edx; - + uint32_t tsc_val; /* * The lfence is to wait (on Intel CPUs) until all previous - * instructions have been executed. + * instructions have been executed. If software requires RDTSC to be + * executed prior to execution of any subsequent instruction, it can + * execute LFENCE immediately after RDTSC */ - __asm__ __volatile__("lfence; rdtsc" : "=a"(eax), "=d"(edx)); - return ((uint64_t)edx) << 32 | eax; + __asm__ __volatile__("lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); + tsc_val = ((uint64_t)edx) << 32 | eax; + return tsc_val; } static inline uint64_t rdtscp(uint32_t *aux) diff --git a/tools/testing/selftests/kvm/include/x86_64/vmx.h b/tools/testing/selftests/kvm/include/x86_64/vmx.h index 3d27069b9ed9..ccff3e6e2704 100644 --- a/tools/testing/selftests/kvm/include/x86_64/vmx.h +++ b/tools/testing/selftests/kvm/include/x86_64/vmx.h @@ -575,6 +575,33 @@ struct vmx_pages { void *eptp; }; +union vmx_basic { + u64 val; + struct { + u32 revision; + u32 size:13, + reserved1:3, + width:1, + dual:1, + type:4, + insouts:1, + ctrl:1, + vm_entry_exception_ctrl:1, + reserved2:7; + }; +}; + +union vmx_ctrl_msr { + u64 val; + struct { + u32 set, clr; + }; +}; + +union vmx_basic basic; +union vmx_ctrl_msr ctrl_pin_rev; +union vmx_ctrl_msr ctrl_exit_rev; + struct vmx_pages *vcpu_alloc_vmx(struct kvm_vm *vm, vm_vaddr_t *p_vmx_gva); bool prepare_for_vmx_operation(struct vmx_pages *vmx); void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip, void *guest_rsp); diff --git a/tools/testing/selftests/kvm/x86_64/vmx_preemption_timer_test.c b/tools/testing/selftests/kvm/x86_64/vmx_preemption_timer_test.c new file mode 100644 index 000000000000..cc72b6188ca7 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/vmx_preemption_timer_test.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * VMX-preemption timer test + * + * Copyright (C) 2020, Google, LLC. + * + * Test to ensure the VM-Enter after migration doesn't + * incorrectly restarts the timer with the full timer + * value instead of partially decayed timer value + * + */ +#define _GNU_SOURCE /* for program_invocation_short_name */ +#include +#include +#include +#include +#include + +#include "test_util.h" + +#include "kvm_util.h" +#include "processor.h" +#include "vmx.h" + +#define VCPU_ID 5 +#define PREEMPTION_TIMER_VALUE 100000000ull +#define PREEMPTION_TIMER_VALUE_THRESHOLD1 80000000ull + +u32 vmx_pt_rate; +bool l2_save_restore_done; +static u64 l2_vmx_pt_start; +volatile u64 l2_vmx_pt_finish; + +void l2_guest_code(void) +{ + u64 vmx_pt_delta; + + vmcall(); + l2_vmx_pt_start = (rdtsc() >> vmx_pt_rate) << vmx_pt_rate; + + /* + * Wait until the 1st threshold has passed + */ + do { + l2_vmx_pt_finish = rdtsc(); + vmx_pt_delta = (l2_vmx_pt_finish - l2_vmx_pt_start) >> + vmx_pt_rate; + } while (vmx_pt_delta < PREEMPTION_TIMER_VALUE_THRESHOLD1); + + /* + * Force L2 through Save and Restore cycle + */ + GUEST_SYNC(1); + + l2_save_restore_done = 1; + + /* + * Now wait for the preemption timer to fire and + * exit to L1 + */ + while ((l2_vmx_pt_finish = rdtsc())) + ; +} + +void l1_guest_code(struct vmx_pages *vmx_pages) +{ +#define L2_GUEST_STACK_SIZE 64 + unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; + u64 l1_vmx_pt_start; + u64 l1_vmx_pt_finish; + u64 l1_tsc_deadline, l2_tsc_deadline; + + GUEST_ASSERT(vmx_pages->vmcs_gpa); + GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + GUEST_ASSERT(load_vmcs(vmx_pages)); + GUEST_ASSERT(vmptrstz() == vmx_pages->vmcs_gpa); + + prepare_vmcs(vmx_pages, l2_guest_code, + &l2_guest_stack[L2_GUEST_STACK_SIZE]); + + /* + * Check for Preemption timer support + */ + basic.val = rdmsr(MSR_IA32_VMX_BASIC); + ctrl_pin_rev.val = rdmsr(basic.ctrl ? MSR_IA32_VMX_TRUE_PINBASED_CTLS + : MSR_IA32_VMX_PINBASED_CTLS); + ctrl_exit_rev.val = rdmsr(basic.ctrl ? MSR_IA32_VMX_TRUE_EXIT_CTLS + : MSR_IA32_VMX_EXIT_CTLS); + + if (!(ctrl_pin_rev.clr & PIN_BASED_VMX_PREEMPTION_TIMER) || + !(ctrl_exit_rev.clr & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)) + return; + + GUEST_ASSERT(!vmlaunch()); + GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + vmwrite(GUEST_RIP, vmreadz(GUEST_RIP) + vmreadz(VM_EXIT_INSTRUCTION_LEN)); + + /* + * Turn on PIN control and resume the guest + */ + GUEST_ASSERT(!vmwrite(PIN_BASED_VM_EXEC_CONTROL, + vmreadz(PIN_BASED_VM_EXEC_CONTROL) | + PIN_BASED_VMX_PREEMPTION_TIMER)); + + GUEST_ASSERT(!vmwrite(VMX_PREEMPTION_TIMER_VALUE, + PREEMPTION_TIMER_VALUE)); + + vmx_pt_rate = rdmsr(MSR_IA32_VMX_MISC) & 0x1F; + + l2_save_restore_done = 0; + + l1_vmx_pt_start = (rdtsc() >> vmx_pt_rate) << vmx_pt_rate; + + GUEST_ASSERT(!vmresume()); + + l1_vmx_pt_finish = rdtsc(); + + /* + * Ensure exit from L2 happens after L2 goes through + * save and restore + */ + GUEST_ASSERT(l2_save_restore_done); + + /* + * Ensure the exit from L2 is due to preemption timer expiry + */ + GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_PREEMPTION_TIMER); + + l1_tsc_deadline = l1_vmx_pt_start + + (PREEMPTION_TIMER_VALUE << vmx_pt_rate); + + l2_tsc_deadline = l2_vmx_pt_start + + (PREEMPTION_TIMER_VALUE << vmx_pt_rate); + + /* + * Sync with the host and pass the l1|l2 pt_expiry_finish times and + * tsc deadlines so that host can verify they are as expected + */ + GUEST_SYNC_ARGS(2, l1_vmx_pt_finish, l1_tsc_deadline, + l2_vmx_pt_finish, l2_tsc_deadline); +} + +void guest_code(struct vmx_pages *vmx_pages) +{ + if (vmx_pages) + l1_guest_code(vmx_pages); + + GUEST_DONE(); +} + +int main(int argc, char *argv[]) +{ + vm_vaddr_t vmx_pages_gva = 0; + + struct kvm_regs regs1, regs2; + struct kvm_vm *vm; + struct kvm_run *run; + struct kvm_x86_state *state; + struct ucall uc; + int stage; + + /* + * AMD currently does not implement any VMX features, so for now we + * just early out. + */ + nested_vmx_check_supported(); + + /* Create VM */ + vm = vm_create_default(VCPU_ID, 0, guest_code); + vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid()); + run = vcpu_state(vm, VCPU_ID); + + vcpu_regs_get(vm, VCPU_ID, ®s1); + + if (kvm_check_cap(KVM_CAP_NESTED_STATE)) { + vcpu_alloc_vmx(vm, &vmx_pages_gva); + vcpu_args_set(vm, VCPU_ID, 1, vmx_pages_gva); + } else { + pr_info("will skip vmx preemption timer checks\n"); + goto done; + } + + for (stage = 1;; stage++) { + _vcpu_run(vm, VCPU_ID); + TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, + "Stage %d: unexpected exit reason: %u (%s),\n", + stage, run->exit_reason, + exit_reason_str(run->exit_reason)); + + switch (get_ucall(vm, VCPU_ID, &uc)) { + case UCALL_ABORT: + TEST_FAIL("%s at %s:%ld", (const char *)uc.args[0], + __FILE__, uc.args[1]); + /* NOT REACHED */ + case UCALL_SYNC: + break; + case UCALL_DONE: + goto done; + default: + TEST_FAIL("Unknown ucall %lu", uc.cmd); + } + + /* UCALL_SYNC is handled here. */ + TEST_ASSERT(!strcmp((const char *)uc.args[0], "hello") && + uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", + stage, (ulong)uc.args[1]); + /* + * If this stage 2 then we should verify the vmx pt expiry + * is as expected. + * From L1's perspective verify Preemption timer hasn't + * expired too early. + * From L2's perspective verify Preemption timer hasn't + * expired too late. + */ + if (stage == 2) { + + pr_info("Stage %d: L1 PT expiry TSC (%lu) , L1 TSC deadline (%lu)\n", + stage, uc.args[2], uc.args[3]); + + pr_info("Stage %d: L2 PT expiry TSC (%lu) , L2 TSC deadline (%lu)\n", + stage, uc.args[4], uc.args[5]); + + TEST_ASSERT(uc.args[2] >= uc.args[3], + "Stage %d: L1 PT expiry TSC (%lu) < L1 TSC deadline (%lu)", + stage, uc.args[2], uc.args[3]); + + TEST_ASSERT(uc.args[4] < uc.args[5], + "Stage %d: L2 PT expiry TSC (%lu) > L2 TSC deadline (%lu)", + stage, uc.args[4], uc.args[5]); + } + + state = vcpu_save_state(vm, VCPU_ID); + memset(®s1, 0, sizeof(regs1)); + vcpu_regs_get(vm, VCPU_ID, ®s1); + + kvm_vm_release(vm); + + /* Restore state in a new VM. */ + kvm_vm_restart(vm, O_RDWR); + vm_vcpu_add(vm, VCPU_ID); + vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid()); + vcpu_load_state(vm, VCPU_ID, state); + run = vcpu_state(vm, VCPU_ID); + free(state); + + memset(®s2, 0, sizeof(regs2)); + vcpu_regs_get(vm, VCPU_ID, ®s2); + TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), + "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", + (ulong) regs2.rdi, (ulong) regs2.rsi); + } + +done: + kvm_vm_free(vm); +} From 22ad0026d0978d4211eb07a2be77f49fb40d86eb Mon Sep 17 00:00:00 2001 From: Jon Doron Date: Fri, 29 May 2020 16:45:39 +0300 Subject: [PATCH 1020/1043] x86/hyper-v: Add synthetic debugger definitions Hyper-V synthetic debugger has two modes, one that uses MSRs and the other that use Hypercalls. Add all the required definitions to both types of synthetic debugger interface. Some of the required new CPUIDs and MSRs are not documented in the TLFS so they are in hyperv.h instead. The reason they are not documented is because they are subjected to be removed in future versions of Windows. Reviewed-by: Michael Kelley Signed-off-by: Jon Doron Message-Id: <20200529134543.1127440-3-arilou@gmail.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/hyperv-tlfs.h | 6 ++++++ arch/x86/kvm/hyperv.h | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h index 29336574d0bc..53ef6b7bd380 100644 --- a/arch/x86/include/asm/hyperv-tlfs.h +++ b/arch/x86/include/asm/hyperv-tlfs.h @@ -131,6 +131,8 @@ #define HV_FEATURE_FREQUENCY_MSRS_AVAILABLE BIT(8) /* Crash MSR available */ #define HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE BIT(10) +/* Support for debug MSRs available */ +#define HV_FEATURE_DEBUG_MSRS_AVAILABLE BIT(11) /* stimer Direct Mode is available */ #define HV_STIMER_DIRECT_MODE_AVAILABLE BIT(19) @@ -376,6 +378,9 @@ struct hv_tsc_emulation_status { #define HVCALL_SEND_IPI_EX 0x0015 #define HVCALL_POST_MESSAGE 0x005c #define HVCALL_SIGNAL_EVENT 0x005d +#define HVCALL_POST_DEBUG_DATA 0x0069 +#define HVCALL_RETRIEVE_DEBUG_DATA 0x006a +#define HVCALL_RESET_DEBUG_SESSION 0x006b #define HVCALL_RETARGET_INTERRUPT 0x007e #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE 0x00af #define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_LIST 0x00b0 @@ -422,6 +427,7 @@ enum HV_GENERIC_SET_FORMAT { #define HV_STATUS_INVALID_HYPERCALL_INPUT 3 #define HV_STATUS_INVALID_ALIGNMENT 4 #define HV_STATUS_INVALID_PARAMETER 5 +#define HV_STATUS_OPERATION_DENIED 8 #define HV_STATUS_INSUFFICIENT_MEMORY 11 #define HV_STATUS_INVALID_PORT_ID 17 #define HV_STATUS_INVALID_CONNECTION_ID 18 diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index 757cb578101c..7f50ff0bad07 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -23,6 +23,33 @@ #include +/* + * The #defines related to the synthetic debugger are required by KDNet, but + * they are not documented in the Hyper-V TLFS because the synthetic debugger + * functionality has been deprecated and is subject to removal in future + * versions of Windows. + */ +#define HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS 0x40000080 +#define HYPERV_CPUID_SYNDBG_INTERFACE 0x40000081 +#define HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES 0x40000082 + +/* + * Hyper-V synthetic debugger platform capabilities + * These are HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES.EAX bits. + */ +#define HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING BIT(1) + +/* Hyper-V Synthetic debug options MSR */ +#define HV_X64_MSR_SYNDBG_CONTROL 0x400000F1 +#define HV_X64_MSR_SYNDBG_STATUS 0x400000F2 +#define HV_X64_MSR_SYNDBG_SEND_BUFFER 0x400000F3 +#define HV_X64_MSR_SYNDBG_RECV_BUFFER 0x400000F4 +#define HV_X64_MSR_SYNDBG_PENDING_BUFFER 0x400000F5 +#define HV_X64_MSR_SYNDBG_OPTIONS 0x400000FF + +/* Hyper-V HV_X64_MSR_SYNDBG_OPTIONS bits */ +#define HV_X64_SYNDBG_OPTION_USE_HCALLS BIT(2) + static inline struct kvm_vcpu_hv *vcpu_to_hv_vcpu(struct kvm_vcpu *vcpu) { return &vcpu->arch.hyperv; From f97f5a56f5977311f3833056a73cdbb0ee56cb1e Mon Sep 17 00:00:00 2001 From: Jon Doron Date: Fri, 29 May 2020 16:45:40 +0300 Subject: [PATCH 1021/1043] x86/kvm/hyper-v: Add support for synthetic debugger interface Add support for Hyper-V synthetic debugger (syndbg) interface. The syndbg interface is using MSRs to emulate a way to send/recv packets data. The debug transport dll (kdvm/kdnet) will identify if Hyper-V is enabled and if it supports the synthetic debugger interface it will attempt to use it, instead of trying to initialize a network adapter. Reviewed-by: Vitaly Kuznetsov Signed-off-by: Jon Doron Message-Id: <20200529134543.1127440-4-arilou@gmail.com> Signed-off-by: Paolo Bonzini --- Documentation/virt/kvm/api.rst | 16 ++++ arch/x86/include/asm/kvm_host.h | 13 +++ arch/x86/kvm/hyperv.c | 158 +++++++++++++++++++++++++++++++- arch/x86/kvm/hyperv.h | 5 + arch/x86/kvm/trace.h | 51 +++++++++++ arch/x86/kvm/x86.c | 8 ++ include/uapi/linux/kvm.h | 10 ++ 7 files changed, 258 insertions(+), 3 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index db22e834ce2a..aad60be4884e 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -5070,6 +5070,7 @@ EOI was received. struct kvm_hyperv_exit { #define KVM_EXIT_HYPERV_SYNIC 1 #define KVM_EXIT_HYPERV_HCALL 2 + #define KVM_EXIT_HYPERV_SYNDBG 3 __u32 type; __u32 pad1; union { @@ -5085,6 +5086,15 @@ EOI was received. __u64 result; __u64 params[2]; } hcall; + struct { + __u32 msr; + __u32 pad2; + __u64 control; + __u64 status; + __u64 send_page; + __u64 recv_page; + __u64 pending_page; + } syndbg; } u; }; /* KVM_EXIT_HYPERV */ @@ -5101,6 +5111,12 @@ Hyper-V SynIC state change. Notification is used to remap SynIC event/message pages and to enable/disable SynIC messages/events processing in userspace. + - KVM_EXIT_HYPERV_SYNDBG -- synchronously notify user-space about + +Hyper-V Synthetic debugger state change. Notification is used to either update +the pending_page location or to send a control command (send the buffer located +in send_page or recv a buffer to recv_page). + :: /* KVM_EXIT_ARM_NISV */ diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index b878fcd164ce..58337a25396a 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -863,6 +863,18 @@ struct kvm_apic_map { struct kvm_lapic *phys_map[]; }; +/* Hyper-V synthetic debugger (SynDbg)*/ +struct kvm_hv_syndbg { + struct { + u64 control; + u64 status; + u64 send_page; + u64 recv_page; + u64 pending_page; + } control; + u64 options; +}; + /* Hyper-V emulation context */ struct kvm_hv { struct mutex hv_lock; @@ -886,6 +898,7 @@ struct kvm_hv { atomic_t num_mismatched_vp_indexes; struct hv_partition_assist_pg *hv_pa_pg; + struct kvm_hv_syndbg hv_syndbg; }; enum kvm_irqchip_mode { diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index f9d3b919823c..c21f99357ad5 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -21,6 +21,7 @@ #include "x86.h" #include "lapic.h" #include "ioapic.h" +#include "cpuid.h" #include "hyperv.h" #include @@ -266,6 +267,123 @@ static int synic_set_msr(struct kvm_vcpu_hv_synic *synic, return ret; } +static bool kvm_hv_is_syndbg_enabled(struct kvm_vcpu *vcpu) +{ + struct kvm_cpuid_entry2 *entry; + + entry = kvm_find_cpuid_entry(vcpu, + HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES, + 0); + if (!entry) + return false; + + return entry->eax & HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING; +} + +static int kvm_hv_syndbg_complete_userspace(struct kvm_vcpu *vcpu) +{ + struct kvm *kvm = vcpu->kvm; + struct kvm_hv *hv = &kvm->arch.hyperv; + + if (vcpu->run->hyperv.u.syndbg.msr == HV_X64_MSR_SYNDBG_CONTROL) + hv->hv_syndbg.control.status = + vcpu->run->hyperv.u.syndbg.status; + return 1; +} + +static void syndbg_exit(struct kvm_vcpu *vcpu, u32 msr) +{ + struct kvm_hv_syndbg *syndbg = vcpu_to_hv_syndbg(vcpu); + struct kvm_vcpu_hv *hv_vcpu = &vcpu->arch.hyperv; + + hv_vcpu->exit.type = KVM_EXIT_HYPERV_SYNDBG; + hv_vcpu->exit.u.syndbg.msr = msr; + hv_vcpu->exit.u.syndbg.control = syndbg->control.control; + hv_vcpu->exit.u.syndbg.send_page = syndbg->control.send_page; + hv_vcpu->exit.u.syndbg.recv_page = syndbg->control.recv_page; + hv_vcpu->exit.u.syndbg.pending_page = syndbg->control.pending_page; + vcpu->arch.complete_userspace_io = + kvm_hv_syndbg_complete_userspace; + + kvm_make_request(KVM_REQ_HV_EXIT, vcpu); +} + +static int syndbg_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) +{ + struct kvm_hv_syndbg *syndbg = vcpu_to_hv_syndbg(vcpu); + + if (!kvm_hv_is_syndbg_enabled(vcpu) && !host) + return 1; + + trace_kvm_hv_syndbg_set_msr(vcpu->vcpu_id, + vcpu_to_hv_vcpu(vcpu)->vp_index, msr, data); + switch (msr) { + case HV_X64_MSR_SYNDBG_CONTROL: + syndbg->control.control = data; + if (!host) + syndbg_exit(vcpu, msr); + break; + case HV_X64_MSR_SYNDBG_STATUS: + syndbg->control.status = data; + break; + case HV_X64_MSR_SYNDBG_SEND_BUFFER: + syndbg->control.send_page = data; + break; + case HV_X64_MSR_SYNDBG_RECV_BUFFER: + syndbg->control.recv_page = data; + break; + case HV_X64_MSR_SYNDBG_PENDING_BUFFER: + syndbg->control.pending_page = data; + if (!host) + syndbg_exit(vcpu, msr); + break; + case HV_X64_MSR_SYNDBG_OPTIONS: + syndbg->options = data; + break; + default: + break; + } + + return 0; +} + +static int syndbg_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) +{ + struct kvm_hv_syndbg *syndbg = vcpu_to_hv_syndbg(vcpu); + + if (!kvm_hv_is_syndbg_enabled(vcpu) && !host) + return 1; + + switch (msr) { + case HV_X64_MSR_SYNDBG_CONTROL: + *pdata = syndbg->control.control; + break; + case HV_X64_MSR_SYNDBG_STATUS: + *pdata = syndbg->control.status; + break; + case HV_X64_MSR_SYNDBG_SEND_BUFFER: + *pdata = syndbg->control.send_page; + break; + case HV_X64_MSR_SYNDBG_RECV_BUFFER: + *pdata = syndbg->control.recv_page; + break; + case HV_X64_MSR_SYNDBG_PENDING_BUFFER: + *pdata = syndbg->control.pending_page; + break; + case HV_X64_MSR_SYNDBG_OPTIONS: + *pdata = syndbg->options; + break; + default: + break; + } + + trace_kvm_hv_syndbg_get_msr(vcpu->vcpu_id, + vcpu_to_hv_vcpu(vcpu)->vp_index, msr, + *pdata); + + return 0; +} + static int synic_get_msr(struct kvm_vcpu_hv_synic *synic, u32 msr, u64 *pdata, bool host) { @@ -800,6 +918,8 @@ static bool kvm_hv_msr_partition_wide(u32 msr) case HV_X64_MSR_REENLIGHTENMENT_CONTROL: case HV_X64_MSR_TSC_EMULATION_CONTROL: case HV_X64_MSR_TSC_EMULATION_STATUS: + case HV_X64_MSR_SYNDBG_OPTIONS: + case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: r = true; break; } @@ -1061,6 +1181,9 @@ static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, if (!host) return 1; break; + case HV_X64_MSR_SYNDBG_OPTIONS: + case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: + return syndbg_set_msr(vcpu, msr, data, host); default: vcpu_unimpl(vcpu, "Hyper-V unhandled wrmsr: 0x%x data 0x%llx\n", msr, data); @@ -1190,7 +1313,8 @@ static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) return 0; } -static int kvm_hv_get_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) +static int kvm_hv_get_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, + bool host) { u64 data = 0; struct kvm *kvm = vcpu->kvm; @@ -1227,6 +1351,9 @@ static int kvm_hv_get_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) case HV_X64_MSR_TSC_EMULATION_STATUS: data = hv->hv_tsc_emulation_status; break; + case HV_X64_MSR_SYNDBG_OPTIONS: + case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: + return syndbg_get_msr(vcpu, msr, pdata, host); default: vcpu_unimpl(vcpu, "Hyper-V unhandled rdmsr: 0x%x\n", msr); return 1; @@ -1316,7 +1443,7 @@ int kvm_hv_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host) int r; mutex_lock(&vcpu->kvm->arch.hyperv.hv_lock); - r = kvm_hv_get_msr_pw(vcpu, msr, pdata); + r = kvm_hv_get_msr_pw(vcpu, msr, pdata, host); mutex_unlock(&vcpu->kvm->arch.hyperv.hv_lock); return r; } else @@ -1795,6 +1922,9 @@ int kvm_vcpu_ioctl_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, { .function = HYPERV_CPUID_FEATURES }, { .function = HYPERV_CPUID_ENLIGHTMENT_INFO }, { .function = HYPERV_CPUID_IMPLEMENT_LIMITS }, + { .function = HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS }, + { .function = HYPERV_CPUID_SYNDBG_INTERFACE }, + { .function = HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES }, { .function = HYPERV_CPUID_NESTED_FEATURES }, }; int i, nent = ARRAY_SIZE(cpuid_entries); @@ -1820,7 +1950,7 @@ int kvm_vcpu_ioctl_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, case HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS: memcpy(signature, "Linux KVM Hv", 12); - ent->eax = HYPERV_CPUID_NESTED_FEATURES; + ent->eax = HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES; ent->ebx = signature[0]; ent->ecx = signature[1]; ent->edx = signature[2]; @@ -1859,6 +1989,10 @@ int kvm_vcpu_ioctl_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, ent->edx |= HV_FEATURE_FREQUENCY_MSRS_AVAILABLE; ent->edx |= HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE; + ent->ebx |= HV_X64_DEBUGGING; + ent->edx |= HV_X64_GUEST_DEBUGGING_AVAILABLE; + ent->edx |= HV_FEATURE_DEBUG_MSRS_AVAILABLE; + /* * Direct Synthetic timers only make sense with in-kernel * LAPIC @@ -1902,6 +2036,24 @@ int kvm_vcpu_ioctl_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, break; + case HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS: + memcpy(signature, "Linux KVM Hv", 12); + + ent->eax = 0; + ent->ebx = signature[0]; + ent->ecx = signature[1]; + ent->edx = signature[2]; + break; + + case HYPERV_CPUID_SYNDBG_INTERFACE: + memcpy(signature, "VS#1\0\0\0\0\0\0\0\0", 12); + ent->eax = signature[0]; + break; + + case HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES: + ent->eax |= HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING; + break; + default: break; } diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index 7f50ff0bad07..e68c6c2e9649 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -73,6 +73,11 @@ static inline struct kvm_vcpu *synic_to_vcpu(struct kvm_vcpu_hv_synic *synic) return hv_vcpu_to_vcpu(container_of(synic, struct kvm_vcpu_hv, synic)); } +static inline struct kvm_hv_syndbg *vcpu_to_hv_syndbg(struct kvm_vcpu *vcpu) +{ + return &vcpu->kvm->arch.hyperv.hv_syndbg; +} + int kvm_hv_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host); int kvm_hv_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host); diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index 54a10c98d746..b66432b015d2 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -1541,6 +1541,57 @@ TRACE_EVENT(kvm_nested_vmenter_failed, __print_symbolic(__entry->err, VMX_VMENTER_INSTRUCTION_ERRORS)) ); +/* + * Tracepoint for syndbg_set_msr. + */ +TRACE_EVENT(kvm_hv_syndbg_set_msr, + TP_PROTO(int vcpu_id, u32 vp_index, u32 msr, u64 data), + TP_ARGS(vcpu_id, vp_index, msr, data), + + TP_STRUCT__entry( + __field(int, vcpu_id) + __field(u32, vp_index) + __field(u32, msr) + __field(u64, data) + ), + + TP_fast_assign( + __entry->vcpu_id = vcpu_id; + __entry->vp_index = vp_index; + __entry->msr = msr; + __entry->data = data; + ), + + TP_printk("vcpu_id %d vp_index %u msr 0x%x data 0x%llx", + __entry->vcpu_id, __entry->vp_index, __entry->msr, + __entry->data) +); + +/* + * Tracepoint for syndbg_get_msr. + */ +TRACE_EVENT(kvm_hv_syndbg_get_msr, + TP_PROTO(int vcpu_id, u32 vp_index, u32 msr, u64 data), + TP_ARGS(vcpu_id, vp_index, msr, data), + + TP_STRUCT__entry( + __field(int, vcpu_id) + __field(u32, vp_index) + __field(u32, msr) + __field(u64, data) + ), + + TP_fast_assign( + __entry->vcpu_id = vcpu_id; + __entry->vp_index = vp_index; + __entry->msr = msr; + __entry->data = data; + ), + + TP_printk("vcpu_id %d vp_index %u msr 0x%x data 0x%llx", + __entry->vcpu_id, __entry->vp_index, __entry->msr, + __entry->data) +); #endif /* _TRACE_KVM_H */ #undef TRACE_INCLUDE_PATH diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index ca8a57312291..9e41b5135340 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1246,6 +1246,10 @@ static const u32 emulated_msrs_all[] = { HV_X64_MSR_VP_ASSIST_PAGE, HV_X64_MSR_REENLIGHTENMENT_CONTROL, HV_X64_MSR_TSC_EMULATION_CONTROL, HV_X64_MSR_TSC_EMULATION_STATUS, + HV_X64_MSR_SYNDBG_OPTIONS, + HV_X64_MSR_SYNDBG_CONTROL, HV_X64_MSR_SYNDBG_STATUS, + HV_X64_MSR_SYNDBG_SEND_BUFFER, HV_X64_MSR_SYNDBG_RECV_BUFFER, + HV_X64_MSR_SYNDBG_PENDING_BUFFER, MSR_KVM_ASYNC_PF_EN, MSR_KVM_STEAL_TIME, MSR_KVM_PV_EOI_EN, MSR_KVM_ASYNC_PF_INT, MSR_KVM_ASYNC_PF_ACK, @@ -3011,6 +3015,8 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) */ break; case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: + case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: + case HV_X64_MSR_SYNDBG_OPTIONS: case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: case HV_X64_MSR_CRASH_CTL: case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT: @@ -3272,6 +3278,8 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) msr_info->data = 0x20000000; break; case HV_X64_MSR_GUEST_OS_ID ... HV_X64_MSR_SINT15: + case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: + case HV_X64_MSR_SYNDBG_OPTIONS: case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: case HV_X64_MSR_CRASH_CTL: case HV_X64_MSR_STIMER0_CONFIG ... HV_X64_MSR_STIMER3_COUNT: diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index cc04aceb8793..6721eb563eda 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -188,6 +188,7 @@ struct kvm_s390_cmma_log { struct kvm_hyperv_exit { #define KVM_EXIT_HYPERV_SYNIC 1 #define KVM_EXIT_HYPERV_HCALL 2 +#define KVM_EXIT_HYPERV_SYNDBG 3 __u32 type; __u32 pad1; union { @@ -203,6 +204,15 @@ struct kvm_hyperv_exit { __u64 result; __u64 params[2]; } hcall; + struct { + __u32 msr; + __u32 pad2; + __u64 control; + __u64 status; + __u64 send_page; + __u64 recv_page; + __u64 pending_page; + } syndbg; } u; }; From 45c38973ed1868b8448079edd48bf24ab8b326fa Mon Sep 17 00:00:00 2001 From: Jon Doron Date: Fri, 29 May 2020 16:45:41 +0300 Subject: [PATCH 1022/1043] x86/kvm/hyper-v: enable hypercalls regardless of hypercall page Microsoft's kdvm.dll dbgtransport module does not respect the hypercall page and simply identifies the CPU being used (AMD/Intel) and according to it simply makes hypercalls with the relevant instruction (vmmcall/vmcall respectively). The relevant function in kdvm is KdHvConnectHypervisor which first checks if the hypercall page has been enabled via HV_X64_MSR_HYPERCALL_ENABLE, and in case it was not it simply sets the HV_X64_MSR_GUEST_OS_ID to 0x1000101010001 which means: build_number = 0x0001 service_version = 0x01 minor_version = 0x01 major_version = 0x01 os_id = 0x00 (Undefined) vendor_id = 1 (Microsoft) os_type = 0 (A value of 0 indicates a proprietary, closed source OS) and starts issuing the hypercall without setting the hypercall page. To resolve this issue simply enable hypercalls also if the guest_os_id is not 0. Reviewed-by: Vitaly Kuznetsov Signed-off-by: Jon Doron Message-Id: <20200529134543.1127440-5-arilou@gmail.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/hyperv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index c21f99357ad5..2fb1464a483f 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1656,7 +1656,7 @@ ret_success: bool kvm_hv_hypercall_enabled(struct kvm *kvm) { - return READ_ONCE(kvm->arch.hyperv.hv_hypercall) & HV_X64_MSR_HYPERCALL_ENABLE; + return READ_ONCE(kvm->arch.hyperv.hv_guest_os_id) != 0; } static void kvm_hv_hypercall_set_result(struct kvm_vcpu *vcpu, u64 result) From b187038b5e3f8404dbafce89fd169e66fe604df4 Mon Sep 17 00:00:00 2001 From: Jon Doron Date: Fri, 29 May 2020 16:45:42 +0300 Subject: [PATCH 1023/1043] x86/kvm/hyper-v: Add support for synthetic debugger via hypercalls There is another mode for the synthetic debugger which uses hypercalls to send/recv network data instead of the MSR interface. This interface is much slower and less recommended since you might get a lot of VMExits while KDVM polling for new packets to recv, rather than simply checking the pending page to see if there is data avialble and then request. Reviewed-by: Vitaly Kuznetsov Signed-off-by: Jon Doron Message-Id: <20200529134543.1127440-6-arilou@gmail.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/hyperv.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 2fb1464a483f..4e1695db788a 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1835,6 +1835,34 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) } ret = kvm_hv_send_ipi(vcpu, ingpa, outgpa, true, false); break; + case HVCALL_POST_DEBUG_DATA: + case HVCALL_RETRIEVE_DEBUG_DATA: + if (unlikely(fast)) { + ret = HV_STATUS_INVALID_PARAMETER; + break; + } + fallthrough; + case HVCALL_RESET_DEBUG_SESSION: { + struct kvm_hv_syndbg *syndbg = vcpu_to_hv_syndbg(vcpu); + + if (!kvm_hv_is_syndbg_enabled(vcpu)) { + ret = HV_STATUS_INVALID_HYPERCALL_CODE; + break; + } + + if (!(syndbg->options & HV_X64_SYNDBG_OPTION_USE_HCALLS)) { + ret = HV_STATUS_OPERATION_DENIED; + break; + } + vcpu->run->exit_reason = KVM_EXIT_HYPERV; + vcpu->run->hyperv.type = KVM_EXIT_HYPERV_HCALL; + vcpu->run->hyperv.u.hcall.input = param; + vcpu->run->hyperv.u.hcall.params[0] = ingpa; + vcpu->run->hyperv.u.hcall.params[1] = outgpa; + vcpu->arch.complete_userspace_io = + kvm_hv_hypercall_complete_userspace; + return 0; + } default: ret = HV_STATUS_INVALID_HYPERCALL_CODE; break; From fb0cb6a8211cfe00178614ae72a0b426bd1ff016 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Fri, 29 May 2020 16:45:43 +0300 Subject: [PATCH 1024/1043] KVM: selftests: update hyperv_cpuid with SynDBG tests Update tests to reflect new CPUID capabilities with SYNDBG. Check that we get the right number of entries and that 0x40000000.EAX always returns the correct max leaf. Signed-off-by: Vitaly Kuznetsov Signed-off-by: Jon Doron Message-Id: <20200529134543.1127440-7-arilou@gmail.com> Signed-off-by: Paolo Bonzini --- .../selftests/kvm/x86_64/hyperv_cpuid.c | 101 ++++++++++-------- 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c b/tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c index 83323f3d7ca0..4a7967cca281 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c @@ -26,18 +26,18 @@ static void guest_code(void) { } -static int smt_possible(void) +static bool smt_possible(void) { char buf[16]; FILE *f; - bool res = 1; + bool res = true; f = fopen("/sys/devices/system/cpu/smt/control", "r"); if (f) { if (fread(buf, sizeof(*buf), sizeof(buf), f) > 0) { if (!strncmp(buf, "forceoff", 8) || !strncmp(buf, "notsupported", 12)) - res = 0; + res = false; } fclose(f); } @@ -46,29 +46,31 @@ static int smt_possible(void) } static void test_hv_cpuid(struct kvm_cpuid2 *hv_cpuid_entries, - int evmcs_enabled) + bool evmcs_enabled) { int i; + int nent = 9; + u32 test_val; - if (!evmcs_enabled) - TEST_ASSERT(hv_cpuid_entries->nent == 6, - "KVM_GET_SUPPORTED_HV_CPUID should return 6 entries" - " when Enlightened VMCS is disabled (returned %d)", - hv_cpuid_entries->nent); - else - TEST_ASSERT(hv_cpuid_entries->nent == 7, - "KVM_GET_SUPPORTED_HV_CPUID should return 7 entries" - " when Enlightened VMCS is enabled (returned %d)", - hv_cpuid_entries->nent); + if (evmcs_enabled) + nent += 1; /* 0x4000000A */ + + TEST_ASSERT(hv_cpuid_entries->nent == nent, + "KVM_GET_SUPPORTED_HV_CPUID should return %d entries" + " with evmcs=%d (returned %d)", + nent, evmcs_enabled, hv_cpuid_entries->nent); for (i = 0; i < hv_cpuid_entries->nent; i++) { struct kvm_cpuid_entry2 *entry = &hv_cpuid_entries->entries[i]; TEST_ASSERT((entry->function >= 0x40000000) && - (entry->function <= 0x4000000A), + (entry->function <= 0x40000082), "function %x is our of supported range", entry->function); + TEST_ASSERT(evmcs_enabled || (entry->function != 0x4000000A), + "0x4000000A leaf should not be reported"); + TEST_ASSERT(entry->index == 0, ".index field should be zero"); @@ -78,12 +80,23 @@ static void test_hv_cpuid(struct kvm_cpuid2 *hv_cpuid_entries, TEST_ASSERT(!entry->padding[0] && !entry->padding[1] && !entry->padding[2], "padding should be zero"); - if (entry->function == 0x40000004) { - int nononarchcs = !!(entry->eax & (1UL << 18)); + switch (entry->function) { + case 0x40000000: + test_val = 0x40000082; - TEST_ASSERT(nononarchcs == !smt_possible(), + TEST_ASSERT(entry->eax == test_val, + "Wrong max leaf report in 0x40000000.EAX: %x" + " (evmcs=%d)", + entry->eax, evmcs_enabled + ); + break; + case 0x40000004: + test_val = entry->eax & (1UL << 18); + + TEST_ASSERT(!!test_val == !smt_possible(), "NoNonArchitecturalCoreSharing bit" " doesn't reflect SMT setting"); + break; } /* @@ -133,8 +146,9 @@ struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(struct kvm_vm *vm) int main(int argc, char *argv[]) { struct kvm_vm *vm; - int rv; + int rv, stage; struct kvm_cpuid2 *hv_cpuid_entries; + bool evmcs_enabled; /* Tell stdout not to buffer its content */ setbuf(stdout, NULL); @@ -145,36 +159,31 @@ int main(int argc, char *argv[]) exit(KSFT_SKIP); } - /* Create VM */ - vm = vm_create_default(VCPU_ID, 0, guest_code); + for (stage = 0; stage < 3; stage++) { + evmcs_enabled = false; - test_hv_cpuid_e2big(vm); + vm = vm_create_default(VCPU_ID, 0, guest_code); + switch (stage) { + case 0: + test_hv_cpuid_e2big(vm); + continue; + case 1: + break; + case 2: + if (!kvm_check_cap(KVM_CAP_HYPERV_ENLIGHTENED_VMCS)) { + print_skip("Enlightened VMCS is unsupported"); + continue; + } + vcpu_enable_evmcs(vm, VCPU_ID); + evmcs_enabled = true; + break; + } - hv_cpuid_entries = kvm_get_supported_hv_cpuid(vm); - if (!hv_cpuid_entries) - return 1; - - test_hv_cpuid(hv_cpuid_entries, 0); - - free(hv_cpuid_entries); - - if (!kvm_check_cap(KVM_CAP_HYPERV_ENLIGHTENED_VMCS)) { - print_skip("Enlightened VMCS is unsupported"); - goto vm_free; + hv_cpuid_entries = kvm_get_supported_hv_cpuid(vm); + test_hv_cpuid(hv_cpuid_entries, evmcs_enabled); + free(hv_cpuid_entries); + kvm_vm_free(vm); } - vcpu_enable_evmcs(vm, VCPU_ID); - - hv_cpuid_entries = kvm_get_supported_hv_cpuid(vm); - if (!hv_cpuid_entries) - return 1; - - test_hv_cpuid(hv_cpuid_entries, 1); - - free(hv_cpuid_entries); - -vm_free: - kvm_vm_free(vm); - return 0; } From 09d952c971a58c897eb7ea7d41516e7d953bb65d Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 1 Jun 2020 04:17:45 -0400 Subject: [PATCH 1025/1043] KVM: check userspace_addr for all memslots The userspace_addr alignment and range checks are not performed for private memory slots that are prepared by KVM itself. This is unnecessary and makes it questionable to use __*_user functions to access memory later on. We also rely on the userspace address being aligned since we have an entire family of functions to map gfn to pfn. Fortunately skipping the check is completely unnecessary. Only x86 uses private memslots and their userspace_addr is obtained from vm_mmap, therefore it must be below PAGE_OFFSET. In fact, any attempt to pass an address above PAGE_OFFSET would have failed because such an address would return true for kvm_is_error_hva. Reported-by: Linus Torvalds Signed-off-by: Paolo Bonzini --- virt/kvm/kvm_main.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index e2af6ffce9c9..26e0ac871a10 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -1225,10 +1225,9 @@ int __kvm_set_memory_region(struct kvm *kvm, if (mem->guest_phys_addr & (PAGE_SIZE - 1)) return -EINVAL; /* We can read the guest memory with __xxx_user() later on. */ - if ((id < KVM_USER_MEM_SLOTS) && - ((mem->userspace_addr & (PAGE_SIZE - 1)) || + if ((mem->userspace_addr & (PAGE_SIZE - 1)) || !access_ok((void __user *)(unsigned long)mem->userspace_addr, - mem->memory_size))) + mem->memory_size)) return -EINVAL; if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM) return -EINVAL; From 13ffbd8db1dd43d63d086517872a4e702a6bf309 Mon Sep 17 00:00:00 2001 From: Vitaly Kuznetsov Date: Mon, 1 Jun 2020 17:47:26 +0200 Subject: [PATCH 1026/1043] KVM: selftests: fix rdtsc() for vmx_tsc_adjust_test vmx_tsc_adjust_test fails with: IA32_TSC_ADJUST is -4294969448 (-1 * TSC_ADJUST_VALUE + -2152). IA32_TSC_ADJUST is -4294969448 (-1 * TSC_ADJUST_VALUE + -2152). IA32_TSC_ADJUST is 281470681738540 (65534 * TSC_ADJUST_VALUE + 4294962476). ==== Test Assertion Failure ==== x86_64/vmx_tsc_adjust_test.c:153: false pid=19738 tid=19738 - Interrupted system call 1 0x0000000000401192: main at vmx_tsc_adjust_test.c:153 2 0x00007fe1ef8583d4: ?? ??:0 3 0x0000000000401201: _start at ??:? Failed guest assert: (adjust <= max) The problem is that is 'tsc_val' should be u64, not u32 or the reading gets truncated. Fixes: 8d7fbf01f9afc ("KVM: selftests: VMX preemption timer migration test") Signed-off-by: Vitaly Kuznetsov Message-Id: <20200601154726.261868-1-vkuznets@redhat.com> Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/include/x86_64/processor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/include/x86_64/processor.h b/tools/testing/selftests/kvm/include/x86_64/processor.h index 7cb19eca6c72..82b7fe16a824 100644 --- a/tools/testing/selftests/kvm/include/x86_64/processor.h +++ b/tools/testing/selftests/kvm/include/x86_64/processor.h @@ -79,7 +79,7 @@ static inline uint64_t get_desc64_base(const struct desc64 *desc) static inline uint64_t rdtsc(void) { uint32_t eax, edx; - uint32_t tsc_val; + uint64_t tsc_val; /* * The lfence is to wait (on Intel CPUs) until all previous * instructions have been executed. If software requires RDTSC to be From 24289f5601182974f20b9e46ce81eb2c51664518 Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Mon, 1 Jun 2020 23:00:52 +0200 Subject: [PATCH 1027/1043] parisc: Kconfig: Update references to parisc website The PA-RISC Linux project web page is now hosted at https://parisc.wiki.kernel.org Signed-off-by: Helge Deller --- arch/parisc/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig index 3801a2ef9bca..92128f9164ce 100644 --- a/arch/parisc/Kconfig +++ b/arch/parisc/Kconfig @@ -68,7 +68,7 @@ config PARISC The PA-RISC microprocessor is designed by Hewlett-Packard and used in many of their workstations & servers (HP9000 700 and 800 series, and later HP3000 series). The PA-RISC Linux project home page is - at . + at . config CPU_BIG_ENDIAN def_bool y From 861e93cf88b3722daf75947aa6c8ac342f238911 Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Mon, 1 Jun 2020 23:02:11 +0200 Subject: [PATCH 1028/1043] parisc: firmware: Update references to parisc website The PA-RISC Linux project web page is now hosted at https://parisc.wiki.kernel.org Signed-off-by: Helge Deller --- arch/parisc/kernel/firmware.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/parisc/kernel/firmware.c b/arch/parisc/kernel/firmware.c index 1d976f2ebff0..665b70086685 100644 --- a/arch/parisc/kernel/firmware.c +++ b/arch/parisc/kernel/firmware.c @@ -4,7 +4,8 @@ * * PDC == Processor Dependent Code * - * See http://www.parisc-linux.org/documentation/index.html + * See PDC documentation at + * https://parisc.wiki.kernel.org/index.php/Technical_Documentation * for documentation describing the entry points and calling * conventions defined below. * From 186cbb1737762b86b79a715662b17e6270f8c4f6 Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Mon, 1 Jun 2020 23:02:39 +0200 Subject: [PATCH 1029/1043] parisc: hardware: Update references to parisc website The PA-RISC Linux project web page is now hosted at https://parisc.wiki.kernel.org Signed-off-by: Helge Deller --- arch/parisc/kernel/hardware.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/parisc/kernel/hardware.c b/arch/parisc/kernel/hardware.c index 98c5203c1ab0..17161e72ea29 100644 --- a/arch/parisc/kernel/hardware.c +++ b/arch/parisc/kernel/hardware.c @@ -6,7 +6,8 @@ * * Based on the document "PA-RISC 1.1 I/O Firmware Architecture * Reference Specification", March 7, 1999, version 0.96. This - * is available at http://parisc-linux.org/documentation/ + * is available at + * https://parisc.wiki.kernel.org/index.php/Technical_Documentation * * Copyright 1999 by Alex deVries * and copyright 1999 The Puffin Group Inc. From 486a77c90346b6145353e69351803e17d91be3f6 Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Mon, 1 Jun 2020 23:03:04 +0200 Subject: [PATCH 1030/1043] parisc: module: Update references to parisc website The PA-RISC Linux project web page is now hosted at https://parisc.wiki.kernel.org Signed-off-by: Helge Deller --- arch/parisc/kernel/module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/parisc/kernel/module.c b/arch/parisc/kernel/module.c index 1c50093e2ebe..fac18c623d16 100644 --- a/arch/parisc/kernel/module.c +++ b/arch/parisc/kernel/module.c @@ -3,9 +3,9 @@ * * The best reference for this stuff is probably the Processor- * Specific ELF Supplement for PA-RISC: - * http://ftp.parisc-linux.org/docs/arch/elf-pa-hp.pdf + * https://parisc.wiki.kernel.org/index.php/File:Elf-pa-hp.pdf * - * Linux/PA-RISC Project (http://www.parisc-linux.org/) + * Linux/PA-RISC Project * Copyright (C) 2003 Randolph Chung * Copyright (C) 2008 Helge Deller * From 775024cf49ff76c30757bac9a21f6e908b63f6b5 Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Mon, 1 Jun 2020 23:03:27 +0200 Subject: [PATCH 1031/1043] parisc: MAINTAINERS: Update references to parisc website The PA-RISC Linux project web page is now hosted at https://parisc.wiki.kernel.org Signed-off-by: Helge Deller --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 2926327e4976..b6f73f02269d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12762,7 +12762,7 @@ M: "James E.J. Bottomley" M: Helge Deller L: linux-parisc@vger.kernel.org S: Maintained -W: http://www.parisc-linux.org/ +W: https://parisc.wiki.kernel.org Q: http://patchwork.kernel.org/project/linux-parisc/list/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/jejb/parisc-2.6.git T: git git://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux.git From 25de110d148666752dc0e0da7a0b69de31cd7098 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Tue, 2 Jun 2020 12:08:39 +0200 Subject: [PATCH 1032/1043] irq_work: Define irq_work_single() on !CONFIG_IRQ_WORK too Some SMP platforms don't have CONFIG_IRQ_WORK defined, resulting in a link error at build time. Define a stub and clean up the prototype definitions. Reported-by: kbuild test robot Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra Cc: linux-kernel@vger.kernel.org Signed-off-by: Ingo Molnar --- include/linux/irq_work.h | 2 ++ kernel/smp.c | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h index f23a359c8f46..2735da5f839e 100644 --- a/include/linux/irq_work.h +++ b/include/linux/irq_work.h @@ -58,9 +58,11 @@ void irq_work_sync(struct irq_work *work); void irq_work_run(void); bool irq_work_needs_cpu(void); +void irq_work_single(void *arg); #else static inline bool irq_work_needs_cpu(void) { return false; } static inline void irq_work_run(void) { } +static inline void irq_work_single(void *arg) { } #endif #endif /* _LINUX_IRQ_WORK_H */ diff --git a/kernel/smp.c b/kernel/smp.c index 4dec04f7fdc5..c80486a7e3b8 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -194,8 +194,6 @@ void generic_smp_call_function_single_interrupt(void) flush_smp_call_function_queue(true); } -extern void irq_work_single(void *); - /** * flush_smp_call_function_queue - Flush pending smp-call-function callbacks * From 1feb48baf2fb2aff02aa734c04a3ba5c3297d2de Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Tue, 19 May 2020 08:44:02 -0700 Subject: [PATCH 1033/1043] kgdboc: Disable all the early code when kgdboc is a module When kgdboc is compiled as a module all of the "ekgdboc" and "kgdb_earlycon" code isn't useful and, in fact, breaks compilation. This is because early_param() isn't defined for modules and that's how this code gets configured. It turns out that this was broken by commit eae3e19ca930 ("kgdboc: Remove useless #ifdef CONFIG_KGDB_SERIAL_CONSOLE in kgdboc") and then made worse by commit 220995622da5 ("kgdboc: Add kgdboc_earlycon to support early kgdb using boot consoles"). I guess the #ifdef wasn't so useless, even if it wasn't obvious why it was useful. When kgdboc was compiled as a module only "CONFIG_KGDB_SERIAL_CONSOLE_MODULE" was defined, not "CONFIG_KGDB_SERIAL_CONSOLE". That meant that the old module. Let's basically do the same thing that the old code (pre-removal of the #ifdef) did but use "IS_BUILTIN(CONFIG_KGDB_SERIAL_CONSOLE)" to make it more obvious what the point of the check is. We'll fix kgdboc_earlycon in a similar way. Fixes: 220995622da5 ("kgdboc: Add kgdboc_earlycon to support early kgdb using boot consoles") Fixes: eae3e19ca930 ("kgdboc: Remove useless #ifdef CONFIG_KGDB_SERIAL_CONSOLE in kgdboc") Reported-by: Stephen Rothwell Signed-off-by: Douglas Anderson Link: https://lore.kernel.org/r/20200519084345.1.I91670accc8a5ddabab227eb63bb4ad3e2e9d2b58@changeid Signed-off-by: Daniel Thompson --- drivers/tty/serial/kgdboc.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/drivers/tty/serial/kgdboc.c b/drivers/tty/serial/kgdboc.c index 34b5e91dd245..fa6f7a3e73b9 100644 --- a/drivers/tty/serial/kgdboc.c +++ b/drivers/tty/serial/kgdboc.c @@ -43,9 +43,11 @@ static int kgdb_tty_line; static struct platform_device *kgdboc_pdev; +#if IS_BUILTIN(CONFIG_KGDB_SERIAL_CONSOLE) static struct kgdb_io kgdboc_earlycon_io_ops; static struct console *earlycon; static int (*earlycon_orig_exit)(struct console *con); +#endif /* IS_BUILTIN(CONFIG_KGDB_SERIAL_CONSOLE) */ #ifdef CONFIG_KDB_KEYBOARD static int kgdboc_reset_connect(struct input_handler *handler, @@ -140,10 +142,19 @@ static void kgdboc_unregister_kbd(void) #define kgdboc_restore_input() #endif /* ! CONFIG_KDB_KEYBOARD */ -static void cleanup_kgdboc(void) +#if IS_BUILTIN(CONFIG_KGDB_SERIAL_CONSOLE) +static void cleanup_earlycon(void) { if (earlycon) kgdb_unregister_io_module(&kgdboc_earlycon_io_ops); +} +#else /* !IS_BUILTIN(CONFIG_KGDB_SERIAL_CONSOLE) */ +static inline void cleanup_earlycon(void) { } +#endif /* !IS_BUILTIN(CONFIG_KGDB_SERIAL_CONSOLE) */ + +static void cleanup_kgdboc(void) +{ + cleanup_earlycon(); if (configured != 1) return; @@ -388,6 +399,7 @@ static struct kgdb_io kgdboc_io_ops = { .post_exception = kgdboc_post_exp_handler, }; +#if IS_BUILTIN(CONFIG_KGDB_SERIAL_CONSOLE) static int kgdboc_option_setup(char *opt) { if (!opt) { @@ -544,6 +556,7 @@ unlock: } early_param("kgdboc_earlycon", kgdboc_earlycon_init); +#endif /* IS_BUILTIN(CONFIG_KGDB_SERIAL_CONSOLE) */ module_init(init_kgdboc); module_exit(exit_kgdboc); From b1350132fef7e1f0ddfd5a985d516a6ed7a329fc Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Tue, 26 May 2020 14:20:06 -0700 Subject: [PATCH 1034/1043] kgdb: Don't call the deinit under spinlock When I combined kgdboc_earlycon with an inflight patch titled ("soc: qcom-geni-se: Add interconnect support to fix earlycon crash") [1] things went boom. Specifically I got a crash during the transition between kgdboc_earlycon and the main kgdboc that looked like this: Call trace: __schedule_bug+0x68/0x6c __schedule+0x75c/0x924 schedule+0x8c/0xbc schedule_timeout+0x9c/0xfc do_wait_for_common+0xd0/0x160 wait_for_completion_timeout+0x54/0x74 rpmh_write_batch+0x1fc/0x23c qcom_icc_bcm_voter_commit+0x1b4/0x388 qcom_icc_set+0x2c/0x3c apply_constraints+0x5c/0x98 icc_set_bw+0x204/0x3bc icc_put+0x30/0xf8 geni_remove_earlycon_icc_vote+0x6c/0x9c qcom_geni_serial_earlycon_exit+0x10/0x1c kgdboc_earlycon_deinit+0x38/0x58 kgdb_register_io_module+0x11c/0x194 configure_kgdboc+0x108/0x174 kgdboc_probe+0x38/0x60 platform_drv_probe+0x90/0xb0 really_probe+0x130/0x2fc ... The problem was that we were holding the "kgdb_registration_lock" while calling into code that didn't expect to be called in spinlock context. Let's slightly defer when we call the deinit code so that it's not done under spinlock. NOTE: this does mean that the "deinit" call of the old kgdb IO module is now made _after_ the init of the new IO module, but presumably that's OK. [1] https://lkml.kernel.org/r/1588919619-21355-3-git-send-email-akashast@codeaurora.org Fixes: 220995622da5 ("kgdboc: Add kgdboc_earlycon to support early kgdb using boot consoles") Signed-off-by: Douglas Anderson Link: https://lore.kernel.org/r/20200526142001.1.I523dc33f96589cb9956f5679976d402c8cda36fa@changeid [daniel.thompson@linaro.org: Resolved merge issues by hand] Signed-off-by: Daniel Thompson --- kernel/debug/debug_core.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c index 4d59aa907fdc..ef94e906f05a 100644 --- a/kernel/debug/debug_core.c +++ b/kernel/debug/debug_core.c @@ -1089,7 +1089,6 @@ int kgdb_register_io_module(struct kgdb_io *new_dbg_io_ops) } pr_info("Replacing I/O driver %s with %s\n", old_dbg_io_ops->name, new_dbg_io_ops->name); - old_dbg_io_ops->deinit(); } if (new_dbg_io_ops->init) { @@ -1104,8 +1103,10 @@ int kgdb_register_io_module(struct kgdb_io *new_dbg_io_ops) spin_unlock(&kgdb_registration_lock); - if (old_dbg_io_ops) + if (old_dbg_io_ops) { + old_dbg_io_ops->deinit(); return 0; + } pr_info("Registered I/O driver %s\n", new_dbg_io_ops->name); From f71fc3bc7b3279cb7d0ab78c1fcc78825779bb65 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Thu, 7 May 2020 13:08:47 -0700 Subject: [PATCH 1035/1043] Documentation: kgdboc: Document new kgdboc_earlycon parameter The recent patch ("kgdboc: Add kgdboc_earlycon to support early kgdb using boot consoles") adds a new kernel command line parameter. Document it. Note that the patch adding the feature does some comparing/contrasting of "kgdboc_earlycon" vs. the existing "ekgdboc". See that patch for more details, but briefly "ekgdboc" can be used _instead_ of "kgdboc" and just makes "kgdboc" do its normal initialization early (only works if your tty driver is already ready). The new "kgdboc_earlycon" works in combination with "kgdboc" and is backed by boot consoles. Signed-off-by: Douglas Anderson Reviewed-by: Greg Kroah-Hartman Reviewed-by: Daniel Thompson Link: https://lore.kernel.org/r/20200507130644.v4.9.I7d5eb42c6180c831d47aef1af44d0b8be3fac559@changeid Signed-off-by: Daniel Thompson --- .../admin-guide/kernel-parameters.txt | 20 ++++++++++++++++ Documentation/dev-tools/kgdb.rst | 24 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 7bc83f3d9bdf..2cbde9ea476d 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1190,6 +1190,11 @@ This is designed to be used in conjunction with the boot argument: earlyprintk=vga + This parameter works in place of the kgdboc parameter + but can only be used if the backing tty is available + very early in the boot process. For early debugging + via a serial port see kgdboc_earlycon instead. + edd= [EDD] Format: {"off" | "on" | "skip[mbr]"} @@ -2105,6 +2110,21 @@ kms, kbd format: kms,kbd kms, kbd and serial format: kms,kbd,[,baud] + kgdboc_earlycon= [KGDB,HW] + If the boot console provides the ability to read + characters and can work in polling mode, you can use + this parameter to tell kgdb to use it as a backend + until the normal console is registered. Intended to + be used together with the kgdboc parameter which + specifies the normal console to transition to. + + The name of the early console should be specified + as the value of this parameter. Note that the name of + the early console might be different than the tty + name passed to kgdboc. It's OK to leave the value + blank and the first boot console that implements + read() will be picked. + kgdbwait [KGDB] Stop kernel execution and enter the kernel debugger at the earliest opportunity. diff --git a/Documentation/dev-tools/kgdb.rst b/Documentation/dev-tools/kgdb.rst index d38be58f872a..61293f40bc6e 100644 --- a/Documentation/dev-tools/kgdb.rst +++ b/Documentation/dev-tools/kgdb.rst @@ -274,6 +274,30 @@ don't like this are to hack gdb to send the :kbd:`SysRq-G` for you as well as on the initial connect, or to use a debugger proxy that allows an unmodified gdb to do the debugging. +Kernel parameter: ``kgdboc_earlycon`` +------------------------------------- + +If you specify the kernel parameter ``kgdboc_earlycon`` and your serial +driver registers a boot console that supports polling (doesn't need +interrupts and implements a nonblocking read() function) kgdb will attempt +to work using the boot console until it can transition to the regular +tty driver specified by the ``kgdboc`` parameter. + +Normally there is only one boot console (especially that implements the +read() function) so just adding ``kgdboc_earlycon`` on its own is +sufficient to make this work. If you have more than one boot console you +can add the boot console's name to differentiate. Note that names that +are registered through the boot console layer and the tty layer are not +the same for the same port. + +For instance, on one board to be explicit you might do:: + + kgdboc_earlycon=qcom_geni kgdboc=ttyMSM0 + +If the only boot console on the device was "qcom_geni", you could simplify:: + + kgdboc_earlycon kgdboc=ttyMSM0 + Kernel parameter: ``kgdbwait`` ------------------------------ From a4912303ac6fede434acf5c23a9108cdaf79844a Mon Sep 17 00:00:00 2001 From: Daniel Thompson Date: Thu, 30 Apr 2020 17:17:41 +0100 Subject: [PATCH 1036/1043] serial: kgdboc: Allow earlycon initialization to be deferred Currently there is no guarantee that an earlycon will be initialized before kgdboc tries to adopt it. Almost the opposite: on systems with ACPI then if earlycon has no arguments then it is guaranteed that earlycon will not be initialized. This patch mitigates the problem by giving kgdboc_earlycon a second chance during console_init(). This isn't quite as good as stopping during early parameter parsing but it is still early in the kernel boot. Signed-off-by: Daniel Thompson Link: https://lore.kernel.org/r/20200430161741.1832050-1-daniel.thompson@linaro.org Reviewed-by: Douglas Anderson --- drivers/tty/serial/kgdboc.c | 39 ++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/drivers/tty/serial/kgdboc.c b/drivers/tty/serial/kgdboc.c index fa6f7a3e73b9..41396982e9e0 100644 --- a/drivers/tty/serial/kgdboc.c +++ b/drivers/tty/serial/kgdboc.c @@ -514,6 +514,10 @@ static struct kgdb_io kgdboc_earlycon_io_ops = { .is_console = true, }; +#define MAX_CONSOLE_NAME_LEN (sizeof((struct console *) 0)->name) +static char kgdboc_earlycon_param[MAX_CONSOLE_NAME_LEN] __initdata; +static bool kgdboc_earlycon_late_enable __initdata; + static int __init kgdboc_earlycon_init(char *opt) { struct console *con; @@ -533,7 +537,23 @@ static int __init kgdboc_earlycon_init(char *opt) } if (!con) { - pr_info("Couldn't find kgdb earlycon\n"); + /* + * Both earlycon and kgdboc_earlycon are initialized during * early parameter parsing. We cannot guarantee earlycon gets + * in first and, in any case, on ACPI systems earlycon may + * defer its own initialization (usually to somewhere within + * setup_arch() ). To cope with either of these situations + * we can defer our own initialization to a little later in + * the boot. + */ + if (!kgdboc_earlycon_late_enable) { + pr_info("No suitable earlycon yet, will try later\n"); + if (opt) + strscpy(kgdboc_earlycon_param, opt, + sizeof(kgdboc_earlycon_param)); + kgdboc_earlycon_late_enable = true; + } else { + pr_info("Couldn't find kgdb earlycon\n"); + } goto unlock; } @@ -556,6 +576,23 @@ unlock: } early_param("kgdboc_earlycon", kgdboc_earlycon_init); + +/* + * This is only intended for the late adoption of an early console. + * + * It is not a reliable way to adopt regular consoles because we can not + * control what order console initcalls are made and, in any case, many + * regular consoles are registered much later in the boot process than + * the console initcalls! + */ +static int __init kgdboc_earlycon_late_init(void) +{ + if (kgdboc_earlycon_late_enable) + kgdboc_earlycon_init(kgdboc_earlycon_param); + return 0; +} +console_initcall(kgdboc_earlycon_late_init); + #endif /* IS_BUILTIN(CONFIG_KGDB_SERIAL_CONSOLE) */ module_init(init_kgdboc); From 205b5bdda2090d4730dabf9c0d9646cb32f2551d Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Thu, 7 May 2020 13:08:48 -0700 Subject: [PATCH 1037/1043] serial: qcom_geni_serial: Support kgdboc_earlycon Implement the read() function in the early console driver. With recent kgdb patches this allows you to use kgdb to debug fairly early into the system boot. We only bother implementing this if polling is enabled since kgdb can't be enabled without that. Signed-off-by: Douglas Anderson Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20200507130644.v4.10.If2deff9679a62c1ce1b8f2558a8635dc837adf8c@changeid Signed-off-by: Daniel Thompson --- drivers/tty/serial/qcom_geni_serial.c | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c index 6119090ce045..6bace1c6bb09 100644 --- a/drivers/tty/serial/qcom_geni_serial.c +++ b/drivers/tty/serial/qcom_geni_serial.c @@ -1090,6 +1090,36 @@ static void qcom_geni_serial_earlycon_write(struct console *con, __qcom_geni_serial_console_write(&dev->port, s, n); } +#ifdef CONFIG_CONSOLE_POLL +static int qcom_geni_serial_earlycon_read(struct console *con, + char *s, unsigned int n) +{ + struct earlycon_device *dev = con->data; + struct uart_port *uport = &dev->port; + int num_read = 0; + int ch; + + while (num_read < n) { + ch = qcom_geni_serial_get_char(uport); + if (ch == NO_POLL_CHAR) + break; + s[num_read++] = ch; + } + + return num_read; +} + +static void __init qcom_geni_serial_enable_early_read(struct geni_se *se, + struct console *con) +{ + geni_se_setup_s_cmd(se, UART_START_READ, 0); + con->read = qcom_geni_serial_earlycon_read; +} +#else +static inline void qcom_geni_serial_enable_early_read(struct geni_se *se, + struct console *con) { } +#endif + static int __init qcom_geni_serial_earlycon_setup(struct earlycon_device *dev, const char *opt) { @@ -1136,6 +1166,8 @@ static int __init qcom_geni_serial_earlycon_setup(struct earlycon_device *dev, dev->con->write = qcom_geni_serial_earlycon_write; dev->con->setup = NULL; + qcom_geni_serial_enable_early_read(&se, dev->con); + return 0; } OF_EARLYCON_DECLARE(qcom_geni, "qcom,geni-debug-uart", From c5e7467d92b849450bd03d9215afb09665d76af6 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Thu, 7 May 2020 13:08:49 -0700 Subject: [PATCH 1038/1043] serial: 8250_early: Support kgdboc_earlycon Implement the read() function in the early console driver. With recent kgdb patches this allows you to use kgdb to debug fairly early into the system boot. We only bother implementing this if polling is enabled since kgdb can't be enabled without that. Signed-off-by: Douglas Anderson Reviewed-by: Greg Kroah-Hartman Reviewed-by: Daniel Thompson Link: https://lore.kernel.org/r/20200507130644.v4.11.I8f668556c244776523320a95b09373a86eda11b7@changeid Signed-off-by: Daniel Thompson --- drivers/tty/serial/8250/8250_early.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/drivers/tty/serial/8250/8250_early.c b/drivers/tty/serial/8250/8250_early.c index 5cd8c36c8fcc..70d7826788f5 100644 --- a/drivers/tty/serial/8250/8250_early.c +++ b/drivers/tty/serial/8250/8250_early.c @@ -109,6 +109,28 @@ static void early_serial8250_write(struct console *console, uart_console_write(port, s, count, serial_putc); } +#ifdef CONFIG_CONSOLE_POLL +static int early_serial8250_read(struct console *console, + char *s, unsigned int count) +{ + struct earlycon_device *device = console->data; + struct uart_port *port = &device->port; + unsigned int status; + int num_read = 0; + + while (num_read < count) { + status = serial8250_early_in(port, UART_LSR); + if (!(status & UART_LSR_DR)) + break; + s[num_read++] = serial8250_early_in(port, UART_RX); + } + + return num_read; +} +#else +#define early_serial8250_read NULL +#endif + static void __init init_port(struct earlycon_device *device) { struct uart_port *port = &device->port; @@ -149,6 +171,7 @@ int __init early_serial8250_setup(struct earlycon_device *device, init_port(device); device->con->write = early_serial8250_write; + device->con->read = early_serial8250_read; return 0; } EARLYCON_DECLARE(uart8250, early_serial8250_setup); From 195867ffea13b755dc727b47eaa5beb0ffa6e0ce Mon Sep 17 00:00:00 2001 From: Sumit Garg Date: Thu, 7 May 2020 13:08:50 -0700 Subject: [PATCH 1039/1043] serial: amba-pl011: Support kgdboc_earlycon Implement the read() function in the early console driver. With recently added kgdboc_earlycon feature, this allows you to use kgdb to debug fairly early into the system boot. We only bother implementing this if polling is enabled since kgdb can't be enabled without that. Signed-off-by: Sumit Garg Reviewed-by: Douglas Anderson Reviewed-by: Daniel Thompson Signed-off-by: Douglas Anderson Link: https://lore.kernel.org/r/20200507130644.v4.12.I8ee0811f0e0816dd8bfe7f2f5540b3dba074fae8@changeid Signed-off-by: Daniel Thompson --- drivers/tty/serial/amba-pl011.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c index 2296bb0f9578..c010f639298d 100644 --- a/drivers/tty/serial/amba-pl011.c +++ b/drivers/tty/serial/amba-pl011.c @@ -2435,6 +2435,37 @@ static void pl011_early_write(struct console *con, const char *s, unsigned n) uart_console_write(&dev->port, s, n, pl011_putc); } +#ifdef CONFIG_CONSOLE_POLL +static int pl011_getc(struct uart_port *port) +{ + if (readl(port->membase + UART01x_FR) & UART01x_FR_RXFE) + return NO_POLL_CHAR; + + if (port->iotype == UPIO_MEM32) + return readl(port->membase + UART01x_DR); + else + return readb(port->membase + UART01x_DR); +} + +static int pl011_early_read(struct console *con, char *s, unsigned int n) +{ + struct earlycon_device *dev = con->data; + int ch, num_read = 0; + + while (num_read < n) { + ch = pl011_getc(&dev->port); + if (ch == NO_POLL_CHAR) + break; + + s[num_read++] = ch; + } + + return num_read; +} +#else +#define pl011_early_read NULL +#endif + /* * On non-ACPI systems, earlycon is enabled by specifying * "earlycon=pl011,
" on the kernel command line. @@ -2454,6 +2485,7 @@ static int __init pl011_early_console_setup(struct earlycon_device *device, return -ENODEV; device->con->write = pl011_early_write; + device->con->read = pl011_early_read; return 0; } From 1b310030bb855b9b13d1c0a9feffdb54883b06ab Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Thu, 7 May 2020 16:11:46 -0700 Subject: [PATCH 1040/1043] kdb: Cleanup math with KDB_CMD_HISTORY_COUNT From code inspection the math in handle_ctrl_cmd() looks super sketchy because it subjects -1 from cmdptr and then does a "% KDB_CMD_HISTORY_COUNT". It turns out that this code works because "cmdptr" is unsigned and KDB_CMD_HISTORY_COUNT is a nice power of 2. Let's make this a little less sketchy. This patch should be a no-op. Signed-off-by: Douglas Anderson Link: https://lore.kernel.org/r/20200507161125.1.I2cce9ac66e141230c3644b8174b6c15d4e769232@changeid Reviewed-by: Sumit Garg Signed-off-by: Daniel Thompson --- kernel/debug/kdb/kdb_main.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/debug/kdb/kdb_main.c b/kernel/debug/kdb/kdb_main.c index 515379cbf209..6865a0f58d38 100644 --- a/kernel/debug/kdb/kdb_main.c +++ b/kernel/debug/kdb/kdb_main.c @@ -1108,7 +1108,8 @@ static int handle_ctrl_cmd(char *cmd) switch (*cmd) { case CTRL_P: if (cmdptr != cmd_tail) - cmdptr = (cmdptr-1) % KDB_CMD_HISTORY_COUNT; + cmdptr = (cmdptr + KDB_CMD_HISTORY_COUNT - 1) % + KDB_CMD_HISTORY_COUNT; strscpy(cmd_cur, cmd_hist[cmdptr], CMD_BUFLEN); return 1; case CTRL_N: From c893de12e1ef17b581eb2cf8fc9018ec0cbd07df Mon Sep 17 00:00:00 2001 From: Wei Li Date: Thu, 21 May 2020 15:21:25 +0800 Subject: [PATCH 1041/1043] kdb: Remove the misfeature 'KDBFLAGS' Currently, 'KDBFLAGS' is an internal variable of kdb, it is combined by 'KDBDEBUG' and state flags. It will be shown only when 'KDBDEBUG' is set, and the user can define an environment variable named 'KDBFLAGS' too. These are puzzling indeed. After communication with Daniel, it seems that 'KDBFLAGS' is a misfeature. So let's replace 'KDBFLAGS' with 'KDBDEBUG' to just show the value we wrote into. After this modification, we can use `md4c1 kdb_flags` instead, to observe the state flags. Suggested-by: Daniel Thompson Signed-off-by: Wei Li Link: https://lore.kernel.org/r/20200521072125.21103-1-liwei391@huawei.com [daniel.thompson@linaro.org: Make kdb_flags unsigned to avoid arithmetic right shift] Signed-off-by: Daniel Thompson --- include/linux/kdb.h | 2 +- kernel/debug/kdb/kdb_main.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/linux/kdb.h b/include/linux/kdb.h index 24cd447659e0..0125a677b67f 100644 --- a/include/linux/kdb.h +++ b/include/linux/kdb.h @@ -125,7 +125,7 @@ extern const char *kdb_diemsg; #define KDB_FLAG_NO_I8042 (1 << 7) /* No i8042 chip is available, do * not use keyboard */ -extern int kdb_flags; /* Global flags, see kdb_state for per cpu state */ +extern unsigned int kdb_flags; /* Global flags, see kdb_state for per cpu state */ extern void kdb_save_flags(void); extern void kdb_restore_flags(void); diff --git a/kernel/debug/kdb/kdb_main.c b/kernel/debug/kdb/kdb_main.c index 6865a0f58d38..ec190569f690 100644 --- a/kernel/debug/kdb/kdb_main.c +++ b/kernel/debug/kdb/kdb_main.c @@ -62,7 +62,7 @@ int kdb_grep_trailing; /* * Kernel debugger state flags */ -int kdb_flags; +unsigned int kdb_flags; /* * kdb_lock protects updates to kdb_initial_cpu. Used to @@ -418,8 +418,7 @@ int kdb_set(int argc, const char **argv) argv[2]); return 0; } - kdb_flags = (kdb_flags & - ~(KDB_DEBUG_FLAG_MASK << KDB_DEBUG_FLAG_SHIFT)) + kdb_flags = (kdb_flags & ~KDB_DEBUG(MASK)) | (debugflags << KDB_DEBUG_FLAG_SHIFT); return 0; @@ -2082,7 +2081,8 @@ static int kdb_env(int argc, const char **argv) } if (KDB_DEBUG(MASK)) - kdb_printf("KDBFLAGS=0x%x\n", kdb_flags); + kdb_printf("KDBDEBUG=0x%x\n", + (kdb_flags & KDB_DEBUG(MASK)) >> KDB_DEBUG_FLAG_SHIFT); return 0; } From 56f2e3b7d819f4fa44857ba81aa6870f18714ea0 Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Tue, 2 Jun 2020 10:17:28 +0100 Subject: [PATCH 1042/1043] capabilities: add description for CAP_SETFCAP Document the purpose of CAP_SETFCAP. For some reason this capability had no description while the others did. Signed-off-by: Stefan Hajnoczi Signed-off-by: James Morris --- include/uapi/linux/capability.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/uapi/linux/capability.h b/include/uapi/linux/capability.h index 240fdb9a60f6..877972fdfda3 100644 --- a/include/uapi/linux/capability.h +++ b/include/uapi/linux/capability.h @@ -331,6 +331,8 @@ struct vfs_ns_cap_data { #define CAP_AUDIT_CONTROL 30 +/* Set or remove capabilities on files */ + #define CAP_SETFCAP 31 /* Override MAC access. From 174e1ea8a2f6140078b6c61068b478cf3c4aa74f Mon Sep 17 00:00:00 2001 From: Al Viro Date: Wed, 3 Jun 2020 16:15:28 -0400 Subject: [PATCH 1043/1043] fix a braino in ia64 uaccess csum changes Fixes: cc03f19cfd45 (ia64: csum_partial_copy_nocheck(): don't abuse csum_partial_copy_from_user()) Reported-by: Guenter Roeck Tested-by: Guenter Roeck Signed-off-by: Al Viro --- arch/ia64/lib/csum_partial_copy.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/ia64/lib/csum_partial_copy.c b/arch/ia64/lib/csum_partial_copy.c index 5d147a33d648..6e82e0be8040 100644 --- a/arch/ia64/lib/csum_partial_copy.c +++ b/arch/ia64/lib/csum_partial_copy.c @@ -12,7 +12,7 @@ #include #include -#include +#include /* * XXX Fixme: those 2 inlines are meant for debugging and will go away